You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by we...@apache.org on 2016/08/11 05:22:32 UTC
hadoop git commit: HDFS-8897. Balancer should handle fs.defaultFS
trailing slash in HA. Contributed by John Zhuge.
Repository: hadoop
Updated Branches:
refs/heads/trunk e7e8aed20 -> a428d4f50
HDFS-8897. Balancer should handle fs.defaultFS trailing slash in HA. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a428d4f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a428d4f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a428d4f5
Branch: refs/heads/trunk
Commit: a428d4f50e8caf553fb19a26200ec58f0b83da5d
Parents: e7e8aed
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Wed Aug 10 22:17:11 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Wed Aug 10 22:17:35 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 60 ++++++++++++--------
.../org/apache/hadoop/hdfs/TestDFSUtil.java | 54 +++++++++++-------
2 files changed, 71 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a428d4f5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 1ff39e1..83870cf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -771,12 +771,7 @@ public class DFSUtil {
Set<URI> nonPreferredUris = new HashSet<URI>();
for (String nsId : nameServices) {
- URI nsUri;
- try {
- nsUri = new URI(HdfsConstants.HDFS_URI_SCHEME + "://" + nsId);
- } catch (URISyntaxException ue) {
- throw new IllegalArgumentException(ue);
- }
+ URI nsUri = createUri(HdfsConstants.HDFS_URI_SCHEME, nsId, -1);
/**
* Determine whether the logical URI of the name service can be resolved
* by the configured failover proxy provider. If not, we should try to
@@ -816,7 +811,8 @@ public class DFSUtil {
for (String key : keys) {
String addr = conf.get(key);
if (addr != null) {
- URI uri = createUri("hdfs", NetUtils.createSocketAddr(addr));
+ URI uri = createUri(HdfsConstants.HDFS_URI_SCHEME,
+ NetUtils.createSocketAddr(addr));
if (!uriFound) {
uriFound = true;
ret.add(uri);
@@ -834,19 +830,21 @@ public class DFSUtil {
// nor the rpc-address (which overrides defaultFS) is given.
if (!uriFound) {
URI defaultUri = FileSystem.getDefaultUri(conf);
+ if (defaultUri != null) {
+ // checks if defaultUri is ip:port format
+ // and convert it to hostname:port format
+ if (defaultUri.getPort() != -1) {
+ defaultUri = createUri(defaultUri.getScheme(),
+ NetUtils.createSocketAddr(defaultUri.getHost(),
+ defaultUri.getPort()));
+ }
- // checks if defaultUri is ip:port format
- // and convert it to hostname:port format
- if (defaultUri != null && (defaultUri.getPort() != -1)) {
- defaultUri = createUri(defaultUri.getScheme(),
- NetUtils.createSocketAddr(defaultUri.getHost(),
- defaultUri.getPort()));
- }
+ defaultUri = trimUri(defaultUri);
- if (defaultUri != null &&
- HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) &&
- !nonPreferredUris.contains(defaultUri)) {
- ret.add(defaultUri);
+ if (HdfsConstants.HDFS_URI_SCHEME.equals(defaultUri.getScheme()) &&
+ !nonPreferredUris.contains(defaultUri)) {
+ ret.add(defaultUri);
+ }
}
}
@@ -1175,16 +1173,30 @@ public class DFSUtil {
public boolean match(InetSocketAddress s);
}
- /** Create a URI from the scheme and address */
+ /** Create an URI from scheme and address. */
public static URI createUri(String scheme, InetSocketAddress address) {
+ return createUri(scheme, address.getHostName(), address.getPort());
+ }
+
+ /** Create an URI from scheme, host, and port. */
+ public static URI createUri(String scheme, String host, int port) {
try {
- return new URI(scheme, null, address.getHostName(), address.getPort(),
- null, null, null);
- } catch (URISyntaxException ue) {
- throw new IllegalArgumentException(ue);
+ return new URI(scheme, null, host, port, null, null, null);
+ } catch (URISyntaxException x) {
+ throw new IllegalArgumentException(x.getMessage(), x);
}
}
-
+
+ /** Remove unnecessary path from HDFS URI. */
+ static URI trimUri(URI uri) {
+ String path = uri.getPath();
+ if (HdfsConstants.HDFS_URI_SCHEME.equals(uri.getScheme()) &&
+ path != null && !path.isEmpty()) {
+ uri = createUri(uri.getScheme(), uri.getHost(), uri.getPort());
+ }
+ return uri;
+ }
+
/**
* Add protobuf based protocol to the {@link org.apache.hadoop.ipc.RPC.Server}
* @param conf configuration
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a428d4f5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index bef5e33..b7db601 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -81,7 +81,11 @@ import org.junit.Test;
import com.google.common.collect.Sets;
public class TestDFSUtil {
-
+
+ static final String NS1_NN_ADDR = "ns1-nn.example.com:9820";
+ static final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820";
+ static final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820";
+
/**
* Reset to default UGI settings since some tests change them.
*/
@@ -587,8 +591,6 @@ public class TestDFSUtil {
@Test
public void testGetHaNnHttpAddresses() throws IOException {
final String LOGICAL_HOST_NAME = "ns1";
- final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820";
- final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820";
Configuration conf = createWebHDFSHAConfiguration(LOGICAL_HOST_NAME, NS1_NN1_ADDR, NS1_NN2_ADDR);
@@ -637,9 +639,6 @@ public class TestDFSUtil {
public void testGetNNUris() throws Exception {
HdfsConfiguration conf = new HdfsConfiguration();
- final String NS1_NN_ADDR = "ns1-nn.example.com:9820";
- final String NS1_NN1_ADDR = "ns1-nn1.example.com:9820";
- final String NS1_NN2_ADDR = "ns1-nn2.example.com:9820";
final String NS2_NN_ADDR = "ns2-nn.example.com:9820";
final String NN1_ADDR = "nn.example.com:9820";
final String NN1_SRVC_ADDR = "nn.example.com:9821";
@@ -648,12 +647,10 @@ public class TestDFSUtil {
conf.set(DFS_NAMESERVICES, "ns1");
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN1_ADDR);
-
conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "hdfs://" + NN2_ADDR);
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN1_ADDR);
Collection<URI> uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 2, uris.size());
assertTrue("Missing URI for name service ns1",
uris.contains(new URI("hdfs://" + NS1_NN1_ADDR)));
@@ -668,15 +665,11 @@ public class TestDFSUtil {
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR);
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR);
-
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN_ADDR);
-
conf.set(DFSUtil.addKeySuffixes(
DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns2"), NS2_NN_ADDR);
-
conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, "hdfs://" + NN1_ADDR);
-
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN2_ADDR);
/**
@@ -714,7 +707,6 @@ public class TestDFSUtil {
+ "ConfiguredFailoverProxyProvider");
uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 3, uris.size());
assertTrue("Missing URI for name service ns1",
uris.contains(new URI("hdfs://ns1")));
@@ -728,7 +720,6 @@ public class TestDFSUtil {
"viewfs://vfs-name.example.com");
uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 3, uris.size());
assertTrue("Missing URI for name service ns1",
uris.contains(new URI("hdfs://ns1")));
@@ -742,7 +733,6 @@ public class TestDFSUtil {
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1");
uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 3, uris.size());
assertTrue("Missing URI for name service ns1",
uris.contains(new URI("hdfs://ns1")));
@@ -756,7 +746,6 @@ public class TestDFSUtil {
conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://" + NN1_ADDR);
uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 1, uris.size());
assertTrue("Missing URI for RPC address (defaultFS)",
uris.contains(new URI("hdfs://" + NN1_ADDR)));
@@ -766,7 +755,6 @@ public class TestDFSUtil {
conf.set(DFS_NAMENODE_RPC_ADDRESS_KEY, NN2_ADDR);
uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 1, uris.size());
assertTrue("Missing URI for RPC address",
uris.contains(new URI("hdfs://" + NN2_ADDR)));
@@ -778,7 +766,6 @@ public class TestDFSUtil {
conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, NN1_ADDR);
uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 1, uris.size());
assertTrue("Missing URI for service ns1",
uris.contains(new URI("hdfs://" + NN1_ADDR)));
@@ -790,12 +777,41 @@ public class TestDFSUtil {
conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, NN1_SRVC_ADDR);
uris = DFSUtil.getInternalNsRpcUris(conf);
-
assertEquals("Incorrect number of URIs returned", 1, uris.size());
assertTrue("Missing URI for service address",
uris.contains(new URI("hdfs://" + NN1_SRVC_ADDR)));
}
+ @Test
+ public void testGetNNUris2() throws Exception {
+ // Make sure that an HA URI plus a slash being the default URI doesn't
+ // result in multiple entries being returned.
+ HdfsConfiguration conf = new HdfsConfiguration();
+ conf.set(DFS_NAMESERVICES, "ns1");
+ conf.set(DFSUtil.addKeySuffixes(DFS_HA_NAMENODES_KEY_PREFIX, "ns1"),
+ "nn1,nn2");
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn1"), NS1_NN1_ADDR);
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_RPC_ADDRESS_KEY, "ns1", "nn2"), NS1_NN2_ADDR);
+
+ conf.set(DFSUtil.addKeySuffixes(
+ DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, "ns1"), NS1_NN_ADDR);
+
+ String proxyProviderKey = HdfsClientConfigKeys.Failover.
+ PROXY_PROVIDER_KEY_PREFIX + ".ns1";
+ conf.set(proxyProviderKey, "org.apache.hadoop.hdfs.server.namenode.ha."
+ + "ConfiguredFailoverProxyProvider");
+
+ conf.set(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, "hdfs://ns1/");
+
+ Collection<URI> uris = DFSUtil.getInternalNsRpcUris(conf);
+
+ assertEquals("Incorrect number of URIs returned", 1, uris.size());
+ assertTrue("Missing URI for name service ns1",
+ uris.contains(new URI("hdfs://ns1")));
+ }
+
@Test (timeout=15000)
public void testLocalhostReverseLookup() {
// 127.0.0.1 -> localhost reverse resolution does not happen on Windows.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org