You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2021/07/30 03:10:23 UTC
[hadoop] branch HADOOP-17800 updated: HDFS-9266.Avoid unsafe split
and append on fields that might be IPv6 literals. Contributed by Nemanja
Matkovic And Hemanth Boyina.
This is an automated email from the ASF dual-hosted git repository.
brahma pushed a commit to branch HADOOP-17800
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/HADOOP-17800 by this push:
new 62e77a5 HDFS-9266.Avoid unsafe split and append on fields that might be IPv6 literals. Contributed by Nemanja Matkovic And Hemanth Boyina.
62e77a5 is described below
commit 62e77a5bc13358b3a6b9092f8f3f4d5c556e11b1
Author: Brahma Reddy Battula <br...@apache.org>
AuthorDate: Fri Jul 30 08:39:51 2021 +0530
HDFS-9266.Avoid unsafe split and append on fields that might be IPv6 literals. Contributed by Nemanja Matkovic And Hemanth Boyina.
---
.../hdfs/client/impl/BlockReaderFactory.java | 3 +-
.../org/apache/hadoop/hdfs/web/JsonUtilClient.java | 2 +-
.../hdfs/qjournal/client/IPCLoggerChannel.java | 5 +-
.../server/blockmanagement/DatanodeManager.java | 9 +-
.../server/datanode/BlockPoolSliceStorage.java | 16 ++-
.../hadoop/hdfs/server/datanode/DataXceiver.java | 2 +-
.../hadoop/hdfs/server/namenode/Checkpointer.java | 6 +-
.../hadoop/hdfs/server/namenode/NNStorage.java | 5 +
.../web/resources/NamenodeWebHdfsMethods.java | 15 +--
.../java/org/apache/hadoop/hdfs/tools/GetConf.java | 3 +-
.../tools/offlineImageViewer/WebImageViewer.java | 5 +-
.../apache/hadoop/hdfs/TestDFSAddressConfig.java | 10 +-
.../java/org/apache/hadoop/hdfs/TestDFSUtil.java | 49 ++++++--
.../org/apache/hadoop/hdfs/TestFileAppend.java | 7 +-
.../org/apache/hadoop/hdfs/TestFileCreation.java | 137 +++++++++++----------
.../hdfs/client/impl/BlockReaderTestUtil.java | 3 +-
.../qjournal/client/TestQuorumJournalManager.java | 6 +-
.../server/datanode/TestBlockPoolSliceStorage.java | 28 ++++-
.../namenode/TestNameNodeRespectsBindHostKeys.java | 79 ++++++++++--
.../server/namenode/TestNameNodeRpcServer.java | 12 +-
20 files changed, 274 insertions(+), 128 deletions(-)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
index f9fd2b1..70545c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.Slot;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
import org.apache.hadoop.hdfs.util.IOUtilsClient;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.unix.DomainSocket;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -876,6 +877,6 @@ public class BlockReaderFactory implements ShortCircuitReplicaCreator {
*/
public static String getFileName(final InetSocketAddress s,
final String poolId, final long blockId) {
- return s.toString() + ":" + poolId + ":" + blockId;
+ return NetUtils.getSocketAddressString(s) + ":" + poolId + ":" + blockId;
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index 6acd062..e302605 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -305,7 +305,7 @@ public class JsonUtilClient {
if (ipAddr == null) {
String name = getString(m, "name", null);
if (name != null) {
- int colonIdx = name.indexOf(':');
+ int colonIdx = name.lastIndexOf(':');
if (colonIdx > 0) {
ipAddr = name.substring(0, colonIdx);
xferPort = Integer.parseInt(name.substring(colonIdx +1));
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
index 9908160..e695790 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java
@@ -54,12 +54,12 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.ipc.ProtobufRpcEngine2;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.StopWatch;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Preconditions;
-import org.apache.hadoop.thirdparty.com.google.common.net.InetAddresses;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.FutureCallback;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.Futures;
import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListenableFuture;
@@ -709,8 +709,7 @@ public class IPCLoggerChannel implements AsyncLogger {
@Override
public String toString() {
- return InetAddresses.toAddrString(addr.getAddress()) + ':' +
- addr.getPort();
+ return NetUtils.getHostPortString(addr);
}
@Override
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 68ee16c..dd3d23b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.net.NetworkTopology.InvalidTopologyException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.http.conn.util.InetAddressUtils;
import org.apache.hadoop.util.Sets;
import org.apache.hadoop.util.Timer;
import org.slf4j.Logger;
@@ -1522,7 +1523,13 @@ public class DatanodeManager {
DatanodeID dnId;
String hostStr;
int port;
- int idx = hostLine.indexOf(':');
+ int idx;
+
+ if (InetAddressUtils.isIPv6StdAddress(hostLine)) {
+ idx = -1;
+ } else {
+ idx = hostLine.lastIndexOf(':');
+ }
if (-1 == idx) {
hostStr = hostLine;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
index e92ae07..40df72e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockPoolSliceStorage.java
@@ -80,11 +80,21 @@ public class BlockPoolSliceStorage extends Storage {
* progress. Do not delete the 'previous' directory.
*/
static final String ROLLING_UPGRADE_MARKER_FILE = "RollingUpgradeInProgress";
+ private static final String BLOCK_POOL_ID_IPV4_PATTERN_BASE =
+ "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}";
+
+ // Because we don't support ":" in path BlockPoolID on IPv6 boxes we replace
+ // ":" with ".".
+ // Also format of IPv6 is less fixed so we surround it with square brackets
+ // and just check that match
+ private static final String BLOCK_POOL_ID_IPV6_PATTERN_BASE =
+ Pattern.quote("[") + "(?:.*)" + Pattern.quote("]");
private static final String BLOCK_POOL_ID_PATTERN_BASE =
- Pattern.quote(File.separator) +
- "BP-\\d+-\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}-\\d+" +
- Pattern.quote(File.separator);
+ Pattern.quote(File.separator) + "BP-\\d+-(?:"
+ + BLOCK_POOL_ID_IPV4_PATTERN_BASE + "|"
+ + BLOCK_POOL_ID_IPV6_PATTERN_BASE + ")-\\d+" + Pattern
+ .quote(File.separator);
private static final Pattern BLOCK_POOL_PATH_PATTERN = Pattern.compile(
"^(.*)(" + BLOCK_POOL_ID_PATTERN_BASE + ")(.*)$");
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index a80d0c4..88ed7e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -147,7 +147,7 @@ class DataXceiver extends Receiver implements Runnable {
this.ioFileBufferSize = DFSUtilClient.getIoFileBufferSize(datanode.getConf());
this.smallBufferSize = DFSUtilClient.getSmallBufferSize(datanode.getConf());
remoteAddress = peer.getRemoteAddressString();
- final int colonIdx = remoteAddress.indexOf(':');
+ final int colonIdx = remoteAddress.lastIndexOf(':');
remoteAddressWithoutPort =
(colonIdx < 0) ? remoteAddress : remoteAddress.substring(0, colonIdx);
localAddress = peer.getLocalAddressString();
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
index 618a372..252d56e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Checkpointer.java
@@ -103,9 +103,9 @@ class Checkpointer extends Daemon {
checkpointConf = new CheckpointConf(conf);
// Pull out exact http address for posting url to avoid ip aliasing issues
- String fullInfoAddr = conf.get(DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
- DFS_NAMENODE_BACKUP_HTTP_ADDRESS_DEFAULT);
- infoBindAddress = fullInfoAddr.substring(0, fullInfoAddr.indexOf(":"));
+ String fullInfoAddr = conf.get(DFS_NAMENODE_BACKUP_HTTP_ADDRESS_KEY,
+ DFS_NAMENODE_BACKUP_HTTP_ADDRESS_DEFAULT);
+ infoBindAddress = fullInfoAddr.substring(0, fullInfoAddr.lastIndexOf(":"));
LOG.info("Checkpoint Period : " +
checkpointConf.getPeriod() + " secs " +
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
index 092885a..e8a346b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNStorage.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.Lists;
import org.apache.hadoop.util.Time;
+import org.apache.http.conn.util.InetAddressUtils;
import org.eclipse.jetty.util.ajax.JSON;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
@@ -1024,6 +1025,10 @@ public class NNStorage extends Storage implements Closeable,
String ip;
try {
ip = DNS.getDefaultIP("default");
+ if (InetAddressUtils.isIPv6StdAddress(ip)) {
+ // HDFS doesn't support ":" in path, replace it with "."
+ ip = "[" + ip.replaceAll(":", ".") + "]";
+ }
} catch (UnknownHostException e) {
LOG.warn("Could not find ip address of \"default\" inteface.");
throw e;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 249bac6..72fa9b5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -117,6 +117,7 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.base.Charsets;
+import org.apache.hadoop.thirdparty.com.google.common.net.HostAndPort;
import com.sun.jersey.spi.container.ResourceFilters;
/** Web-hdfs NameNode implementation. */
@@ -273,22 +274,22 @@ public class NamenodeWebHdfsMethods {
HashSet<Node> excludes = new HashSet<Node>();
if (excludeDatanodes != null) {
- for (String host : StringUtils
+ for (String hostAndPort : StringUtils
.getTrimmedStringCollection(excludeDatanodes)) {
- int idx = host.indexOf(":");
+ HostAndPort hp = HostAndPort.fromString(hostAndPort);
Node excludeNode = null;
- if (idx != -1) {
- excludeNode = bm.getDatanodeManager().getDatanodeByXferAddr(
- host.substring(0, idx), Integer.parseInt(host.substring(idx + 1)));
+ if (hp.hasPort()) {
+ excludeNode = bm.getDatanodeManager()
+ .getDatanodeByXferAddr(hp.getHost(), hp.getPort());
} else {
- excludeNode = bm.getDatanodeManager().getDatanodeByHost(host);
+ excludeNode = bm.getDatanodeManager().getDatanodeByHost(hostAndPort);
}
if (excludeNode != null) {
excludes.add(excludeNode);
} else {
LOG.debug("DataNode {} was requested to be excluded, "
- + "but it was not found.", host);
+ + "but it was not found.", hostAndPort);
}
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
index aaa1038..448d993 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@@ -245,7 +246,7 @@ public class GetConf extends Configured implements Tool {
if (!cnnlist.isEmpty()) {
for (ConfiguredNNAddress cnn : cnnlist) {
InetSocketAddress rpc = cnn.getAddress();
- tool.printOut(rpc.getHostName()+":"+rpc.getPort());
+ tool.printOut(NetUtils.getHostPortString(rpc));
}
return 0;
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java
index 29ac759..b4c3dcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/WebImageViewer.java
@@ -34,6 +34,7 @@ import io.netty.handler.codec.string.StringEncoder;
import io.netty.util.concurrent.GlobalEventExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.security.UserGroupInformation;
@@ -122,7 +123,9 @@ public class WebImageViewer implements Closeable {
allChannels.add(channel);
address = (InetSocketAddress) channel.localAddress();
- LOG.info("WebImageViewer started. Listening on " + address.toString() + ". Press Ctrl+C to stop the viewer.");
+ LOG.info("WebImageViewer started. Listening on " + NetUtils
+ .getSocketAddressString(address) +
+ ". Press Ctrl+C to stop the viewer.");
}
/**
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java
index c61c0b1..266c0fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSAddressConfig.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.net.NetUtils;
import org.junit.Test;
@@ -55,7 +56,7 @@ public class TestDFSAddressConfig {
ArrayList<DataNode> dns = cluster.getDataNodes();
DataNode dn = dns.get(0);
- String selfSocketAddr = dn.getXferAddress().toString();
+ String selfSocketAddr = NetUtils.getSocketAddressString(dn.getXferAddress());
System.out.println("DN Self Socket Addr == " + selfSocketAddr);
assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
@@ -80,7 +81,7 @@ public class TestDFSAddressConfig {
dns = cluster.getDataNodes();
dn = dns.get(0);
- selfSocketAddr = dn.getXferAddress().toString();
+ selfSocketAddr = NetUtils.getSocketAddressString(dn.getXferAddress());
System.out.println("DN Self Socket Addr == " + selfSocketAddr);
// assert that default self socket address is 127.0.0.1
assertTrue(selfSocketAddr.contains("/127.0.0.1:"));
@@ -105,10 +106,11 @@ public class TestDFSAddressConfig {
dns = cluster.getDataNodes();
dn = dns.get(0);
- selfSocketAddr = dn.getXferAddress().toString();
+ selfSocketAddr = NetUtils.getSocketAddressString(dn.getXferAddress());
System.out.println("DN Self Socket Addr == " + selfSocketAddr);
// assert that default self socket address is 0.0.0.0
- assertTrue(selfSocketAddr.contains("/0.0.0.0:"));
+ assertTrue(selfSocketAddr.contains("/0.0.0.0:") ||
+ selfSocketAddr.contains("/[0:0:0:0:0:0:0:0]:"));
cluster.shutdown();
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index 9a024c3..43bd44e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -541,7 +541,7 @@ public class TestDFSUtil {
DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn1"),
NS2_NN1_HOST);
conf.set(DFSUtil.addKeySuffixes(
- DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn2"),
+ DFS_NAMENODE_RPC_ADDRESS_KEY, "ns2", "ns2-nn2"),
NS2_NN2_HOST);
Map<String, Map<String, InetSocketAddress>> map =
@@ -550,17 +550,21 @@ public class TestDFSUtil {
assertTrue(HAUtil.isHAEnabled(conf, "ns1"));
assertTrue(HAUtil.isHAEnabled(conf, "ns2"));
assertFalse(HAUtil.isHAEnabled(conf, "ns3"));
-
- assertEquals(NS1_NN1_HOST, map.get("ns1").get("ns1-nn1").toString());
- assertEquals(NS1_NN2_HOST, map.get("ns1").get("ns1-nn2").toString());
- assertEquals(NS2_NN1_HOST, map.get("ns2").get("ns2-nn1").toString());
- assertEquals(NS2_NN2_HOST, map.get("ns2").get("ns2-nn2").toString());
-
- assertEquals(NS1_NN1_HOST,
+
+ assertEquals(NS1_NN1_HOST,
+ NetUtils.getHostPortString(map.get("ns1").get("ns1-nn1")));
+ assertEquals(NS1_NN2_HOST,
+ NetUtils.getHostPortString(map.get("ns1").get("ns1-nn2")));
+ assertEquals(NS2_NN1_HOST,
+ NetUtils.getHostPortString(map.get("ns2").get("ns2-nn1")));
+ assertEquals(NS2_NN2_HOST,
+ NetUtils.getHostPortString(map.get("ns2").get("ns2-nn2")));
+
+ assertEquals(NS1_NN1_HOST,
DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn1"));
- assertEquals(NS1_NN2_HOST,
+ assertEquals(NS1_NN2_HOST,
DFSUtil.getNamenodeServiceAddr(conf, "ns1", "ns1-nn2"));
- assertEquals(NS2_NN1_HOST,
+ assertEquals(NS2_NN1_HOST,
DFSUtil.getNamenodeServiceAddr(conf, "ns2", "ns2-nn1"));
// No nameservice was given and we can't determine which service addr
@@ -630,8 +634,29 @@ public class TestDFSUtil {
Map<String, Map<String, InetSocketAddress>> map =
DFSUtilClient.getHaNnWebHdfsAddresses(conf, "webhdfs");
- assertEquals(NS1_NN1_ADDR, map.get("ns1").get("nn1").toString());
- assertEquals(NS1_NN2_ADDR, map.get("ns1").get("nn2").toString());
+ assertEquals(NS1_NN1_ADDR,
+ NetUtils.getHostPortString(map.get("ns1").get("nn1")));
+ assertEquals(NS1_NN2_ADDR,
+ NetUtils.getHostPortString(map.get("ns1").get("nn2")));
+ }
+
+ @Test
+ public void testIPv6GetHaNnHttpAddresses() throws IOException {
+ final String logicalHostName = "ns1";
+ final String ns1Nn1Addr = "[0:0:0:0:0:b00c:c0a8:12a]:8020";
+ final String ns1Nn2Addr = "[::face:a0b:182a]:8020";
+
+ Configuration conf =
+ createWebHDFSHAConfiguration(logicalHostName, ns1Nn1Addr,
+ ns1Nn2Addr);
+
+ Map<String, Map<String, InetSocketAddress>> map =
+ DFSUtilClient.getHaNnWebHdfsAddresses(conf, "webhdfs");
+
+ assertEquals(ns1Nn1Addr,
+ NetUtils.getHostPortString(map.get("ns1").get("nn1")));
+ assertEquals(ns1Nn2Addr.replace("::", "0:0:0:0:0:"),
+ NetUtils.getHostPortString(map.get("ns1").get("nn2")));
}
private static Configuration createWebHDFSHAConfiguration(String logicalHostName, String nnaddr1, String nnaddr2) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
index b65301f8..1b5b6cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetUtil;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.DataChecksum;
import org.apache.hadoop.util.Time;
@@ -554,7 +555,8 @@ public class TestFileAppend{
// stop one datanode
DataNodeProperties dnProp = cluster.stopDataNode(0);
- String dnAddress = dnProp.datanode.getXferAddress().toString();
+ String dnAddress = NetUtils.getSocketAddressString(
+ dnProp.datanode.getXferAddress());
if (dnAddress.startsWith("/")) {
dnAddress = dnAddress.substring(1);
}
@@ -609,7 +611,8 @@ public class TestFileAppend{
// stop one datanode
DataNodeProperties dnProp = cluster.stopDataNode(0);
- String dnAddress = dnProp.datanode.getXferAddress().toString();
+ String dnAddress = NetUtils
+ .getSocketAddressString(dnProp.datanode.getXferAddress());
if (dnAddress.startsWith("/")) {
dnAddress = dnAddress.substring(1);
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
index a7cf68b..0c50a89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileCreation.java
@@ -92,11 +92,14 @@ import org.apache.hadoop.util.Time;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.event.Level;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
/**
* This class tests various cases during file creation.
*/
public class TestFileCreation {
+ public static final Log LOG = LogFactory.getLog(TestFileCreation.class);
static final String DIR = "/" + TestFileCreation.class.getSimpleName() + "/";
{
@@ -125,7 +128,7 @@ public class TestFileCreation {
// creates a file but does not close it
public static FSDataOutputStream createFile(FileSystem fileSys, Path name, int repl)
throws IOException {
- System.out.println("createFile: Created " + name + " with " + repl + " replica.");
+ LOG.info("createFile: Created " + name + " with " + repl + " replica.");
FSDataOutputStream stm = fileSys.create(name, true, fileSys.getConf()
.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096),
(short) repl, blockSize);
@@ -305,8 +308,8 @@ public class TestFileCreation {
public void testFileCreationSetLocalInterface() throws IOException {
assumeTrue(System.getProperty("os.name").startsWith("Linux"));
- // The mini cluster listens on the loopback so we can use it here
- checkFileCreation("lo", false);
+ // Use wildcard address to force interface to be used
+ checkFileCreation("0.0.0.0", false);
try {
checkFileCreation("bogus-interface", false);
@@ -348,9 +351,9 @@ public class TestFileCreation {
// check that / exists
//
Path path = new Path("/");
- System.out.println("Path : \"" + path.toString() + "\"");
- System.out.println(fs.getFileStatus(path).isDirectory());
- assertTrue("/ should be a directory",
+ LOG.info("Path : \"" + path.toString() + "\"");
+ LOG.info(fs.getFileStatus(path).isDirectory());
+ assertTrue("/ should be a directory",
fs.getFileStatus(path).isDirectory());
//
@@ -358,8 +361,8 @@ public class TestFileCreation {
//
Path dir1 = new Path("/test_dir");
fs.mkdirs(dir1);
- System.out.println("createFile: Creating " + dir1.getName() +
- " for overwrite of existing directory.");
+ LOG.info("createFile: Creating " + dir1.getName()
+ + " for overwrite of existing directory.");
try {
fs.create(dir1, true); // Create path, overwrite=true
fs.close();
@@ -379,9 +382,9 @@ public class TestFileCreation {
FSDataOutputStream stm = createFile(fs, file1, 1);
// verify that file exists in FS namespace
- assertTrue(file1 + " should be a file",
+ assertTrue(file1 + " should be a file",
fs.getFileStatus(file1).isFile());
- System.out.println("Path : \"" + file1 + "\"");
+ LOG.info("Path : \"" + file1 + "\"");
// write to file
writeFile(stm);
@@ -393,13 +396,13 @@ public class TestFileCreation {
assertTrue(file1 + " should be of size " + fileSize +
" but found to be of size " + len,
len == fileSize);
-
+
// verify the disk space the file occupied
long diskSpace = dfs.getContentSummary(file1.getParent()).getLength();
assertEquals(file1 + " should take " + fileSize + " bytes disk space " +
"but found to take " + diskSpace + " bytes", fileSize, diskSpace);
-
- // Check storage usage
+
+ // Check storage usage
// can't check capacities for real storage since the OS file system may be changing under us.
if (simulatedStorage) {
DataNode dn = cluster.getDataNodes().get(0);
@@ -436,7 +439,7 @@ public class TestFileCreation {
FSDataOutputStream stm1 = createFile(fs, file1, 1);
FSDataOutputStream stm2 = createFile(fs, file2, 1);
FSDataOutputStream stm3 = createFile(localfs, file3, 1);
- System.out.println("DeleteOnExit: Created files.");
+ LOG.info("DeleteOnExit: Created files.");
// write to files and close. Purposely, do not close file2.
writeFile(stm1);
@@ -467,7 +470,7 @@ public class TestFileCreation {
!fs.exists(file2));
assertTrue(file3 + " still exists inspite of deletOnExit set.",
!localfs.exists(file3));
- System.out.println("DeleteOnExit successful.");
+ LOG.info("DeleteOnExit successful.");
} finally {
IOUtils.closeStream(fs);
@@ -563,7 +566,7 @@ public class TestFileCreation {
// verify that file exists in FS namespace
assertTrue(file1 + " should be a file",
fs.getFileStatus(file1).isFile());
- System.out.println("Path : \"" + file1 + "\"");
+ LOG.info("Path : \"" + file1 + "\"");
// kill the datanode
cluster.shutdownDataNodes();
@@ -575,7 +578,7 @@ public class TestFileCreation {
if (info.length == 0) {
break;
}
- System.out.println("testFileCreationError1: waiting for datanode " +
+ LOG.info("testFileCreationError1: waiting for datanode " +
" to die.");
try {
Thread.sleep(1000);
@@ -597,7 +600,7 @@ public class TestFileCreation {
// bad block allocations were cleaned up earlier.
LocatedBlocks locations = client.getNamenode().getBlockLocations(
file1.toString(), 0, Long.MAX_VALUE);
- System.out.println("locations = " + locations.locatedBlockCount());
+ LOG.info("locations = " + locations.locatedBlockCount());
assertTrue("Error blocks were not cleaned up",
locations.locatedBlockCount() == 0);
} finally {
@@ -613,7 +616,7 @@ public class TestFileCreation {
@Test
public void testFileCreationError2() throws IOException {
long leasePeriod = 1000;
- System.out.println("testFileCreationError2 start");
+ LOG.info("testFileCreationError2 start");
Configuration conf = new HdfsConfiguration();
conf.setInt(DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
conf.setInt(DFS_HEARTBEAT_INTERVAL_KEY, 1);
@@ -632,24 +635,24 @@ public class TestFileCreation {
//
Path file1 = new Path("/filestatus.dat");
createFile(dfs, file1, 1);
- System.out.println("testFileCreationError2: "
+ LOG.info("testFileCreationError2: "
+ "Created file filestatus.dat with one replicas.");
LocatedBlocks locations = client.getNamenode().getBlockLocations(
file1.toString(), 0, Long.MAX_VALUE);
- System.out.println("testFileCreationError2: "
+ LOG.info("testFileCreationError2: "
+ "The file has " + locations.locatedBlockCount() + " blocks.");
// add one block to the file
LocatedBlock location = client.getNamenode().addBlock(file1.toString(),
client.clientName, null, null, HdfsConstants.GRANDFATHER_INODE_ID, null, null);
- System.out.println("testFileCreationError2: "
+ LOG.info("testFileCreationError2: "
+ "Added block " + location.getBlock());
locations = client.getNamenode().getBlockLocations(file1.toString(),
0, Long.MAX_VALUE);
int count = locations.locatedBlockCount();
- System.out.println("testFileCreationError2: "
+ LOG.info("testFileCreationError2: "
+ "The file now has " + count + " blocks.");
// set the soft and hard limit to be 1 second so that the
@@ -665,10 +668,10 @@ public class TestFileCreation {
// verify that the last block was synchronized.
locations = client.getNamenode().getBlockLocations(file1.toString(),
0, Long.MAX_VALUE);
- System.out.println("testFileCreationError2: "
+ LOG.info("testFileCreationError2: "
+ "locations = " + locations.locatedBlockCount());
assertEquals(0, locations.locatedBlockCount());
- System.out.println("testFileCreationError2 successful");
+ LOG.info("testFileCreationError2 successful");
} finally {
IOUtils.closeStream(dfs);
cluster.shutdown();
@@ -678,7 +681,7 @@ public class TestFileCreation {
/** test addBlock(..) when replication<min and excludeNodes==null. */
@Test
public void testFileCreationError3() throws IOException {
- System.out.println("testFileCreationError3 start");
+ LOG.info("testFileCreationError3 start");
Configuration conf = new HdfsConfiguration();
// create cluster
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
@@ -699,7 +702,7 @@ public class TestFileCreation {
FileSystem.LOG.info("GOOD!", ioe);
}
- System.out.println("testFileCreationError3 successful");
+ LOG.info("testFileCreationError3 successful");
} finally {
IOUtils.closeStream(dfs);
cluster.shutdown();
@@ -732,7 +735,7 @@ public class TestFileCreation {
// create a new file.
Path file1 = new Path("/filestatus.dat");
HdfsDataOutputStream stm = create(fs, file1, 1);
- System.out.println("testFileCreationNamenodeRestart: "
+ LOG.info("testFileCreationNamenodeRestart: "
+ "Created file " + file1);
assertEquals(file1 + " should be replicated to 1 datanode.", 1,
stm.getCurrentBlockReplication());
@@ -746,7 +749,7 @@ public class TestFileCreation {
// rename file wile keeping it open.
Path fileRenamed = new Path("/filestatusRenamed.dat");
fs.rename(file1, fileRenamed);
- System.out.println("testFileCreationNamenodeRestart: "
+ LOG.info("testFileCreationNamenodeRestart: "
+ "Renamed file " + file1 + " to " +
fileRenamed);
file1 = fileRenamed;
@@ -755,7 +758,7 @@ public class TestFileCreation {
//
Path file2 = new Path("/filestatus2.dat");
FSDataOutputStream stm2 = createFile(fs, file2, 1);
- System.out.println("testFileCreationNamenodeRestart: "
+ LOG.info("testFileCreationNamenodeRestart: "
+ "Created file " + file2);
// create yet another new file with full path name.
@@ -763,21 +766,21 @@ public class TestFileCreation {
//
Path file3 = new Path("/user/home/fullpath.dat");
FSDataOutputStream stm3 = createFile(fs, file3, 1);
- System.out.println("testFileCreationNamenodeRestart: "
+ LOG.info("testFileCreationNamenodeRestart: "
+ "Created file " + file3);
Path file4 = new Path("/user/home/fullpath4.dat");
FSDataOutputStream stm4 = createFile(fs, file4, 1);
- System.out.println("testFileCreationNamenodeRestart: "
+ LOG.info("testFileCreationNamenodeRestart: "
+ "Created file " + file4);
fs.mkdirs(new Path("/bin"));
fs.rename(new Path("/user/home"), new Path("/bin"));
Path file3new = new Path("/bin/home/fullpath.dat");
- System.out.println("testFileCreationNamenodeRestart: "
+ LOG.info("testFileCreationNamenodeRestart: "
+ "Renamed file " + file3 + " to " +
file3new);
Path file4new = new Path("/bin/home/fullpath4.dat");
- System.out.println("testFileCreationNamenodeRestart: "
+ LOG.info("testFileCreationNamenodeRestart: "
+ "Renamed file " + file4 + " to " +
file4new);
@@ -837,14 +840,14 @@ public class TestFileCreation {
DFSClient client = fs.dfs;
LocatedBlocks locations = client.getNamenode().getBlockLocations(
file1.toString(), 0, Long.MAX_VALUE);
- System.out.println("locations = " + locations.locatedBlockCount());
+ LOG.info("locations = " + locations.locatedBlockCount());
assertTrue("Error blocks were not cleaned up for file " + file1,
locations.locatedBlockCount() == 3);
// verify filestatus2.dat
locations = client.getNamenode().getBlockLocations(
file2.toString(), 0, Long.MAX_VALUE);
- System.out.println("locations = " + locations.locatedBlockCount());
+ LOG.info("locations = " + locations.locatedBlockCount());
assertTrue("Error blocks were not cleaned up for file " + file2,
locations.locatedBlockCount() == 1);
} finally {
@@ -859,7 +862,7 @@ public class TestFileCreation {
@Test
public void testDFSClientDeath() throws IOException, InterruptedException {
Configuration conf = new HdfsConfiguration();
- System.out.println("Testing adbornal client death.");
+ LOG.info("Testing adbornal client death.");
if (simulatedStorage) {
SimulatedFSDataset.setFactory(conf);
}
@@ -873,7 +876,7 @@ public class TestFileCreation {
//
Path file1 = new Path("/clienttest.dat");
FSDataOutputStream stm = createFile(fs, file1, 1);
- System.out.println("Created file clienttest.dat");
+ LOG.info("Created file clienttest.dat");
// write to file
writeFile(stm);
@@ -889,7 +892,7 @@ public class TestFileCreation {
cluster.shutdown();
}
}
-
+
/**
* Test file creation using createNonRecursive().
*/
@@ -970,7 +973,7 @@ public class TestFileCreation {
static IOException createNonRecursive(FileSystem fs, Path name,
int repl, EnumSet<CreateFlag> flag) throws IOException {
try {
- System.out.println("createNonRecursive: Attempting to create " + name +
+ LOG.info("createNonRecursive: Attempting to create " + name +
" with " + repl + " replica.");
int bufferSize = fs.getConf()
.getInt(CommonConfigurationKeys.IO_FILE_BUFFER_SIZE_KEY, 4096);
@@ -1004,9 +1007,9 @@ public class TestFileCreation {
try {
FileSystem fs = cluster.getFileSystem();
-
+
Path[] p = {new Path("/foo"), new Path("/bar")};
-
+
//write 2 files at the same time
FSDataOutputStream[] out = {fs.create(p[0]), fs.create(p[1])};
int i = 0;
@@ -1038,9 +1041,9 @@ public class TestFileCreation {
try {
FileSystem fs = cluster.getFileSystem();
-
+
Path[] p = {new Path("/foo"), new Path("/bar")};
-
+
//write 2 files at the same time
FSDataOutputStream[] out = {fs.create(p[0]), fs.create(p[1])};
int i = 0;
@@ -1068,7 +1071,7 @@ public class TestFileCreation {
*/
@Test
public void testLeaseExpireHardLimit() throws Exception {
- System.out.println("testLeaseExpireHardLimit start");
+ LOG.info("testLeaseExpireHardLimit start");
final long leasePeriod = 1000;
final int DATANODE_NUM = 3;
@@ -1113,20 +1116,20 @@ public class TestFileCreation {
successcount++;
}
}
- System.out.println("successcount=" + successcount);
- assertTrue(successcount > 0);
+ LOG.info("successcount=" + successcount);
+ assertTrue(successcount > 0);
} finally {
IOUtils.closeStream(dfs);
cluster.shutdown();
}
- System.out.println("testLeaseExpireHardLimit successful");
+ LOG.info("testLeaseExpireHardLimit successful");
}
// test closing file system before all file handles are closed.
@Test
public void testFsClose() throws Exception {
- System.out.println("test file system close start");
+ LOG.info("test file system close start");
final int DATANODE_NUM = 3;
Configuration conf = new HdfsConfiguration();
@@ -1147,7 +1150,7 @@ public class TestFileCreation {
// close file system without closing file
dfs.close();
} finally {
- System.out.println("testFsClose successful");
+ LOG.info("testFsClose successful");
cluster.shutdown();
}
}
@@ -1155,7 +1158,7 @@ public class TestFileCreation {
// test closing file after cluster is shutdown
@Test
public void testFsCloseAfterClusterShutdown() throws IOException {
- System.out.println("test testFsCloseAfterClusterShutdown start");
+ LOG.info("test testFsCloseAfterClusterShutdown start");
final int DATANODE_NUM = 3;
Configuration conf = new HdfsConfiguration();
@@ -1186,13 +1189,13 @@ public class TestFileCreation {
boolean hasException = false;
try {
out.close();
- System.out.println("testFsCloseAfterClusterShutdown: Error here");
+ LOG.info("testFsCloseAfterClusterShutdown: Error here");
} catch (IOException e) {
hasException = true;
}
assertTrue("Failed to close file after cluster shutdown", hasException);
} finally {
- System.out.println("testFsCloseAfterClusterShutdown successful");
+ LOG.info("testFsCloseAfterClusterShutdown successful");
if (cluster != null) {
cluster.shutdown();
}
@@ -1211,7 +1214,7 @@ public class TestFileCreation {
public void testCreateNonCanonicalPathAndRestartRpc() throws Exception {
doCreateTest(CreationMethod.DIRECT_NN_RPC);
}
-
+
/**
* Another regression test for HDFS-3626. This one creates files using
* a Path instantiated from a string object.
@@ -1231,7 +1234,7 @@ public class TestFileCreation {
throws Exception {
doCreateTest(CreationMethod.PATH_FROM_URI);
}
-
+
private enum CreationMethod {
DIRECT_NN_RPC,
PATH_FROM_URI,
@@ -1246,7 +1249,7 @@ public class TestFileCreation {
NamenodeProtocols nnrpc = cluster.getNameNodeRpc();
for (String pathStr : NON_CANONICAL_PATHS) {
- System.out.println("Creating " + pathStr + " by " + method);
+ LOG.info("Creating " + pathStr + " by " + method);
switch (method) {
case DIRECT_NN_RPC:
try {
@@ -1261,7 +1264,7 @@ public class TestFileCreation {
// So, we expect all of them to fail.
}
break;
-
+
case PATH_FROM_URI:
case PATH_FROM_STRING:
// Unlike the above direct-to-NN case, we expect these to succeed,
@@ -1279,7 +1282,7 @@ public class TestFileCreation {
throw new AssertionError("bad method: " + method);
}
}
-
+
cluster.restartNameNode();
} finally {
@@ -1336,7 +1339,7 @@ public class TestFileCreation {
dfs.mkdirs(new Path("/foo/dir"));
String file = "/foo/dir/file";
Path filePath = new Path(file);
-
+
// Case 1: Create file with overwrite, check the blocks of old file
// are cleaned after creating with overwrite
NameNode nn = cluster.getNameNode();
@@ -1350,7 +1353,7 @@ public class TestFileCreation {
} finally {
out.close();
}
-
+
LocatedBlocks oldBlocks = NameNodeAdapter.getBlockLocations(
nn, file, 0, fileSize);
assertBlocks(bm, oldBlocks, true);
@@ -1363,7 +1366,7 @@ public class TestFileCreation {
out.close();
}
dfs.deleteOnExit(filePath);
-
+
LocatedBlocks newBlocks = NameNodeAdapter.getBlockLocations(
nn, file, 0, fileSize);
assertBlocks(bm, newBlocks, true);
@@ -1377,7 +1380,7 @@ public class TestFileCreation {
in.close();
}
Assert.assertArrayEquals(newData, result);
-
+
// Case 2: Restart NN, check the file
cluster.restartNameNode();
nn = cluster.getNameNode();
@@ -1388,13 +1391,13 @@ public class TestFileCreation {
in.close();
}
Assert.assertArrayEquals(newData, result);
-
+
// Case 3: Save new checkpoint and restart NN, check the file
NameNodeAdapter.enterSafeMode(nn, false);
NameNodeAdapter.saveNamespace(nn);
cluster.restartNameNode();
nn = cluster.getNameNode();
-
+
in = dfs.open(filePath);
try {
result = readAll(in);
@@ -1411,8 +1414,8 @@ public class TestFileCreation {
}
}
}
-
- private void assertBlocks(BlockManager bm, LocatedBlocks lbs,
+
+ private void assertBlocks(BlockManager bm, LocatedBlocks lbs,
boolean exist) {
for (LocatedBlock locatedBlock : lbs.getLocatedBlocks()) {
if (exist) {
@@ -1424,7 +1427,7 @@ public class TestFileCreation {
}
}
}
-
+
private byte[] readAll(FSDataInputStream in) throws IOException {
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/BlockReaderTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/BlockReaderTestUtil.java
index 826299b..40ad502 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/BlockReaderTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/client/impl/BlockReaderTestUtil.java
@@ -195,7 +195,8 @@ public class BlockReaderTestUtil {
return new BlockReaderFactory(fs.getClient().getConf()).
setInetSocketAddress(targetAddr).
setBlock(block).
- setFileName(targetAddr.toString()+ ":" + block.getBlockId()).
+ setFileName(NetUtils.getSocketAddressString(targetAddr) + ":" + block
+ .getBlockId()).
setBlockToken(testBlock.getBlockToken()).
setStartOffset(offset).
setLength(lenToRead).
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
index c4760a0..a135251 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java
@@ -921,14 +921,14 @@ public class TestQuorumJournalManager {
GenericTestUtils.assertGlobEquals(paxosDir, "\\d+",
"3");
}
-
+
@Test
public void testToString() throws Exception {
GenericTestUtils.assertMatches(
qjm.toString(),
- "QJM to \\[127.0.0.1:\\d+, 127.0.0.1:\\d+, 127.0.0.1:\\d+\\]");
+ "QJM to \\[localhost:\\d+, localhost:\\d+, localhost:\\d+\\]");
}
-
+
@Test
public void testSelectInputStreamsNotOnBoundary() throws Exception {
final int txIdsPerSegment = 10;
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java
index 5e850b9..a9b315e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockPoolSliceStorage.java
@@ -22,6 +22,8 @@ import org.junit.Test;
import org.mockito.Mockito;
import java.io.File;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.util.Random;
import java.util.UUID;
import org.slf4j.Logger;
@@ -56,13 +58,37 @@ public class TestBlockPoolSliceStorage {
}
}
- private String makeRandomIpAddress() {
+ private String makeRandomIpv4Address() {
return rand.nextInt(256) + "." +
rand.nextInt(256) + "." +
rand.nextInt(256) + "." +
rand.nextInt(256);
}
+ private String makeRandomIpv6Address() {
+ byte[] bytes = new byte[16];
+ rand.nextBytes(bytes);
+ InetAddress adr = null;
+ try {
+ adr = InetAddress.getByAddress("unused", bytes);
+ } catch (UnknownHostException uhe) {
+ // Should never happen
+ LOG.error("UnknownHostException " + uhe);
+ assertThat(true, is(false));
+ }
+ String addrString = adr.getHostAddress().replaceAll(":", ".");
+
+ return "[" + addrString + "]";
+ }
+
+ private String makeRandomIpAddress() {
+ if (rand.nextBoolean()) {
+ return makeRandomIpv4Address();
+ } else {
+ return makeRandomIpv6Address();
+ }
+ }
+
private String makeRandomBlockpoolId() {
return "BP-" + rand.nextInt(Integer.MAX_VALUE) +
"-" + makeRandomIpAddress() +
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java
index 594b07b..4b9a0a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRespectsBindHostKeys.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.AnyOf.anyOf;
import static org.hamcrest.core.IsNot.not;
+import org.apache.hadoop.net.NetUtils;
import org.junit.Test;
import org.apache.hadoop.fs.FileUtil;
@@ -30,6 +32,8 @@ import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.hadoop.test.GenericTestUtils;
+import java.net.InetAddress;
+import java.net.Inet6Address;
import java.io.File;
import java.io.IOException;
@@ -55,6 +59,7 @@ public class TestNameNodeRespectsBindHostKeys {
public static final Logger LOG =
LoggerFactory.getLogger(TestNameNodeRespectsBindHostKeys.class);
private static final String WILDCARD_ADDRESS = "0.0.0.0";
+ private static final String IPV6_WILDCARD_ADDRESS = "0:0:0:0:0:0:0:0";
private static final String LOCALHOST_SERVER_ADDRESS = "127.0.0.1:0";
private static String keystoresDir;
private static String sslConfDir;
@@ -79,9 +84,9 @@ public class TestNameNodeRespectsBindHostKeys {
public void testRpcBindHostKey() throws IOException {
Configuration conf = new HdfsConfiguration();
MiniDFSCluster cluster = null;
-
+
LOG.info("Testing without " + DFS_NAMENODE_RPC_BIND_HOST_KEY);
-
+
// NN should not bind the wildcard address by default.
try {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
@@ -97,7 +102,7 @@ public class TestNameNodeRespectsBindHostKeys {
}
LOG.info("Testing with " + DFS_NAMENODE_RPC_BIND_HOST_KEY);
-
+
// Tell NN to bind the wildcard address.
conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY, WILDCARD_ADDRESS);
@@ -106,13 +111,36 @@ public class TestNameNodeRespectsBindHostKeys {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
String address = getRpcServerAddress(cluster);
- assertThat("Bind address " + address + " is not wildcard.",
- address, is("/" + WILDCARD_ADDRESS));
+ assertThat("Bind address " + address + " is not wildcard.", address,
+ anyOf(is("/" + WILDCARD_ADDRESS), is("/" + IPV6_WILDCARD_ADDRESS)));
} finally {
if (cluster != null) {
cluster.shutdown();
}
- }
+ }
+
+ InetAddress localAddr = InetAddress.getLocalHost();
+ if (localAddr instanceof Inet6Address) {
+ // Tell NN to bind the IPv6 wildcard address.
+ conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY, IPV6_WILDCARD_ADDRESS);
+
+ // Verify that NN binds wildcard address now.
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ String address = getRpcServerAddress(cluster);
+ assertThat("Bind address " + address + " is not wildcard.",
+ address, anyOf(
+ is("/" + WILDCARD_ADDRESS),
+ is("/" + IPV6_WILDCARD_ADDRESS)));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ } else {
+ LOG.info("Not testing IPv6 binding as IPv6 us not supported");
+ }
}
@Test (timeout=300000)
@@ -121,7 +149,7 @@ public class TestNameNodeRespectsBindHostKeys {
MiniDFSCluster cluster = null;
LOG.info("Testing without " + DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY);
-
+
conf.set(DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS);
// NN should not bind the wildcard address by default.
@@ -140,6 +168,27 @@ public class TestNameNodeRespectsBindHostKeys {
LOG.info("Testing with " + DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY);
+ InetAddress localAddr = InetAddress.getLocalHost();
+ if (localAddr instanceof Inet6Address) {
+ // Tell NN to bind the IPv6 wildcard address.
+ conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY, IPV6_WILDCARD_ADDRESS);
+
+ // Verify that NN binds wildcard address now.
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
+ cluster.waitActive();
+ String address = getRpcServerAddress(cluster);
+ assertThat("Bind address " + address + " is not wildcard.", address,
+ anyOf(is("/" + WILDCARD_ADDRESS), is("/" + IPV6_WILDCARD_ADDRESS)));
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ } else {
+ LOG.info("Not testing IPv6 binding as IPv6 us not supported");
+ }
+
// Tell NN to bind the wildcard address.
conf.set(DFS_NAMENODE_SERVICE_RPC_BIND_HOST_KEY, WILDCARD_ADDRESS);
@@ -148,8 +197,8 @@ public class TestNameNodeRespectsBindHostKeys {
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
String address = getServiceRpcServerAddress(cluster);
- assertThat("Bind address " + address + " is not wildcard.",
- address, is("/" + WILDCARD_ADDRESS));
+ assertThat("Bind address " + address + " is not wildcard.", address,
+ anyOf(is("/" + WILDCARD_ADDRESS), is("/" + IPV6_WILDCARD_ADDRESS)));
} finally {
if (cluster != null) {
cluster.shutdown();
@@ -211,7 +260,8 @@ public class TestNameNodeRespectsBindHostKeys {
conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
- String address = cluster.getNameNode().getHttpAddress().toString();
+ String address = NetUtils.getSocketAddressString(
+ cluster.getNameNode().getHttpAddress());
assertFalse("HTTP Bind address not expected to be wildcard by default.",
address.startsWith(WILDCARD_ADDRESS));
} finally {
@@ -231,7 +281,8 @@ public class TestNameNodeRespectsBindHostKeys {
conf.set(DFS_NAMENODE_HTTP_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
- String address = cluster.getNameNode().getHttpAddress().toString();
+ String address = NetUtils.getSocketAddressString(
+ cluster.getNameNode().getHttpAddress());
assertTrue("HTTP Bind address " + address + " is not wildcard.",
address.startsWith(WILDCARD_ADDRESS));
} finally {
@@ -285,7 +336,8 @@ public class TestNameNodeRespectsBindHostKeys {
conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
- String address = cluster.getNameNode().getHttpsAddress().toString();
+ String address = NetUtils.getSocketAddressString(
+ cluster.getNameNode().getHttpsAddress());
assertFalse("HTTP Bind address not expected to be wildcard by default.",
address.startsWith(WILDCARD_ADDRESS));
} finally {
@@ -305,7 +357,8 @@ public class TestNameNodeRespectsBindHostKeys {
conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, LOCALHOST_SERVER_ADDRESS);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0).build();
cluster.waitActive();
- String address = cluster.getNameNode().getHttpsAddress().toString();
+ String address = NetUtils
+ .getSocketAddressString(cluster.getNameNode().getHttpsAddress());
assertTrue("HTTP Bind address " + address + " is not wildcard.",
address.startsWith(WILDCARD_ADDRESS));
} finally {
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
index ada93e8..5000ce0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNameNodeRpcServer.java
@@ -25,7 +25,9 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RPC_BIND_HOST_KEY;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.hamcrest.core.Is.is;
+import static org.hamcrest.core.AnyOf.anyOf;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
@@ -43,14 +45,18 @@ public class TestNameNodeRpcServer {
// The name node in MiniDFSCluster only binds to 127.0.0.1.
// We can set the bind address to 0.0.0.0 to make it listen
// to all interfaces.
+ // On IPv4-only machines it will return that it is listening on 0.0.0.0
+ // On dual-stack or IPv6-only machines it will return 0:0:0:0:0:0:0:0
conf.set(DFS_NAMENODE_RPC_BIND_HOST_KEY, "0.0.0.0");
MiniDFSCluster cluster = null;
try {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
- assertEquals("0.0.0.0", ((NameNodeRpcServer)cluster.getNameNodeRpc())
- .getClientRpcServer().getListenerAddress().getHostName());
+ String listenerAddress = ((NameNodeRpcServer)cluster.getNameNodeRpc())
+ .getClientRpcServer().getListenerAddress().getHostName();
+ assertThat("Bind address " + listenerAddress + " is not wildcard.",
+ listenerAddress, anyOf(is("0.0.0.0"), is("0:0:0:0:0:0:0:0")));
} finally {
if (cluster != null) {
cluster.shutdown();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org