You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ec...@apache.org on 2015/10/23 21:01:41 UTC
[74/75] [abbrv] hadoop git commit: HADOOP-12491 Avoid unsafe split
and append on fields that might be IPv6 literals
HADOOP-12491 Avoid unsafe split and append on fields that might be IPv6 literals
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd776461
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd776461
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd776461
Branch: refs/heads/HADOOP-11890
Commit: dd7764611e3ca00c5d34bae5456df032d1c05495
Parents: d89cecf
Author: Elliott Clark <ec...@apache.org>
Authored: Tue Oct 20 12:40:12 2015 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Fri Oct 23 11:59:34 2015 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/conf/hadoop-env.sh | 3 +-
.../org/apache/hadoop/conf/Configuration.java | 2 +-
.../crypto/key/kms/KMSClientProvider.java | 13 ++--
.../main/java/org/apache/hadoop/ipc/Client.java | 13 ++--
.../main/java/org/apache/hadoop/net/DNS.java | 64 ++++++++++++++++----
.../apache/hadoop/net/SocksSocketFactory.java | 19 +++---
.../apache/hadoop/ha/ClientBaseWithFixes.java | 15 +----
.../java/org/apache/hadoop/net/TestDNS.java | 17 ++++++
.../org/apache/hadoop/net/TestNetUtils.java | 25 ++++++++
9 files changed, 126 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index ae18542..fd454cc 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -78,8 +78,7 @@
# memory size.
# export HADOOP_HEAPSIZE_MIN=
-# Extra Java runtime options for all Hadoop commands. We don't support
-# IPv6 yet/still, so by default the preference is set to IPv4.
+# Extra Java runtime options for all Hadoop commands.
# export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"
# Some parts of the shell code may do special things dependent upon
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 8801c6c..320028c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2131,7 +2131,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return updateConnectAddr(addressProperty, addr);
}
- final String connectHost = connectHostPort.split(":")[0];
+ final String connectHost = NetUtils.getHostFromHostPort(connectHostPort);
// Create connect address using client address hostname and server port.
return updateConnectAddr(addressProperty, NetUtils.createSocketAddrForHost(
connectHost, addr.getPort()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 1ffc44d..8c2c85d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.CryptoExtension;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import com.google.common.net.HostAndPort;
/**
* KMS client <code>KeyProvider</code> implementation.
@@ -253,16 +254,20 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
// In the current scheme, all hosts have to run on the same port
int port = -1;
String hostsPart = authority;
+
if (authority.contains(":")) {
- String[] t = authority.split(":");
try {
- port = Integer.parseInt(t[1]);
- } catch (Exception e) {
+ HostAndPort hp = HostAndPort.fromString(hostsPart);
+ if (hp.hasPort()) {
+ port = hp.getPort();
+ hostsPart = hp.getHostText();
+ }
+ } catch (IllegalArgumentException e) {
throw new IOException(
"Could not parse port in kms uri [" + origUrl + "]");
}
- hostsPart = t[0];
}
+
return createProvider(providerUri, conf, origUrl, port, hostsPart);
}
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index f067d59..885d449 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ipc;
-import static org.apache.hadoop.ipc.RpcConstants.*;
-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayOutputStream;
@@ -98,6 +96,8 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.protobuf.CodedOutputStream;
+import static org.apache.hadoop.ipc.RpcConstants.CONNECTION_CONTEXT_CALL_ID;
+import static org.apache.hadoop.ipc.RpcConstants.PING_CALL_ID;
/** A client for an IPC service. IPC calls take a single {@link Writable} as a
* parameter, and return a {@link Writable} as their value. A service runs on
@@ -447,7 +447,7 @@ public class Client {
this.authProtocol = trySasl ? AuthProtocol.SASL : AuthProtocol.NONE;
this.setName("IPC Client (" + socketFactory.hashCode() +") connection to " +
- server.toString() +
+ NetUtils.getSocketAddressString(server) +
" from " + ((ticket==null)?"an unknown user":ticket.getUserName()));
this.setDaemon(true);
}
@@ -578,8 +578,9 @@ public class Client {
server.getHostName(), server.getPort());
if (!server.equals(currentAddr)) {
- LOG.warn("Address change detected. Old: " + server.toString() +
- " New: " + currentAddr.toString());
+ LOG.warn("Address change detected. Old: " + NetUtils
+ .getSocketAddressString(server) +
+ " New: " + NetUtils.getSocketAddressString(currentAddr));
server = currentAddr;
return true;
}
@@ -1691,7 +1692,7 @@ public class Client {
@Override
public String toString() {
- return address.toString();
+ return NetUtils.getSocketAddressString(address);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
index a6dc8e3..17845b1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/DNS.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.net;
import com.google.common.net.InetAddresses;
+import com.google.common.annotations.VisibleForTesting;
import com.sun.istack.Nullable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -26,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import java.net.InetAddress;
+import java.net.Inet6Address;
import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
@@ -74,21 +76,22 @@ public class DNS {
*/
public static String reverseDns(InetAddress hostIp, @Nullable String ns)
throws NamingException {
- //
- // Builds the reverse IP lookup form
- // This is formed by reversing the IP numbers and appending in-addr.arpa
- //
- String[] parts = hostIp.getHostAddress().split("\\.");
- String reverseIP = parts[3] + "." + parts[2] + "." + parts[1] + "."
- + parts[0] + ".in-addr.arpa";
+ String dnsQueryAddress;
+ if (hostIp instanceof Inet6Address) {
+ dnsQueryAddress = getIPv6DnsAddr((Inet6Address)hostIp, ns);
+ } else {
+ dnsQueryAddress = getIPv4DnsAddr(hostIp, ns);
+ }
+ LOG.info("Querying using DNS address: " + dnsQueryAddress);
DirContext ictx = new InitialDirContext();
Attributes attribute;
try {
- attribute = ictx.getAttributes("dns://" // Use "dns:///" if the default
+ // Use "dns:///" if the default
+ // nameserver is to be used
+ attribute = ictx.getAttributes("dns://"
+ ((ns == null) ? "" : ns) +
- // nameserver is to be used
- "/" + reverseIP, new String[] { "PTR" });
+ "/" + dnsQueryAddress, new String[] { "PTR" });
} finally {
ictx.close();
}
@@ -101,17 +104,52 @@ public class DNS {
return hostname;
}
+ private static String getIPv4DnsAddr(InetAddress hostIp, @Nullable String ns)
+ throws NamingException {
+ String ipString = hostIp.getHostAddress();
+ LOG.info("Doing reverse DNS lookup for IPv4 address: " + ipString);
+ String[] parts = ipString.split("\\.");
+ if (parts.length != 4) {
+ throw new NamingException("Invalid IPv4 address " + ipString);
+ }
+
+ return parts[3] + "." + parts[2] + "." + parts[1] + "."
+ + parts[0] + ".in-addr.arpa";
+ }
+
+ @VisibleForTesting
+ public static String getIPv6DnsAddr(Inet6Address hostIp, @Nullable String ns)
+ throws NamingException {
+ LOG.info("Doing reverse DNS lookup for IPv6 address: " +
+ hostIp.getHostAddress());
+
+ // bytes need to be converted to hex string and reversed to get IPv6
+ // reverse resolution address
+ byte[] bytes = hostIp.getAddress();
+ StringBuilder sb = new StringBuilder();
+ for(int pos = bytes.length - 1; pos >= 0; pos--) {
+ byte b = bytes[pos];
+ String hexStr = String.format("%02x", b);
+ sb.append(hexStr.charAt(1));
+ sb.append(".");
+ sb.append(hexStr.charAt(0));
+ sb.append(".");
+ }
+ sb.append("ip6.arpa");
+ return sb.toString();
+ }
+
/**
* @return NetworkInterface for the given subinterface name (eg eth0:0)
* or null if no interface with the given name can be found
*/
private static NetworkInterface getSubinterface(String strInterface)
throws SocketException {
- Enumeration<NetworkInterface> nifs =
+ Enumeration<NetworkInterface> nifs =
NetworkInterface.getNetworkInterfaces();
-
+
while (nifs.hasMoreElements()) {
- Enumeration<NetworkInterface> subNifs =
+ Enumeration<NetworkInterface> subNifs =
nifs.nextElement().getSubInterfaces();
while (subNifs.hasMoreElements()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocksSocketFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocksSocketFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocksSocketFactory.java
index 6b84f9d..8864104 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocksSocketFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/SocksSocketFactory.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import com.google.common.net.HostAndPort;
+
/**
* Specialized SocketFactory to create sockets with a SOCKS proxy
*/
@@ -146,13 +148,16 @@ public class SocksSocketFactory extends SocketFactory implements
* @param proxyStr the proxy address using the format "host:port"
*/
private void setProxy(String proxyStr) {
- String[] strs = proxyStr.split(":", 2);
- if (strs.length != 2)
- throw new RuntimeException("Bad SOCKS proxy parameter: " + proxyStr);
- String host = strs[0];
- int port = Integer.parseInt(strs[1]);
- this.proxy =
+ try {
+ HostAndPort hp = HostAndPort.fromString(proxyStr);
+ if (!hp.hasPort())
+ throw new RuntimeException("Bad SOCKS proxy parameter: " + proxyStr);
+ String host = hp.getHostText();
+ this.proxy =
new Proxy(Proxy.Type.SOCKS, InetSocketAddress.createUnresolved(host,
- port));
+ hp.getPort()));
+ } catch (IllegalArgumentException e) {
+ throw new RuntimeException("Bad SOCKS proxy parameter: " + proxyStr);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java
index b1ce1d1..3ad4aed 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/ClientBaseWithFixes.java
@@ -31,6 +31,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.util.Time;
import org.apache.zookeeper.TestableZooKeeper;
@@ -321,22 +322,12 @@ public abstract class ClientBaseWithFixes extends ZKTestCase {
return tmpDir;
}
- private static int getPort(String hostPort) {
- String[] split = hostPort.split(":");
- String portstr = split[split.length-1];
- String[] pc = portstr.split("/");
- if (pc.length > 1) {
- portstr = pc[0];
- }
- return Integer.parseInt(portstr);
- }
-
static ServerCnxnFactory createNewServerInstance(File dataDir,
ServerCnxnFactory factory, String hostPort, int maxCnxns)
throws IOException, InterruptedException
{
ZooKeeperServer zks = new ZooKeeperServer(dataDir, dataDir, 3000);
- final int PORT = getPort(hostPort);
+ final int PORT = NetUtils.getPortFromHostPort(hostPort);
if (factory == null) {
factory = ServerCnxnFactory.createFactory(PORT, maxCnxns);
}
@@ -364,7 +355,7 @@ public abstract class ClientBaseWithFixes extends ZKTestCase {
} catch (IOException ie) {
LOG.warn("Error closing logs ", ie);
}
- final int PORT = getPort(hostPort);
+ final int PORT = NetUtils.getPortFromHostPort(hostPort);
Assert.assertTrue("waiting for server down",
ClientBaseWithFixes.waitForServerDown("127.0.0.1:" + PORT,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
index b26c7ca..bb82abf 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestDNS.java
@@ -24,6 +24,7 @@ import java.net.NetworkInterface;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.net.InetAddress;
+import java.net.Inet6Address;
import javax.naming.CommunicationException;
import javax.naming.NameNotFoundException;
@@ -261,4 +262,20 @@ public class TestDNS {
assertNotNull("localhost is null", localhost);
LOG.info("Localhost IPAddr is " + localhost.toString());
}
+
+ /**
+ * Test that dns query address is calculated correctly for ipv6 addresses
+ */
+ @Test
+ public void testIPv6ReverseDNSAddress() throws Exception {
+ Inet6Address adr = (Inet6Address)InetAddress.getByName("::");
+ assertEquals(
+ "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa",
+ DNS.getIPv6DnsAddr(adr, null));
+
+ adr = (Inet6Address)InetAddress.getByName("fe80::62eb:69ff:fe9b:bade");
+ assertEquals(
+ "e.d.a.b.b.9.e.f.f.f.9.6.b.e.2.6.0.0.0.0.0.0.0.0.0.0.0.0.0.8.e.f.ip6.arpa",
+ DNS.getIPv6DnsAddr(adr, null));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd776461/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
index 55937a6..ddb1f83 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
@@ -57,6 +57,11 @@ public class TestNetUtils {
private static final String DEST_PORT_NAME = Integer.toString(DEST_PORT);
private static final int LOCAL_PORT = 8080;
private static final String LOCAL_PORT_NAME = Integer.toString(LOCAL_PORT);
+ private final String IPV6_LOOPBACK_LONG_STRING = "0:0:0:0:0:0:0:1";
+ private final String IPV6_SAMPLE_ADDRESS = "2a03:2880:2130:cf05:face:b00c:0:1";
+ private final String IPV6_LOOPBACK_SHORT_STRING = "::1";
+ private final String IPV6_LOOPBACK_WITH_PORT = "["+IPV6_LOOPBACK_LONG_STRING+"]:10";
+ private final String IPV6_SAMPLE_WITH_PORT = "[" + IPV6_SAMPLE_ADDRESS + "]:10";
/**
* Some slop around expected times when making sure timeouts behave
@@ -658,6 +663,15 @@ public class TestNetUtils {
}
@Test
+ public void testGetHostNameOfIPworksWithIPv6() {
+ assertNotNull(NetUtils.getHostNameOfIP(IPV6_LOOPBACK_LONG_STRING));
+ assertNotNull(NetUtils.getHostNameOfIP(IPV6_LOOPBACK_SHORT_STRING));
+ assertNotNull(NetUtils.getHostNameOfIP(IPV6_SAMPLE_ADDRESS));
+ assertNotNull(NetUtils.getHostNameOfIP(IPV6_SAMPLE_WITH_PORT));
+ assertNotNull(NetUtils.getHostNameOfIP(IPV6_LOOPBACK_WITH_PORT));
+ }
+
+ @Test
public void testTrimCreateSocketAddress() {
Configuration conf = new Configuration();
NetUtils.addStaticResolution("host", "127.0.0.1");
@@ -668,6 +682,17 @@ public class TestNetUtils {
assertEquals(defaultAddr.trim(), NetUtils.getHostPortString(addr));
}
+ @Test
+ public void testTrimCreateSocketAddressIPv6() {
+ Configuration conf = new Configuration();
+ NetUtils.addStaticResolution("hostIPv6", IPV6_LOOPBACK_LONG_STRING);
+ final String defaultAddr = "hostIPv6:1 ";
+
+ InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);
+ conf.setSocketAddr("myAddress", addr);
+ assertEquals(defaultAddr.trim(), NetUtils.getHostPortString(addr));
+ }
+
private <T> void assertBetterArrayEquals(T[] expect, T[]got) {
String expectStr = StringUtils.join(expect, ", ");
String gotStr = StringUtils.join(got, ", ");