You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cu...@apache.org on 2006/09/07 22:24:46 UTC
svn commit: r441204 - in /lucene/hadoop/trunk: ./ conf/
src/java/org/apache/hadoop/dfs/ src/java/org/apache/hadoop/mapred/
src/java/org/apache/hadoop/net/
Author: cutting
Date: Thu Sep 7 13:24:45 2006
New Revision: 441204
URL: http://svn.apache.org/viewvc?view=rev&rev=441204
Log:
HADOOP-497. Permit the specification of the network interface and nameserver to be used when determining the local hostname advertised by datanodes and tasktrackers. Contributed by Lorenzo Thione.
Added:
lucene/hadoop/trunk/src/java/org/apache/hadoop/net/
lucene/hadoop/trunk/src/java/org/apache/hadoop/net/DNS.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/net/package.html
Modified:
lucene/hadoop/trunk/CHANGES.txt
lucene/hadoop/trunk/conf/hadoop-default.xml
lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
Modified: lucene/hadoop/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?view=diff&rev=441204&r1=441203&r2=441204
==============================================================================
--- lucene/hadoop/trunk/CHANGES.txt (original)
+++ lucene/hadoop/trunk/CHANGES.txt Thu Sep 7 13:24:45 2006
@@ -143,6 +143,11 @@
connection is closed, conserving resources on both client and
server. (Devaraj Das via cutting)
+36. HADOOP-497. Permit the specification of the network interface and
+ nameserver to be used when determining the local hostname
+ advertised by datanodes and tasktrackers.
+ (Lorenzo Thione via cutting)
+
Release 0.5.0 - 2006-08-04
Modified: lucene/hadoop/trunk/conf/hadoop-default.xml
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/conf/hadoop-default.xml?view=diff&rev=441204&r1=441203&r2=441204
==============================================================================
--- lucene/hadoop/trunk/conf/hadoop-default.xml (original)
+++ lucene/hadoop/trunk/conf/hadoop-default.xml Thu Sep 7 13:24:45 2006
@@ -111,6 +111,23 @@
</description>
</property>
+ <property>
+ <name>dfs.datanode.dns.interface</name>
+ <value>default</value>
+ <description>The name of the Network Interface from which a data node should
+ report its IP address.
+ </description>
+ </property>
+
+<property>
+ <name>dfs.datanode.dns.nameserver</name>
+ <value>default</value>
+ <description>The host name or IP address of the name server (DNS)
+ which a DataNode should use to determine the host name used by the
+ NameNode for communication and display purposes.
+ </description>
+ </property>
+
<property>
<name>dfs.default.chunk.view.size</name>
<value>2048</value>
@@ -372,6 +389,24 @@
</description>
</property>
+
+<property>
+ <name>mapred.tasktracker.dns.interface</name>
+ <value>default</value>
+ <description>The name of the Network Interface from which a task
+ tracker should report its IP address.
+ </description>
+ </property>
+
+<property>
+ <name>mapred.tasktracker.dns.nameserver</name>
+ <value>default</value>
+ <description>The host name or IP address of the name server (DNS)
+ which a TaskTracker should use to determine the host name used by
+ the JobTracker for communication and display purposes.
+ </description>
+ </property>
+
<property>
<name>tasktracker.http.threads</name>
<value>40</value>
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java?view=diff&rev=441204&r1=441203&r2=441204
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/dfs/DataNode.java Thu Sep 7 13:24:45 2006
@@ -20,6 +20,7 @@
import org.apache.hadoop.ipc.*;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.metrics.Metrics;
+import org.apache.hadoop.net.DNS;
import org.apache.hadoop.util.*;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException;
@@ -182,6 +183,13 @@
for (int idx = 0; idx < dataDirs.length; idx++) {
volumes[idx] = new File(dataDirs[idx]);
}
+
+ // use configured nameserver & interface to get local hostname
+ machineName =
+ DNS.getDefaultHost
+ (conf.get("dfs.datanode.dns.interface","default"),
+ conf.get("dfs.datanode.dns.nameserver","default"));
+
// get storage info and lock the data dirs
storage = new DataStorage( volumes );
int numDirs = storage.getNumLocked();
Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java?view=diff&rev=441204&r1=441203&r2=441204
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java (original)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/mapred/TaskTracker.java Thu Sep 7 13:24:45 2006
@@ -31,6 +31,7 @@
import org.apache.hadoop.metrics.ContextFactory;
import org.apache.hadoop.metrics.MetricsContext;
import org.apache.hadoop.metrics.MetricsRecord;
+import org.apache.hadoop.net.DNS;
/*******************************************************
* TaskTracker is a process that starts and tracks MR Tasks
@@ -153,8 +154,12 @@
* close().
*/
synchronized void initialize() throws IOException {
- this.localHostname = InetAddress.getLocalHost().getHostName();
-
+ // use configured nameserver & interface to get local hostname
+ this.localHostname =
+ DNS.getDefaultHost
+ (fConf.get("mapred.tasktracker.dns.interface","default"),
+ fConf.get("mapred.tasktracker.dns.nameserver","default"));
+
//check local disk
checkLocalDirs(this.fConf.getLocalDirs());
fConf.deleteLocalFiles(SUBDIR);
Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/net/DNS.java
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/net/DNS.java?view=auto&rev=441204
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/net/DNS.java (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/net/DNS.java Thu Sep 7 13:24:45 2006
@@ -0,0 +1,194 @@
+package org.apache.hadoop.net;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.net.SocketException;
+import java.net.UnknownHostException;
+import java.util.Enumeration;
+import java.util.Vector;
+
+import javax.naming.NamingException;
+import javax.naming.directory.Attributes;
+import javax.naming.directory.DirContext;
+import javax.naming.directory.InitialDirContext;
+
+/**
+ *
+ * A class that provides direct and reverse lookup functionalities, allowing
+ * the querying of specific network interfaces or nameservers.
+ *
+ * @author Lorenzo Thione
+ *
+ */
+public class DNS {
+
+ /**
+ * Returns the hostname associated with the specified IP address by the
+ * provided nameserver.
+ *
+ * @param hostIp
+ * The address to reverse lookup
+ * @param ns
+ * The host name of a reachable DNS server
+ * @return The host name associated with the provided IP
+ * @throws NamingException
+ * If a NamingException is encountered
+ */
+ public static String reverseDns(InetAddress hostIp, String ns)
+ throws NamingException {
+ //
+ // Builds the reverse IP lookup form
+ // This is formed by reversing the IP numbers and appending in-addr.arpa
+ //
+ String[] parts = hostIp.getHostAddress().split("\\.");
+ String reverseIP = parts[3] + "." + parts[2] + "." + parts[1] + "."
+ + parts[0] + ".in-addr.arpa";
+
+ DirContext ictx = new InitialDirContext();
+ Attributes attribute =
+ ictx.getAttributes("dns://" // Use "dns:///" if the default
+ + ((ns == null) ? "" : ns) +
+ // nameserver is to be used
+ "/" + reverseIP, new String[] { "PTR" });
+ ictx.close();
+
+ return attribute.get("PTR").get().toString();
+ }
+
+ /**
+ * Returns all the IPs associated with the provided interface, if any, in
+ * textual form.
+ *
+ * @param strInterface
+ * The name of the network interface to query (e.g. eth0)
+ * @return A string vector of all the IPs associated with the provided
+ * interface
+ * @throws UnknownHostException
+ * If an UnknownHostException is encountered in querying the
+ * default interface
+ *
+ */
+ public static String[] getIPs(String strInterface)
+ throws UnknownHostException {
+ try {
+ NetworkInterface netIF = NetworkInterface.getByName(strInterface);
+ if (netIF == null)
+ return new String[] { InetAddress.getLocalHost()
+ .getHostAddress() };
+ else {
+ Vector ips = new Vector();
+ Enumeration e = netIF.getInetAddresses();
+ while (e.hasMoreElements())
+ ips.add(((InetAddress) e.nextElement()).getHostAddress());
+ return (String[]) ips.toArray(new String[] {});
+ }
+ } catch (SocketException e) {
+ return new String[] { InetAddress.getLocalHost().getHostAddress() };
+ }
+ }
+
+ /**
+ * Returns the first available IP address associated with the provided
+ * network interface
+ *
+ * @param strInterface
+ * The name of the network interface to query (e.g. eth0)
+ * @return The IP address in text form
+ * @throws UnknownHostException
+ * If one is encountered in querying the default interface
+ */
+ public static String getDefaultIP(String strInterface)
+ throws UnknownHostException {
+ String[] ips = getIPs(strInterface);
+ return ips[0];
+ }
+
+ /**
+ * Returns all the host names associated by the provided nameserver with the
+ * address bound to the specified network interface
+ *
+ * @param strInterface
+ * The name of the network interface to query (e.g. eth0)
+ * @param nameserver
+ * The DNS host name
+ * @return A string vector of all host names associated with the IPs tied to
+ * the specified interface
+ * @throws UnknownHostException
+ */
+ public static String[] getHosts(String strInterface, String nameserver)
+ throws UnknownHostException {
+ String[] ips = getIPs(strInterface);
+ Vector hosts = new Vector();
+ for (int ctr = 0; ctr < ips.length; ctr++)
+ try {
+ hosts.add(reverseDns(InetAddress.getByName(ips[ctr]),
+ nameserver));
+ } catch (Exception e) {
+ }
+
+ if (hosts.size() == 0)
+ return new String[] { InetAddress.getLocalHost().getHostName() };
+ else
+ return (String[]) hosts.toArray(new String[] {});
+ }
+
+ /**
+ * Returns all the host names associated by the default nameserver with the
+ * address bound to the specified network interface
+ *
+ * @param strInterface
+ * The name of the network interface to query (e.g. eth0)
+ * @return The list of host names associated with IPs bound to the network
+ * interface
+ * @throws UnknownHostException
+ * If one is encountered while querying the deault interface
+ *
+ */
+ public static String[] getHosts(String strInterface)
+ throws UnknownHostException {
+ return getHosts(strInterface, null);
+ }
+
+ /**
+ * Returns the default (first) host name associated by the provided
+ * nameserver with the address bound to the specified network interface
+ *
+ * @param strInterface
+ * The name of the network interface to query (e.g. eth0)
+ * @param nameserver
+ * The DNS host name
+ * @return The default host names associated with IPs bound to the network
+ * interface
+ * @throws UnknownHostException
+ * If one is encountered while querying the deault interface
+ */
+ public static String getDefaultHost(String strInterface, String nameserver)
+ throws UnknownHostException {
+ if (strInterface.equals("default"))
+ return InetAddress.getLocalHost().getHostName();
+
+ if (nameserver.equals("default"))
+ return getDefaultHost(strInterface);
+
+ String[] hosts = getHosts(strInterface, nameserver);
+ return hosts[0];
+ }
+
+ /**
+ * Returns the default (first) host name associated by the default
+ * nameserver with the address bound to the specified network interface
+ *
+ * @param strInterface
+ * The name of the network interface to query (e.g. eth0)
+ * @return The default host name associated with IPs bound to the network
+ * interface
+ * @throws UnknownHostException
+ * If one is encountered while querying the deault interface
+ */
+ public static String getDefaultHost(String strInterface)
+ throws UnknownHostException {
+ return getDefaultHost(strInterface, null);
+ }
+
+}
Added: lucene/hadoop/trunk/src/java/org/apache/hadoop/net/package.html
URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/net/package.html?view=auto&rev=441204
==============================================================================
--- lucene/hadoop/trunk/src/java/org/apache/hadoop/net/package.html (added)
+++ lucene/hadoop/trunk/src/java/org/apache/hadoop/net/package.html Thu Sep 7 13:24:45 2006
@@ -0,0 +1,5 @@
+<html>
+<body>
+Network-related classes.
+</body>
+</html>