You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by cn...@apache.org on 2013/08/06 07:01:52 UTC
svn commit: r1510855 - in /hadoop/common/branches/branch-1: ./
src/core/org/apache/hadoop/util/ src/hdfs/ src/hdfs/org/apache/hadoop/hdfs/
src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/
src/hdfs/org/apache/hadoop/hdfs/server/namenode/ src/h...
Author: cnauroth
Date: Tue Aug 6 05:01:51 2013
New Revision: 1510855
URL: http://svn.apache.org/r1510855
Log:
HDFS-4963. Improve multihoming support in namenode. Contributed by Arpit Agarwal.
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java
hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java
hadoop/common/branches/branch-1/src/webapps/hdfs/corrupt_files.jsp
hadoop/common/branches/branch-1/src/webapps/hdfs/dfshealth.jsp
hadoop/common/branches/branch-1/src/webapps/hdfs/dfsnodelist.jsp
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Aug 6 05:01:51 2013
@@ -31,6 +31,9 @@ Release 1.3.0 - unreleased
MAPREDUCE-5408. Backport MAPREDUCE-336 to branch-1. (acmurthy)
+ HDFS-4963. Improve multihoming support in namenode. (Arpit Agarwal via
+ cnauroth)
+
BUG FIXES
MAPREDUCE-5047. keep.failed.task.files=true causes job failure on
Modified: hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java (original)
+++ hadoop/common/branches/branch-1/src/core/org/apache/hadoop/util/StringUtils.java Tue Aug 6 05:01:51 2013
@@ -539,7 +539,20 @@ public class StringUtils {
*/
public static void startupShutdownMessage(Class<?> clazz, String[] args,
final org.apache.commons.logging.Log LOG) {
- final String hostname = getHostname();
+ startupShutdownMessage(clazz, getHostname(), args, LOG);
+ }
+
+ /**
+ * Print a log message for starting up and shutting down
+ * @param clazz the class of the server
+ * @param hostname the hostname that the server is listening on.
+ * @param args arguments
+ * @param LOG the target log object
+ */
+ public static void startupShutdownMessage(Class<?> clazz,
+ final String hostname,
+ String[] args,
+ final org.apache.commons.logging.Log LOG) {
final String classname = clazz.getSimpleName();
LOG.info(
toStartupShutdownString("STARTUP_MSG: ", new String[] {
Modified: hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml (original)
+++ hadoop/common/branches/branch-1/src/hdfs/hdfs-default.xml Tue Aug 6 05:01:51 2013
@@ -439,6 +439,15 @@ creations/deletions), or "all".</descrip
</property>
<property>
+ <name>dfs.namenode.master.name</name>
+ <description>
+ The hostname that the NameNode will bind to. This setting should be used
+ when the NameNode is multihomed to force the NameNode to bind to a
+ specific hostname or IP address.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.delegation.key.update-interval</name>
<value>86400000</value>
<description>The update interval for master key for delegation tokens
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSConfigKeys.java Tue Aug 6 05:01:51 2013
@@ -122,6 +122,7 @@ public class DFSConfigKeys extends Commo
public static final boolean DFS_DATANODE_USE_DN_HOSTNAME_DEFAULT = false;
public static final String DFS_DATANODE_SYNCONCLOSE_KEY = "dfs.datanode.synconclose";
public static final boolean DFS_DATANODE_SYNCONCLOSE_DEFAULT = false;
+ public static final String DFS_NAMENODE_MASTER_NAME = "dfs.namenode.master.name";
//Delegation token related keys
public static final String DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_KEY = "dfs.namenode.delegation.key.update-interval";
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DFSUtil.java Tue Aug 6 05:01:51 2013
@@ -116,7 +116,24 @@ public class DFSUtil {
return blkLocations;
}
- /** Create a URI from the scheme and address */
+ /** Create a URI from the scheme, host and port */
+ public static URI createUri(String scheme,
+ String hostName,
+ int port) {
+ try {
+ return new URI(scheme, null, hostName, port, null, null, null);
+ } catch (URISyntaxException ue) {
+ throw new IllegalArgumentException(ue);
+ }
+ }
+
+ /** Create a URI from the scheme and address.
+ *
+ * Marked as deprecated because querying the hostname from the socket
+ * address does not work reliably when dealing with multi-homed hosts.
+ * Prefer using the hostname explicitly instead (when available).
+ */
+ @Deprecated
public static URI createUri(String scheme, InetSocketAddress address) {
try {
return new URI(scheme, null, address.getHostName(), address.getPort(),
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/DistributedFileSystem.java Tue Aug 6 05:01:51 2013
@@ -79,7 +79,7 @@ public class DistributedFileSystem exten
/** @deprecated */
public DistributedFileSystem(InetSocketAddress namenode,
Configuration conf) throws IOException {
- initialize(NameNode.getUri(namenode), conf);
+ initialize(NameNode.getUri(conf, namenode), conf);
}
/** @deprecated */
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java Tue Aug 6 05:01:51 2013
@@ -48,6 +48,7 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
import org.apache.hadoop.hdfs.server.namenode.JspHelper;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.StreamFile;
import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher;
import org.apache.hadoop.io.Text;
@@ -84,6 +85,7 @@ public class HftpFileSystem extends File
}
public static final Text TOKEN_KIND = new Text("HFTP delegation");
+ private String masterHostName = null;
protected UserGroupInformation ugi;
private boolean remoteIsInsecure = false;
@@ -136,7 +138,17 @@ public class HftpFileSystem extends File
super.initialize(name, conf);
this.ugi = UserGroupInformation.getCurrentUser();
this.nnAddr = getNamenodeAddr(name);
- this.hftpURI = DFSUtil.createUri(name.getScheme(), nnAddr);
+ this.masterHostName = conf.get(DFSConfigKeys.DFS_NAMENODE_MASTER_NAME);
+ this.hftpURI = DFSUtil.createUri(
+ name.getScheme(), getMasterHostName(), nnAddr.getPort());
+ }
+
+ String getMasterHostName() {
+ if (masterHostName == null || masterHostName.isEmpty()) {
+ return nnAddr.getHostName();
+ } else {
+ return masterHostName;
+ }
}
protected void initDelegationToken() throws IOException {
@@ -281,7 +293,7 @@ public class HftpFileSystem extends File
try {
query = updateQuery(query);
final URL url = new URI(getUnderlyingProtocol(), null,
- nnAddr.getHostName(),
+ getMasterHostName(),
nnAddr.getPort(), path, query, null).toURL();
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
@@ -680,6 +692,7 @@ public class HftpFileSystem extends File
InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
return DelegationTokenFetcher.
renewDelegationToken(getUnderlyingProtocol(),
+ NameNode.getNamenodeHostName(conf, serviceAddr),
serviceAddr,
(Token<DelegationTokenIdentifier>) token,
conf);
@@ -694,6 +707,7 @@ public class HftpFileSystem extends File
InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
DelegationTokenFetcher.
cancelDelegationToken(getUnderlyingProtocol(),
+ NameNode.getNamenodeHostName(conf, serviceAddr),
serviceAddr,
(Token<DelegationTokenIdentifier>) token,
conf);
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSelector.java Tue Aug 6 05:01:51 2013
@@ -52,11 +52,12 @@ public class DelegationTokenSelector
int nnRpcPort = NameNode.DEFAULT_PORT;
if (nnServiceName != null) {
- nnRpcPort = NetUtils.createSocketAddr(nnServiceName, nnRpcPort).getPort();
+ nnRpcPort = NetUtils.createSocketAddr(nnServiceName, nnRpcPort).getPort();
}
-
+
+ final String nameNodeHostName = NameNode.getNamenodeHostName(conf, nnAddr);
final Text serviceName = SecurityUtil.buildTokenService(
- NetUtils.makeSocketAddr(nnAddr.getHostName(), nnRpcPort));
+ NetUtils.makeSocketAddr(nameNodeHostName, nnRpcPort));
return INSTANCE.selectToken(serviceName, ugi.getTokens());
}
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Aug 6 05:01:51 2013
@@ -467,9 +467,7 @@ public class FSNamesystem implements FSC
dnsToSwitchMapping.resolve(new ArrayList<String>(hostsReader.getHosts()));
}
- InetSocketAddress socAddr = NameNode.getAddress(conf);
- this.nameNodeHostName = socAddr.getHostName();
-
+ this.nameNodeHostName = NameNode.getNamenodeHostName(conf);
registerWith(DefaultMetricsSystem.INSTANCE);
}
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Tue Aug 6 05:01:51 2013
@@ -22,6 +22,7 @@ import static org.apache.hadoop.hdfs.DFS
import java.io.File;
import java.io.IOException;
+import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
@@ -218,7 +219,7 @@ public class NameNode implements ClientP
*/
public static void setServiceAddress(Configuration conf,
String address) {
- LOG.info("Setting ADDRESS " + address);
+ LOG.info("Setting RPC Service address: " + address);
conf.set(DFSConfigKeys.DFS_NAMENODE_SERVICE_RPC_ADDRESS_KEY, address);
}
@@ -238,18 +239,50 @@ public class NameNode implements ClientP
return getAddress(addr);
}
- public static InetSocketAddress getAddress(Configuration conf) {
+ private static String getAddressString(Configuration conf) {
String addr = conf.get(DFSConfigKeys.DFS_NAMENODE_RPC_ADDRESS_KEY);
if (addr == null || addr.isEmpty()) {
- return getAddress(FileSystem.getDefaultUri(conf).toString());
+ addr = FileSystem.getDefaultUri(conf).toString();
}
- return getAddress(addr);
+ return addr;
+ }
+
+ public static InetSocketAddress getAddress(Configuration conf) {
+ return getAddress(getAddressString(conf));
}
- public static URI getUri(InetSocketAddress namenode) {
+ /**
+ * If DFS_NAMENODE_MASTER_NAME is set then use that else query the
+ * hostname from the socket address. The latter behavior is legacy but
+ * wrong since it can behave unexpectedly when the namenode is multi-
+ * homed.
+ *
+ * @param conf configuration database
+ * @param address socket address for the local endpoint.
+ * @return The hostname to connect to the namenode.
+ */
+ public static String getNamenodeHostName(Configuration conf,
+ InetSocketAddress address) {
+ String masterHostName =
+ conf.get(DFSConfigKeys.DFS_NAMENODE_MASTER_NAME);
+
+ if (masterHostName == null || masterHostName.isEmpty()) {
+ return address.getHostName();
+ } else {
+ return masterHostName;
+ }
+ }
+
+ public static String getNamenodeHostName(Configuration conf) {
+ return getNamenodeHostName(conf, getAddress(conf));
+ }
+
+ public static URI getUri(Configuration conf,
+ InetSocketAddress namenode) {
int port = namenode.getPort();
- String portString = port == DEFAULT_PORT ? "" : (":"+port);
- return URI.create("hdfs://"+ namenode.getHostName()+portString);
+ String portString = port == DEFAULT_PORT ? "" : (":" + port);
+ String hostname = getNamenodeHostName(conf, namenode);
+ return URI.create("hdfs://"+ hostname + portString);
}
/**
@@ -266,8 +299,8 @@ public class NameNode implements ClientP
* Modifies the configuration passed to contain the service rpc address setting
*/
protected void setRpcServiceServerAddress(Configuration conf) {
- String address = serviceRPCAddress.getHostName() + ":"
- + serviceRPCAddress.getPort();
+ String hostname = getNamenodeHostName(conf, serviceRPCAddress);
+ String address = hostname + ":" + serviceRPCAddress.getPort();
setServiceAddress(conf, address);
}
@@ -279,8 +312,10 @@ public class NameNode implements ClientP
private void initialize(Configuration conf) throws IOException {
InetSocketAddress socAddr = NameNode.getAddress(conf);
UserGroupInformation.setConfiguration(conf);
- SecurityUtil.login(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, socAddr.getHostName());
+ SecurityUtil.login(conf,
+ DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY,
+ getNamenodeHostName(conf, socAddr));
int handlerCount = conf.getInt("dfs.namenode.handler.count", 10);
// set service-level authorization security policy
@@ -313,26 +348,29 @@ public class NameNode implements ClientP
InetSocketAddress dnSocketAddr = getServiceRpcServerAddress(conf);
if (dnSocketAddr != null) {
int serviceHandlerCount =
- conf.getInt(DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
- DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
- this.serviceRpcServer = RPC.getServer(this, dnSocketAddr.getHostName(),
- dnSocketAddr.getPort(), serviceHandlerCount,
- false, conf, namesystem.getDelegationTokenSecretManager());
+ conf.getInt(DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
+ DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
+ this.serviceRpcServer = RPC.getServer(this,
+ getNamenodeHostName(conf, dnSocketAddr), dnSocketAddr.getPort(),
+ serviceHandlerCount, false, conf,
+ namesystem.getDelegationTokenSecretManager());
this.serviceRPCAddress = this.serviceRpcServer.getListenerAddress();
setRpcServiceServerAddress(conf);
}
- this.server = RPC.getServer(this, socAddr.getHostName(),
+ this.server = RPC.getServer(this, getNamenodeHostName(conf, socAddr),
socAddr.getPort(), handlerCount, false, conf, namesystem
.getDelegationTokenSecretManager());
// Set terse exception whose stack trace won't be logged
this.server.addTerseExceptions(SafeModeException.class);
-
+
// The rpc-server port can be ephemeral... ensure we have the correct info
- this.serverAddress = this.server.getListenerAddress();
- FileSystem.setDefaultUri(conf, getUri(serverAddress));
- LOG.info("Namenode up at: " + this.serverAddress);
+ // The ephemeral port is primarily used when testing.
+ this.serverAddress = this.server.getListenerAddress();
+ URI serverUri = getUri(conf, serverAddress);
+ FileSystem.setDefaultUri(conf, serverUri);
+ LOG.info("Namenode up at: " + serverUri);
+
-
startHttpServer(conf);
this.server.start(); //start RPC server
@@ -380,14 +418,14 @@ public class NameNode implements ClientP
final InetSocketAddress infoSocAddr = NetUtils.createSocketAddr(infoAddr);
if (SecurityUtil.useKsslAuth()) {
- String httpsUser = SecurityUtil.getServerPrincipal(conf
- .get(DFSConfigKeys.DFS_NAMENODE_KRB_HTTPS_USER_NAME_KEY), infoSocAddr
- .getHostName());
+ String httpsUser = SecurityUtil.getServerPrincipal(
+ conf.get(DFSConfigKeys.DFS_NAMENODE_KRB_HTTPS_USER_NAME_KEY),
+ getNamenodeHostName(conf, infoSocAddr));
// Kerberized SSL servers must be run from the host principal...
LOG.info("Logging in as " + httpsUser + " to start http server.");
SecurityUtil.login(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
- DFSConfigKeys.DFS_NAMENODE_KRB_HTTPS_USER_NAME_KEY, infoSocAddr
- .getHostName());
+ DFSConfigKeys.DFS_NAMENODE_KRB_HTTPS_USER_NAME_KEY,
+ getNamenodeHostName(conf, infoSocAddr));
}
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
@@ -395,7 +433,7 @@ public class NameNode implements ClientP
this.httpServer = ugi.doAs(new PrivilegedExceptionAction<HttpServer>() {
@Override
public HttpServer run() throws IOException, InterruptedException {
- String infoHost = infoSocAddr.getHostName();
+ String infoHost = getNamenodeHostName(conf, infoSocAddr);
int infoPort = infoSocAddr.getPort();
httpServer = new HttpServer("hdfs", infoHost, infoPort,
infoPort == 0, conf,
@@ -409,8 +447,9 @@ public class NameNode implements ClientP
DFSConfigKeys.DFS_NAMENODE_INTERNAL_SPENGO_USER_NAME_KEY);
if (principalInConf != null && !principalInConf.isEmpty()) {
params.put("kerberos.principal",
- SecurityUtil.getServerPrincipal(principalInConf,
- serverAddress.getHostName()));
+ SecurityUtil.getServerPrincipal(
+ principalInConf,
+ getNamenodeHostName(conf, serverAddress)));
}
String httpKeytab = conf.get(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
@@ -455,7 +494,7 @@ public class NameNode implements ClientP
.put(
DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY,
SecurityUtil.getServerPrincipal(principalInConf,
- serverAddress.getHostName()));
+ getNamenodeHostName(conf, serverAddress)));
}
String httpKeytab = conf
.get(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY);
@@ -529,13 +568,15 @@ public class NameNode implements ClientP
} finally {
if (SecurityUtil.useKsslAuth()) {
// Go back to being the correct Namenode principal
- LOG.info("Logging back in as "
- + SecurityUtil.getServerPrincipal(conf
- .get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY), serverAddress
- .getHostName()) + " following http server start.");
- SecurityUtil.login(conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
- DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, serverAddress
- .getHostName());
+ LOG.info("Logging back in as " +
+ SecurityUtil.getServerPrincipal(
+ conf.get(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY),
+ getNamenodeHostName(conf, serverAddress)) +
+ " following http server start.");
+ SecurityUtil.login(
+ conf, DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY,
+ DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY,
+ getNamenodeHostName(conf, serverAddress));
}
}
}
@@ -921,7 +962,7 @@ public class NameNode implements ClientP
@Override
public DirectoryListing getListing(String src, byte[] startAfter)
- throws IOException {
+ throws IOException {
DirectoryListing files = namesystem.getListing(src, startAfter);
myMetrics.incrNumGetListingOps();
if (files != null) {
@@ -1481,10 +1522,32 @@ public class NameNode implements ClientP
}
/**
+ * Get the hostname that the namenode is listening on.
+ *
+ * @return
+ */
+ private static String getHostname(Configuration conf) throws Exception {
+ if (conf == null) {
+ conf = new Configuration();
+ }
+
+ String masterHostName =
+ conf.get(DFSConfigKeys.DFS_NAMENODE_MASTER_NAME);
+
+ if (masterHostName == null || masterHostName.isEmpty()) {
+ return InetAddress.getLocalHost().toString();
+ } else {
+ InetAddress address = InetAddress.getByName(masterHostName);
+ return address.toString();
+ }
+ }
+
+ /**
*/
public static void main(String argv[]) throws Exception {
try {
- StringUtils.startupShutdownMessage(NameNode.class, argv, LOG);
+ StringUtils.startupShutdownMessage(
+ NameNode.class, getHostname(null), argv, LOG);
NameNode namenode = createNameNode(argv, null);
if (namenode != null)
namenode.join();
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Tue Aug 6 05:01:51 2013
@@ -200,7 +200,32 @@ public class DelegationTokenFetcher {
String renewer,
Configuration conf
) throws IOException {
- final String renewAddress = getRenewAddress(protocol, nnAddr, conf);
+ return getDTfromRemote(
+ protocol,
+ nnAddr.getHostName(),
+ nnAddr,
+ renewer,
+ conf);
+ }
+
+ /**
+ * Utility method to obtain a delegation token over http
+ * @param protocol whether to use http or https
+ * @param hostname the NameNode's hostname. This may be different from the
+ * hostname looked up via in the address if the namenode is
+ * multihomed.
+ * @param nnAddr the address for the NameNode
+ * @param renewer User that is renewing the ticket in such a request
+ * @param conf the configuration
+ */
+ static public Credentials getDTfromRemote(String protocol,
+ String hostname,
+ final InetSocketAddress nnAddr,
+ String renewer,
+ Configuration conf
+ ) throws IOException {
+ final String renewAddress =
+ getRenewAddress(protocol, hostname, nnAddr, conf);
final boolean https = "https".equals(protocol);
try {
@@ -252,6 +277,7 @@ public class DelegationTokenFetcher {
* https and the NN's https port.
*/
protected static String getRenewAddress(String protocol,
+ String hostname,
InetSocketAddress addr,
Configuration conf) {
if (SecurityUtil.useKsslAuth() && "http".equals(protocol)) {
@@ -261,12 +287,15 @@ public class DelegationTokenFetcher {
DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT);
addr = new InetSocketAddress(addr.getAddress(), port);
}
- return DFSUtil.createUri(protocol, addr).toString();
+ return DFSUtil.createUri(protocol, hostname, addr.getPort()).toString();
}
/**
* Renew a Delegation Token.
* @param protocol The protocol to renew over (http or https)
+ * @param hostname the NameNode's hostname. This may be different from the
+ * hostname looked up via in the address if the namenode is
+ * multihomed.
* @param addr the address of the NameNode
* @param tok the token to renew
* @param conf the configuration
@@ -274,11 +303,13 @@ public class DelegationTokenFetcher {
* @throws IOException
*/
static public long renewDelegationToken(String protocol,
+ String hostname,
InetSocketAddress addr,
Token<DelegationTokenIdentifier> tok,
Configuration conf
) throws IOException {
- final String renewAddress = getRenewAddress(protocol, addr, conf);
+ final String renewAddress =
+ getRenewAddress(protocol, hostname, addr, conf);
final StringBuilder buf = new StringBuilder(renewAddress);
final String service = tok.getService().toString();
buf.append(RenewDelegationTokenServlet.PATH_SPEC);
@@ -353,16 +384,20 @@ public class DelegationTokenFetcher {
/**
* Cancel a Delegation Token.
- * @param nnAddr the NameNode's address
+ * @param hostname the NameNode's hostname. This may be different from the
+ * hostname looked up via in the address if the namenode is
+ * multihomed.
+ * @param addr the NameNode's address
* @param tok the token to cancel
* @throws IOException
*/
static public void cancelDelegationToken(String protocol,
+ String hostname,
InetSocketAddress addr,
Token<DelegationTokenIdentifier> tok,
Configuration conf
) throws IOException {
- final String renewAddress = getRenewAddress(protocol, addr, conf);
+ final String renewAddress = getRenewAddress(protocol, hostname, addr, conf);
StringBuilder buf = new StringBuilder(renewAddress);
buf.append(CancelDelegationTokenServlet.PATH_SPEC);
buf.append("?");
Modified: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Tue Aug 6 05:01:51 2013
@@ -57,6 +57,7 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenRenewer;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
import org.apache.hadoop.hdfs.server.namenode.JspHelper;
+import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.web.resources.AccessTimeParam;
import org.apache.hadoop.hdfs.web.resources.BlockSizeParam;
@@ -144,6 +145,7 @@ public class WebHdfsFileSystem extends F
private final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
private RetryPolicy retryPolicy = null;
private Path workingDir;
+ private String masterHostName = null;
{
try {
@@ -172,11 +174,21 @@ public class WebHdfsFileSystem extends F
);
this.workingDir = getHomeDirectory();
+ masterHostName = conf.get(DFSConfigKeys.DFS_NAMENODE_MASTER_NAME);
+
if (UserGroupInformation.isSecurityEnabled()) {
initDelegationToken();
}
}
+ String getMasterHostName() {
+ if (masterHostName == null || masterHostName.isEmpty()) {
+ return nnAddr.getHostName();
+ } else {
+ return masterHostName;
+ }
+ }
+
protected void initDelegationToken() throws IOException {
// look for webhdfs token, then try hdfs
final Text serviceName = SecurityUtil.buildTokenService(nnAddr);
@@ -214,7 +226,7 @@ public class WebHdfsFileSystem extends F
@Override
public URI getUri() {
try {
- return new URI(SCHEME, null, nnAddr.getHostName(), nnAddr.getPort(),
+ return new URI(SCHEME, null, getMasterHostName(), nnAddr.getPort(),
null, null, null);
} catch (URISyntaxException e) {
return null;
@@ -337,7 +349,7 @@ public class WebHdfsFileSystem extends F
* @throws IOException on error constructing the URL
*/
private URL getNamenodeURL(String path, String query) throws IOException {
- final URL url = new URL("http", nnAddr.getHostName(),
+ final URL url = new URL("http", getMasterHostName(),
nnAddr.getPort(), path + '?' + query);
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
@@ -903,7 +915,10 @@ public class WebHdfsFileSystem extends F
final Token<?> token, final Configuration conf) throws IOException {
final InetSocketAddress nnAddr = SecurityUtil.getTokenServiceAddr(token);
- final URI uri = DFSUtil.createUri(WebHdfsFileSystem.SCHEME, nnAddr);
+ final URI uri = DFSUtil.createUri(
+ WebHdfsFileSystem.SCHEME,
+ NameNode.getNamenodeHostName(conf, nnAddr),
+ nnAddr.getPort());
return (WebHdfsFileSystem)FileSystem.get(uri, conf);
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSCluster.java Tue Aug 6 05:01:51 2013
@@ -347,7 +347,8 @@ public class MiniDFSCluster {
InetSocketAddress nnAddr = nameNode.getNameNodeAddress();
int nameNodePort = nnAddr.getPort();
FileSystem.setDefaultUri(conf,
- "hdfs://"+ nnAddr.getHostName() +
+ "hdfs://" +
+ nameNode.getNamenodeHostName(conf, nnAddr) +
":" + Integer.toString(nameNodePort));
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java Tue Aug 6 05:01:51 2013
@@ -83,7 +83,8 @@ public class MiniDFSClusterWithNodeGroup
InetSocketAddress nnAddr = nameNode.getNameNodeAddress();
int nameNodePort = nnAddr.getPort();
FileSystem.setDefaultUri(conf,
- "hdfs://"+ nnAddr.getHostName() +
+ "hdfs://" +
+ nameNode.getNamenodeHostName(conf, nnAddr) +
":" + Integer.toString(nameNodePort));
}
if (racks != null && numDataNodes > racks.length ) {
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/TestDefaultNameNodePort.java Tue Aug 6 05:01:51 2013
@@ -59,9 +59,10 @@ public class TestDefaultNameNodePort {
}
public void testGetUri() {
- assertEquals(NameNode.getUri(new InetSocketAddress("foo", 555)),
+ Configuration conf = new Configuration();
+ assertEquals(NameNode.getUri(conf, new InetSocketAddress("foo", 555)),
URI.create("hdfs://foo:555"));
- assertEquals(NameNode.getUri(new InetSocketAddress("foo",
+ assertEquals(NameNode.getUri(conf, new InetSocketAddress("foo",
NameNode.DEFAULT_PORT)),
URI.create("hdfs://foo"));
}
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapred/TestMiniMRWithDFS.java Tue Aug 6 05:01:51 2013
@@ -336,7 +336,7 @@ public class TestMiniMRWithDFS extends T
TestResult result;
final Path inDir = new Path("./wc/input");
final Path outDir = new Path("hdfs://" +
- dfs.getNameNode().getNameNodeAddress().getHostName() +
+ dfs.getNameNode().getNamenodeHostName(conf) +
":" + NameNode.DEFAULT_PORT +"/./wc/output");
String input = "The quick brown fox\nhas many silly\nred fox sox\n";
result = launchWordCount(jobConf, inDir, outDir, input, 3, 1);
Modified: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java (original)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/mapreduce/security/TestTokenCache.java Tue Aug 6 05:01:51 2013
@@ -129,6 +129,7 @@ public class TestTokenCache {
}
private static MiniMRCluster mrCluster;
+ private static Configuration conf;
private static MiniDFSCluster dfsCluster;
private static final Path TEST_DIR =
new Path(System.getProperty("test.build.data","/tmp"), "sleepTest");
@@ -141,7 +142,7 @@ public class TestTokenCache {
@BeforeClass
public static void setUp() throws Exception {
- Configuration conf = new Configuration();
+ conf = new Configuration();
dfsCluster = new MiniDFSCluster(conf, numSlaves, true, null);
jConf = new JobConf(conf);
mrCluster = new MiniMRCluster(0, 0, numSlaves,
@@ -215,7 +216,7 @@ public class TestTokenCache {
// provide namenodes names for the job to get the delegation tokens for
//String nnUri = dfsCluster.getNameNode().getUri(namenode).toString();
NameNode nn = dfsCluster.getNameNode();
- URI nnUri = NameNode.getUri(nn.getNameNodeAddress());
+ URI nnUri = NameNode.getUri(conf, nn.getNameNodeAddress());
jConf.set(JobContext.JOB_NAMENODES, nnUri + "," + nnUri.toString());
// job tracker principle id..
jConf.set(JobTracker.JT_USER_NAME, "jt_id");
Modified: hadoop/common/branches/branch-1/src/webapps/hdfs/corrupt_files.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/webapps/hdfs/corrupt_files.jsp?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/webapps/hdfs/corrupt_files.jsp (original)
+++ hadoop/common/branches/branch-1/src/webapps/hdfs/corrupt_files.jsp Tue Aug 6 05:01:51 2013
@@ -25,14 +25,17 @@
import="org.apache.hadoop.fs.Path"
import="java.util.Collection"
import="java.util.Arrays" %>
+<%@ page import="org.apache.hadoop.hdfs.server.namenode.NameNode" %>
+<%@ page import="org.apache.hadoop.hdfs.server.namenode.FSNamesystem" %>
+<%@ page import="org.apache.hadoop.hdfs.server.namenode.JspHelper" %>
+<%@ page import="org.apache.hadoop.conf.Configuration" %>
<%!//for java.io.Serializable
private static final long serialVersionUID = 1L;%>
<%
NameNode nn = (NameNode) application.getAttribute("name.node");
FSNamesystem fsn = nn.getNamesystem();
- // String namenodeRole = nn.getRole().toString();
- String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":"
- + nn.getNameNodeAddress().getPort();
+ Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
+ String namenodeLabel = nn.getNamenodeHostName(conf) + ":" + nn.getNameNodeAddress().getPort();
Collection<FSNamesystem.CorruptFileBlockInfo> corruptFileBlocks =
fsn.listCorruptFileBlocks();
int corruptFileCount = corruptFileBlocks.size();
Modified: hadoop/common/branches/branch-1/src/webapps/hdfs/dfshealth.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/webapps/hdfs/dfshealth.jsp?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/webapps/hdfs/dfshealth.jsp (original)
+++ hadoop/common/branches/branch-1/src/webapps/hdfs/dfshealth.jsp Tue Aug 6 05:01:51 2013
@@ -32,6 +32,7 @@
import="java.lang.Math"
import="java.net.URLEncoder"
%>
+<%@ page import="org.apache.hadoop.conf.Configuration" %>
<%!
JspHelper jspHelper = new JspHelper();
@@ -248,9 +249,10 @@
}%>
<%
- NameNode nn = (NameNode)application.getAttribute("name.node");
- FSNamesystem fsn = nn.getNamesystem();
- String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":" + nn.getNameNodeAddress().getPort();
+ NameNode nn = (NameNode)application.getAttribute("name.node");
+ Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
+ FSNamesystem fsn = nn.getNamesystem();
+ String namenodeLabel = nn.getNamenodeHostName(conf) + ":" + nn.getNameNodeAddress().getPort();
%>
<!DOCTYPE html>
Modified: hadoop/common/branches/branch-1/src/webapps/hdfs/dfsnodelist.jsp
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/webapps/hdfs/dfsnodelist.jsp?rev=1510855&r1=1510854&r2=1510855&view=diff
==============================================================================
--- hadoop/common/branches/branch-1/src/webapps/hdfs/dfsnodelist.jsp (original)
+++ hadoop/common/branches/branch-1/src/webapps/hdfs/dfsnodelist.jsp Tue Aug 6 05:01:51 2013
@@ -31,6 +31,7 @@
import="java.lang.Math"
import="java.net.URLEncoder"
%>
+<%@ page import="org.apache.hadoop.conf.Configuration" %>
<%!
JspHelper jspHelper = new JspHelper();
@@ -324,8 +325,9 @@ throws IOException {
<%
NameNode nn = (NameNode)application.getAttribute("name.node");
+Configuration conf = (Configuration) getServletContext().getAttribute(JspHelper.CURRENT_CONF);
FSNamesystem fsn = nn.getNamesystem();
-String namenodeLabel = nn.getNameNodeAddress().getHostName() + ":" + nn.getNameNodeAddress().getPort();
+String namenodeLabel = nn.getNamenodeHostName(conf) + ":" + nn.getNameNodeAddress().getPort();
%>
<!DOCTYPE html>