You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2012/02/02 20:56:14 UTC
svn commit: r1239797 - in
/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/security/token/delegation/
src/main/java/org/apache/hadoop/hdfs/server/common/ ...
Author: jitendra
Date: Thu Feb 2 19:56:13 2012
New Revision: 1239797
URL: http://svn.apache.org/viewvc?rev=1239797&view=rev
Log:
Merged r1239763 from trunk for HDFS-2784.
Added:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/ssl-client.xml
- copied unchanged from r1239763, hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/ssl-client.xml
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Feb 2 19:56:13 2012
@@ -101,6 +101,9 @@ Release 0.23.1 - UNRELEASED
HDFS-2814. NamenodeMXBean does not account for svn revision in the version
information. (Hitesh Shah via jitendra)
+ HDFS-2784. Update hftp and hdfs for host-based token support.
+ (Kihwal Lee via jitendra)
+
OPTIMIZATIONS
HDFS-2130. Switch default checksum to CRC32C. (todd)
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Thu Feb 2 19:56:13 2012
@@ -613,7 +613,7 @@ public class DFSClient implements java.i
DelegationTokenIdentifier.stringifyToken(delToken));
ClientProtocol nn =
DFSUtil.createRPCNamenode
- (NameNode.getAddress(token.getService().toString()),
+ (SecurityUtil.getTokenServiceAddr(delToken),
conf, UserGroupInformation.getCurrentUser());
try {
return nn.renewDelegationToken(delToken);
@@ -631,7 +631,7 @@ public class DFSClient implements java.i
LOG.info("Cancelling " +
DelegationTokenIdentifier.stringifyToken(delToken));
ClientProtocol nn = DFSUtil.createRPCNamenode(
- NameNode.getAddress(token.getService().toString()), conf,
+ SecurityUtil.getTokenServiceAddr(delToken), conf,
UserGroupInformation.getCurrentUser());
try {
nn.cancelDelegationToken(delToken);
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Thu Feb 2 19:56:13 2012
@@ -108,45 +108,10 @@ public class DistributedFileSystem exten
InetSocketAddress namenode = NameNode.getAddress(uri.getAuthority());
this.dfs = new DFSClient(namenode, conf, statistics);
- this.uri = URI.create(HdfsConstants.HDFS_URI_SCHEME + "://" + uri.getAuthority());
+ this.uri = URI.create(uri.getScheme()+"://"+uri.getAuthority());
this.workingDir = getHomeDirectory();
}
- /** Permit paths which explicitly specify the default port. */
- @Override
- protected void checkPath(Path path) {
- URI thisUri = this.getUri();
- URI thatUri = path.toUri();
- String thatAuthority = thatUri.getAuthority();
- if (thatUri.getScheme() != null
- && thatUri.getScheme().equalsIgnoreCase(thisUri.getScheme())
- && thatUri.getPort() == NameNode.DEFAULT_PORT
- && (thisUri.getPort() == -1 ||
- thisUri.getPort() == NameNode.DEFAULT_PORT)
- && thatAuthority.substring(0,thatAuthority.indexOf(":"))
- .equalsIgnoreCase(thisUri.getAuthority()))
- return;
- super.checkPath(path);
- }
-
- /** Normalize paths that explicitly specify the default port. */
- @Override
- public Path makeQualified(Path path) {
- URI thisUri = this.getUri();
- URI thatUri = path.toUri();
- String thatAuthority = thatUri.getAuthority();
- if (thatUri.getScheme() != null
- && thatUri.getScheme().equalsIgnoreCase(thisUri.getScheme())
- && thatUri.getPort() == NameNode.DEFAULT_PORT
- && thisUri.getPort() == -1
- && thatAuthority.substring(0,thatAuthority.indexOf(":"))
- .equalsIgnoreCase(thisUri.getAuthority())) {
- path = new Path(thisUri.getScheme(), thisUri.getAuthority(),
- thatUri.getPath());
- }
- return super.makeQualified(path);
- }
-
@Override
public Path getWorkingDirectory() {
return workingDir;
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Thu Feb 2 19:56:13 2012
@@ -59,6 +59,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.security.token.TokenRenewer;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSelector;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ServletUtil;
import org.xml.sax.Attributes;
@@ -89,17 +90,20 @@ public class HftpFileSystem extends File
public static final Text TOKEN_KIND = new Text("HFTP delegation");
- private String nnHttpUrl;
- private Text hdfsServiceName;
+ protected UserGroupInformation ugi;
private URI hftpURI;
+
protected InetSocketAddress nnAddr;
- protected UserGroupInformation ugi;
+ protected InetSocketAddress nnSecureAddr;
public static final String HFTP_TIMEZONE = "UTC";
public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
+
private Token<?> delegationToken;
private Token<?> renewToken;
-
+ private static final HftpDelegationTokenSelector hftpTokenSelector =
+ new HftpDelegationTokenSelector();
+
public static final SimpleDateFormat getDateFormat() {
final SimpleDateFormat df = new SimpleDateFormat(HFTP_DATE_FORMAT);
df.setTimeZone(TimeZone.getTimeZone(HFTP_TIMEZONE));
@@ -115,11 +119,8 @@ public class HftpFileSystem extends File
@Override
protected int getDefaultPort() {
- return getDefaultSecurePort();
-
- //TODO: un-comment the following once HDFS-7510 is committed.
-// return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
-// DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
+ return getConf().getInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY,
+ DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT);
}
protected int getDefaultSecurePort() {
@@ -127,16 +128,22 @@ public class HftpFileSystem extends File
DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT);
}
- @Override
- public String getCanonicalServiceName() {
- return SecurityUtil.buildDTServiceName(hftpURI, getDefaultPort());
+ protected InetSocketAddress getNamenodeAddr(URI uri) {
+ // use authority so user supplied uri can override port
+ return NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
}
-
- private String buildUri(String schema, String host, int port) {
- StringBuilder sb = new StringBuilder(schema);
- return sb.append(host).append(":").append(port).toString();
+
+ protected InetSocketAddress getNamenodeSecureAddr(URI uri) {
+ // must only use the host and the configured https port
+ return NetUtils.createSocketAddrForHost(uri.getHost(), getDefaultSecurePort());
}
+ @Override
+ public String getCanonicalServiceName() {
+ // unlike other filesystems, hftp's service is the secure port, not the
+ // actual port in the uri
+ return SecurityUtil.buildTokenService(nnSecureAddr).toString();
+ }
@Override
public void initialize(final URI name, final Configuration conf)
@@ -144,95 +151,51 @@ public class HftpFileSystem extends File
super.initialize(name, conf);
setConf(conf);
this.ugi = UserGroupInformation.getCurrentUser();
- nnAddr = NetUtils.createSocketAddr(name.toString());
-
- // in case we open connection to hftp of a different cluster
- // we need to know this cluster https port
- // if it is not set we assume it is the same cluster or same port
- int urlPort = conf.getInt("dfs.hftp.https.port", -1);
- if(urlPort == -1)
- urlPort = conf.getInt(DFSConfigKeys.DFS_HTTPS_PORT_KEY,
- DFSConfigKeys.DFS_HTTPS_PORT_DEFAULT);
-
- String normalizedNN = NetUtils.normalizeHostName(name.getHost());
- nnHttpUrl = buildUri("https://", normalizedNN ,urlPort);
- LOG.debug("using url to get DT:" + nnHttpUrl);
- try {
- hftpURI = new URI(buildUri("hftp://", normalizedNN, urlPort));
- } catch (URISyntaxException ue) {
- throw new IOException("bad uri for hdfs", ue);
- }
-
- // if one uses RPC port different from the Default one,
- // one should specify what is the setvice name for this delegation token
- // otherwise it is hostname:RPC_PORT
- String key = DelegationTokenSelector.SERVICE_NAME_KEY
- + SecurityUtil.buildDTServiceName(name,
- DFSConfigKeys.DFS_HTTPS_PORT_DEFAULT);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Trying to find DT for " + name + " using key=" + key +
- "; conf=" + conf.get(key, ""));
- }
- String nnServiceName = conf.get(key);
- int nnPort = NameNode.DEFAULT_PORT;
- if (nnServiceName != null) { // get the real port
- nnPort = NetUtils.createSocketAddr(nnServiceName,
- NameNode.DEFAULT_PORT).getPort();
- }
+ this.nnAddr = getNamenodeAddr(name);
+ this.nnSecureAddr = getNamenodeSecureAddr(name);
try {
- URI hdfsURI = new URI("hdfs://" + normalizedNN + ":" + nnPort);
- hdfsServiceName = new Text(SecurityUtil.buildDTServiceName(hdfsURI,
- nnPort));
- } catch (URISyntaxException ue) {
- throw new IOException("bad uri for hdfs", ue);
+ this.hftpURI = new URI(name.getScheme(), name.getAuthority(),
+ null, null, null);
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(e);
}
if (UserGroupInformation.isSecurityEnabled()) {
- //try finding a token for this namenode (esp applicable for tasks
- //using hftp). If there exists one, just set the delegationField
- String hftpServiceName = getCanonicalServiceName();
- for (Token<? extends TokenIdentifier> t : ugi.getTokens()) {
- Text kind = t.getKind();
- if (DelegationTokenIdentifier.HDFS_DELEGATION_KIND.equals(kind)) {
- if (t.getService().equals(hdfsServiceName)) {
- setDelegationToken(t);
- break;
- }
- } else if (TOKEN_KIND.equals(kind)) {
- if (hftpServiceName
- .equals(normalizeService(t.getService().toString()))) {
- setDelegationToken(t);
- break;
- }
- }
- }
-
- //since we don't already have a token, go get one over https
- if (delegationToken == null) {
- setDelegationToken(getDelegationToken(null));
+ initDelegationToken();
+ }
+ }
+
+ protected void initDelegationToken() throws IOException {
+ // look for hftp token, then try hdfs
+ Token<?> token = selectHftpDelegationToken();
+ if (token == null) {
+ token = selectHdfsDelegationToken();
+ }
+
+ // if we don't already have a token, go get one over https
+ boolean createdToken = false;
+ if (token == null) {
+ token = getDelegationToken(null);
+ createdToken = (token != null);
+ }
+
+ // we already had a token or getDelegationToken() didn't fail.
+ if (token != null) {
+ setDelegationToken(token);
+ if (createdToken) {
dtRenewer.addRenewAction(this);
+ LOG.debug("Created new DT for " + token.getService());
+ } else {
+ LOG.debug("Found existing DT for " + token.getService());
}
}
}
- private String normalizeService(String service) {
- int colonIndex = service.indexOf(':');
- if (colonIndex == -1) {
- throw new IllegalArgumentException("Invalid service for hftp token: " +
- service);
- }
- String hostname =
- NetUtils.normalizeHostName(service.substring(0, colonIndex));
- String port = service.substring(colonIndex + 1);
- return hostname + ":" + port;
- }
-
- //TODO: un-comment the following once HDFS-7510 is committed.
-// protected Token<DelegationTokenIdentifier> selectHftpDelegationToken() {
-// Text serviceName = SecurityUtil.buildTokenService(nnSecureAddr);
-// return hftpTokenSelector.selectToken(serviceName, ugi.getTokens());
-// }
-
+ protected Token<DelegationTokenIdentifier> selectHftpDelegationToken() {
+ Text serviceName = SecurityUtil.buildTokenService(nnSecureAddr);
+ return hftpTokenSelector.selectToken(serviceName, ugi.getTokens());
+ }
+
protected Token<DelegationTokenIdentifier> selectHdfsDelegationToken() {
return DelegationTokenSelector.selectHdfsDelegationToken(
nnAddr, ugi, getConf());
@@ -245,13 +208,17 @@ public class HftpFileSystem extends File
}
@Override
- public <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
+ public synchronized <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
renewToken = token;
// emulate the 203 usage of the tokens
// by setting the kind and service as if they were hdfs tokens
delegationToken = new Token<T>(token);
+ // NOTE: the remote nn must be configured to use hdfs
delegationToken.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
- delegationToken.setService(hdfsServiceName);
+ // no need to change service because we aren't exactly sure what it
+ // should be. we can guess, but it might be wrong if the local conf
+ // value is incorrect. the service is a client side field, so the remote
+ // end does not care about the value
}
@Override
@@ -262,6 +229,7 @@ public class HftpFileSystem extends File
ugi.reloginFromKeytab();
return ugi.doAs(new PrivilegedExceptionAction<Token<?>>() {
public Token<?> run() throws IOException {
+ final String nnHttpUrl = DFSUtil.createUri("https", nnSecureAddr).toString();
Credentials c;
try {
c = DelegationTokenFetcher.getDTfromRemote(nnHttpUrl, renewer);
@@ -291,12 +259,7 @@ public class HftpFileSystem extends File
@Override
public URI getUri() {
- try {
- return new URI("hftp", null, nnAddr.getHostName(), nnAddr.getPort(),
- null, null, null);
- } catch (URISyntaxException e) {
- return null;
- }
+ return hftpURI;
}
/**
@@ -722,11 +685,12 @@ public class HftpFileSystem extends File
public long renew(Token<?> token,
Configuration conf) throws IOException {
// update the kerberos credentials, if they are coming from a keytab
- UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+ UserGroupInformation.getLoginUser().reloginFromKeytab();
// use https to renew the token
+ InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
return
DelegationTokenFetcher.renewDelegationToken
- ("https://" + token.getService().toString(),
+ (DFSUtil.createUri("https", serviceAddr).toString(),
(Token<DelegationTokenIdentifier>) token);
}
@@ -737,10 +701,18 @@ public class HftpFileSystem extends File
// update the kerberos credentials, if they are coming from a keytab
UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
// use https to cancel the token
+ InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
DelegationTokenFetcher.cancelDelegationToken
- ("https://" + token.getService().toString(),
+ (DFSUtil.createUri("https", serviceAddr).toString(),
(Token<DelegationTokenIdentifier>) token);
+ }
+ }
+
+ private static class HftpDelegationTokenSelector
+ extends AbstractDelegationTokenSelector<DelegationTokenIdentifier> {
+
+ public HftpDelegationTokenSelector() {
+ super(TOKEN_KIND);
}
-
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java Thu Feb 2 19:56:13 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
import java.io.FileInputStream;
import java.io.IOException;
import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
@@ -121,6 +122,16 @@ public class HsftpFileSystem extends Hft
}
@Override
+ protected int getDefaultPort() {
+ return getDefaultSecurePort();
+ }
+
+ @Override
+ protected InetSocketAddress getNamenodeSecureAddr(URI uri) {
+ return getNamenodeAddr(uri);
+ }
+
+ @Override
protected HttpURLConnection openConnection(String path, String query)
throws IOException {
query = addDelegationTokenParam(query);
@@ -161,16 +172,6 @@ public class HsftpFileSystem extends Hft
return (HttpURLConnection) conn;
}
- @Override
- public URI getUri() {
- try {
- return new URI("hsftp", null, nnAddr.getHostName(), nnAddr.getPort(),
- null, null, null);
- } catch (URISyntaxException e) {
- return null;
- }
- }
-
/**
* Dummy hostname verifier that is used to bypass hostname checking
*/
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/delegation/DelegationTokenSecretManager.java Thu Feb 2 19:56:13 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
@@ -296,8 +297,7 @@ public class DelegationTokenSecretManage
}
final InetSocketAddress addr = namenode.getNameNodeAddress();
- final String s = addr.getAddress().getHostAddress() + ":" + addr.getPort();
- token.setService(new Text(s));
+ SecurityUtil.setTokenService(token, addr);
final Credentials c = new Credentials();
c.addToken(new Text(ugi.getShortUserName()), token);
return c;
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java Thu Feb 2 19:56:13 2012
@@ -62,6 +62,7 @@ import org.apache.hadoop.http.HtmlQuotin
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.util.KerberosName;
@@ -492,7 +493,7 @@ public class JspHelper {
return UserGroupInformation.createRemoteUser(strings[0]);
}
- private static String getNNServiceAddress(ServletContext context,
+ private static InetSocketAddress getNNServiceAddress(ServletContext context,
HttpServletRequest request) {
String namenodeAddressInUrl = request.getParameter(NAMENODE_ADDRESS);
InetSocketAddress namenodeAddress = null;
@@ -503,8 +504,7 @@ public class JspHelper {
context);
}
if (namenodeAddress != null) {
- return (namenodeAddress.getAddress().getHostAddress() + ":"
- + namenodeAddress.getPort());
+ return namenodeAddress;
}
return null;
}
@@ -547,9 +547,9 @@ public class JspHelper {
Token<DelegationTokenIdentifier> token =
new Token<DelegationTokenIdentifier>();
token.decodeFromUrlString(tokenString);
- String serviceAddress = getNNServiceAddress(context, request);
+ InetSocketAddress serviceAddress = getNNServiceAddress(context, request);
if (serviceAddress != null) {
- token.setService(new Text(serviceAddress));
+ SecurityUtil.setTokenService(token, serviceAddress);
token.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
}
ByteArrayInputStream buf = new ByteArrayInputStream(token
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Thu Feb 2 19:56:13 2012
@@ -25,6 +25,7 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
import java.net.URL;
import java.net.URLConnection;
import java.security.PrivilegedExceptionAction;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.server.nam
import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -201,7 +203,8 @@ public class DelegationTokenFetcher {
static public Credentials getDTfromRemote(String nnAddr,
String renewer) throws IOException {
DataInputStream dis = null;
-
+ InetSocketAddress serviceAddr = NetUtils.createSocketAddr(nnAddr);
+
try {
StringBuffer url = new StringBuffer();
if (renewer != null) {
@@ -221,9 +224,7 @@ public class DelegationTokenFetcher {
ts.readFields(dis);
for(Token<?> token: ts.getAllTokens()) {
token.setKind(HftpFileSystem.TOKEN_KIND);
- token.setService(new Text(SecurityUtil.buildDTServiceName
- (remoteURL.toURI(),
- DFSConfigKeys.DFS_HTTPS_PORT_DEFAULT)));
+ SecurityUtil.setTokenService(token, serviceAddr);
}
return ts;
} catch (Exception e) {
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java Thu Feb 2 19:56:13 2012
@@ -902,6 +902,8 @@ public class MiniDFSCluster {
if(dn == null)
throw new IOException("Cannot start DataNode in "
+ dnConf.get(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY));
+ //NOTE: the following is true if and only if:
+ // hadoop.security.token.service.use_ip=true
//since the HDFS does things based on IP:port, we need to add the mapping
//for IP:port to rackId
String ipAddr = dn.getSelfAddr().getAddress().getHostAddress();
Modified: hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1239797&r1=1239796&r2=1239797&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpFileSystem.java Thu Feb 2 19:56:13 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import java.io.IOException;
import java.net.URISyntaxException;
+import java.net.URI;
import java.net.URL;
import java.net.HttpURLConnection;
import java.util.Random;
@@ -232,4 +233,164 @@ public class TestHftpFileSystem {
in.seek(7);
assertEquals('7', in.read());
}
+
+ public void resetFileSystem() throws IOException {
+ // filesystem caching has a quirk/bug that it caches based on the user's
+ // given uri. the result is if a filesystem is instantiated with no port,
+ // it gets the default port. then if the default port is changed,
+ // and another filesystem is instantiated with no port, the prior fs
+ // is returned, not a new one using the changed port. so let's flush
+ // the cache between tests...
+ FileSystem.closeAll();
+ }
+
+ @Test
+ public void testHftpDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ URI uri = URI.create("hftp://localhost");
+ HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, fs.getDefaultPort());
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:"+DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT,
+ fs.getCanonicalServiceName()
+ );
+ }
+
+ @Test
+ public void testHftpCustomDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ conf.setInt("dfs.http.port", 123);
+ conf.setInt("dfs.https.port", 456);
+
+ URI uri = URI.create("hftp://localhost");
+ HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(123, fs.getDefaultPort());
+ assertEquals(456, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:456",
+ fs.getCanonicalServiceName()
+ );
+ }
+
+ @Test
+ public void testHftpCustomUriPortWithDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ URI uri = URI.create("hftp://localhost:123");
+ HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_DEFAULT, fs.getDefaultPort());
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:"+DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT,
+ fs.getCanonicalServiceName()
+ );
+ }
+
+ @Test
+ public void testHftpCustomUriPortWithCustomDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ conf.setInt("dfs.http.port", 123);
+ conf.setInt("dfs.https.port", 456);
+
+ URI uri = URI.create("hftp://localhost:789");
+ HftpFileSystem fs = (HftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(123, fs.getDefaultPort());
+ assertEquals(456, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:456",
+ fs.getCanonicalServiceName()
+ );
+ }
+
+ ///
+
+ @Test
+ public void testHsftpDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ URI uri = URI.create("hsftp://localhost");
+ HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultPort());
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:"+DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT,
+ fs.getCanonicalServiceName()
+ );
+ }
+
+ @Test
+ public void testHsftpCustomDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ conf.setInt("dfs.http.port", 123);
+ conf.setInt("dfs.https.port", 456);
+
+ URI uri = URI.create("hsftp://localhost");
+ HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(456, fs.getDefaultPort());
+ assertEquals(456, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:456",
+ fs.getCanonicalServiceName()
+ );
+ }
+
+ @Test
+ public void testHsftpCustomUriPortWithDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ URI uri = URI.create("hsftp://localhost:123");
+ HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultPort());
+ assertEquals(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_DEFAULT, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:123",
+ fs.getCanonicalServiceName()
+ );
+ }
+
+ @Test
+ public void testHsftpCustomUriPortWithCustomDefaultPorts() throws IOException {
+ resetFileSystem();
+ Configuration conf = new Configuration();
+ conf.setInt("dfs.http.port", 123);
+ conf.setInt("dfs.https.port", 456);
+
+ URI uri = URI.create("hsftp://localhost:789");
+ HsftpFileSystem fs = (HsftpFileSystem) FileSystem.get(uri, conf);
+
+ assertEquals(456, fs.getDefaultPort());
+ assertEquals(456, fs.getDefaultSecurePort());
+
+ assertEquals(uri, fs.getUri());
+ assertEquals(
+ "127.0.0.1:789",
+ fs.getCanonicalServiceName()
+ );
+ }
}