You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ji...@apache.org on 2011/10/14 03:24:21 UTC
svn commit: r1183187 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/security/token/block/
src/main/java/org/apache/hadoop/hdfs...
Author: jitendra
Date: Fri Oct 14 01:24:20 2011
New Revision: 1183187
URL: http://svn.apache.org/viewvc?rev=1183187&view=rev
Log:
MAPREDUCE-2764. Fix renewal of dfs delegation tokens. Contributed by Owen.
Added:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Oct 14 01:24:20 2011
@@ -123,6 +123,8 @@ Trunk (unreleased changes)
HDFS-2424. Added a root element "HdfsFileStatuses" for the response
of webhdfs listStatus. (szetszwo)
+ MAPREDUCE-2764. Fix renewal of dfs delegation tokens. (Owen via jitendra)
+
Release 0.23.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/Hdfs.java Fri Oct 14 01:24:20 2011
@@ -409,6 +409,7 @@ public class Hdfs extends AbstractFileSy
* @return the new expiration time
* @throws InvalidToken
* @throws IOException
+ * @deprecated Use Token.renew instead.
*/
@SuppressWarnings("unchecked")
public long renewDelegationToken(
@@ -423,6 +424,7 @@ public class Hdfs extends AbstractFileSy
* @param token delegation token
* @throws InvalidToken
* @throws IOException
+ * @deprecated Use Token.cancel instead.
*/
@SuppressWarnings("unchecked")
public void cancelDelegationToken(
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java Fri Oct 14 01:24:20 2011
@@ -93,9 +93,11 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.util.Progressable;
/********************************************************
@@ -115,6 +117,7 @@ public class DFSClient implements java.i
public static final long SERVER_DEFAULTS_VALIDITY_PERIOD = 60 * 60 * 1000L; // 1 hour
static final int TCP_WINDOW_SIZE = 128 * 1024; // 128 KB
final ClientProtocol namenode;
+ private final InetSocketAddress nnAddress;
final UserGroupInformation ugi;
volatile boolean clientRunning = true;
private volatile FsServerDefaults serverDefaults;
@@ -241,6 +244,7 @@ public class DFSClient implements java.i
this.dfsClientConf = new Conf(conf);
this.conf = conf;
this.stats = stats;
+ this.nnAddress = nameNodeAddr;
this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
@@ -442,18 +446,26 @@ public class DFSClient implements java.i
throws IOException {
Token<DelegationTokenIdentifier> result =
namenode.getDelegationToken(renewer);
+ SecurityUtil.setTokenService(result, nnAddress);
LOG.info("Created " + DelegationTokenIdentifier.stringifyToken(result));
return result;
}
/**
- * @see ClientProtocol#renewDelegationToken(Token)
+ * Renew a delegation token
+ * @param token the token to renew
+ * @return the new expiration time
+ * @throws InvalidToken
+ * @throws IOException
+ * @deprecated Use Token.renew instead.
*/
public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
throws InvalidToken, IOException {
LOG.info("Renewing " + DelegationTokenIdentifier.stringifyToken(token));
try {
- return namenode.renewDelegationToken(token);
+ return token.renew(conf);
+ } catch (InterruptedException ie) {
+ throw new RuntimeException("caught interrupted", ie);
} catch (RemoteException re) {
throw re.unwrapRemoteException(InvalidToken.class,
AccessControlException.class);
@@ -461,19 +473,77 @@ public class DFSClient implements java.i
}
/**
- * @see ClientProtocol#cancelDelegationToken(Token)
+ * Cancel a delegation token
+ * @param token the token to cancel
+ * @throws InvalidToken
+ * @throws IOException
+ * @deprecated Use Token.cancel instead.
*/
public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
throws InvalidToken, IOException {
LOG.info("Cancelling " + DelegationTokenIdentifier.stringifyToken(token));
try {
- namenode.cancelDelegationToken(token);
+ token.cancel(conf);
+ } catch (InterruptedException ie) {
+ throw new RuntimeException("caught interrupted", ie);
} catch (RemoteException re) {
throw re.unwrapRemoteException(InvalidToken.class,
AccessControlException.class);
}
}
+ @InterfaceAudience.Private
+ public static class Renewer extends TokenRenewer {
+
+ @Override
+ public boolean handleKind(Text kind) {
+ return DelegationTokenIdentifier.HDFS_DELEGATION_KIND.equals(kind);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public long renew(Token<?> token, Configuration conf) throws IOException {
+ Token<DelegationTokenIdentifier> delToken =
+ (Token<DelegationTokenIdentifier>) token;
+ LOG.info("Renewing " +
+ DelegationTokenIdentifier.stringifyToken(delToken));
+ ClientProtocol nn =
+ DFSUtil.createNamenode
+ (NameNode.getAddress(token.getService().toString()),
+ conf, UserGroupInformation.getCurrentUser());
+ try {
+ return nn.renewDelegationToken(delToken);
+ } catch (RemoteException re) {
+ throw re.unwrapRemoteException(InvalidToken.class,
+ AccessControlException.class);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void cancel(Token<?> token, Configuration conf) throws IOException {
+ Token<DelegationTokenIdentifier> delToken =
+ (Token<DelegationTokenIdentifier>) token;
+ LOG.info("Cancelling " +
+ DelegationTokenIdentifier.stringifyToken(delToken));
+ ClientProtocol nn = DFSUtil.createNamenode(
+ NameNode.getAddress(token.getService().toString()), conf,
+ UserGroupInformation.getCurrentUser());
+ try {
+ nn.cancelDelegationToken(delToken);
+ } catch (RemoteException re) {
+ throw re.unwrapRemoteException(InvalidToken.class,
+ AccessControlException.class);
+ }
+ }
+
+ @Override
+ public boolean isManaged(Token<?> token) throws IOException {
+ return true;
+ }
+
+ }
+
/**
* Report corrupt blocks that were discovered by the client.
* @see ClientProtocol#reportBadBlocks(LocatedBlock[])
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java Fri Oct 14 01:24:20 2011
@@ -811,7 +811,6 @@ public class DistributedFileSystem exten
) throws IOException {
Token<DelegationTokenIdentifier> result =
dfs.getDelegationToken(renewer == null ? null : new Text(renewer));
- result.setService(new Text(getCanonicalServiceName()));
return result;
}
@@ -831,7 +830,7 @@ public class DistributedFileSystem exten
@Deprecated
public Token<DelegationTokenIdentifier> getDelegationToken(Text renewer)
throws IOException {
- return dfs.getDelegationToken(renewer);
+ return getDelegationToken(renewer.toString());
}
@Override // FileSystem
@@ -848,10 +847,15 @@ public class DistributedFileSystem exten
* @param token delegation token obtained earlier
* @return the new expiration time
* @throws IOException
+ * @deprecated Use Token.renew instead.
*/
public long renewDelegationToken(Token<DelegationTokenIdentifier> token)
throws InvalidToken, IOException {
- return dfs.renewDelegationToken(token);
+ try {
+ return token.renew(getConf());
+ } catch (InterruptedException ie) {
+ throw new RuntimeException("Caught interrupted", ie);
+ }
}
/**
@@ -859,10 +863,15 @@ public class DistributedFileSystem exten
*
* @param token delegation token
* @throws IOException
+ * @deprecated Use Token.cancel instead.
*/
public void cancelDelegationToken(Token<DelegationTokenIdentifier> token)
throws IOException {
- dfs.cancelDelegationToken(token);
+ try {
+ token.cancel(getConf());
+ } catch (InterruptedException ie) {
+ throw new RuntimeException("Caught interrupted", ie);
+ }
}
/**
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Fri Oct 14 01:24:20 2011
@@ -60,6 +60,7 @@ import org.apache.hadoop.security.Securi
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenRenewer;
import org.apache.hadoop.util.Progressable;
import org.apache.hadoop.util.ServletUtil;
import org.xml.sax.Attributes;
@@ -83,14 +84,18 @@ public class HftpFileSystem extends File
HttpURLConnection.setFollowRedirects(true);
}
+ public static final Text TOKEN_KIND = new Text("HFTP delegation");
+
private String nnHttpUrl;
- private URI hdfsURI;
+ private Text hdfsServiceName;
+ private URI hftpURI;
protected InetSocketAddress nnAddr;
protected UserGroupInformation ugi;
public static final String HFTP_TIMEZONE = "UTC";
public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
- private Token<DelegationTokenIdentifier> delegationToken;
+ private Token<?> delegationToken;
+ private Token<?> renewToken;
public static final String HFTP_SERVICE_NAME_KEY = "hdfs.service.host_";
public static final SimpleDateFormat getDateFormat() {
@@ -118,7 +123,7 @@ public class HftpFileSystem extends File
@Override
public String getCanonicalServiceName() {
- return SecurityUtil.buildDTServiceName(hdfsURI, getDefaultPort());
+ return SecurityUtil.buildDTServiceName(hftpURI, getDefaultPort());
}
private String buildUri(String schema, String host, int port) {
@@ -144,17 +149,21 @@ public class HftpFileSystem extends File
urlPort = conf.getInt(DFSConfigKeys.DFS_HTTPS_PORT_KEY,
DFSConfigKeys.DFS_HTTPS_PORT_DEFAULT);
- nnHttpUrl =
- buildUri("https://", NetUtils.normalizeHostName(name.getHost()), urlPort);
+ String normalizedNN = NetUtils.normalizeHostName(name.getHost());
+ nnHttpUrl = buildUri("https://", normalizedNN ,urlPort);
LOG.debug("using url to get DT:" + nnHttpUrl);
+ try {
+ hftpURI = new URI(buildUri("hftp://", normalizedNN, urlPort));
+ } catch (URISyntaxException ue) {
+ throw new IOException("bad uri for hdfs", ue);
+ }
-
-
// if one uses RPC port different from the Default one,
// one should specify what is the setvice name for this delegation token
// otherwise it is hostname:RPC_PORT
- String key = HftpFileSystem.HFTP_SERVICE_NAME_KEY+
- SecurityUtil.buildDTServiceName(name, DFSConfigKeys.DFS_HTTPS_PORT_DEFAULT);
+ String key = HftpFileSystem.HFTP_SERVICE_NAME_KEY
+ + SecurityUtil.buildDTServiceName(name,
+ DFSConfigKeys.DFS_HTTPS_PORT_DEFAULT);
if(LOG.isDebugEnabled()) {
LOG.debug("Trying to find DT for " + name + " using key=" + key +
"; conf=" + conf.get(key, ""));
@@ -165,9 +174,10 @@ public class HftpFileSystem extends File
nnPort = NetUtils.createSocketAddr(nnServiceName,
NameNode.DEFAULT_PORT).getPort();
}
-
try {
- hdfsURI = new URI(buildUri("hdfs://", nnAddr.getHostName(), nnPort));
+ URI hdfsURI = new URI("hdfs://" + normalizedNN + ":" + nnPort);
+ hdfsServiceName = new Text(SecurityUtil.buildDTServiceName(hdfsURI,
+ nnPort));
} catch (URISyntaxException ue) {
throw new IOException("bad uri for hdfs", ue);
}
@@ -175,30 +185,55 @@ public class HftpFileSystem extends File
if (UserGroupInformation.isSecurityEnabled()) {
//try finding a token for this namenode (esp applicable for tasks
//using hftp). If there exists one, just set the delegationField
- String canonicalName = getCanonicalServiceName();
+ String hftpServiceName = getCanonicalServiceName();
for (Token<? extends TokenIdentifier> t : ugi.getTokens()) {
- if (DelegationTokenIdentifier.HDFS_DELEGATION_KIND.equals(t.getKind()) &&
- t.getService().toString().equals(canonicalName)) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Found existing DT for " + name);
+ Text kind = t.getKind();
+ if (DelegationTokenIdentifier.HDFS_DELEGATION_KIND.equals(kind)) {
+ if (t.getService().toString().equals(hdfsServiceName)) {
+ setDelegationToken(t);
+ break;
+ }
+ } else if (TOKEN_KIND.equals(kind)) {
+ if (hftpServiceName
+ .equals(normalizeService(t.getService().toString()))) {
+ setDelegationToken(t);
+ break;
}
- delegationToken = (Token<DelegationTokenIdentifier>) t;
- break;
}
}
//since we don't already have a token, go get one over https
if (delegationToken == null) {
- delegationToken =
- (Token<DelegationTokenIdentifier>) getDelegationToken(null);
+ setDelegationToken(getDelegationToken(null));
renewer.addTokenToRenew(this);
}
}
}
-
+
+ private String normalizeService(String service) {
+ int colonIndex = service.indexOf(':');
+ if (colonIndex == -1) {
+ throw new IllegalArgumentException("Invalid service for hftp token: " +
+ service);
+ }
+ String hostname =
+ NetUtils.normalizeHostName(service.substring(0, colonIndex));
+ String port = service.substring(colonIndex + 1);
+ return hostname + ":" + port;
+ }
+
+ private <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
+ renewToken = token;
+ // emulate the 203 usage of the tokens
+ // by setting the kind and service as if they were hdfs tokens
+ delegationToken = new Token<T>(token);
+ delegationToken.setKind(DelegationTokenIdentifier.HDFS_DELEGATION_KIND);
+ delegationToken.setService(hdfsServiceName);
+ }
@Override
- public synchronized Token<?> getDelegationToken(final String renewer) throws IOException {
+ public synchronized Token<?> getDelegationToken(final String renewer
+ ) throws IOException {
try {
//Renew TGT if needed
ugi.reloginFromKeytab();
@@ -221,7 +256,6 @@ public class HftpFileSystem extends File
LOG.debug("Got dt for " + getUri() + ";t.service="
+t.getService());
}
- t.setService(new Text(getCanonicalServiceName()));
return t;
}
return null;
@@ -625,7 +659,8 @@ public class HftpFileSystem extends File
@Override
public int compareTo(Delayed o) {
if (o.getClass() != RenewAction.class) {
- throw new IllegalArgumentException("Illegal comparision to non-RenewAction");
+ throw new IllegalArgumentException
+ ("Illegal comparision to non-RenewAction");
}
RenewAction other = (RenewAction) o;
return timestamp < other.timestamp ? -1 :
@@ -662,31 +697,20 @@ public class HftpFileSystem extends File
* @return
* @throws IOException
*/
- @SuppressWarnings("unchecked")
public boolean renew() throws IOException, InterruptedException {
final HftpFileSystem fs = weakFs.get();
if (fs != null) {
synchronized (fs) {
- fs.ugi.reloginFromKeytab();
- fs.ugi.doAs(new PrivilegedExceptionAction<Void>() {
-
- @Override
- public Void run() throws Exception {
- try {
- DelegationTokenFetcher.renewDelegationToken(fs.nnHttpUrl,
- fs.delegationToken);
- } catch (IOException ie) {
- try {
- fs.delegationToken =
- (Token<DelegationTokenIdentifier>) fs.getDelegationToken(null);
- } catch (IOException ie2) {
- throw new IOException("Can't renew or get new delegation token ",
- ie);
- }
- }
- return null;
- }
- });
+ try {
+ fs.renewToken.renew(fs.getConf());
+ } catch (IOException ie) {
+ try {
+ fs.setDelegationToken(fs.getDelegationToken(null));
+ } catch (IOException ie2) {
+ throw new IOException("Can't renew or get new delegation "
+ + "token ", ie);
+ }
+ }
}
}
return fs != null;
@@ -722,7 +746,7 @@ public class HftpFileSystem extends File
}
public void addTokenToRenew(HftpFileSystem fs) {
- queue.add(new RenewAction(RENEW_CYCLE + System.currentTimeMillis(),fs));
+ queue.add(new RenewAction(RENEW_CYCLE + System.currentTimeMillis(), fs));
}
public void run() {
@@ -747,4 +771,44 @@ public class HftpFileSystem extends File
}
}
}
+
+ @InterfaceAudience.Private
+ public static class TokenManager extends TokenRenewer {
+
+ @Override
+ public boolean handleKind(Text kind) {
+ return kind.equals(TOKEN_KIND);
+ }
+
+ @Override
+ public boolean isManaged(Token<?> token) throws IOException {
+ return true;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public long renew(Token<?> token,
+ Configuration conf) throws IOException {
+ // update the kerberos credentials, if they are coming from a keytab
+ UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+ // use https to renew the token
+ return
+ DelegationTokenFetcher.renewDelegationToken
+ ("https://" + token.getService().toString(),
+ (Token<DelegationTokenIdentifier>) token);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void cancel(Token<?> token,
+ Configuration conf) throws IOException {
+ // update the kerberos credentials, if they are coming from a keytab
+ UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+ // use https to cancel the token
+ DelegationTokenFetcher.cancelDelegationToken
+ ("https://" + token.getService().toString(),
+ (Token<DelegationTokenIdentifier>) token);
+ }
+
+ }
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/security/token/block/BlockTokenIdentifier.java Fri Oct 14 01:24:20 2011
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@InterfaceAudience.Private
@@ -171,4 +172,12 @@ public class BlockTokenIdentifier extend
return cache;
}
+
+ @InterfaceAudience.Private
+ public static class Renewer extends Token.TrivialRenewer {
+ @Override
+ protected Text getKind() {
+ return KIND_NAME;
+ }
+ }
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Fri Oct 14 01:24:20 2011
@@ -39,14 +39,17 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.HftpFileSystem;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
import org.apache.hadoop.hdfs.server.namenode.CancelDelegationTokenServlet;
import org.apache.hadoop.hdfs.server.namenode.GetDelegationTokenServlet;
import org.apache.hadoop.hdfs.server.namenode.RenewDelegationTokenServlet;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -149,34 +152,31 @@ public class DelegationTokenFetcher {
DataInputStream in = new DataInputStream(
new ByteArrayInputStream(token.getIdentifier()));
id.readFields(in);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Token (" + id + ") for " + token.getService());
- }
+ System.out.println("Token (" + id + ") for " +
+ token.getService());
}
- return null;
- }
-
- if (webUrl != null) {
- if (renew) {
- long result;
- for (Token<?> token : readTokens(tokenFile, conf)) {
- result = renewDelegationToken(webUrl,
- (Token<DelegationTokenIdentifier>) token);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Renewed token via " + webUrl + " for "
- + token.getService() + " until: " + new Date(result));
+ } else if (cancel) {
+ for(Token<?> token: readTokens(tokenFile, conf)) {
+ if (token.isManaged()) {
+ token.cancel(conf);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cancelled token for " + token.getService());
}
}
- } else if (cancel) {
- for (Token<?> token : readTokens(tokenFile, conf)) {
- cancelDelegationToken(webUrl,
- (Token<DelegationTokenIdentifier>) token);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Cancelled token via " + webUrl + " for "
- + token.getService());
+ }
+ } else if (renew) {
+ for (Token<?> token : readTokens(tokenFile, conf)) {
+ if (token.isManaged()) {
+ long result = token.renew(conf);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Renewed token for " + token.getService()
+ + " until: " + new Date(result));
}
}
- } else {
+ }
+ } else {
+ // otherwise we are fetching
+ if (webUrl != null) {
Credentials creds = getDTfromRemote(webUrl, renewer);
creds.writeTokenStorageFile(tokenFile, conf);
for (Token<?> token : creds.getAllTokens()) {
@@ -185,29 +185,8 @@ public class DelegationTokenFetcher {
+ token.getService() + " into " + tokenFile);
}
}
- }
- } else {
- FileSystem fs = FileSystem.get(conf);
- if (cancel) {
- for (Token<?> token : readTokens(tokenFile, conf)) {
- ((DistributedFileSystem) fs)
- .cancelDelegationToken((Token<DelegationTokenIdentifier>) token);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Cancelled token for "
- + token.getService());
- }
- }
- } else if (renew) {
- long result;
- for (Token<?> token : readTokens(tokenFile, conf)) {
- result = ((DistributedFileSystem) fs)
- .renewDelegationToken((Token<DelegationTokenIdentifier>) token);
- if(LOG.isDebugEnabled()) {
- LOG.debug("Renewed token for " + token.getService()
- + " until: " + new Date(result));
- }
- }
} else {
+ FileSystem fs = FileSystem.get(conf);
Token<?> token = fs.getDelegationToken(renewer);
Credentials cred = new Credentials();
cred.addToken(token.getService(), token);
@@ -230,8 +209,9 @@ public class DelegationTokenFetcher {
try {
StringBuffer url = new StringBuffer();
if (renewer != null) {
- url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC).append("?").
- append(GetDelegationTokenServlet.RENEWER).append("=").append(renewer);
+ url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC)
+ .append("?").append(GetDelegationTokenServlet.RENEWER).append("=")
+ .append(renewer);
} else {
url.append(nnAddr).append(GetDelegationTokenServlet.PATH_SPEC);
}
@@ -248,6 +228,12 @@ public class DelegationTokenFetcher {
Credentials ts = new Credentials();
dis = new DataInputStream(in);
ts.readFields(dis);
+ for(Token<?> token: ts.getAllTokens()) {
+ token.setKind(HftpFileSystem.TOKEN_KIND);
+ token.setService(new Text(SecurityUtil.buildDTServiceName
+ (remoteURL.toURI(),
+ DFSConfigKeys.DFS_HTTPS_PORT_DEFAULT)));
+ }
return ts;
} catch (Exception e) {
throw new IOException("Unable to obtain remote token", e);
@@ -295,7 +281,8 @@ public class DelegationTokenFetcher {
IOUtils.cleanup(LOG, in);
if(e!=null) {
- LOG.info("rethrowing exception from HTTP request: " + e.getLocalizedMessage());
+ LOG.info("rethrowing exception from HTTP request: " +
+ e.getLocalizedMessage());
throw e;
}
throw ie;
@@ -383,7 +370,8 @@ public class DelegationTokenFetcher {
IOUtils.cleanup(LOG, in);
if(e!=null) {
- LOG.info("rethrowing exception from HTTP request: " + e.getLocalizedMessage());
+ LOG.info("rethrowing exception from HTTP request: " +
+ e.getLocalizedMessage());
throw e;
}
throw ie;
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer?rev=1183187&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer Fri Oct 14 01:24:20 2011
@@ -0,0 +1,3 @@
+org.apache.hadoop.hdfs.DFSClient$Renewer
+org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier$Renewer
+org.apache.hadoop.hdfs.HftpFileSystem$TokenManager
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestResolveHdfsSymlink.java Fri Oct 14 01:24:20 2011
@@ -105,7 +105,7 @@ public class TestResolveHdfsSymlink {
* @throws IOException
* @throws InterruptedException
*/
- @SuppressWarnings("unchecked")
+ @SuppressWarnings({ "unchecked", "deprecation" })
@Test
public void testFcDelegationToken() throws UnsupportedFileSystemException,
IOException, InterruptedException {
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestDelegationToken.java Fri Oct 14 01:24:20 2011
@@ -183,6 +183,7 @@ public class TestDelegationToken {
dtSecretManager.renewToken(token, "JobTracker");
}
+ @SuppressWarnings("deprecation")
@Test
public void testDelegationTokenWithDoAs() throws Exception {
final DistributedFileSystem dfs = (DistributedFileSystem) cluster.getFileSystem();
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java Fri Oct 14 01:24:20 2011
@@ -203,11 +203,9 @@ public class OfflineEditsViewerHelper {
"JobTracker/foo.com@FOO.COM");
try {
longUgi.doAs(new PrivilegedExceptionAction<Object>() {
- public Object run() throws IOException {
- final DistributedFileSystem dfs =
- (DistributedFileSystem) cluster.getFileSystem();
- dfs.renewDelegationToken(token);
- dfs.cancelDelegationToken(token);
+ public Object run() throws IOException, InterruptedException {
+ token.renew(config);
+ token.cancel(config);
return null;
}
});
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java?rev=1183187&r1=1183186&r2=1183187&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestDelegationTokenFetcher.java Fri Oct 14 01:24:20 2011
@@ -20,7 +20,6 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
@@ -37,7 +36,9 @@ import org.apache.hadoop.hdfs.security.t
import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenRenewer;
import org.junit.Before;
import org.junit.Test;
@@ -46,6 +47,7 @@ public class TestDelegationTokenFetcher
private Configuration conf;
private URI uri;
private static final String SERVICE_VALUE = "localhost:2005";
+ private static final Text KIND = new Text("TESTING-TOKEN-KIND");
private static String tokenFile = "file.dta";
@Before
@@ -56,25 +58,59 @@ public class TestDelegationTokenFetcher
FileSystemTestHelper.addFileSystemForTesting(uri, conf, dfs);
}
+ public static class FakeRenewer extends TokenRenewer {
+ static Token<?> lastRenewed = null;
+ static Token<?> lastCanceled = null;
+
+ @Override
+ public boolean handleKind(Text kind) {
+ return KIND.equals(kind);
+ }
+
+ @Override
+ public boolean isManaged(Token<?> token) throws IOException {
+ return true;
+ }
+
+ @Override
+ public long renew(Token<?> token, Configuration conf) {
+ lastRenewed = token;
+ return 0;
+ }
+
+ @Override
+ public void cancel(Token<?> token, Configuration conf) {
+ lastCanceled = token;
+ }
+
+ public static void reset() {
+ lastRenewed = null;
+ lastCanceled = null;
+ }
+ }
+
/**
* Verify that when the DelegationTokenFetcher runs, it talks to the Namenode,
* pulls out the correct user's token and successfully serializes it to disk.
*/
+ @SuppressWarnings("deprecation")
@Test
public void expectedTokenIsRetrievedFromDFS() throws Exception {
final byte[] ident = new DelegationTokenIdentifier(new Text("owner"),
new Text("renewer"), new Text("realuser")).getBytes();
final byte[] pw = new byte[] { 42 };
- final Text kind = new Text("MY-KIND");
final Text service = new Text(uri.toString());
+ final String user =
+ UserGroupInformation.getCurrentUser().getShortUserName();
// Create a token for the fetcher to fetch, wire NN to return it when asked
// for this particular user.
- Token<DelegationTokenIdentifier> t = new Token<DelegationTokenIdentifier>(
- ident, pw, kind, service);
- when(dfs.getDelegationToken((String) null)).thenReturn(t);
+ Token<DelegationTokenIdentifier> t =
+ new Token<DelegationTokenIdentifier>(ident, pw, KIND, service);
+ when(dfs.getDelegationToken(eq((String) null))).thenReturn(t);
when(dfs.renewDelegationToken(eq(t))).thenReturn(1000L);
when(dfs.getUri()).thenReturn(uri);
+ FakeRenewer.reset();
FileSystem fileSys = FileSystem.getLocal(conf);
try {
@@ -88,14 +124,13 @@ public class TestDelegationTokenFetcher
assertEquals(t, itr.next());
assertTrue(!itr.hasNext());
- DelegationTokenFetcher.main(new String[] { "-fs", uri.toString(),
- "--print", tokenFile });
- DelegationTokenFetcher.main(new String[] { "-fs", uri.toString(),
- "--renew", tokenFile });
- DelegationTokenFetcher.main(new String[] { "-fs", uri.toString(),
- "--cancel", tokenFile });
- verify(dfs).renewDelegationToken(eq(t));
- verify(dfs).cancelDelegationToken(eq(t));
+ DelegationTokenFetcher.main(new String[] { "--print", tokenFile });
+ DelegationTokenFetcher.main(new String[] { "--renew", tokenFile });
+ assertEquals(t, FakeRenewer.lastRenewed);
+ FakeRenewer.reset();
+
+ DelegationTokenFetcher.main(new String[] { "--cancel", tokenFile });
+ assertEquals(t, FakeRenewer.lastCanceled);
} finally {
fileSys.delete(new Path(tokenFile), true);
}
Added: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer?rev=1183187&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (added)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer Fri Oct 14 01:24:20 2011
@@ -0,0 +1 @@
+org.apache.hadoop.tools.TestDelegationTokenFetcher$FakeRenewer