You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bo...@apache.org on 2011/04/05 20:50:16 UTC
svn commit: r1089157 - in /hadoop/common/branches/branch-0.20-security: ./
src/hdfs/org/apache/hadoop/hdfs/server/namenode/
src/hdfs/org/apache/hadoop/hdfs/tools/
src/mapred/org/apache/hadoop/mapreduce/security/token/
Author: boryas
Date: Tue Apr 5 18:50:15 2011
New Revision: 1089157
URL: http://svn.apache.org/viewvc?rev=1089157&view=rev
Log:
MAPREDUCE-2420. JobTracker should be able to renew delegation token over HTTP
Modified:
hadoop/common/branches/branch-0.20-security/CHANGES.txt
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java
hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java
Modified: hadoop/common/branches/branch-0.20-security/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/CHANGES.txt?rev=1089157&r1=1089156&r2=1089157&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security/CHANGES.txt Tue Apr 5 18:50:15 2011
@@ -40,7 +40,8 @@ Release 0.20.204.0 - unreleased
HDFS-1692. In secure mode, Datanode process doesn't exit when disks
fail. (bharathm via boryas)
-
+ MAPREDUCE-2420. JobTracker should be able to renew delegation token
+ over HTTP (boryas)
IMPROVEMENTS
HDFS-1541. Not marking datanodes dead when namenode in safemode.
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java?rev=1089157&r1=1089156&r2=1089157&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/server/namenode/RenewDelegationTokenServlet.java Tue Apr 5 18:50:15 2011
@@ -77,9 +77,14 @@ public class RenewDelegationTokenServlet
os.println(result);
os.close();
} catch(Exception e) {
- LOG.info("Exception while renewing token. Re-throwing. ", e);
+ // transfer exception over the http
+ String exceptionClass = e.getClass().getCanonicalName();
+ String exceptionMsg = e.getLocalizedMessage();
+ String strException = exceptionClass + ";" + exceptionMsg;
+ LOG.info("Exception while renewing token. Re-throwing. s=" + strException, e);
+
resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR,
- e.getMessage());
+ strException);
}
}
}
Modified: hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1089157&r1=1089156&r2=1089157&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/hdfs/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Tue Apr 5 18:50:15 2011
@@ -198,21 +198,56 @@ public class DelegationTokenFetcher {
buf.append("=");
buf.append(tok.encodeToUrlString());
BufferedReader in = null;
+ HttpURLConnection connection = null;
try {
URL url = new URL(buf.toString());
SecurityUtil.fetchServiceTicket(url);
- URLConnection connection = url.openConnection();
+ connection = (HttpURLConnection)url.openConnection();
in = new BufferedReader(new InputStreamReader
(connection.getInputStream()));
long result = Long.parseLong(in.readLine());
in.close();
return result;
} catch (IOException ie) {
+ LOG.info("error in renew over HTTP", ie);
+ IOException e = null;
+ if(connection != null) {
+ String resp = connection.getResponseMessage();
+ e = getExceptionFromResponse(resp);
+ }
+
IOUtils.cleanup(LOG, in);
+ if(e!=null) {
+ LOG.info("rethrowing exception from HTTP request: " + e.getLocalizedMessage());
+ throw e;
+ }
throw ie;
}
}
+ static private IOException getExceptionFromResponse(String resp) {
+ String exceptionClass = "", exceptionMsg = "";
+ if(resp != null && !resp.isEmpty()) {
+ String[] rs = resp.split(";");
+ exceptionClass = rs[0];
+ exceptionMsg = rs[1];
+ }
+ LOG.info("Error response from HTTP request=" + resp +
+ ";ec=" + exceptionClass + ";em="+exceptionMsg);
+ IOException e = null;
+ if(exceptionClass != null && !exceptionClass.isEmpty()) {
+ if(exceptionClass.contains("InvalidToken")) {
+ e = new org.apache.hadoop.security.token.SecretManager.InvalidToken(exceptionMsg);
+ e.setStackTrace(new StackTraceElement[0]); // stack is not relevant
+ } else if(exceptionClass.contains("AccessControlException")) {
+ e = new org.apache.hadoop.security.AccessControlException(exceptionMsg);
+ e.setStackTrace(new StackTraceElement[0]); // stack is not relevant
+ }
+ }
+ LOG.info("Exception from HTTP response=" + e.getLocalizedMessage());
+ return e;
+ }
+
/**
* Cancel a Delegation Token.
* @param nnAddr the NameNode's address
Modified: hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java?rev=1089157&r1=1089156&r2=1089157&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java (original)
+++ hadoop/common/branches/branch-0.20-security/src/mapred/org/apache/hadoop/mapreduce/security/token/DelegationTokenRenewal.java Tue Apr 5 18:50:15 2011
@@ -25,13 +25,11 @@ import org.apache.hadoop.security.Access
import org.apache.hadoop.security.UserGroupInformation;
import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
@@ -42,7 +40,9 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.VersionMismatchException;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.token.Token;
@@ -143,6 +143,21 @@ public class DelegationTokenRenewal {
}
}
+ protected static long renewDelegationTokenOverHttps(
+ final Token<DelegationTokenIdentifier> token, final Configuration conf)
+ throws InterruptedException, IOException{
+ final String httpAddress = getHttpAddressForToken(token, conf);
+
+ Long expDate = (Long) UserGroupInformation.getLoginUser().doAs(
+ new PrivilegedExceptionAction<Long>() {
+ public Long run() throws IOException {
+ return DelegationTokenFetcher.renewDelegationToken(httpAddress, token);
+ }
+ });
+ LOG.info("Renew over HTTP done. addr="+httpAddress+";res="+expDate);
+ return expDate;
+ }
+
private static long renewDelegationToken(DelegationTokenToRenew dttr)
throws Exception {
long newExpirationDate=System.currentTimeMillis()+3600*1000;
@@ -151,10 +166,17 @@ public class DelegationTokenRenewal {
if(token.getKind().equals(kindHdfs)) {
try {
- DistributedFileSystem dfs = getDFSForToken(token, conf);
- newExpirationDate = dfs.renewDelegationToken(token);
- } catch (InvalidToken ite) {
- LOG.warn("invalid token - not scheduling for renew");
+ try {
+ DistributedFileSystem dfs = getDFSForToken(token, conf);
+ newExpirationDate = dfs.renewDelegationToken(token);
+ } catch(VersionMismatchException vme) {
+ // if there is a version mismatch we try over https
+ LOG.info("Delegation token renew for t=" + token.getService() +
+ " failed with VersionMissmaptch:" + vme.toString()+". Trying over https");
+ renewDelegationTokenOverHttps(token, conf);
+ }
+ }catch (InvalidToken ite) {
+ LOG.warn("invalid token - not scheduling for renew: " + ite.getLocalizedMessage());
removeFailedDelegationToken(dttr);
throw new IOException("failed to renew token", ite);
} catch (AccessControlException ioe) {
@@ -166,7 +188,7 @@ public class DelegationTokenRenewal {
// returns default expiration date
}
} else {
- throw new Exception("unknown token type to renew+"+token.getKind());
+ throw new Exception("unknown token type to renew: "+token.getKind());
}
return newExpirationDate;
}
@@ -200,6 +222,20 @@ public class DelegationTokenRenewal {
}
}
+ private static String getHttpAddressForToken(
+ Token<DelegationTokenIdentifier> token, final Configuration conf)
+ throws IOException {
+
+ String[] ipaddr = token.getService().toString().split(":");
+
+ InetAddress iaddr = InetAddress.getByName(ipaddr[0]);
+ String dnsName = iaddr.getCanonicalHostName();
+ String httpsPort = conf.get("dfs.https.port", "50470");
+
+ // always use https (it is for security only)
+ return "https://" + dnsName+":"+httpsPort;
+ }
+
private static DistributedFileSystem getDFSForToken(
Token<DelegationTokenIdentifier> token, final Configuration conf)
throws Exception {