You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by wa...@apache.org on 2014/01/30 01:30:50 UTC
svn commit: r1562654 - in
/hadoop/common/branches/branch-2.3/hadoop-hdfs-project: ./ hadoop-hdfs/
hadoop-hdfs/src/main/java/
hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/
hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/ hadoop-hdfs/src/m...
Author: wang
Date: Thu Jan 30 00:30:49 2014
New Revision: 1562654
URL: http://svn.apache.org/r1562654
Log:
HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs on a secure cluster. (jing9)
Modified:
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/ (props changed)
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/native/ (props changed)
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/ (props changed)
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/ (props changed)
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/ (props changed)
hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/ (props changed)
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project:r1562608
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs:r1562608
Modified: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1562654&r1=1562653&r2=1562654&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Thu Jan 30 00:30:49 2014
@@ -526,6 +526,9 @@ Release 2.3.0 - UNRELEASED
HDFS-5845. SecondaryNameNode dies when checkpointing with cache pools.
(wang)
+ HDFS-5842. Cannot create hftp filesystem when using a proxy user ugi and a doAs
+ on a secure cluster. (jing9)
+
BREAKDOWN OF HDFS-2832 SUBTASKS AND RELATED JIRAS
HDFS-4985. Add storage type to the protocol and expose it in block report
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1562608
Modified: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1562654&r1=1562653&r2=1562654&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java (original)
+++ hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java Thu Jan 30 00:30:49 2014
@@ -186,8 +186,8 @@ public class DelegationTokenFetcher {
} else {
// otherwise we are fetching
if (webUrl != null) {
- Credentials creds = getDTfromRemote(connectionFactory, new URI(webUrl),
- renewer);
+ Credentials creds = getDTfromRemote(connectionFactory, new URI(
+ webUrl), renewer, null);
creds.writeTokenStorageFile(tokenFile, conf);
for (Token<?> token : creds.getAllTokens()) {
System.out.println("Fetched token via " + webUrl + " for "
@@ -210,12 +210,17 @@ public class DelegationTokenFetcher {
}
static public Credentials getDTfromRemote(URLConnectionFactory factory,
- URI nnUri, String renewer) throws IOException {
+ URI nnUri, String renewer, String proxyUser) throws IOException {
StringBuilder buf = new StringBuilder(nnUri.toString())
.append(GetDelegationTokenServlet.PATH_SPEC);
+ String separator = "?";
if (renewer != null) {
buf.append("?").append(GetDelegationTokenServlet.RENEWER).append("=")
.append(renewer);
+ separator = "&";
+ }
+ if (proxyUser != null) {
+ buf.append(separator).append("doas=").append(proxyUser);
}
boolean isHttps = nnUri.getScheme().equals("https");
Modified: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java?rev=1562654&r1=1562653&r2=1562654&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java (original)
+++ hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/HftpFileSystem.java Thu Jan 30 00:30:49 2014
@@ -57,7 +57,6 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Progressable;
@@ -234,17 +233,23 @@ public class HftpFileSystem extends File
}
@Override
- public synchronized Token<?> getDelegationToken(final String renewer
- ) throws IOException {
+ public synchronized Token<?> getDelegationToken(final String renewer)
+ throws IOException {
try {
- //Renew TGT if needed
- ugi.checkTGTAndReloginFromKeytab();
- return ugi.doAs(new PrivilegedExceptionAction<Token<?>>() {
+ // Renew TGT if needed
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ final String proxyUser = connectUgi == null ? null : ugi
+ .getShortUserName();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
+ return connectUgi.doAs(new PrivilegedExceptionAction<Token<?>>() {
@Override
public Token<?> run() throws IOException {
Credentials c;
try {
- c = DelegationTokenFetcher.getDTfromRemote(connectionFactory, nnUri, renewer);
+ c = DelegationTokenFetcher.getDTfromRemote(connectionFactory,
+ nnUri, renewer, proxyUser);
} catch (IOException e) {
if (e.getCause() instanceof ConnectException) {
LOG.warn("Couldn't connect to " + nnUri +
@@ -299,13 +304,13 @@ public class HftpFileSystem extends File
* @return user_shortname,group1,group2...
*/
private String getEncodedUgiParameter() {
- StringBuilder ugiParamenter = new StringBuilder(
+ StringBuilder ugiParameter = new StringBuilder(
ServletUtil.encodeQueryValue(ugi.getShortUserName()));
for(String g: ugi.getGroupNames()) {
- ugiParamenter.append(",");
- ugiParamenter.append(ServletUtil.encodeQueryValue(g));
+ ugiParameter.append(",");
+ ugiParameter.append(ServletUtil.encodeQueryValue(g));
}
- return ugiParamenter.toString();
+ return ugiParameter.toString();
}
/**
@@ -675,30 +680,48 @@ public class HftpFileSystem extends File
@SuppressWarnings("unchecked")
@Override
- public long renewDelegationToken(Token<?> token) throws IOException {
+ public long renewDelegationToken(final Token<?> token) throws IOException {
// update the kerberos credentials, if they are coming from a keytab
- UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
- InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
try {
- return DelegationTokenFetcher.renewDelegationToken(connectionFactory,
- DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
- (Token<DelegationTokenIdentifier>) token);
- } catch (AuthenticationException e) {
+ return connectUgi.doAs(new PrivilegedExceptionAction<Long>() {
+ @Override
+ public Long run() throws Exception {
+ InetSocketAddress serviceAddr = SecurityUtil
+ .getTokenServiceAddr(token);
+ return DelegationTokenFetcher.renewDelegationToken(connectionFactory,
+ DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
+ (Token<DelegationTokenIdentifier>) token);
+ }
+ });
+ } catch (InterruptedException e) {
throw new IOException(e);
}
}
@SuppressWarnings("unchecked")
@Override
- public void cancelDelegationToken(Token<?> token) throws IOException {
- // update the kerberos credentials, if they are coming from a keytab
- UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
- InetSocketAddress serviceAddr = SecurityUtil.getTokenServiceAddr(token);
+ public void cancelDelegationToken(final Token<?> token) throws IOException {
+ UserGroupInformation connectUgi = ugi.getRealUser();
+ if (connectUgi == null) {
+ connectUgi = ugi;
+ }
try {
- DelegationTokenFetcher.cancelDelegationToken(connectionFactory, DFSUtil
- .createUri(getUnderlyingProtocol(), serviceAddr),
- (Token<DelegationTokenIdentifier>) token);
- } catch (AuthenticationException e) {
+ connectUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ InetSocketAddress serviceAddr = SecurityUtil
+ .getTokenServiceAddr(token);
+ DelegationTokenFetcher.cancelDelegationToken(connectionFactory,
+ DFSUtil.createUri(getUnderlyingProtocol(), serviceAddr),
+ (Token<DelegationTokenIdentifier>) token);
+ return null;
+ }
+ });
+ } catch (InterruptedException e) {
throw new IOException(e);
}
}
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/native/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/native:r1562608
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode:r1562608
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs:r1562608
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/secondary:r1562608
Propchange: hadoop/common/branches/branch-2.3/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/hdfs:r1562608