You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2011/10/04 08:55:51 UTC
svn commit: r1178703 - in /hadoop/common/branches/branch-0.20-security-205:
CHANGES.txt src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java
Author: jitendra
Date: Tue Oct 4 06:55:51 2011
New Revision: 1178703
URL: http://svn.apache.org/viewvc?rev=1178703&view=rev
Log:
Merged change r1178698 from branch-0.20-security for HDFS-2392.
Modified:
hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/branches/branch-0.20-security-205/src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java
Modified: hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/CHANGES.txt?rev=1178703&r1=1178702&r2=1178703&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-205/CHANGES.txt Tue Oct 4 06:55:51 2011
@@ -254,6 +254,8 @@ Release 0.20.205.0 - 2011.09.28
HADOOP-7715. Removed unnecessary security logger configuration. (Eric Yang)
+ HDFS-2392. Dist with hftp is failing again. (Daryn Sharp via jitendra)
+
IMPROVEMENTS
MAPREDUCE-2928. MR-2413 improvements (Eli Collins via mattf)
Modified: hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1178703&r1=1178702&r2=1178703&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/branch-0.20-security-205/src/hdfs/org/apache/hadoop/hdfs/HftpFileSystem.java Tue Oct 4 06:55:51 2011
@@ -155,31 +155,42 @@ public class HftpFileSystem extends File
this.hftpURI = createUri(name.getScheme(), nnAddr);
if (UserGroupInformation.isSecurityEnabled()) {
- Token<?> token = selectHftpDelegationToken();
- if (token == null) {
- token = selectHdfsDelegationToken();
- }
- //since we don't already have a token, go get one over https
- if (token == null) {
- token = getDelegationToken(null);
- // security might be disabled
- if (token != null) {
- setDelegationToken(token);
- renewer.addTokenToRenew(this);
- LOG.debug("Created new DT for " + token.getService());
- }
+ initDelegationToken();
+ }
+ }
+
+ protected void initDelegationToken() throws IOException {
+ // look for hftp token, then try hdfs
+ Token<?> token = selectHftpDelegationToken();
+ if (token == null) {
+ token = selectHdfsDelegationToken();
+ }
+
+ //since we don't already have a token, go get one over https
+ boolean createdToken = false;
+ if (token == null) {
+ token = getDelegationToken(null);
+ createdToken = (token != null);
+ }
+
+ // security might be disabled
+ if (token != null) {
+ setDelegationToken(token);
+ if (createdToken) {
+ renewer.addTokenToRenew(this);
+ LOG.debug("Created new DT for " + token.getService());
} else {
LOG.debug("Found existing DT for " + token.getService());
}
}
}
- private Token<DelegationTokenIdentifier> selectHftpDelegationToken() {
+ protected Token<DelegationTokenIdentifier> selectHftpDelegationToken() {
Text serviceName = SecurityUtil.buildTokenService(nnSecureAddr);
return hftpTokenSelector.selectToken(serviceName, ugi.getTokens());
}
- private Token<DelegationTokenIdentifier> selectHdfsDelegationToken() {
+ protected Token<DelegationTokenIdentifier> selectHdfsDelegationToken() {
// this guesses the remote cluster's rpc service port.
// the current token design assumes it's the same as the local cluster's
// rpc port unless a config key is set. there should be a way to automatic
@@ -212,7 +223,7 @@ public class HftpFileSystem extends File
return uri;
}
- private <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
+ protected <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
renewToken = token;
// emulate the 203 usage of the tokens
// by setting the kind and service as if they were hdfs tokens
Modified: hadoop/common/branches/branch-0.20-security-205/src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java?rev=1178703&r1=1178702&r2=1178703&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java (original)
+++ hadoop/common/branches/branch-0.20-security-205/src/test/org/apache/hadoop/hdfs/TestHftpFileSystem.java Tue Oct 4 06:55:51 2011
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs;
+import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import java.io.IOException;
@@ -25,6 +26,11 @@ import java.net.URI;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.junit.Before;
import org.junit.Test;
@@ -206,4 +212,74 @@ public class TestHftpFileSystem {
fs.getCanonicalServiceName()
);
}
+
+ Token<DelegationTokenIdentifier> hftpToken;
+ Token<DelegationTokenIdentifier> hdfsToken;
+ Token<DelegationTokenIdentifier> gotToken;
+
+ class StubbedHftpFileSystem extends HftpFileSystem {
+ @Override
+ protected Token<DelegationTokenIdentifier> selectHftpDelegationToken() {
+ return hftpToken;
+ }
+
+ @Override
+ protected Token<DelegationTokenIdentifier> selectHdfsDelegationToken() {
+ return hdfsToken;
+ }
+
+ @Override
+ public Token<DelegationTokenIdentifier> getDelegationToken(String renewer) {
+ return makeDummyToken("new");
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ protected <T extends TokenIdentifier> void setDelegationToken(Token<T> token) {
+ gotToken = (Token<DelegationTokenIdentifier>) token;
+ }
+ }
+
+ static Token<DelegationTokenIdentifier> makeDummyToken(String kind) {
+ Token<DelegationTokenIdentifier> token = new Token();
+ token.setKind(new Text(kind));
+ return token;
+ }
+
+ @Before
+ public void resetTokens() {
+ hftpToken = hdfsToken = gotToken = null;
+ }
+
+ @Test
+ public void testHftpWithNoTokens() throws IOException {
+ new StubbedHftpFileSystem().initDelegationToken();
+ assertNotNull(gotToken);
+ assertEquals(new Text("new"), gotToken.getKind());
+
+ }
+ @Test
+ public void testHftpWithHftpToken() throws IOException {
+ hftpToken = makeDummyToken("hftp");
+ new StubbedHftpFileSystem().initDelegationToken();
+ assertNotNull(gotToken);
+ assertEquals(gotToken, hftpToken);
+ }
+
+ @Test
+ public void testHftpWithHdfsToken() throws IOException {
+ hdfsToken = makeDummyToken("hdfs");
+ new StubbedHftpFileSystem().initDelegationToken();
+ assertNotNull(gotToken);
+ assertEquals(gotToken, hdfsToken);
+ }
+
+ @Test
+ public void testHftpWithHftpAndHdfsToken() throws IOException {
+ hftpToken = makeDummyToken("hftp");
+ hdfsToken = makeDummyToken("hdfs");
+ new StubbedHftpFileSystem().initDelegationToken();
+ assertNotNull(gotToken);
+ assertEquals(gotToken, hftpToken);
+ }
}
\ No newline at end of file