You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/04/21 02:01:59 UTC
svn commit: r1328542 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/web/
src/test/java/org/apache/hadoop/hdfs/ src/test/java/org/apache...
Author: szetszwo
Date: Sat Apr 21 00:01:59 2012
New Revision: 1328542
URL: http://svn.apache.org/viewvc?rev=1328542&view=rev
Log:
svn merge -c 1328541 from trunk for HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem and WebHdfsFileSystem.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1328541
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1328542&r1=1328541&r2=1328542&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Sat Apr 21 00:01:59 2012
@@ -764,6 +764,9 @@ Release 0.23.3 - UNRELEASED
HDFS-2652. Add support for host-based delegation tokens. (Daryn Sharp via
szetszwo)
+ HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem
+ and WebHdfsFileSystem. (Daryn Sharp via szetszwo)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1328541
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1328542&r1=1328541&r2=1328542&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Sat Apr 21 00:01:59 2012
@@ -168,7 +168,7 @@ public class HftpFileSystem extends File
protected void initDelegationToken() throws IOException {
// look for hftp token, then try hdfs
- Token<?> token = selectDelegationToken();
+ Token<?> token = selectDelegationToken(ugi);
// if we don't already have a token, go get one over https
boolean createdToken = false;
@@ -189,8 +189,9 @@ public class HftpFileSystem extends File
}
}
- protected Token<DelegationTokenIdentifier> selectDelegationToken() {
- return hftpTokenSelector.selectToken(getUri(), ugi.getTokens(), getConf());
+ protected Token<DelegationTokenIdentifier> selectDelegationToken(
+ UserGroupInformation ugi) {
+ return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java?rev=1328542&r1=1328541&r2=1328542&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java Sat Apr 21 00:01:59 2012
@@ -165,7 +165,7 @@ public class WebHdfsFileSystem extends F
} catch (URISyntaxException e) {
throw new IllegalArgumentException(e);
}
- this.nnAddr = NetUtils.createSocketAddrForHost(uri.getHost(), uri.getPort());
+ this.nnAddr = NetUtils.createSocketAddr(uri.getAuthority(), getDefaultPort());
this.workingDir = getHomeDirectory();
if (UserGroupInformation.isSecurityEnabled()) {
@@ -175,7 +175,7 @@ public class WebHdfsFileSystem extends F
protected void initDelegationToken() throws IOException {
// look for webhdfs token, then try hdfs
- Token<?> token = selectDelegationToken();
+ Token<?> token = selectDelegationToken(ugi);
//since we don't already have a token, go get one
boolean createdToken = false;
@@ -196,8 +196,9 @@ public class WebHdfsFileSystem extends F
}
}
- protected Token<DelegationTokenIdentifier> selectDelegationToken() {
- return DT_SELECTOR.selectToken(getUri(), ugi.getTokens(), getConf());
+ protected Token<DelegationTokenIdentifier> selectDelegationToken(
+ UserGroupInformation ugi) {
+ return DT_SELECTOR.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
}
@Override
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1328542&r1=1328541&r2=1328542&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Sat Apr 21 00:01:59 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs;
import static
org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION;
+import java.io.IOException;
import java.lang.reflect.Field;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
@@ -73,34 +74,58 @@ public class TestHftpDelegationToken {
SecurityUtilTestHelper.setTokenServiceUseIp(true);
Configuration conf = new Configuration();
- URI hftpUri = URI.create("hftp://localhost:0");
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- Token<?> token = null;
+ conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class);
+ // test with implicit default port
+ URI fsUri = URI.create("hftp://localhost");
+ MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
+ checkTokenSelection(fs, conf);
+
+ // test with explicit default port
+ fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort());
+ fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
+ checkTokenSelection(fs, conf);
+
+ // test with non-default port
+ fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1));
+ fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
+ checkTokenSelection(fs, conf);
+
+ }
+
+ private void checkTokenSelection(MyHftpFileSystem fs,
+ Configuration conf) throws IOException {
+ int port = fs.getCanonicalUri().getPort();
+ UserGroupInformation ugi =
+ UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
+
+ // use ip-based tokens
+ SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
// test fallback to hdfs token
Token<?> hdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0],
DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("127.0.0.1:8020"));
ugi.addToken(hdfsToken);
-
- HftpFileSystem fs = (HftpFileSystem) FileSystem.get(hftpUri, conf);
- token = fs.selectDelegationToken();
+
+ // test fallback to hdfs token
+ Token<?> token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(hdfsToken, token);
-
+
// test hftp is favored over hdfs
Token<?> hftpToken = new Token<TokenIdentifier>(
new byte[0], new byte[0],
- HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:0"));
+ HftpFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port));
ugi.addToken(hftpToken);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(hftpToken, token);
// switch to using host-based tokens, no token should match
SecurityUtilTestHelper.setTokenServiceUseIp(false);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNull(token);
// test fallback to hdfs token
@@ -109,17 +134,31 @@ public class TestHftpDelegationToken {
DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("localhost:8020"));
ugi.addToken(hdfsToken);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(hdfsToken, token);
// test hftp is favored over hdfs
hftpToken = new Token<TokenIdentifier>(
new byte[0], new byte[0],
- HftpFileSystem.TOKEN_KIND, new Text("localhost:0"));
+ HftpFileSystem.TOKEN_KIND, new Text("localhost:"+port));
ugi.addToken(hftpToken);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(hftpToken, token);
}
+
+ static class MyHftpFileSystem extends HftpFileSystem {
+ @Override
+ public URI getCanonicalUri() {
+ return super.getCanonicalUri();
+ }
+ @Override
+ public int getDefaultPort() {
+ return super.getDefaultPort();
+ }
+ // don't automatically get a token
+ @Override
+ protected void initDelegationToken() throws IOException {}
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java?rev=1328542&r1=1328541&r2=1328542&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java Sat Apr 21 00:01:59 2012
@@ -98,38 +98,63 @@ public class TestWebHdfsUrl {
}
@Test
- public void testSelectDelegationToken() throws Exception {
+ public void testSelectHdfsDelegationToken() throws Exception {
SecurityUtilTestHelper.setTokenServiceUseIp(true);
Configuration conf = new Configuration();
- URI webHdfsUri = URI.create("webhdfs://localhost:0");
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- Token<?> token = null;
+ conf.setClass("fs.webhdfs.impl", MyWebHdfsFileSystem.class, FileSystem.class);
+ // test with implicit default port
+ URI fsUri = URI.create("webhdfs://localhost");
+ MyWebHdfsFileSystem fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
+ checkTokenSelection(fs, conf);
+
+ // test with explicit default port
+ fsUri = URI.create("webhdfs://localhost:"+fs.getDefaultPort());
+ fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
+ checkTokenSelection(fs, conf);
+
+ // test with non-default port
+ fsUri = URI.create("webhdfs://localhost:"+(fs.getDefaultPort()-1));
+ fs = (MyWebHdfsFileSystem) FileSystem.get(fsUri, conf);
+ checkTokenSelection(fs, conf);
+
+ }
+
+ private void checkTokenSelection(MyWebHdfsFileSystem fs,
+ Configuration conf) throws IOException {
+ int port = fs.getCanonicalUri().getPort();
+ // can't clear tokens from ugi, so create a new user everytime
+ UserGroupInformation ugi =
+ UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
+
+ // use ip-based tokens
+ SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
// test fallback to hdfs token
Token<?> hdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0],
DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("127.0.0.1:8020"));
ugi.addToken(hdfsToken);
-
- WebHdfsFileSystem fs = (WebHdfsFileSystem) FileSystem.get(webHdfsUri, conf);
- token = fs.selectDelegationToken();
+
+ // test fallback to hdfs token
+ Token<?> token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(hdfsToken, token);
-
+
// test webhdfs is favored over hdfs
Token<?> webHdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0],
- WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:0"));
+ WebHdfsFileSystem.TOKEN_KIND, new Text("127.0.0.1:"+port));
ugi.addToken(webHdfsToken);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(webHdfsToken, token);
// switch to using host-based tokens, no token should match
SecurityUtilTestHelper.setTokenServiceUseIp(false);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNull(token);
// test fallback to hdfs token
@@ -138,18 +163,31 @@ public class TestWebHdfsUrl {
DelegationTokenIdentifier.HDFS_DELEGATION_KIND,
new Text("localhost:8020"));
ugi.addToken(hdfsToken);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(hdfsToken, token);
// test webhdfs is favored over hdfs
webHdfsToken = new Token<TokenIdentifier>(
new byte[0], new byte[0],
- WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:0"));
+ WebHdfsFileSystem.TOKEN_KIND, new Text("localhost:"+port));
ugi.addToken(webHdfsToken);
- token = fs.selectDelegationToken();
+ token = fs.selectDelegationToken(ugi);
assertNotNull(token);
assertEquals(webHdfsToken, token);
}
-
-}
+
+ static class MyWebHdfsFileSystem extends WebHdfsFileSystem {
+ @Override
+ public URI getCanonicalUri() {
+ return super.getCanonicalUri();
+ }
+ @Override
+ public int getDefaultPort() {
+ return super.getDefaultPort();
+ }
+ // don't automatically get a token
+ @Override
+ protected void initDelegationToken() throws IOException {}
+ }
+}
\ No newline at end of file