You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/04/23 23:27:02 UTC
svn commit: r1329462 - in
/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/org/apache/hadoop/hdfs/ src/test/java/org/apache/hadoop/hdfs/
Author: szetszwo
Date: Mon Apr 23 21:27:01 2012
New Revision: 1329462
URL: http://svn.apache.org/viewvc?rev=1329462&view=rev
Log:
HDFS-3312. In HftpFileSystem, the namenode URI is non-secure but the delegation tokens have to use secure URI. Contributed by Daryn Sharp
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1329462&r1=1329461&r2=1329462&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Apr 23 21:27:01 2012
@@ -890,6 +890,9 @@ Release 0.23.3 - UNRELEASED
HDFS-3308. Uses canonical URI to select delegation tokens in HftpFileSystem
and WebHdfsFileSystem. (Daryn Sharp via szetszwo)
+ HDFS-3312. In HftpFileSystem, the namenode URI is non-secure but the
+ delegation tokens have to use secure URI. (Daryn Sharp via szetszwo)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java?rev=1329462&r1=1329461&r2=1329462&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HftpFileSystem.java Mon Apr 23 21:27:01 2012
@@ -94,8 +94,8 @@ public class HftpFileSystem extends File
protected UserGroupInformation ugi;
private URI hftpURI;
- protected InetSocketAddress nnAddr;
- protected InetSocketAddress nnSecureAddr;
+ protected URI nnUri;
+ protected URI nnSecureUri;
public static final String HFTP_TIMEZONE = "UTC";
public static final String HFTP_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
@@ -139,11 +139,19 @@ public class HftpFileSystem extends File
return NetUtils.createSocketAddrForHost(uri.getHost(), getDefaultSecurePort());
}
+ protected URI getNamenodeUri(URI uri) {
+ return DFSUtil.createUri("http", getNamenodeAddr(uri));
+ }
+
+ protected URI getNamenodeSecureUri(URI uri) {
+ return DFSUtil.createUri("https", getNamenodeSecureAddr(uri));
+ }
+
@Override
public String getCanonicalServiceName() {
// unlike other filesystems, hftp's service is the secure port, not the
// actual port in the uri
- return SecurityUtil.buildTokenService(nnSecureAddr).toString();
+ return SecurityUtil.buildTokenService(nnSecureUri).toString();
}
@Override
@@ -152,8 +160,8 @@ public class HftpFileSystem extends File
super.initialize(name, conf);
setConf(conf);
this.ugi = UserGroupInformation.getCurrentUser();
- this.nnAddr = getNamenodeAddr(name);
- this.nnSecureAddr = getNamenodeSecureAddr(name);
+ this.nnUri = getNamenodeUri(name);
+ this.nnSecureUri = getNamenodeSecureUri(name);
try {
this.hftpURI = new URI(name.getScheme(), name.getAuthority(),
null, null, null);
@@ -191,7 +199,7 @@ public class HftpFileSystem extends File
protected Token<DelegationTokenIdentifier> selectDelegationToken(
UserGroupInformation ugi) {
- return hftpTokenSelector.selectToken(getCanonicalUri(), ugi.getTokens(), getConf());
+ return hftpTokenSelector.selectToken(nnSecureUri, ugi.getTokens(), getConf());
}
@@ -222,7 +230,7 @@ public class HftpFileSystem extends File
ugi.reloginFromKeytab();
return ugi.doAs(new PrivilegedExceptionAction<Token<?>>() {
public Token<?> run() throws IOException {
- final String nnHttpUrl = DFSUtil.createUri("https", nnSecureAddr).toString();
+ final String nnHttpUrl = nnSecureUri.toString();
Credentials c;
try {
c = DelegationTokenFetcher.getDTfromRemote(nnHttpUrl, renewer);
@@ -264,8 +272,8 @@ public class HftpFileSystem extends File
* @throws IOException on error constructing the URL
*/
protected URL getNamenodeURL(String path, String query) throws IOException {
- final URL url = new URL("http", nnAddr.getHostName(),
- nnAddr.getPort(), path + '?' + query);
+ final URL url = new URL("http", nnUri.getHost(),
+ nnUri.getPort(), path + '?' + query);
if (LOG.isTraceEnabled()) {
LOG.trace("url=" + url);
}
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java?rev=1329462&r1=1329461&r2=1329462&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/HsftpFileSystem.java Mon Apr 23 21:27:01 2012
@@ -133,11 +133,16 @@ public class HsftpFileSystem extends Hft
}
@Override
+ protected URI getNamenodeUri(URI uri) {
+ return getNamenodeSecureUri(uri);
+ }
+
+ @Override
protected HttpURLConnection openConnection(String path, String query)
throws IOException {
query = addDelegationTokenParam(query);
- final URL url = new URL("https", nnAddr.getHostName(),
- nnAddr.getPort(), path + '?' + query);
+ final URL url = new URL("https", nnUri.getHost(),
+ nnUri.getPort(), path + '?' + query);
HttpsURLConnection conn = (HttpsURLConnection)URLUtils.openConnection(url);
// bypass hostname verification
conn.setHostnameVerifier(new DummyHostnameVerifier());
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1329462&r1=1329461&r2=1329462&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java Mon Apr 23 21:27:01 2012
@@ -70,32 +70,74 @@ public class TestHftpDelegationToken {
}
@Test
- public void testSelectHdfsDelegationToken() throws Exception {
+ public void testSelectHftpDelegationToken() throws Exception {
SecurityUtilTestHelper.setTokenServiceUseIp(true);
Configuration conf = new Configuration();
conf.setClass("fs.hftp.impl", MyHftpFileSystem.class, FileSystem.class);
+ int httpPort = 80;
+ int httpsPort = 443;
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, httpPort);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, httpsPort);
+
// test with implicit default port
URI fsUri = URI.create("hftp://localhost");
- MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
- checkTokenSelection(fs, conf);
+ MyHftpFileSystem fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf);
+ assertEquals(httpPort, fs.getCanonicalUri().getPort());
+ checkTokenSelection(fs, httpsPort, conf); // should still use secure port
// test with explicit default port
- fsUri = URI.create("hftp://localhost:"+fs.getDefaultPort());
- fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
- checkTokenSelection(fs, conf);
+ fsUri = URI.create("hftp://localhost:"+httpPort);
+ fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf);
+ assertEquals(httpPort, fs.getCanonicalUri().getPort());
+ checkTokenSelection(fs, httpsPort, conf); // should still use secure port
// test with non-default port
- fsUri = URI.create("hftp://localhost:"+(fs.getDefaultPort()-1));
- fs = (MyHftpFileSystem) FileSystem.get(fsUri, conf);
- checkTokenSelection(fs, conf);
+ fsUri = URI.create("hftp://localhost:"+(httpPort+1));
+ fs = (MyHftpFileSystem) FileSystem.newInstance(fsUri, conf);
+ assertEquals(httpPort+1, fs.getCanonicalUri().getPort());
+ checkTokenSelection(fs, httpsPort, conf); // should still use secure port
+
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5);
+ }
+
+ @Test
+ public void testSelectHsftpDelegationToken() throws Exception {
+ SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
+ Configuration conf = new Configuration();
+ conf.setClass("fs.hsftp.impl", MyHsftpFileSystem.class, FileSystem.class);
+
+ int httpPort = 80;
+ int httpsPort = 443;
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTP_PORT_KEY, httpPort);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, httpsPort);
+
+ // test with implicit default port
+ URI fsUri = URI.create("hsftp://localhost");
+ MyHsftpFileSystem fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf);
+ assertEquals(httpsPort, fs.getCanonicalUri().getPort());
+ checkTokenSelection(fs, httpsPort, conf);
+ // test with explicit default port
+ fsUri = URI.create("hsftp://localhost:"+httpsPort);
+ fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf);
+ assertEquals(httpsPort, fs.getCanonicalUri().getPort());
+ checkTokenSelection(fs, httpsPort, conf);
+
+ // test with non-default port
+ fsUri = URI.create("hsftp://localhost:"+(httpsPort+1));
+ fs = (MyHsftpFileSystem) FileSystem.newInstance(fsUri, conf);
+ assertEquals(httpsPort+1, fs.getCanonicalUri().getPort());
+ checkTokenSelection(fs, httpsPort+1, conf);
+
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HTTPS_PORT_KEY, 5);
}
- private void checkTokenSelection(MyHftpFileSystem fs,
+ private void checkTokenSelection(HftpFileSystem fs,
+ int port,
Configuration conf) throws IOException {
- int port = fs.getCanonicalUri().getPort();
UserGroupInformation ugi =
UserGroupInformation.createUserForTesting(fs.getUri().getAuthority(), new String[]{});
@@ -161,4 +203,18 @@ public class TestHftpDelegationToken {
@Override
protected void initDelegationToken() throws IOException {}
}
+
+ static class MyHsftpFileSystem extends HsftpFileSystem {
+ @Override
+ public URI getCanonicalUri() {
+ return super.getCanonicalUri();
+ }
+ @Override
+ public int getDefaultPort() {
+ return super.getDefaultPort();
+ }
+ // don't automatically get a token
+ @Override
+ protected void initDelegationToken() throws IOException {}
+ }
}
\ No newline at end of file