You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2016/01/25 18:26:14 UTC
[39/50] [abbrv] hadoop git commit: HDFS-9525. hadoop utilities need
to support provided delegation tokens (HeeSoo Kim via aw)
HDFS-9525. hadoop utilities need to support provided delegation tokens (HeeSoo Kim via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d22c4239
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d22c4239
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d22c4239
Branch: refs/heads/YARN-1011
Commit: d22c4239a40a1c7ed49c06038138f0e3f387b4a0
Parents: 95363bc
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jan 22 12:15:22 2016 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Fri Jan 22 12:15:22 2016 -0800
----------------------------------------------------------------------
.../fs/CommonConfigurationKeysPublic.java | 3 ++
.../hadoop/security/UserGroupInformation.java | 22 +++++++++
.../src/main/resources/core-default.xml | 6 +++
.../security/TestUserGroupInformation.java | 48 +++++++++++++++++++-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 14 +++---
.../hdfs/web/resources/DelegationParam.java | 5 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 5 +-
8 files changed, 92 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index c9f758b..648ad59 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -311,6 +311,9 @@ public class CommonConfigurationKeysPublic {
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_SECURITY_DNS_NAMESERVER_KEY =
"hadoop.security.dns.nameserver";
+ /** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
+ public static final String HADOOP_TOKEN_FILES =
+ "hadoop.token.files";
/** See <a href="{@docRoot}/../core-default.html">core-default.xml</a> */
public static final String HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 28014bf..d33e1aa 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.security;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
import java.io.File;
@@ -70,6 +71,7 @@ import org.apache.hadoop.security.authentication.util.KerberosUtil;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
@@ -821,6 +823,26 @@ public class UserGroupInformation {
}
loginUser = proxyUser == null ? realUser : createProxyUser(proxyUser, realUser);
+ String tokenFileLocation = System.getProperty(HADOOP_TOKEN_FILES);
+ if (tokenFileLocation == null) {
+ tokenFileLocation = conf.get(HADOOP_TOKEN_FILES);
+ }
+ if (tokenFileLocation != null) {
+ for (String tokenFileName:
+ StringUtils.getTrimmedStrings(tokenFileLocation)) {
+ if (tokenFileName.length() > 0) {
+ File tokenFile = new File(tokenFileName);
+ if (tokenFile.exists() && tokenFile.isFile()) {
+ Credentials cred = Credentials.readTokenStorageFile(
+ tokenFile, conf);
+ loginUser.addCredentials(cred);
+ } else {
+ LOG.info("tokenFile("+tokenFileName+") does not exist");
+ }
+ }
+ }
+ }
+
String fileLocation = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
if (fileLocation != null) {
// Load the token storage file and put all of the tokens into the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index bf60a25..c25f49e 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -466,6 +466,12 @@ for ldap providers in the same way as above does.
<description>Maps kerberos principals to local user names</description>
</property>
+<property>
+ <name>hadoop.token.files</name>
+ <value></value>
+ <description>List of token cache files that have delegation tokens for hadoop service</description>
+</property>
+
<!-- i/o properties -->
<property>
<name>io.file.buffer.size</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index 54cfc2d..6abe78c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -18,6 +18,7 @@ package org.apache.hadoop.security;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@@ -35,6 +36,7 @@ import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
import java.io.BufferedReader;
+import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Method;
@@ -848,7 +850,9 @@ public class TestUserGroupInformation {
*/
@Test
public void testPrivateTokenExclusion() throws Exception {
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation ugi =
+ UserGroupInformation.createUserForTesting(
+ "privateUser", new String[] { "PRIVATEUSERS" });
TestTokenIdentifier tokenId = new TestTokenIdentifier();
Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(
tokenId.getBytes(), "password".getBytes(),
@@ -928,4 +932,46 @@ public class TestUserGroupInformation {
}
}
}
+
+ @Test
+ public void testExternalTokenFiles() throws Exception {
+ StringBuilder tokenFullPathnames = new StringBuilder();
+ String tokenFilenames = "token1,token2";
+ String tokenFiles[] = StringUtils.getTrimmedStrings(tokenFilenames);
+ final File testDir = new File("target",
+ TestUserGroupInformation.class.getName() + "-tmpDir").getAbsoluteFile();
+ String testDirPath = testDir.getAbsolutePath();
+
+ // create path for token files
+ for (String tokenFile: tokenFiles) {
+ if (tokenFullPathnames.length() > 0) {
+ tokenFullPathnames.append(",");
+ }
+ tokenFullPathnames.append(testDirPath).append("/").append(tokenFile);
+ }
+
+ // create new token and store it
+ TestTokenIdentifier tokenId = new TestTokenIdentifier();
+ Credentials cred1 = new Credentials();
+ Token<TestTokenIdentifier> token1 = new Token<TestTokenIdentifier>(
+ tokenId.getBytes(), "password".getBytes(),
+ tokenId.getKind(), new Text("token-service1"));
+ cred1.addToken(token1.getService(), token1);
+ cred1.writeTokenStorageFile(new Path(testDirPath, tokenFiles[0]), conf);
+
+ Credentials cred2 = new Credentials();
+ Token<TestTokenIdentifier> token2 = new Token<TestTokenIdentifier>(
+ tokenId.getBytes(), "password".getBytes(),
+ tokenId.getKind(), new Text("token-service2"));
+ cred2.addToken(token2.getService(), token2);
+ cred2.writeTokenStorageFile(new Path(testDirPath, tokenFiles[1]), conf);
+
+ // set property for token external token files
+ System.setProperty("hadoop.token.files", tokenFullPathnames.toString());
+ UserGroupInformation.setLoginUser(null);
+ UserGroupInformation tokenUgi = UserGroupInformation.getLoginUser();
+ Collection<Token<?>> credsugiTokens = tokenUgi.getTokens();
+ assertTrue(credsugiTokens.contains(token1));
+ assertTrue(credsugiTokens.contains(token2));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index f9c2c6e..d806d55 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -238,7 +238,7 @@ public class WebHdfsFileSystem extends FileSystem
// the first getAuthParams() for a non-token op will either get the
// internal token from the ugi or lazy fetch one
protected synchronized Token<?> getDelegationToken() throws IOException {
- if (canRefreshDelegationToken && delegationToken == null) {
+ if (delegationToken == null) {
Token<?> token = tokenSelector.selectToken(
new Text(getCanonicalServiceName()), ugi.getTokens());
// ugi tokens are usually indicative of a task which can't
@@ -248,11 +248,13 @@ public class WebHdfsFileSystem extends FileSystem
LOG.debug("Using UGI token: {}", token);
canRefreshDelegationToken = false;
} else {
- token = getDelegationToken(null);
- if (token != null) {
- LOG.debug("Fetched new token: {}", token);
- } else { // security is disabled
- canRefreshDelegationToken = false;
+ if (canRefreshDelegationToken) {
+ token = getDelegationToken(null);
+ if (token != null) {
+ LOG.debug("Fetched new token: {}", token);
+ } else { // security is disabled
+ canRefreshDelegationToken = false;
+ }
}
}
setDelegationToken(token);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
index 5329580..fda14438 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/DelegationParam.java
@@ -17,8 +17,6 @@
*/
package org.apache.hadoop.hdfs.web.resources;
-import org.apache.hadoop.security.UserGroupInformation;
-
/** Represents delegation token used for authentication. */
public class DelegationParam extends StringParam {
/** Parameter name. */
@@ -33,8 +31,7 @@ public class DelegationParam extends StringParam {
* @param str a string representation of the parameter value.
*/
public DelegationParam(final String str) {
- super(DOMAIN, UserGroupInformation.isSecurityEnabled()
- && str != null && !str.equals(DEFAULT)? str: null);
+ super(DOMAIN, str != null && !str.equals(DEFAULT)? str: null);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 940fa90..74b8aac 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -55,6 +55,9 @@ Trunk (Unreleased)
HDFS-9057. allow/disallow snapshots via webhdfs
(Bramma Reddy Battula via vinayakumarb)
+ HDFS-9525. hadoop utilities need to support provided delegation
+ tokens (HeeSoo Kim via aw)
+
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d22c4239/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
index 2913a97..24c13af 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
@@ -195,7 +195,7 @@ public class TestWebHdfsUrl {
checkQueryParams(
new String[]{
GetOpParam.Op.GETFILESTATUS.toQueryString(),
- new UserParam(ugi.getShortUserName()).toString()
+ new DelegationParam(tokenString).toString()
},
fileStatusUrl);
}
@@ -280,8 +280,7 @@ public class TestWebHdfsUrl {
checkQueryParams(
new String[]{
GetOpParam.Op.GETFILESTATUS.toQueryString(),
- new UserParam(ugi.getRealUser().getShortUserName()).toString(),
- new DoAsParam(ugi.getShortUserName()).toString()
+ new DelegationParam(tokenString).toString()
},
fileStatusUrl);
}