You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/02/22 00:07:03 UTC
[35/50] [abbrv] hadoop git commit: HADOOP-13805. UGI.getCurrentUser()
fails if user does not have a keytab associated. Contributed by Xiao Chen,
Wei-Chiu Chuang, Yongjun Zhang.
HADOOP-13805. UGI.getCurrentUser() fails if user does not have a keytab associated. Contributed by Xiao Chen, Wei-Chiu Chuang, Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c26c241
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c26c241
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c26c241
Branch: refs/heads/YARN-5734
Commit: 4c26c241ad2b907dc02cecefa9846cbe2b0465ba
Parents: 02c5494
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Thu Feb 16 22:25:37 2017 -0800
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Fri Feb 17 09:18:50 2017 -0800
----------------------------------------------------------------------
.../hadoop/fs/CommonConfigurationKeys.java | 11 ++
.../hadoop/security/UserGroupInformation.java | 129 +++++++++++++++----
.../hadoop/security/TestUGIWithMiniKdc.java | 1 +
.../security/TestUserGroupInformation.java | 16 ++-
4 files changed, 127 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
index b8a60d6..e53f71e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeys.java
@@ -353,6 +353,17 @@ public class CommonConfigurationKeys extends CommonConfigurationKeysPublic {
public static final String HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS =
"hadoop.user.group.metrics.percentiles.intervals";
+ /* When creating UGI with UserGroupInformation(Subject), treat the passed
+ * subject external if set to true, and assume the owner of the subject
+ * should do the credential renewal.
+ *
+ * This is a temporary config to solve the compatibility issue with
+ * HADOOP-13558 and HADOOP-13805 fix, see the jiras for discussions.
+ */
+ public static final String HADOOP_TREAT_SUBJECT_EXTERNAL_KEY =
+ "hadoop.treat.subject.external";
+ public static final boolean HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT = false;
+
public static final String RPC_METRICS_QUANTILE_ENABLE =
"rpc.metrics.quantile.enable";
public static final boolean RPC_METRICS_QUANTILE_ENABLE_DEFAULT = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 6574e55..a5c6226 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.security;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN_DEFAULT;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_TOKEN_FILES;
@@ -79,6 +81,7 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -273,6 +276,29 @@ public class UserGroupInformation {
/** Min time (in seconds) before relogin for Kerberos */
private static long kerberosMinSecondsBeforeRelogin;
/** The configuration to use */
+
+ /*
+ * This config is a temporary one for backward compatibility.
+ * It means whether to treat the subject passed to
+ * UserGroupInformation(Subject) as external. If true,
+ * - no renewal thread will be created to do the renew credential
+ * - reloginFromKeytab() and reloginFromTicketCache will not renew
+ * credential.
+ * and it assumes that the owner of the subject to renew; if false, it means
+ * to retain the old behavior prior to fixing HADOOP-13558 and HADOOP-13805.
+ * The default is false.
+ */
+ private static boolean treatSubjectExternal = false;
+
+ /*
+ * Some test need the renewal thread to be created even if it does
+ * UserGroupInformation.loginUserFromSubject(subject);
+ * The test code may set this variable to true via
+ * setEnableRenewThreadCreationForTest(boolean)
+ * method.
+ */
+ private static boolean enableRenewThreadCreationForTest = false;
+
private static Configuration conf;
@@ -338,6 +364,15 @@ public class UserGroupInformation {
metrics.getGroupsQuantiles = getGroupsQuantiles;
}
}
+
+ treatSubjectExternal = conf.getBoolean(HADOOP_TREAT_SUBJECT_EXTERNAL_KEY,
+ HADOOP_TREAT_SUBJECT_EXTERNAL_DEFAULT);
+ if (treatSubjectExternal) {
+ LOG.info("Config " + HADOOP_TREAT_SUBJECT_EXTERNAL_KEY + " is set to "
+ + "true, the owner of the subject passed to "
+ + " UserGroupInformation(Subject) is supposed to renew the "
+ + "credential.");
+ }
}
/**
@@ -351,7 +386,19 @@ public class UserGroupInformation {
public static void setConfiguration(Configuration conf) {
initialize(conf, true);
}
-
+
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ static void setEnableRenewThreadCreationForTest(boolean b) {
+ enableRenewThreadCreationForTest = b;
+ }
+
+ @InterfaceAudience.Private
+ @VisibleForTesting
+ static boolean getEnableRenewThreadCreationForTest() {
+ return enableRenewThreadCreationForTest;
+ }
+
@InterfaceAudience.Private
@VisibleForTesting
public static void reset() {
@@ -361,6 +408,7 @@ public class UserGroupInformation {
kerberosMinSecondsBeforeRelogin = 0;
setLoginUser(null);
HadoopKerberosName.setRules(null);
+ setEnableRenewThreadCreationForTest(false);
}
/**
@@ -392,6 +440,7 @@ public class UserGroupInformation {
private final User user;
private final boolean isKeytab;
private final boolean isKrbTkt;
+ private final boolean isLoginExternal;
private static String OS_LOGIN_MODULE_NAME;
private static Class<? extends Principal> OS_PRINCIPAL_CLASS;
@@ -644,28 +693,28 @@ public class UserGroupInformation {
/**
* Create a UserGroupInformation for the given subject.
* This does not change the subject or acquire new credentials.
+ *
+ * The creator of subject is responsible for renewing credentials.
* @param subject the user's subject
*/
UserGroupInformation(Subject subject) {
- this(subject, false);
+ this(subject, treatSubjectExternal);
}
/**
* Create a UGI from the given subject.
* @param subject the subject
- * @param externalKeyTab if the subject's keytab is managed by the user.
+ * @param isLoginExternal if the subject's keytab is managed by other UGI.
* Setting this to true will prevent UGI from attempting
* to login the keytab, or to renew it.
*/
- private UserGroupInformation(Subject subject, final boolean externalKeyTab) {
+ private UserGroupInformation(Subject subject, final boolean isLoginExternal) {
this.subject = subject;
this.user = subject.getPrincipals(User.class).iterator().next();
- if (externalKeyTab) {
- this.isKeytab = false;
- } else {
- this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject);
- }
+
+ this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject);
this.isKrbTkt = KerberosUtil.hasKerberosTicket(subject);
+ this.isLoginExternal = isLoginExternal;
}
/**
@@ -766,7 +815,7 @@ public class UserGroupInformation {
User ugiUser = new User(loginPrincipals.iterator().next().getName(),
AuthenticationMethod.KERBEROS, login);
loginSubject.getPrincipals().add(ugiUser);
- UserGroupInformation ugi = new UserGroupInformation(loginSubject);
+ UserGroupInformation ugi = new UserGroupInformation(loginSubject, false);
ugi.setLogin(login);
ugi.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
return ugi;
@@ -782,7 +831,9 @@ public class UserGroupInformation {
/**
* Create a UserGroupInformation from a Subject with Kerberos principal.
*
- * @param subject The KerberosPrincipal to use in UGI
+ * @param subject The KerberosPrincipal to use in UGI.
+ * The creator of subject is responsible for
+ * renewing credentials.
*
* @throws IOException
* @throws KerberosAuthException if the kerberos login fails
@@ -843,6 +894,10 @@ public class UserGroupInformation {
* Log in a user using the given subject
* @param subject the subject to use when logging in a user, or null to
* create a new subject.
+ *
+ * If subject is not null, the creator of subject is responsible for renewing
+ * credentials.
+ *
* @throws IOException if login fails
*/
@InterfaceAudience.Public
@@ -850,17 +905,25 @@ public class UserGroupInformation {
public synchronized
static void loginUserFromSubject(Subject subject) throws IOException {
ensureInitialized();
+ boolean externalSubject = false;
try {
if (subject == null) {
subject = new Subject();
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Treat subject external: " + treatSubjectExternal
+ + ". When true, assuming keytab is managed extenally since "
+ + " logged in from subject");
+ }
+ externalSubject = treatSubjectExternal;
}
LoginContext login =
newLoginContext(authenticationMethod.getLoginAppName(),
subject, new HadoopConfiguration());
login.login();
- LOG.debug("Assuming keytab is managed externally since logged in from"
- + " subject.");
- UserGroupInformation realUser = new UserGroupInformation(subject, true);
+
+ UserGroupInformation realUser =
+ new UserGroupInformation(subject, externalSubject);
realUser.setLogin(login);
realUser.setAuthenticationMethod(authenticationMethod);
// If the HADOOP_PROXY_USER environment variable or property
@@ -959,11 +1022,23 @@ public class UserGroupInformation {
return start + (long) ((end - start) * TICKET_RENEW_WINDOW);
}
+ /**
+ * Should relogin if security is enabled using Kerberos, and
+ * the Subject is not owned by another UGI.
+ * @return true if this UGI should relogin
+ */
+ private boolean shouldRelogin() {
+ return isSecurityEnabled()
+ && user.getAuthenticationMethod() == AuthenticationMethod.KERBEROS
+ && !isLoginExternal;
+ }
+
/**Spawn a thread to do periodic renewals of kerberos credentials*/
private void spawnAutoRenewalThreadForUserCreds() {
- if (!isSecurityEnabled()
- || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS
- || isKeytab) {
+ if (getEnableRenewThreadCreationForTest()) {
+ LOG.warn("Spawning thread to auto renew user credential since " +
+ " enableRenewThreadCreationForTest was set to true.");
+ } else if (!shouldRelogin() || isKeytab) {
return;
}
@@ -1092,7 +1167,7 @@ public class UserGroupInformation {
start = Time.now();
login.login();
metrics.loginSuccess.add(Time.now() - start);
- loginUser = new UserGroupInformation(subject);
+ loginUser = new UserGroupInformation(subject, false);
loginUser.setLogin(login);
loginUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
} catch (LoginException le) {
@@ -1156,8 +1231,9 @@ public class UserGroupInformation {
public synchronized void checkTGTAndReloginFromKeytab() throws IOException {
if (!isSecurityEnabled()
|| user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS
- || !isKeytab)
+ || !isKeytab) {
return;
+ }
KerberosTicket tgt = getTGT();
if (tgt != null && !shouldRenewImmediatelyForTests &&
Time.now() < getRefreshTime(tgt)) {
@@ -1210,9 +1286,7 @@ public class UserGroupInformation {
@InterfaceAudience.Public
@InterfaceStability.Evolving
public synchronized void reloginFromKeytab() throws IOException {
- if (!isSecurityEnabled()
- || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS
- || !isKeytab) {
+ if (!shouldRelogin() || !isKeytab) {
return;
}
@@ -1281,9 +1355,7 @@ public class UserGroupInformation {
@InterfaceAudience.Public
@InterfaceStability.Evolving
public synchronized void reloginFromTicketCache() throws IOException {
- if (!isSecurityEnabled()
- || user.getAuthenticationMethod() != AuthenticationMethod.KERBEROS
- || !isKrbTkt) {
+ if (!shouldRelogin() || !isKrbTkt) {
return;
}
LoginContext login = getLogin();
@@ -1354,7 +1426,8 @@ public class UserGroupInformation {
start = Time.now();
login.login();
metrics.loginSuccess.add(Time.now() - start);
- UserGroupInformation newLoginUser = new UserGroupInformation(subject);
+ UserGroupInformation newLoginUser =
+ new UserGroupInformation(subject, false);
newLoginUser.setLogin(login);
newLoginUser.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
@@ -1427,7 +1500,7 @@ public class UserGroupInformation {
}
Subject subject = new Subject();
subject.getPrincipals().add(new User(user));
- UserGroupInformation result = new UserGroupInformation(subject);
+ UserGroupInformation result = new UserGroupInformation(subject, false);
result.setAuthenticationMethod(authMethod);
return result;
}
@@ -1504,7 +1577,7 @@ public class UserGroupInformation {
Set<Principal> principals = subject.getPrincipals();
principals.add(new User(user));
principals.add(new RealUser(realUser));
- UserGroupInformation result =new UserGroupInformation(subject);
+ UserGroupInformation result =new UserGroupInformation(subject, false);
result.setAuthenticationMethod(AuthenticationMethod.PROXY);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java
index 0016a65..2c6c7e4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUGIWithMiniKdc.java
@@ -75,6 +75,7 @@ public class TestUGIWithMiniKdc {
SecurityUtil.setAuthenticationMethod(
UserGroupInformation.AuthenticationMethod.KERBEROS, conf);
UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.setEnableRenewThreadCreationForTest(true);
LoginContext loginContext = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c26c241/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index ed0f58c..510987e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -61,6 +61,7 @@ import java.util.LinkedHashSet;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_TREAT_SUBJECT_EXTERNAL_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_KERBEROS_MIN_SECONDS_BEFORE_RELOGIN;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
@@ -1020,8 +1021,7 @@ public class TestUserGroupInformation {
assertTrue(credsugiTokens.contains(token2));
}
- @Test
- public void testCheckTGTAfterLoginFromSubject() throws Exception {
+ private void testCheckTGTAfterLoginFromSubjectHelper() throws Exception {
// security on, default is remove default realm
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
UserGroupInformation.setConfiguration(conf);
@@ -1031,6 +1031,7 @@ public class TestUserGroupInformation {
KeyTab keytab = KeyTab.getInstance();
subject.getPrivateCredentials().add(keytab);
UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws IOException {
@@ -1042,6 +1043,17 @@ public class TestUserGroupInformation {
});
}
+ @Test(expected = KerberosAuthException.class)
+ public void testCheckTGTAfterLoginFromSubject() throws Exception {
+ testCheckTGTAfterLoginFromSubjectHelper();
+ }
+
+ @Test
+ public void testCheckTGTAfterLoginFromSubjectFix() throws Exception {
+ conf.setBoolean(HADOOP_TREAT_SUBJECT_EXTERNAL_KEY, true);
+ testCheckTGTAfterLoginFromSubjectHelper();
+ }
+
@Test
public void testGetNextRetryTime() throws Exception {
GenericTestUtils.setLogLevel(UserGroupInformation.LOG, Level.DEBUG);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org