You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/03/13 00:04:43 UTC
[24/49] hadoop git commit: YARN-3287. Made TimelineClient put methods
do as the correct login context. Contributed by Daryn Sharp and Jonathan
Eagles.
YARN-3287. Made TimelineClient put methods do as the correct login context. Contributed by Daryn Sharp and Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d6e05c5e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d6e05c5e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d6e05c5e
Branch: refs/heads/YARN-2928
Commit: d6e05c5ee26feefc17267b7c9db1e2a3dbdef117
Parents: 3241fc2
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Mar 9 13:54:36 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Mar 9 13:54:36 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../client/api/impl/TimelineClientImpl.java | 82 +++----
.../TestTimelineAuthenticationFilter.java | 221 +++++++++----------
3 files changed, 135 insertions(+), 171 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6e05c5e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index da8b02e..1894552 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -737,6 +737,9 @@ Release 2.7.0 - UNRELEASED
YARN-3275. CapacityScheduler: Preemption happening on non-preemptable
queues (Eric Payne via jlowe)
+ YARN-3287. Made TimelineClient put methods do as the correct login context.
+ (Daryn Sharp and Jonathan Eagles via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6e05c5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index c05d65b..df6c7a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
@@ -108,6 +109,8 @@ public class TimelineClientImpl extends TimelineClient {
private DelegationTokenAuthenticator authenticator;
private DelegationTokenAuthenticatedURL.Token token;
private URI resURI;
+ private UserGroupInformation authUgi;
+ private String doAsUser;
@Private
@VisibleForTesting
@@ -252,6 +255,15 @@ public class TimelineClientImpl extends TimelineClient {
}
protected void serviceInit(Configuration conf) throws Exception {
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ UserGroupInformation realUgi = ugi.getRealUser();
+ if (realUgi != null) {
+ authUgi = realUgi;
+ doAsUser = ugi.getShortUserName();
+ } else {
+ authUgi = ugi;
+ doAsUser = null;
+ }
ClientConfig cc = new DefaultClientConfig();
cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
connConfigurator = newConnConfigurator(conf);
@@ -301,16 +313,20 @@ public class TimelineClientImpl extends TimelineClient {
doPosting(domain, "domain");
}
- private ClientResponse doPosting(Object obj, String path) throws IOException, YarnException {
+ private ClientResponse doPosting(final Object obj, final String path)
+ throws IOException, YarnException {
ClientResponse resp;
try {
- resp = doPostingObject(obj, path);
- } catch (RuntimeException re) {
- // runtime exception is expected if the client cannot connect the server
- String msg =
- "Failed to get the response from the timeline server.";
- LOG.error(msg, re);
- throw re;
+ resp = authUgi.doAs(new PrivilegedExceptionAction<ClientResponse>() {
+ @Override
+ public ClientResponse run() throws Exception {
+ return doPostingObject(obj, path);
+ }
+ });
+ } catch (UndeclaredThrowableException e) {
+ throw new IOException(e.getCause());
+ } catch (InterruptedException ie) {
+ throw new IOException(ie);
}
if (resp == null ||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
@@ -331,11 +347,6 @@ public class TimelineClientImpl extends TimelineClient {
@Override
public Token<TimelineDelegationTokenIdentifier> getDelegationToken(
final String renewer) throws IOException, YarnException {
- boolean isProxyAccess =
- UserGroupInformation.getCurrentUser().getAuthenticationMethod()
- == UserGroupInformation.AuthenticationMethod.PROXY;
- final String doAsUser = isProxyAccess ?
- UserGroupInformation.getCurrentUser().getShortUserName() : null;
PrivilegedExceptionAction<Token<TimelineDelegationTokenIdentifier>> getDTAction =
new PrivilegedExceptionAction<Token<TimelineDelegationTokenIdentifier>>() {
@@ -357,11 +368,6 @@ public class TimelineClientImpl extends TimelineClient {
public long renewDelegationToken(
final Token<TimelineDelegationTokenIdentifier> timelineDT)
throws IOException, YarnException {
- boolean isProxyAccess =
- UserGroupInformation.getCurrentUser().getAuthenticationMethod()
- == UserGroupInformation.AuthenticationMethod.PROXY;
- final String doAsUser = isProxyAccess ?
- UserGroupInformation.getCurrentUser().getShortUserName() : null;
boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
final String scheme = useHttps ? "https" : "http";
final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
@@ -393,11 +399,6 @@ public class TimelineClientImpl extends TimelineClient {
public void cancelDelegationToken(
final Token<TimelineDelegationTokenIdentifier> timelineDT)
throws IOException, YarnException {
- boolean isProxyAccess =
- UserGroupInformation.getCurrentUser().getAuthenticationMethod()
- == UserGroupInformation.AuthenticationMethod.PROXY;
- final String doAsUser = isProxyAccess ?
- UserGroupInformation.getCurrentUser().getShortUserName() : null;
boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
final String scheme = useHttps ? "https" : "http";
final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
@@ -433,15 +434,9 @@ public class TimelineClientImpl extends TimelineClient {
@Override
public Object run() throws IOException {
// Try pass the request, if fail, keep retrying
- boolean isProxyAccess =
- UserGroupInformation.getCurrentUser().getAuthenticationMethod()
- == UserGroupInformation.AuthenticationMethod.PROXY;
- UserGroupInformation callerUGI = isProxyAccess ?
- UserGroupInformation.getCurrentUser().getRealUser()
- : UserGroupInformation.getCurrentUser();
- callerUGI.checkTGTAndReloginFromKeytab();
+ authUgi.checkTGTAndReloginFromKeytab();
try {
- return callerUGI.doAs(action);
+ return authUgi.doAs(action);
} catch (UndeclaredThrowableException e) {
throw new IOException(e.getCause());
} catch (InterruptedException e) {
@@ -481,28 +476,15 @@ public class TimelineClientImpl extends TimelineClient {
@Override
public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
- boolean isProxyAccess =
- UserGroupInformation.getCurrentUser().getAuthenticationMethod()
- == UserGroupInformation.AuthenticationMethod.PROXY;
- UserGroupInformation callerUGI = isProxyAccess ?
- UserGroupInformation.getCurrentUser().getRealUser()
- : UserGroupInformation.getCurrentUser();
- final String doAsUser = isProxyAccess ?
- UserGroupInformation.getCurrentUser().getShortUserName() : null;
- callerUGI.checkTGTAndReloginFromKeytab();
+ authUgi.checkTGTAndReloginFromKeytab();
try {
- return callerUGI.doAs(new PrivilegedExceptionAction<HttpURLConnection>() {
- @Override
- public HttpURLConnection run() throws Exception {
- return new DelegationTokenAuthenticatedURL(
- authenticator, connConfigurator).openConnection(url, token,
- doAsUser);
- }
- });
+ return new DelegationTokenAuthenticatedURL(
+ authenticator, connConfigurator).openConnection(url, token,
+ doAsUser);
} catch (UndeclaredThrowableException e) {
throw new IOException(e.getCause());
- } catch (InterruptedException e) {
- throw new IOException(e);
+ } catch (AuthenticationException ae) {
+ throw new IOException(ae);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d6e05c5e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
index 53d8c81..c93e8f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/security/TestTimelineAuthenticationFilter.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authentication.KerberosTestUtils;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
@@ -47,9 +48,9 @@ import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
-import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -77,20 +78,19 @@ public class TestTimelineAuthenticationFilter {
return Arrays.asList(new Object[][] { { false }, { true } });
}
- private MiniKdc testMiniKDC;
- private String keystoresDir;
- private String sslConfDir;
- private ApplicationHistoryServer testTimelineServer;
- private Configuration conf;
- private TimelineClient client;
- private boolean withSsl;
+ private static MiniKdc testMiniKDC;
+ private static String keystoresDir;
+ private static String sslConfDir;
+ private static ApplicationHistoryServer testTimelineServer;
+ private static Configuration conf;
+ private static boolean withSsl;
public TestTimelineAuthenticationFilter(boolean withSsl) {
- this.withSsl = withSsl;
+ TestTimelineAuthenticationFilter.withSsl = withSsl;
}
- @Before
- public void setup() {
+ @BeforeClass
+ public static void setup() {
try {
testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
testMiniKDC.start();
@@ -127,6 +127,7 @@ public class TestTimelineAuthenticationFilter {
"localhost:8190");
conf.set("hadoop.proxyuser.HTTP.hosts", "*");
conf.set("hadoop.proxyuser.HTTP.users", FOO_USER);
+ conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES, 1);
if (withSsl) {
conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY,
@@ -146,14 +147,17 @@ public class TestTimelineAuthenticationFilter {
} catch (Exception e) {
assertTrue("Couldn't setup TimelineServer", false);
}
+ }
- client = TimelineClient.createTimelineClient();
+ private TimelineClient createTimelineClientForUGI() {
+ TimelineClient client = TimelineClient.createTimelineClient();
client.init(conf);
client.start();
+ return client;
}
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDown() throws Exception {
if (testMiniKDC != null) {
testMiniKDC.stop();
}
@@ -162,10 +166,6 @@ public class TestTimelineAuthenticationFilter {
testTimelineServer.stop();
}
- if (client != null) {
- client.stop();
- }
-
if (withSsl) {
KeyStoreTestUtil.cleanupSSLConfig(keystoresDir, sslConfDir);
File base = new File(BASEDIR);
@@ -178,6 +178,7 @@ public class TestTimelineAuthenticationFilter {
KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
@Override
public Void call() throws Exception {
+ TimelineClient client = createTimelineClientForUGI();
TimelineEntity entityToStore = new TimelineEntity();
entityToStore.setEntityType(
TestTimelineAuthenticationFilter.class.getName());
@@ -199,6 +200,7 @@ public class TestTimelineAuthenticationFilter {
KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
@Override
public Void call() throws Exception {
+ TimelineClient client = createTimelineClientForUGI();
TimelineDomain domainToStore = new TimelineDomain();
domainToStore.setId(TestTimelineAuthenticationFilter.class.getName());
domainToStore.setReaders("*");
@@ -215,119 +217,96 @@ public class TestTimelineAuthenticationFilter {
@Test
public void testDelegationTokenOperations() throws Exception {
- KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- // Let HTTP user to get the delegation for itself
- Token<TimelineDelegationTokenIdentifier> token =
- client.getDelegationToken(
- UserGroupInformation.getCurrentUser().getShortUserName());
- Assert.assertNotNull(token);
- TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier();
- Assert.assertNotNull(tDT);
- Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner());
+ TimelineClient httpUserClient =
+ KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<TimelineClient>() {
+ @Override
+ public TimelineClient call() throws Exception {
+ return createTimelineClientForUGI();
+ }
+ });
+ UserGroupInformation httpUser =
+ KerberosTestUtils.doAs(HTTP_USER + "/localhost", new Callable<UserGroupInformation>() {
+ @Override
+ public UserGroupInformation call() throws Exception {
+ return UserGroupInformation.getCurrentUser();
+ }
+ });
+ // Let HTTP user to get the delegation for itself
+ Token<TimelineDelegationTokenIdentifier> token =
+ httpUserClient.getDelegationToken(httpUser.getShortUserName());
+ Assert.assertNotNull(token);
+ TimelineDelegationTokenIdentifier tDT = token.decodeIdentifier();
+ Assert.assertNotNull(tDT);
+ Assert.assertEquals(new Text(HTTP_USER), tDT.getOwner());
- // Renew token
- long renewTime1 = client.renewDelegationToken(token);
- Thread.sleep(100);
- long renewTime2 = client.renewDelegationToken(token);
- Assert.assertTrue(renewTime1 < renewTime2);
+ // Renew token
+ long renewTime1 = httpUserClient.renewDelegationToken(token);
+ Thread.sleep(100);
+ long renewTime2 = httpUserClient.renewDelegationToken(token);
+ Assert.assertTrue(renewTime1 < renewTime2);
- // Cancel token
- client.cancelDelegationToken(token);
- // Renew should not be successful because the token is canceled
- try {
- client.renewDelegationToken(token);
- Assert.fail();
- } catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains(
- "Renewal request for unknown token"));
- }
+ // Cancel token
+ httpUserClient.cancelDelegationToken(token);
+ // Renew should not be successful because the token is canceled
+ try {
+ httpUserClient.renewDelegationToken(token);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Renewal request for unknown token"));
+ }
- // Let HTTP user to get the delegation token for FOO user
- UserGroupInformation fooUgi = UserGroupInformation.createProxyUser(
- FOO_USER, UserGroupInformation.getCurrentUser());
- token = fooUgi.doAs(
- new PrivilegedExceptionAction<Token<TimelineDelegationTokenIdentifier>>() {
+ // Let HTTP user to get the delegation token for FOO user
+ UserGroupInformation fooUgi = UserGroupInformation.createProxyUser(
+ FOO_USER, httpUser);
+ TimelineClient fooUserClient = fooUgi.doAs(
+ new PrivilegedExceptionAction<TimelineClient>() {
@Override
- public Token<TimelineDelegationTokenIdentifier> run()
- throws Exception {
- return client.getDelegationToken(
- UserGroupInformation.getCurrentUser().getShortUserName());
+ public TimelineClient run() throws Exception {
+ return createTimelineClientForUGI();
}
});
- Assert.assertNotNull(token);
- tDT = token.decodeIdentifier();
- Assert.assertNotNull(tDT);
- Assert.assertEquals(new Text(FOO_USER), tDT.getOwner());
- Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser());
+ token = fooUserClient.getDelegationToken(httpUser.getShortUserName());
+ Assert.assertNotNull(token);
+ tDT = token.decodeIdentifier();
+ Assert.assertNotNull(tDT);
+ Assert.assertEquals(new Text(FOO_USER), tDT.getOwner());
+ Assert.assertEquals(new Text(HTTP_USER), tDT.getRealUser());
- // Renew token
- final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token;
- renewTime1 = fooUgi.doAs(
- new PrivilegedExceptionAction<Long>() {
- @Override
- public Long run() throws Exception {
- return client.renewDelegationToken(tokenToRenew);
- }
- });
- renewTime2 = fooUgi.doAs(
- new PrivilegedExceptionAction<Long>() {
- @Override
- public Long run() throws Exception {
- return client.renewDelegationToken(tokenToRenew);
- }
- });
- Assert.assertTrue(renewTime1 < renewTime2);
+ // Renew token as the renewer
+ final Token<TimelineDelegationTokenIdentifier> tokenToRenew = token;
+ renewTime1 = httpUserClient.renewDelegationToken(tokenToRenew);
+ renewTime2 = httpUserClient.renewDelegationToken(tokenToRenew);
+ Assert.assertTrue(renewTime1 < renewTime2);
- // Cancel token
- fooUgi.doAs(
- new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- client.cancelDelegationToken(tokenToRenew);
- return null;
- }
- });
- // Renew should not be successful because the token is canceled
- try {
- fooUgi.doAs(
- new PrivilegedExceptionAction<Void>() {
- @Override
- public Void run() throws Exception {
- client.renewDelegationToken(tokenToRenew);
- return null;
- }
- });
- Assert.fail();
- } catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains(
- "Renewal request for unknown token"));
- }
+ // Cancel token
+ fooUserClient.cancelDelegationToken(tokenToRenew);
+
+ // Renew should not be successful because the token is canceled
+ try {
+ httpUserClient.renewDelegationToken(tokenToRenew);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(
+ e.getMessage().contains("Renewal request for unknown token"));
+ }
- // Let HTTP user to get the delegation token for BAR user
- UserGroupInformation barUgi = UserGroupInformation.createProxyUser(
- BAR_USER, UserGroupInformation.getCurrentUser());
- token = barUgi.doAs(
- new PrivilegedExceptionAction<Token<TimelineDelegationTokenIdentifier>>() {
+ // Let HTTP user to get the delegation token for BAR user
+ UserGroupInformation barUgi = UserGroupInformation.createProxyUser(
+ BAR_USER, httpUser);
+ TimelineClient barUserClient = barUgi.doAs(
+ new PrivilegedExceptionAction<TimelineClient>() {
@Override
- public Token<TimelineDelegationTokenIdentifier> run()
- throws Exception {
- try {
- Token<TimelineDelegationTokenIdentifier> token =
- client.getDelegationToken(
- UserGroupInformation.getCurrentUser().getShortUserName());
- Assert.fail();
- return token;
- } catch (Exception e) {
- Assert.assertTrue(e instanceof AuthorizationException);
- return null;
- }
+ public TimelineClient run() {
+ return createTimelineClientForUGI();
}
});
- return null;
- }
- });
- }
+ try {
+ barUserClient.getDelegationToken(httpUser.getShortUserName());
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getCause() instanceof AuthorizationException || e.getCause() instanceof AuthenticationException);
+ }
+ }
}