You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2014/09/19 01:01:35 UTC
[05/12] git commit: YARN-2563. Fixed YarnClient to call
getTimeLineDelegationToken only if the Token is not present. Contributed by
Zhijie Shen
YARN-2563. Fixed YarnClient to call getTimeLineDelegationToken only if the Token is not present. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb92cc67
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb92cc67
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb92cc67
Branch: refs/heads/HDFS-6581
Commit: eb92cc67dfaa51212fc5315b8db99effd046a154
Parents: a337f0e
Author: Jian He <ji...@apache.org>
Authored: Thu Sep 18 14:36:23 2014 -0700
Committer: Jian He <ji...@apache.org>
Committed: Thu Sep 18 14:40:24 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/client/api/impl/YarnClientImpl.java | 24 ++++---
.../yarn/client/api/impl/TestYarnClient.java | 76 ++++++++++++--------
3 files changed, 66 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb92cc67/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 836248b..759f2bf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -394,6 +394,9 @@ Release 2.6.0 - UNRELEASED
YARN-2561. MR job client cannot reconnect to AM after NM restart. (Junping
Du via jlowe)
+ YARN-2563. Fixed YarnClient to call getTimeLineDelegationToken only if the
+ Token is not present. (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb92cc67/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index f1a3b6e..def6da5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -110,7 +110,8 @@ public class YarnClientImpl extends YarnClient {
private AHSClient historyClient;
private boolean historyServiceEnabled;
protected TimelineClient timelineClient;
- protected Text timelineService;
+ @VisibleForTesting
+ Text timelineService;
protected boolean timelineServiceEnabled;
private static final String ROOT = "root";
@@ -272,12 +273,6 @@ public class YarnClientImpl extends YarnClient {
private void addTimelineDelegationToken(
ContainerLaunchContext clc) throws YarnException, IOException {
- org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier> timelineDelegationToken =
- timelineClient.getDelegationToken(
- UserGroupInformation.getCurrentUser().getUserName());
- if (timelineDelegationToken == null) {
- return;
- }
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = clc.getTokens();
@@ -290,11 +285,15 @@ public class YarnClientImpl extends YarnClient {
// one more
for (org.apache.hadoop.security.token.Token<? extends TokenIdentifier> token : credentials
.getAllTokens()) {
- TokenIdentifier tokenIdentifier = token.decodeIdentifier();
- if (tokenIdentifier instanceof TimelineDelegationTokenIdentifier) {
+ if (token.getKind().equals(TimelineDelegationTokenIdentifier.KIND_NAME)) {
return;
}
}
+ org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
+ timelineDelegationToken = getTimelineDelegationToken();
+ if (timelineDelegationToken == null) {
+ return;
+ }
credentials.addToken(timelineService, timelineDelegationToken);
if (LOG.isDebugEnabled()) {
LOG.debug("Add timline delegation token into credentials: "
@@ -306,6 +305,13 @@ public class YarnClientImpl extends YarnClient {
clc.setTokens(tokens);
}
+ @VisibleForTesting
+ org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
+ getTimelineDelegationToken() throws IOException, YarnException {
+ return timelineClient.getDelegationToken(
+ UserGroupInformation.getCurrentUser().getUserName());
+ }
+
@Private
@VisibleForTesting
protected boolean isSecurityEnabled() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb92cc67/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 8259893..3c1b1c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.client.api.impl;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -39,6 +41,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@@ -745,10 +748,13 @@ public class TestYarnClient {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ TimelineDelegationTokenIdentifier timelineDT =
+ new TimelineDelegationTokenIdentifier();
final Token<TimelineDelegationTokenIdentifier> dToken =
- new Token<TimelineDelegationTokenIdentifier>();
+ new Token<TimelineDelegationTokenIdentifier>(
+ timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
// crate a mock client
- YarnClientImpl client = new YarnClientImpl() {
+ YarnClientImpl client = spy(new YarnClientImpl() {
@Override
protected void serviceInit(Configuration conf) throws Exception {
if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
@@ -784,34 +790,48 @@ public class TestYarnClient {
public boolean isSecurityEnabled() {
return true;
}
- };
+ });
client.init(conf);
client.start();
- ApplicationSubmissionContext context =
- mock(ApplicationSubmissionContext.class);
- ApplicationId applicationId = ApplicationId.newInstance(0, 1);
- when(context.getApplicationId()).thenReturn(applicationId);
- DataOutputBuffer dob = new DataOutputBuffer();
- Credentials credentials = new Credentials();
- credentials.writeTokenStorageToStream(dob);
- ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
- null, null, null, null, tokens, null);
- when(context.getAMContainerSpec()).thenReturn(clc);
- client.submitApplication(context);
- // Check whether token is added or not
- credentials = new Credentials();
- DataInputByteBuffer dibb = new DataInputByteBuffer();
- tokens = clc.getTokens();
- if (tokens != null) {
- dibb.reset(tokens);
- credentials.readTokenStorageStream(dibb);
- tokens.rewind();
+ try {
+ // when i == 0, timeline DT already exists, no need to get one more
+ // when i == 1, timeline DT doesn't exist, need to get one more
+ for (int i = 0; i < 2; ++i) {
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ ApplicationId applicationId = ApplicationId.newInstance(0, i + 1);
+ when(context.getApplicationId()).thenReturn(applicationId);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Credentials credentials = new Credentials();
+ if (i == 0) {
+ credentials.addToken(client.timelineService, dToken);
+ }
+ credentials.writeTokenStorageToStream(dob);
+ ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+ null, null, null, null, tokens, null);
+ when(context.getAMContainerSpec()).thenReturn(clc);
+ client.submitApplication(context);
+ if (i == 0) {
+ // GetTimelineDelegationToken shouldn't be called
+ verify(client, never()).getTimelineDelegationToken();
+ }
+ // In either way, token should be there
+ credentials = new Credentials();
+ DataInputByteBuffer dibb = new DataInputByteBuffer();
+ tokens = clc.getTokens();
+ if (tokens != null) {
+ dibb.reset(tokens);
+ credentials.readTokenStorageStream(dibb);
+ tokens.rewind();
+ }
+ Collection<Token<? extends TokenIdentifier>> dTokens =
+ credentials.getAllTokens();
+ Assert.assertEquals(1, dTokens.size());
+ Assert.assertEquals(dToken, dTokens.iterator().next());
+ }
+ } finally {
+ client.stop();
}
- Collection<Token<? extends TokenIdentifier>> dTokens =
- credentials.getAllTokens();
- Assert.assertEquals(1, dTokens.size());
- Assert.assertEquals(dToken, dTokens.iterator().next());
- client.stop();
}
}