You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jl...@apache.org on 2015/02/06 21:55:37 UTC
hadoop git commit: YARN-3144. Configuration for making delegation
token failures to timeline server not-fatal. Contributed by Jonathan Eagles
Repository: hadoop
Updated Branches:
refs/heads/trunk 4c484320b -> 6f10434a5
YARN-3144. Configuration for making delegation token failures to timeline server not-fatal. Contributed by Jonathan Eagles
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f10434a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f10434a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f10434a
Branch: refs/heads/trunk
Commit: 6f10434a5ad965d50352602ce31a9fce353cb90c
Parents: 4c48432
Author: Jason Lowe <jl...@apache.org>
Authored: Fri Feb 6 20:55:00 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Fri Feb 6 20:55:00 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 7 +++
.../yarn/client/api/impl/YarnClientImpl.java | 22 +++++++-
.../yarn/client/api/impl/TestYarnClient.java | 56 +++++++++++++++-----
.../src/main/resources/yarn-default.xml | 6 +++
5 files changed, 80 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f10434a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 51ed31f..79e66a7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -252,6 +252,9 @@ Release 2.7.0 - UNRELEASED
ApplicationHistoryClientService are uniform when application-attempt is
not found. (zjshen via acmurthy)
+ YARN-3144. Configuration for making delegation token failures to timeline
+ server not-fatal (Jonathan Eagles via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f10434a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 015baa1..d6f6dee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1385,6 +1385,13 @@ public class YarnConfiguration extends Configuration {
public static final long
DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000;
+ /** Timeline client policy for whether connections are fatal */
+ public static final String TIMELINE_SERVICE_CLIENT_BEST_EFFORT =
+ TIMELINE_SERVICE_CLIENT_PREFIX + "best-effort";
+
+ public static final boolean
+ DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT = false;
+
/** Flag to enable recovery of timeline service */
public static final String TIMELINE_SERVICE_RECOVERY_ENABLED =
TIMELINE_SERVICE_PREFIX + "recovery.enabled";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f10434a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 975c267..91fbd00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -129,6 +129,7 @@ public class YarnClientImpl extends YarnClient {
@VisibleForTesting
String timelineDTRenewer;
protected boolean timelineServiceEnabled;
+ protected boolean timelineServiceBestEffort;
private static final String ROOT = "root";
@@ -163,14 +164,22 @@ public class YarnClientImpl extends YarnClient {
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
timelineServiceEnabled = true;
- timelineClient = TimelineClient.createTimelineClient();
+ timelineClient = createTimelineClient();
timelineClient.init(conf);
timelineDTRenewer = getTimelineDelegationTokenRenewer(conf);
timelineService = TimelineUtils.buildTimelineTokenService(conf);
}
+
+ timelineServiceBestEffort = conf.getBoolean(
+ YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
super.serviceInit(conf);
}
+ TimelineClient createTimelineClient() throws IOException, YarnException {
+ return TimelineClient.createTimelineClient();
+ }
+
@Override
protected void serviceStart() throws Exception {
try {
@@ -325,7 +334,16 @@ public class YarnClientImpl extends YarnClient {
@VisibleForTesting
org.apache.hadoop.security.token.Token<TimelineDelegationTokenIdentifier>
getTimelineDelegationToken() throws IOException, YarnException {
- return timelineClient.getDelegationToken(timelineDTRenewer);
+ try {
+ return timelineClient.getDelegationToken(timelineDTRenewer);
+ } catch (Exception e ) {
+ if (timelineServiceBestEffort) {
+ LOG.warn("Failed to get delegation token from the timeline server: "
+ + e.getMessage());
+ return null;
+ }
+ throw e;
+ }
}
private static String getTimelineDelegationTokenRenewer(Configuration conf)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f10434a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index eb8ca48..73e06af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -854,6 +854,42 @@ public class TestYarnClient {
}
@Test
+ public void testBestEffortTimelineDelegationToken()
+ throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+
+ YarnClientImpl client = spy(new YarnClientImpl() {
+
+ @Override
+ TimelineClient createTimelineClient() throws IOException, YarnException {
+ timelineClient = mock(TimelineClient.class);
+ when(timelineClient.getDelegationToken(any(String.class)))
+ .thenThrow(new IOException("Best effort test exception"));
+ return timelineClient;
+ }
+ });
+
+ client.init(conf);
+ try {
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, true);
+ client.serviceInit(conf);
+ client.getTimelineDelegationToken();
+ } catch (Exception e) {
+ Assert.fail("Should not have thrown an exception");
+ }
+ try {
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT, false);
+ client.serviceInit(conf);
+ client.getTimelineDelegationToken();
+ Assert.fail("Get delegation token should have thrown an exception");
+ } catch (Exception e) {
+ // Success
+ }
+ }
+
+ @Test
public void testAutomaticTimelineDelegationTokenLoading()
throws Exception {
Configuration conf = new YarnConfiguration();
@@ -864,22 +900,18 @@ public class TestYarnClient {
final Token<TimelineDelegationTokenIdentifier> dToken =
new Token<TimelineDelegationTokenIdentifier>(
timelineDT.getBytes(), new byte[0], timelineDT.getKind(), new Text());
- // crate a mock client
+ // create a mock client
YarnClientImpl client = spy(new YarnClientImpl() {
+
@Override
- protected void serviceInit(Configuration conf) throws Exception {
- if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
- timelineServiceEnabled = true;
- timelineClient = mock(TimelineClient.class);
- when(timelineClient.getDelegationToken(any(String.class)))
- .thenReturn(dToken);
- timelineClient.init(getConfig());
- timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
- }
- this.setConfig(conf);
+ TimelineClient createTimelineClient() throws IOException, YarnException {
+ timelineClient = mock(TimelineClient.class);
+ when(timelineClient.getDelegationToken(any(String.class)))
+ .thenReturn(dToken);
+ return timelineClient;
}
+
@Override
protected void serviceStart() throws Exception {
rmClient = mock(ApplicationClientProtocol.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f10434a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 349e57b..971ce0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1371,6 +1371,12 @@
</property>
<property>
+ <description>Client policy for whether timeline operations are non-fatal</description>
+ <name>yarn.timeline-service.client.best-effort</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>
Default retry time interval for timeline servive client.
</description>