You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/04/27 23:26:07 UTC
[04/50] [abbrv] hadoop git commit: YARN-3522. Fixed DistributedShell
to instantiate TimeLineClient as the correct user. Contributed by Zhijie Shen
YARN-3522. Fixed DistributedShell to instantiate TimeLineClient as the correct user. Contributed by Zhijie Shen
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78ac5d3f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78ac5d3f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78ac5d3f
Branch: refs/heads/YARN-2928
Commit: 78ac5d3f9e5a7b3e4fc8fa898f6b02cea45c00b9
Parents: d274d3c
Author: Jian He <ji...@apache.org>
Authored: Thu Apr 23 11:07:26 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Apr 27 14:18:32 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 165 +++++++++----------
.../distributedshell/TestDSAppMaster.java | 41 +++++
.../distributedshell/TestDSFailedAppMaster.java | 2 +-
.../hadoop/yarn/client/api/TimelineClient.java | 10 ++
.../client/api/impl/TimelineClientImpl.java | 5 +
6 files changed, 135 insertions(+), 91 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ac5d3f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 34ca05d..5df2cc7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -341,6 +341,9 @@ Release 2.7.1 - UNRELEASED
YARN-2605. [RM HA] Rest api endpoints doing redirect incorrectly.
(Xuan Gong via stevel)
+ YARN-3522. Fixed DistributedShell to instantiate TimeLineClient as the
+ correct user. (Zhijie Shen via jianhe)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ac5d3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index c37fdd0..2470235 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -192,13 +192,14 @@ public class ApplicationMaster {
private AMRMClientAsync amRMClient;
// In both secure and non-secure modes, this points to the job-submitter.
- private UserGroupInformation appSubmitterUgi;
+ @VisibleForTesting
+ UserGroupInformation appSubmitterUgi;
// Handle to communicate with the Node Manager
private NMClientAsync nmClientAsync;
// Listen to process the response from the Node Manager
private NMCallbackHandler containerListener;
-
+
// Application Attempt Id ( combination of attemptId and fail count )
@VisibleForTesting
protected ApplicationAttemptId appAttemptID;
@@ -283,7 +284,8 @@ public class ApplicationMaster {
private List<Thread> launchThreads = new ArrayList<Thread>();
// Timeline Client
- private TimelineClient timelineClient;
+ @VisibleForTesting
+ TimelineClient timelineClient;
private final String linux_bash_command = "bash";
private final String windows_command = "cmd /c";
@@ -531,7 +533,7 @@ public class ApplicationMaster {
.getOptionValue("priority", "0"));
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
if (cliParser.hasOption("timeline_service_version")) {
String timelineServiceVersion =
cliParser.getOptionValue("timeline_service_version", "v1");
@@ -544,24 +546,12 @@ public class ApplicationMaster {
"timeline_service_version is not set properly, should be 'v1' or 'v2'");
}
}
- // Creating the Timeline Client
- if (newTimelineService) {
- timelineClient = TimelineClient.createTimelineClient(
- appAttemptID.getApplicationId());
- } else {
- timelineClient = TimelineClient.createTimelineClient();
- }
- timelineClient.init(conf);
- timelineClient.start();
} else {
- timelineClient = null;
- LOG.warn("Timeline service is not enabled");
if (cliParser.hasOption("timeline_service_version")) {
throw new IllegalArgumentException(
"Timeline service is not enabled");
}
}
-
return true;
}
@@ -581,7 +571,7 @@ public class ApplicationMaster {
* @throws IOException
*/
@SuppressWarnings({ "unchecked" })
- public void run() throws YarnException, IOException {
+ public void run() throws YarnException, IOException, InterruptedException {
LOG.info("Starting ApplicationMaster");
// Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
@@ -608,23 +598,10 @@ public class ApplicationMaster {
appSubmitterUgi =
UserGroupInformation.createRemoteUser(appSubmitterUserName);
appSubmitterUgi.addCredentials(credentials);
-
- if(timelineClient != null) {
- if (newTimelineService) {
- publishApplicationAttemptEventOnNewTimelineService(timelineClient,
- appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId,
- appSubmitterUgi);
- } else {
- publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
- DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
- }
- }
AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
amRMClient.init(conf);
- // need to bind timelineClient before start.
- amRMClient.registerTimelineClient(timelineClient);
amRMClient.start();
containerListener = createNMCallbackHandler();
@@ -632,6 +609,20 @@ public class ApplicationMaster {
nmClientAsync.init(conf);
nmClientAsync.start();
+ startTimelineClient(conf);
+ // need to bind timelineClient
+ amRMClient.registerTimelineClient(timelineClient);
+ if(timelineClient != null) {
+ if (newTimelineService) {
+ publishApplicationAttemptEventOnNewTimelineService(timelineClient,
+ appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId,
+ appSubmitterUgi);
+ } else {
+ publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
+ DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
+ }
+ }
+
// Setup local RPC Server to accept status requests directly from clients
// TODO need to setup a protocol for client to be able to communicate to
// the RPC server
@@ -686,16 +677,35 @@ public class ApplicationMaster {
amRMClient.addContainerRequest(containerAsk);
}
numRequestedContainers.set(numTotalContainers);
+ }
- if(timelineClient != null) {
- if (newTimelineService) {
- publishApplicationAttemptEventOnNewTimelineService(timelineClient,
- appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId,
- appSubmitterUgi);
- } else {
- publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
- DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
- }
+ @VisibleForTesting
+ void startTimelineClient(final Configuration conf)
+ throws YarnException, IOException, InterruptedException {
+ try {
+ appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+ // Creating the Timeline Client
+ if (newTimelineService) {
+ timelineClient = TimelineClient.createTimelineClient(
+ appAttemptID.getApplicationId());
+ } else {
+ timelineClient = TimelineClient.createTimelineClient();
+ }
+ timelineClient.init(conf);
+ timelineClient.start();
+ } else {
+ timelineClient = null;
+ LOG.warn("Timeline service is not enabled");
+ }
+ return null;
+ }
+ });
+ } catch (UndeclaredThrowableException e) {
+ throw new YarnException(e.getCause());
}
}
@@ -714,6 +724,17 @@ public class ApplicationMaster {
} catch (InterruptedException ex) {}
}
+ if(timelineClient != null) {
+ if (newTimelineService) {
+ publishApplicationAttemptEventOnNewTimelineService(timelineClient,
+ appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId,
+ appSubmitterUgi);
+ } else {
+ publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
+ DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
+ }
+ }
+
// Join all launched threads
// needed for when we time out
// and we need to release containers
@@ -1183,18 +1204,11 @@ public class ApplicationMaster {
event.addEventInfo("State", container.getState().name());
event.addEventInfo("Exit Status", container.getExitStatus());
entity.addEvent(event);
-
try {
- ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
- @Override
- public TimelinePutResponse run() throws Exception {
- return timelineClient.putEntities(entity);
- }
- });
- } catch (Exception e) {
+ timelineClient.putEntities(entity);
+ } catch (YarnException | IOException e) {
LOG.error("Container end event could not be published for "
- + container.getContainerId().toString(),
- e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ + container.getContainerId().toString(), e);
}
}
@@ -1210,20 +1224,13 @@ public class ApplicationMaster {
event.setEventType(appEvent.toString());
event.setTimestamp(System.currentTimeMillis());
entity.addEvent(event);
-
try {
- ugi.doAs(new PrivilegedExceptionAction<TimelinePutResponse>() {
- @Override
- public TimelinePutResponse run() throws Exception {
- return timelineClient.putEntities(entity);
- }
- });
- } catch (Exception e) {
+ timelineClient.putEntities(entity);
+ } catch (YarnException | IOException e) {
LOG.error("App Attempt "
+ (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
+ " event could not be published for "
- + appAttemptId.toString(),
- e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ + appAttemptId.toString(), e);
}
}
@@ -1258,17 +1265,10 @@ public class ApplicationMaster {
entity.addEvent(event);
try {
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public TimelinePutResponse run() throws Exception {
- timelineClient.putEntities(entity);
- return null;
- }
- });
- } catch (Exception e) {
+ timelineClient.putEntities(entity);
+ } catch (YarnException | IOException e) {
LOG.error("Container start event could not be published for "
- + container.getId().toString(),
- e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ + container.getId().toString(), e);
}
}
@@ -1300,19 +1300,11 @@ public class ApplicationMaster {
event.addInfo("State", container.getState().name());
event.addInfo("Exit Status", container.getExitStatus());
entity.addEvent(event);
-
try {
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public TimelinePutResponse run() throws Exception {
- timelineClient.putEntities(entity);
- return null;
- }
- });
- } catch (Exception e) {
+ timelineClient.putEntities(entity);
+ } catch (YarnException | IOException e) {
LOG.error("Container end event could not be published for "
- + container.getContainerId().toString(),
- e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ + container.getContainerId().toString(), e);
}
}
@@ -1346,19 +1338,12 @@ public class ApplicationMaster {
entity.addEvent(event);
try {
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public TimelinePutResponse run() throws Exception {
- timelineClient.putEntities(entity);
- return null;
- }
- });
- } catch (Exception e) {
+ timelineClient.putEntities(entity);
+ } catch (YarnException | IOException e) {
LOG.error("App Attempt "
+ (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
+ " event could not be published for "
- + appAttemptId.toString(),
- e instanceof UndeclaredThrowableException ? e.getCause() : e);
+ + appAttemptId.toString(), e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ac5d3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
new file mode 100644
index 0000000..11e840a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSAppMaster.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.applications.distributedshell;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestDSAppMaster {
+
+ @Test
+ public void testTimelineClientInDSAppMaster() throws Exception {
+ ApplicationMaster appMaster = new ApplicationMaster();
+ appMaster.appSubmitterUgi =
+ UserGroupInformation.createUserForTesting("foo", new String[]{"bar"});
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ appMaster.startTimelineClient(conf);
+ Assert.assertEquals(appMaster.appSubmitterUgi,
+ ((TimelineClientImpl)appMaster.timelineClient).getUgi());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ac5d3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java
index f3ab4b7..26022d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDSFailedAppMaster.java
@@ -29,7 +29,7 @@ public class TestDSFailedAppMaster extends ApplicationMaster {
private static final Log LOG = LogFactory.getLog(TestDSFailedAppMaster.class);
@Override
- public void run() throws YarnException, IOException {
+ public void run() throws YarnException, IOException, InterruptedException {
super.run();
// for the 2nd attempt.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ac5d3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index b4c3980..09dd5dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -43,6 +44,15 @@ public abstract class TimelineClient extends AbstractService {
protected ApplicationId contextAppId;
+ /**
+ * Create a timeline client. The current UGI when the user initialize the
+ * client will be used to do the put and the delegation token operations. The
+ * current user may use {@link UserGroupInformation#doAs} another user to
+ * construct and initialize a timeline client if the following operations are
+ * supposed to be conducted by that user.
+ *
+ * @return a timeline client
+ */
@Public
public static TimelineClient createTimelineClient() {
TimelineClient client = new TimelineClientImpl();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78ac5d3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index c54ea1e..5012e7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -823,4 +823,9 @@ public class TimelineClientImpl extends TimelineClient {
new HelpFormatter().printHelp("TimelineClient", opts);
}
+ @VisibleForTesting
+ @Private
+ public UserGroupInformation getUgi() {
+ return authUgi;
+ }
}