You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/09/23 16:07:43 UTC
svn commit: r1174785 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/or...
Author: vinodkv
Date: Fri Sep 23 14:07:42 2011
New Revision: 1174785
URL: http://svn.apache.org/viewvc?rev=1174785&view=rev
Log:
MAPREDUCE-3055. Simplified ApplicationAttemptId passing to ApplicationMaster via environment variable. (vinodkv)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
Removed:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationMasterLauncher.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Sep 23 14:07:42 2011
@@ -310,6 +310,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2726. Added job-file to the AM and JobHistoryServer web
interfaces. (Jeffrey Naisbitt via vinodkv)
+ MAPREDUCE-3055. Simplified ApplicationAttemptId passing to
+ ApplicationMaster via environment variable. (vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Fri Sep 23 14:07:42 2011
@@ -77,6 +77,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.SystemClock;
import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -87,6 +88,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
+import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* The Map-Reduce Application Master.
@@ -647,13 +649,18 @@ public class MRAppMaster extends Composi
public static void main(String[] args) {
try {
- //Configuration.addDefaultResource("job.xml");
- ApplicationId applicationId = RecordFactoryProvider
- .getRecordFactory(null).newRecordInstance(ApplicationId.class);
- applicationId.setClusterTimestamp(Long.valueOf(args[0]));
- applicationId.setId(Integer.valueOf(args[1]));
- int failCount = Integer.valueOf(args[2]);
- MRAppMaster appMaster = new MRAppMaster(applicationId, failCount);
+ String applicationAttemptIdStr = System
+ .getenv(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV);
+ if (applicationAttemptIdStr == null) {
+ String msg = ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV
+ + " is null";
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ ApplicationAttemptId applicationAttemptId = ConverterUtils
+ .toApplicationAttemptId(applicationAttemptIdStr);
+ MRAppMaster appMaster = new MRAppMaster(applicationAttemptId
+ .getApplicationId(), applicationAttemptId.getAttemptId());
Runtime.getRuntime().addShutdownHook(
new CompositeServiceShutdownHook(appMaster));
YarnConfiguration conf = new YarnConfiguration(new JobConf());
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java Fri Sep 23 14:07:42 2011
@@ -321,9 +321,6 @@ public class YARNRunner implements Clien
MRJobConfig.DEFAULT_MR_AM_COMMAND_OPTS));
vargs.add(MRJobConfig.APPLICATION_MASTER_CLASS);
- vargs.add(String.valueOf(applicationId.getClusterTimestamp()));
- vargs.add(String.valueOf(applicationId.getId()));
- vargs.add(ApplicationConstants.AM_FAIL_COUNT_STRING);
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Path.SEPARATOR + ApplicationConstants.STDOUT);
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR +
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java Fri Sep 23 14:07:42 2011
@@ -37,8 +37,11 @@ public interface ApplicationConstants {
public static final String APPLICATION_CLIENT_SECRET_ENV_NAME =
"AppClientTokenEnv";
- // TODO: Weird. This is part of AM command line. Instead it should be a env.
- public static final String AM_FAIL_COUNT_STRING = "<FAILCOUNT>";
+ /**
+ * The environmental variable for APPLICATION_ATTEMPT_ID. Set in
+ * ApplicationMaster's environment only.
+ */
+ public static final String APPLICATION_ATTEMPT_ID_ENV = "APPLICATION_ATTEMPT_ID";
public static final String CONTAINER_TOKEN_FILE_ENV_NAME =
UserGroupInformation.HADOOP_TOKEN_FILE_LOCATION;
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Fri Sep 23 14:07:42 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.util;
import static org.apache.hadoop.yarn.util.StringHelper._split;
+import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.text.NumberFormat;
@@ -45,6 +46,8 @@ import org.apache.hadoop.yarn.factory.pr
public class ConverterUtils {
public static final String APPLICATION_PREFIX = "application";
+ public static final String CONTAINER_PREFIX = "container";
+ public static final String APPLICATION_ATTEMPT_PREFIX = "appattempt";
/**
* return a hadoop path from a given url
@@ -132,14 +135,12 @@ public class ConverterUtils {
}
private static ApplicationAttemptId toApplicationAttemptId(
- RecordFactory recordFactory,
- Iterator<String> it) {
- ApplicationId appId =
- recordFactory.newRecordInstance(ApplicationId.class);
+ Iterator<String> it) throws NumberFormatException {
+ ApplicationId appId = Records.newRecord(ApplicationId.class);
appId.setClusterTimestamp(Long.parseLong(it.next()));
appId.setId(Integer.parseInt(it.next()));
- ApplicationAttemptId appAttemptId =
- recordFactory.newRecordInstance(ApplicationAttemptId.class);
+ ApplicationAttemptId appAttemptId = Records
+ .newRecord(ApplicationAttemptId.class);
appAttemptId.setApplicationId(appId);
appAttemptId.setAttemptId(Integer.parseInt(it.next()));
return appAttemptId;
@@ -149,16 +150,35 @@ public class ConverterUtils {
return cId.toString();
}
- public static ContainerId toContainerId(RecordFactory recordFactory,
- String containerIdStr) {
+ public static ContainerId toContainerId(String containerIdStr)
+ throws IOException {
Iterator<String> it = _split(containerIdStr).iterator();
- it.next(); // prefix. TODO: Validate container prefix
- ApplicationAttemptId appAttemptID =
- toApplicationAttemptId(recordFactory, it);
- ContainerId containerId =
- recordFactory.newRecordInstance(ContainerId.class);
- containerId.setApplicationAttemptId(appAttemptID);
- containerId.setId(Integer.parseInt(it.next()));
- return containerId;
+ if (!it.next().equals(CONTAINER_PREFIX)) {
+ throw new IOException("Invalid ContainerId prefix: " + containerIdStr);
+ }
+ try {
+ ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
+ ContainerId containerId = Records.newRecord(ContainerId.class);
+ containerId.setApplicationAttemptId(appAttemptID);
+ containerId.setId(Integer.parseInt(it.next()));
+ return containerId;
+ } catch (NumberFormatException n) {
+ throw new IOException("Invalid ContainerId: " + containerIdStr, n);
+ }
+ }
+
+ public static ApplicationAttemptId toApplicationAttemptId(
+ String applicationAttmeptIdStr) throws IOException {
+ Iterator<String> it = _split(applicationAttmeptIdStr).iterator();
+ if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) {
+ throw new IOException("Invalid AppAttemptId prefix: "
+ + applicationAttmeptIdStr);
+ }
+ try {
+ return toApplicationAttemptId(it);
+ } catch (NumberFormatException n) {
+ throw new IOException("Invalid AppAttemptId: "
+ + applicationAttmeptIdStr, n);
+ }
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Fri Sep 23 14:07:42 2011
@@ -31,8 +31,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
@@ -56,22 +54,26 @@ public class ContainerLogsPage extends N
private final Configuration conf;
private final LocalDirAllocator logsSelector;
private final Context nmContext;
- private final RecordFactory recordFactory;
@Inject
public ContainersLogsBlock(Configuration conf, Context context) {
this.conf = conf;
this.logsSelector = new LocalDirAllocator(YarnConfiguration.NM_LOG_DIRS);
this.nmContext = context;
- this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
}
@Override
protected void render(Block html) {
DIV<Hamlet> div = html.div("#content");
- ContainerId containerId =
- ConverterUtils.toContainerId(this.recordFactory, $(CONTAINER_ID));
+ ContainerId containerId;
+ try {
+ containerId = ConverterUtils.toContainerId($(CONTAINER_ID));
+ } catch (IOException e) {
+ div.h1("Invalid containerId " + $(CONTAINER_ID))._();
+ return;
+ }
+
Container container = this.nmContext.getContainers().get(containerId);
if (container == null) {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Fri Sep 23 14:07:42 2011
@@ -18,16 +18,15 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import static org.apache.hadoop.yarn.util.StringHelper.ujoin;
-import org.apache.hadoop.conf.Configuration;
+import java.io.IOException;
+
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -53,21 +52,23 @@ public class ContainerPage extends NMVie
public static class ContainerBlock extends HtmlBlock implements NMWebParams {
- private final Configuration conf;
private final Context nmContext;
- private final RecordFactory recordFactory;
@Inject
- public ContainerBlock(Configuration conf, Context nmContext) {
- this.conf = conf;
+ public ContainerBlock(Context nmContext) {
this.nmContext = nmContext;
- this.recordFactory = RecordFactoryProvider.getRecordFactory(this.conf);
}
@Override
protected void render(Block html) {
- ContainerId containerID =
- ConverterUtils.toContainerId(this.recordFactory, $(CONTAINER_ID));
+ ContainerId containerID;
+ try {
+ containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
+ } catch (IOException e) {
+ html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
+ return;
+ }
+
Container container = this.nmContext.getContainers().get(containerID);
ContainerStatus containerData = container.cloneAndGetContainerStatus();
int exitCode = containerData.getExitStatus();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Fri Sep 23 14:07:42 2011
@@ -136,7 +136,7 @@ public class AMLauncher implements Runna
containerMgrProxy.stopContainer(stopRequest);
}
- private ContainerManager getContainerMgrProxy(
+ protected ContainerManager getContainerMgrProxy(
final ApplicationId applicationID) throws IOException {
Container container = application.getMasterContainer();
@@ -173,23 +173,11 @@ public class AMLauncher implements Runna
// Construct the actual Container
ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
- StringBuilder mergedCommand = new StringBuilder();
- String failCount = Integer.toString(application.getAppAttemptId()
- .getAttemptId());
- List<String> commandList = new ArrayList<String>();
- for (String str : container.getCommands()) {
- // This is out-right wrong. AM FAIL count should be passed via env.
- String result =
- str.replaceFirst(ApplicationConstants.AM_FAIL_COUNT_STRING,
- failCount);
- mergedCommand.append(result).append(" ");
- commandList.add(result);
- }
- container.setCommands(commandList);
- /** add the failed count to the app master command line */
-
- LOG.info("Command to launch container " +
- containerID + " : " + mergedCommand);
+ LOG.info("Command to launch container "
+ + containerID
+ + " : "
+ + StringUtils.arrayToString(container.getCommands().toArray(
+ new String[0])));
// Finalize the container
container.setContainerId(containerID);
@@ -203,6 +191,11 @@ public class AMLauncher implements Runna
ContainerLaunchContext container)
throws IOException {
Map<String, String> environment = container.getEnvironment();
+
+ // Set the AppAttemptId to be consumable by the AM.
+ environment.put(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV,
+ application.getAppAttemptId().toString());
+
if (UserGroupInformation.isSecurityEnabled()) {
// TODO: Security enabled/disabled info should come from RM.
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java Fri Sep 23 14:07:42 2011
@@ -42,9 +42,9 @@ public class ApplicationMasterLauncher e
private final BlockingQueue<Runnable> masterEvents
= new LinkedBlockingQueue<Runnable>();
- private ApplicationTokenSecretManager applicationTokenSecretManager;
+ protected ApplicationTokenSecretManager applicationTokenSecretManager;
private ClientToAMSecretManager clientToAMSecretManager;
- private final RMContext context;
+ protected final RMContext context;
public ApplicationMasterLauncher(ApplicationTokenSecretManager
applicationTokenSecretManager, ClientToAMSecretManager clientToAMSecretManager,
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1174785&r1=1174784&r2=1174785&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Fri Sep 23 14:07:42 2011
@@ -195,6 +195,7 @@ public class MockRM extends ResourceMana
};
}
+ @Override
protected AdminService createAdminService() {
return new AdminService(getConfig(), scheduler, getRMContext(),
this.nodesListManager){
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1174785&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Fri Sep 23 14:07:42 2011
@@ -0,0 +1,159 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ContainerManager;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestApplicationMasterLauncher {
+
+ private static final Log LOG = LogFactory
+ .getLog(TestApplicationMasterLauncher.class);
+
+ private static final class MyContainerManagerImpl implements
+ ContainerManager {
+
+ boolean launched = false;
+ boolean cleanedup = false;
+ String attemptIdAtContainerManager = null;
+
+ @Override
+ public StartContainerResponse
+ startContainer(StartContainerRequest request)
+ throws YarnRemoteException {
+ LOG.info("Container started by MyContainerManager: " + request);
+ launched = true;
+ attemptIdAtContainerManager = request.getContainerLaunchContext()
+ .getEnvironment().get(
+ ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV);
+ return null;
+ }
+
+ @Override
+ public StopContainerResponse stopContainer(StopContainerRequest request)
+ throws YarnRemoteException {
+ LOG.info("Container cleaned up by MyContainerManager");
+ cleanedup = true;
+ return null;
+ }
+
+ @Override
+ public GetContainerStatusResponse getContainerStatus(
+ GetContainerStatusRequest request) throws YarnRemoteException {
+ return null;
+ }
+
+ }
+
+ private static final class MockRMWithCustomAMLauncher extends MockRM {
+
+ private final ContainerManager containerManager;
+
+ public MockRMWithCustomAMLauncher(ContainerManager containerManager) {
+ this.containerManager = containerManager;
+ }
+
+ @Override
+ protected ApplicationMasterLauncher createAMLauncher() {
+ return new ApplicationMasterLauncher(super.appTokenSecretManager,
+ super.clientToAMSecretManager, getRMContext()) {
+ @Override
+ protected Runnable createRunnableLauncher(RMAppAttempt application,
+ AMLauncherEventType event) {
+ return new AMLauncher(context, application, event,
+ applicationTokenSecretManager, clientToAMSecretManager,
+ getConfig()) {
+ @Override
+ protected ContainerManager getContainerMgrProxy(
+ ApplicationId applicationID) throws IOException {
+ return containerManager;
+ }
+ };
+ }
+ };
+ }
+ }
+
+ @Test
+ public void testAMLaunchAndCleanup() throws Exception {
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+ MyContainerManagerImpl containerManager = new MyContainerManagerImpl();
+ MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(
+ containerManager);
+ rm.start();
+ MockNM nm1 = rm.registerNode("h1:1234", 5120);
+
+ RMApp app = rm.submitApp(2000);
+
+ // kick the scheduling
+ nm1.nodeHeartbeat(true);
+
+ int waitCount = 0;
+ while (containerManager.launched == false && waitCount++ < 20) {
+ LOG.info("Waiting for AM Launch to happen..");
+ Thread.sleep(1000);
+ }
+ Assert.assertTrue(containerManager.launched);
+
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ ApplicationAttemptId appAttemptId = attempt.getAppAttemptId();
+ Assert.assertEquals(appAttemptId.toString(),
+ containerManager.attemptIdAtContainerManager);
+
+ MockAM am = new MockAM(rm.getRMContext(), rm
+ .getApplicationMasterService(), appAttemptId);
+ am.registerAppAttempt();
+ am.unregisterAppAttempt();
+
+ waitCount = 0;
+ while (containerManager.cleanedup == false && waitCount++ < 20) {
+ LOG.info("Waiting for AM Cleanup to happen..");
+ Thread.sleep(1000);
+ }
+ Assert.assertTrue(containerManager.cleanedup);
+
+ am.waitForState(RMAppAttemptState.FINISHED);
+ rm.stop();
+ }
+}