You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/10/19 07:21:19 UTC
svn commit: r1185976 [2/2] - in
/hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/jav...
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java Wed Oct 19 05:21:18 2011
@@ -126,7 +126,7 @@ public interface ApplicationSubmissionCo
@Public
@Stable
public void setUser(String user);
-
+
/**
* Get the <code>ContainerLaunchContext</code> to describe the
* <code>Container</code> with which the <code>ApplicationMaster</code> is
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Wed Oct 19 05:21:18 2011
@@ -287,7 +287,7 @@ public class ApplicationMaster {
Map<String, String> envs = System.getenv();
appAttemptID = Records.newRecord(ApplicationAttemptId.class);
- if (!envs.containsKey(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV)) {
+ if (!envs.containsKey(ApplicationConstants.AM_CONTAINER_ID_ENV)) {
if (cliParser.hasOption("app_attempt_id")) {
String appIdStr = cliParser.getOptionValue("app_attempt_id", "");
appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr);
@@ -296,7 +296,8 @@ public class ApplicationMaster {
throw new IllegalArgumentException("Application Attempt Id not set in the environment");
}
} else {
- appAttemptID = ConverterUtils.toApplicationAttemptId(envs.get(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV));
+ ContainerId containerId = ConverterUtils.toContainerId(envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV));
+ appAttemptID = containerId.getApplicationAttemptId();
}
LOG.info("Application master for app"
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ConverterUtils.java Wed Oct 19 05:21:18 2011
@@ -20,10 +20,8 @@ package org.apache.hadoop.yarn.util;
import static org.apache.hadoop.yarn.util.StringHelper._split;
-import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
-import java.text.NumberFormat;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
@@ -33,9 +31,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.URL;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -97,27 +93,8 @@ public class ConverterUtils {
return url;
}
- // TODO: Why thread local?
- // ^ NumberFormat instances are not threadsafe
- private static final ThreadLocal<NumberFormat> appIdFormat =
- new ThreadLocal<NumberFormat>() {
- @Override
- public NumberFormat initialValue() {
- NumberFormat fmt = NumberFormat.getInstance();
- fmt.setGroupingUsed(false);
- fmt.setMinimumIntegerDigits(4);
- return fmt;
- }
- };
-
-
-
public static String toString(ApplicationId appId) {
- StringBuilder sb = new StringBuilder();
- sb.append(APPLICATION_PREFIX + "_").append(appId.getClusterTimestamp())
- .append("_");
- sb.append(appIdFormat.get().format(appId.getId()));
- return sb.toString();
+ return appId.toString();
}
public static ApplicationId toApplicationId(RecordFactory recordFactory,
@@ -152,11 +129,11 @@ public class ConverterUtils {
return cId.toString();
}
- public static ContainerId toContainerId(String containerIdStr)
- throws IOException {
+ public static ContainerId toContainerId(String containerIdStr) {
Iterator<String> it = _split(containerIdStr).iterator();
if (!it.next().equals(CONTAINER_PREFIX)) {
- throw new IOException("Invalid ContainerId prefix: " + containerIdStr);
+ throw new IllegalArgumentException("Invalid ContainerId prefix: "
+ + containerIdStr);
}
try {
ApplicationAttemptId appAttemptID = toApplicationAttemptId(it);
@@ -165,21 +142,22 @@ public class ConverterUtils {
containerId.setId(Integer.parseInt(it.next()));
return containerId;
} catch (NumberFormatException n) {
- throw new IOException("Invalid ContainerId: " + containerIdStr, n);
+ throw new IllegalArgumentException("Invalid ContainerId: "
+ + containerIdStr, n);
}
}
public static ApplicationAttemptId toApplicationAttemptId(
- String applicationAttmeptIdStr) throws IOException {
+ String applicationAttmeptIdStr) {
Iterator<String> it = _split(applicationAttmeptIdStr).iterator();
if (!it.next().equals(APPLICATION_ATTEMPT_PREFIX)) {
- throw new IOException("Invalid AppAttemptId prefix: "
+ throw new IllegalArgumentException("Invalid AppAttemptId prefix: "
+ applicationAttmeptIdStr);
}
try {
return toApplicationAttemptId(it);
} catch (NumberFormatException n) {
- throw new IOException("Invalid AppAttemptId: "
+ throw new IllegalArgumentException("Invalid AppAttemptId: "
+ applicationAttmeptIdStr, n);
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Wed Oct 19 05:21:18 2011
@@ -69,7 +69,7 @@ public class ContainerLogsPage extends N
ContainerId containerId;
try {
containerId = ConverterUtils.toContainerId($(CONTAINER_ID));
- } catch (IOException e) {
+ } catch (IllegalArgumentException e) {
div.h1("Invalid containerId " + $(CONTAINER_ID))._();
return;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Wed Oct 19 05:21:18 2011
@@ -22,8 +22,6 @@ import static org.apache.hadoop.yarn.uti
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import java.io.IOException;
-
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -66,7 +64,7 @@ public class ContainerPage extends NMVie
ContainerId containerID;
try {
containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
- } catch (IOException e) {
+ } catch (IllegalArgumentException e) {
html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
return;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Wed Oct 19 05:21:18 2011
@@ -206,11 +206,12 @@ public class ClientRMService extends Abs
// Safety
submissionContext.setUser(user);
-
+
// This needs to be synchronous as the client can query
// immediately following the submission to get the application status.
// So call handle directly and do not send an event.
- rmAppManager.handle(new RMAppManagerSubmitEvent(submissionContext));
+ rmAppManager.handle(new RMAppManagerSubmitEvent(submissionContext, System
+ .currentTimeMillis()));
LOG.info("Application with id " + applicationId.getId() +
" submitted by user " + user + " with " + submissionContext);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Wed Oct 19 05:21:18 2011
@@ -213,7 +213,7 @@ public class RMAppManager implements Eve
@SuppressWarnings("unchecked")
protected synchronized void submitApplication(
- ApplicationSubmissionContext submissionContext) {
+ ApplicationSubmissionContext submissionContext, long submitTime) {
ApplicationId applicationId = submissionContext.getApplicationId();
RMApp application = null;
try {
@@ -241,13 +241,13 @@ public class RMAppManager implements Eve
ApplicationStore appStore = rmContext.getApplicationsStore()
.createApplicationStore(submissionContext.getApplicationId(),
submissionContext);
-
+
// Create RMApp
application = new RMAppImpl(applicationId, rmContext,
this.conf, submissionContext.getApplicationName(), user,
submissionContext.getQueue(), submissionContext, clientTokenStr,
appStore, this.scheduler,
- this.masterService);
+ this.masterService, submitTime);
if (rmContext.getRMApps().putIfAbsent(applicationId, application) !=
null) {
@@ -284,8 +284,9 @@ public class RMAppManager implements Eve
case APP_SUBMIT:
{
ApplicationSubmissionContext submissionContext =
- ((RMAppManagerSubmitEvent)event).getSubmissionContext();
- submitApplication(submissionContext);
+ ((RMAppManagerSubmitEvent)event).getSubmissionContext();
+ long submitTime = ((RMAppManagerSubmitEvent)event).getSubmitTime();
+ submitApplication(submissionContext, submitTime);
}
break;
default:
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManagerSubmitEvent.java Wed Oct 19 05:21:18 2011
@@ -23,13 +23,21 @@ import org.apache.hadoop.yarn.api.record
public class RMAppManagerSubmitEvent extends RMAppManagerEvent {
private final ApplicationSubmissionContext submissionContext;
+ private final long submitTime;
- public RMAppManagerSubmitEvent(ApplicationSubmissionContext submissionContext) {
- super(submissionContext.getApplicationId(), RMAppManagerEventType.APP_SUBMIT);
+ public RMAppManagerSubmitEvent(
+ ApplicationSubmissionContext submissionContext, long submitTime) {
+ super(submissionContext.getApplicationId(),
+ RMAppManagerEventType.APP_SUBMIT);
this.submissionContext = submissionContext;
+ this.submitTime = submitTime;
}
public ApplicationSubmissionContext getSubmissionContext() {
return this.submissionContext;
}
+
+ public long getSubmitTime() {
+ return this.submitTime;
+ }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java Wed Oct 19 05:21:18 2011
@@ -22,8 +22,6 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Map;
import javax.crypto.SecretKey;
@@ -37,7 +35,6 @@ import org.apache.hadoop.io.DataOutputBu
import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.SecurityInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
@@ -58,7 +55,6 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
-import org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -83,6 +79,7 @@ public class AMLauncher implements Runna
private final ApplicationTokenSecretManager applicationTokenSecretManager;
private final ClientToAMSecretManager clientToAMSecretManager;
private final AMLauncherEventType eventType;
+ private final RMContext rmContext;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -96,6 +93,7 @@ public class AMLauncher implements Runna
this.applicationTokenSecretManager = applicationTokenSecretManager;
this.clientToAMSecretManager = clientToAMSecretManager;
this.eventType = eventType;
+ this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
}
@@ -189,9 +187,18 @@ public class AMLauncher implements Runna
throws IOException {
Map<String, String> environment = container.getEnvironment();
- // Set the AppAttemptId to be consumable by the AM.
- environment.put(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV,
- application.getAppAttemptId().toString());
+ // Set the AppAttemptId, containerId, NMHTTPAdress, AppSubmitTime to be
+ // consumable by the AM.
+ environment.put(ApplicationConstants.AM_CONTAINER_ID_ENV, container
+ .getContainerId().toString());
+ environment.put(ApplicationConstants.NM_HTTP_ADDRESS_ENV, application
+ .getMasterContainer().getNodeHttpAddress());
+ environment.put(
+ ApplicationConstants.APP_SUBMIT_TIME_ENV,
+ String.valueOf(rmContext.getRMApps()
+ .get(application.getAppAttemptId().getApplicationId())
+ .getSubmitTime()));
+
if (UserGroupInformation.isSecurityEnabled()) {
// TODO: Security enabled/disabled info should come from RM.
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java Wed Oct 19 05:21:18 2011
@@ -116,6 +116,12 @@ public interface RMApp extends EventHand
long getStartTime();
/**
+ * the submit time of the application.
+ * @return the submit time of the application.
+ */
+ long getSubmitTime();
+
+ /**
* The tracking url for the application master.
* @return the tracking url for the application master.
*/
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Wed Oct 19 05:21:18 2011
@@ -84,6 +84,7 @@ public class RMAppImpl implements RMApp
private final WriteLock writeLock;
private final Map<ApplicationAttemptId, RMAppAttempt> attempts
= new LinkedHashMap<ApplicationAttemptId, RMAppAttempt>();
+ private final long submitTime;
// Mutable fields
private long startTime;
@@ -163,7 +164,8 @@ public class RMAppImpl implements RMApp
Configuration config, String name, String user, String queue,
ApplicationSubmissionContext submissionContext, String clientTokenStr,
ApplicationStore appStore,
- YarnScheduler scheduler, ApplicationMasterService masterService) {
+ YarnScheduler scheduler, ApplicationMasterService masterService,
+ long submitTime) {
this.applicationId = applicationId;
this.name = name;
@@ -178,6 +180,7 @@ public class RMAppImpl implements RMApp
this.appStore = appStore;
this.scheduler = scheduler;
this.masterService = masterService;
+ this.submitTime = submitTime;
this.startTime = System.currentTimeMillis();
this.maxRetries = conf.getInt(YarnConfiguration.RM_AM_MAX_RETRIES,
@@ -368,6 +371,11 @@ public class RMAppImpl implements RMApp
}
@Override
+ public long getSubmitTime() {
+ return this.submitTime;
+ }
+
+ @Override
public String getTrackingUrl() {
this.readLock.lock();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java Wed Oct 19 05:21:18 2011
@@ -42,6 +42,7 @@ public class MockNM {
private final String nodeIdStr;
private final int memory;
private final ResourceTrackerService resourceTracker;
+ private final int httpPort = 2;
MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTracker) {
this.nodeIdStr = nodeIdStr;
@@ -53,6 +54,10 @@ public class MockNM {
return nodeId;
}
+ public String getHttpAddress() {
+ return nodeId.getHost() + ":" + String.valueOf(httpPort);
+ }
+
public void containerStatus(Container container) throws Exception {
Map<ApplicationId, List<ContainerStatus>> conts =
new HashMap<ApplicationId, List<ContainerStatus>>();
@@ -69,7 +74,7 @@ public class MockNM {
RegisterNodeManagerRequest req = Records.newRecord(
RegisterNodeManagerRequest.class);
req.setNodeId(nodeId);
- req.setHttpPort(2);
+ req.setHttpPort(httpPort);
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(memory);
req.setResource(resource);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java Wed Oct 19 05:21:18 2011
@@ -163,7 +163,7 @@ public class TestAppManager{
}
public void submitApplication(
ApplicationSubmissionContext submissionContext) {
- super.submitApplication(submissionContext);
+ super.submitApplication(submissionContext, System.currentTimeMillis());
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Wed Oct 19 05:21:18 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
@@ -39,6 +40,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -56,6 +58,9 @@ public class TestApplicationMasterLaunch
boolean launched = false;
boolean cleanedup = false;
String attemptIdAtContainerManager = null;
+ String containerIdAtContainerManager = null;
+ String nmAddressAtContainerManager = null;
+ long submitTimeAtContainerManager;
@Override
public StartContainerResponse
@@ -63,9 +68,20 @@ public class TestApplicationMasterLaunch
throws YarnRemoteException {
LOG.info("Container started by MyContainerManager: " + request);
launched = true;
- attemptIdAtContainerManager = request.getContainerLaunchContext()
- .getEnvironment().get(
- ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV);
+ containerIdAtContainerManager =
+ request.getContainerLaunchContext().getEnvironment()
+ .get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+ ContainerId containerId =
+ ConverterUtils.toContainerId(containerIdAtContainerManager);
+ attemptIdAtContainerManager =
+ containerId.getApplicationAttemptId().toString();
+ nmAddressAtContainerManager =
+ request.getContainerLaunchContext().getEnvironment()
+ .get(ApplicationConstants.NM_HTTP_ADDRESS_ENV);
+ submitTimeAtContainerManager =
+ Long.parseLong(request.getContainerLaunchContext().getEnvironment()
+ .get(ApplicationConstants.APP_SUBMIT_TIME_ENV));
+
return null;
}
@@ -140,6 +156,13 @@ public class TestApplicationMasterLaunch
ApplicationAttemptId appAttemptId = attempt.getAppAttemptId();
Assert.assertEquals(appAttemptId.toString(),
containerManager.attemptIdAtContainerManager);
+ Assert.assertEquals(app.getSubmitTime(),
+ containerManager.submitTimeAtContainerManager);
+ Assert.assertEquals(app.getRMAppAttempt(appAttemptId)
+ .getSubmissionContext().getAMContainerSpec().getContainerId()
+ .toString(), containerManager.containerIdAtContainerManager);
+ Assert.assertEquals(nm1.getHttpAddress(),
+ containerManager.nmAddressAtContainerManager);
MockAM am = new MockAM(rm.getRMContext(), rm
.getApplicationMasterService(), appAttemptId);
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Wed Oct 19 05:21:18 2011
@@ -167,6 +167,11 @@ public abstract class MockAsm extends Mo
}
@Override
+ public long getSubmitTime() {
+ throw new UnsupportedOperationException("Not supported yet.");
+ }
+
+ @Override
public long getFinishTime() {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java Wed Oct 19 05:21:18 2011
@@ -33,6 +33,7 @@ public class MockRMApp implements RMApp
String name = MockApps.newAppName();
String queue = MockApps.newQueue();
long start = System.currentTimeMillis() - (int) (Math.random() * DT);
+ long submit = start - (int) (Math.random() * DT);
long finish = 0;
RMAppState state = RMAppState.NEW;
int failCount = 0;
@@ -141,6 +142,11 @@ public class MockRMApp implements RMApp
return start;
}
+ @Override
+ public long getSubmitTime() {
+ return submit;
+ }
+
public void setStartTime(long time) {
this.start = time;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Wed Oct 19 05:21:18 2011
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.re
import static org.mockito.Mockito.mock;
import java.io.IOException;
-import java.util.List;
import junit.framework.Assert;
@@ -51,7 +50,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.junit.Before;
-import org.junit.After;
import org.junit.Test;
@@ -61,7 +59,7 @@ public class TestRMAppTransitions {
private RMContext rmContext;
private static int maxRetries = 4;
private static int appId = 1;
- private AsyncDispatcher rmDispatcher;
+// private AsyncDispatcher rmDispatcher;
// ignore all the RM application attempt events
private static final class TestApplicationAttemptEventDispatcher implements
@@ -152,7 +150,7 @@ public class TestRMAppTransitions {
conf, name, user,
queue, submissionContext, clientTokenStr,
appStore, scheduler,
- masterService);
+ masterService, System.currentTimeMillis());
testAppStartState(applicationId, user, name, queue, application);
return application;
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm?rev=1185976&r1=1185975&r2=1185976&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/WritingYarnApplications.apt.vm Wed Oct 19 05:21:18 2011
@@ -323,25 +323,29 @@ Hadoop MapReduce Next Generation - Writi
multi-tenancy nature, amongst other issues, it cannot make any assumptions
of things like pre-configured ports that it can listen on.
+ * When the ApplicationMaster starts up, several parameters are made available
+ to it via the environment. These include the ContainerId for the
+ ApplicationMaster container, the application submission time and the HTTP
+ address of the NodeManager running the container. Ref ApplicationConstants
+ for parameter names.
+
* All interactions with the ResourceManager require an ApplicationAttemptId
- (there can be multiple attempts per application in case of failures). When
- the ApplicationMaster starts up, the ApplicationAttemptId associated with
- this particular instance will be set in the environment. There are helper
- apis to convert the value obtained from the environment into an
- ApplicationAttemptId object.
+ (there can be multiple attempts per application in case of failures). The
+ ApplicationAttemptId can be obtained from the ApplicationMaster
+ containerId. There are helper apis to convert the value obtained from the
+ environment into objects.
+---+
Map<String, String> envs = System.getenv();
- ApplicationAttemptId appAttemptID =
- Records.newRecord(ApplicationAttemptId.class);
- if (!envs.containsKey(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV)) {
- // app attempt id should always be set in the env by the framework
+ String containerIdString =
+ envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+ if (containerIdString == null) {
+ // container id should always be set in the env by the framework
throw new IllegalArgumentException(
- "ApplicationAttemptId not set in the environment");
- }
- appAttemptID =
- ConverterUtils.toApplicationAttemptId(
- envs.get(ApplicationConstants.APPLICATION_ATTEMPT_ID_ENV));
+ "ContainerId not set in the environment");
+ }
+ ContainerId containerId = ConverterUtils.toContainerId(containerIdString);
+ ApplicationAttemptId appAttemptID = containerId.getApplicationAttemptId();
+---+
* After an ApplicationMaster has initialized itself completely, it needs to