You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/06/16 09:39:19 UTC
svn commit: r1136306 - in /hadoop/common/branches/MR-279/mapreduce: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app...
Author: vinodkv
Date: Thu Jun 16 07:39:18 2011
New Revision: 1136306
URL: http://svn.apache.org/viewvc?rev=1136306&view=rev
Log:
Task diagnostic info made available on the AM UI. Contributed by Vinod Kumar Vavilapalli.
Modified:
hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java
Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/CHANGES.txt?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/MR-279/mapreduce/CHANGES.txt Thu Jun 16 07:39:18 2011
@@ -5,6 +5,8 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Task diagnostic info made available on the AM UI. (vinodkv)
+
MAPREDUCE-2598. Fix NPE and UI for JobHistory. (Siddharth Seth via llu)
Fix NPE when killing/failing already killed/failed tasks. (llu)
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java Thu Jun 16 07:39:18 2011
@@ -65,8 +65,10 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.Task;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
+import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
@@ -296,8 +298,11 @@ public class MRClientService extends Abs
public KillJobResponse killJob(KillJobRequest request)
throws YarnRemoteException {
JobId jobId = request.getJobId();
- LOG.info("Kill Job received from client " + jobId);
- verifyAndGetJob(jobId, true);
+ String message = "Kill Job received from client " + jobId;
+ LOG.info(message);
+ verifyAndGetJob(jobId, true);
+ appContext.getEventHandler().handle(
+ new JobDiagnosticsUpdateEvent(jobId, message));
appContext.getEventHandler().handle(
new JobEvent(jobId, JobEventType.JOB_KILL));
KillJobResponse response =
@@ -309,7 +314,8 @@ public class MRClientService extends Abs
public KillTaskResponse killTask(KillTaskRequest request)
throws YarnRemoteException {
TaskId taskId = request.getTaskId();
- LOG.info("Kill task received from client " + taskId);
+ String message = "Kill task received from client " + taskId;
+ LOG.info(message);
verifyAndGetTask(taskId, true);
appContext.getEventHandler().handle(
new TaskEvent(taskId, TaskEventType.T_KILL));
@@ -322,9 +328,12 @@ public class MRClientService extends Abs
public KillTaskAttemptResponse killTaskAttempt(
KillTaskAttemptRequest request) throws YarnRemoteException {
TaskAttemptId taskAttemptId = request.getTaskAttemptId();
- LOG.info("Kill task attempt received from client " + taskAttemptId);
+ String message = "Kill task attempt received from client " + taskAttemptId;
+ LOG.info(message);
verifyAndGetAttempt(taskAttemptId, true);
appContext.getEventHandler().handle(
+ new TaskAttemptDiagnosticsUpdateEvent(taskAttemptId, message));
+ appContext.getEventHandler().handle(
new TaskAttemptEvent(taskAttemptId,
TaskAttemptEventType.TA_KILL));
KillTaskAttemptResponse response =
@@ -348,9 +357,12 @@ public class MRClientService extends Abs
public FailTaskAttemptResponse failTaskAttempt(
FailTaskAttemptRequest request) throws YarnRemoteException {
TaskAttemptId taskAttemptId = request.getTaskAttemptId();
- LOG.info("Fail task attempt received from client " + taskAttemptId);
+ String message = "Fail task attempt received from client " + taskAttemptId;
+ LOG.info(message);
verifyAndGetAttempt(taskAttemptId, true);
appContext.getEventHandler().handle(
+ new TaskAttemptDiagnosticsUpdateEvent(taskAttemptId, message));
+ appContext.getEventHandler().handle(
new TaskAttemptEvent(taskAttemptId,
TaskAttemptEventType.TA_FAILMSG));
FailTaskAttemptResponse response = recordFactory.
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Thu Jun 16 07:39:18 2011
@@ -36,7 +36,6 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Thu Jun 16 07:39:18 2011
@@ -151,6 +151,11 @@ public abstract class TaskAttemptImpl im
private static final CleanupContainerTransition CLEANUP_CONTAINER_TRANSITION =
new CleanupContainerTransition();
+
+ private static final DiagnosticInformationUpdater
+ DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION
+ = new DiagnosticInformationUpdater();
+
private static final StateMachineFactory
<TaskAttemptImpl, TaskAttemptState, TaskAttemptEventType, TaskAttemptEvent>
stateMachineFactory
@@ -198,7 +203,7 @@ public abstract class TaskAttemptImpl im
TaskAttemptEventType.TA_UPDATE, new StatusUpdater())
.addTransition(TaskAttemptState.RUNNING, TaskAttemptState.RUNNING,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
- new DiagnosticInformationUpdater())
+ DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
// If no commit is required, task directly goes to success
.addTransition(TaskAttemptState.RUNNING,
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
@@ -232,7 +237,7 @@ public abstract class TaskAttemptImpl im
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.COMMIT_PENDING,
TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
- new DiagnosticInformationUpdater())
+ DIAGNOSTIC_INFORMATION_UPDATE_TRANSITION)
.addTransition(TaskAttemptState.COMMIT_PENDING,
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
TaskAttemptEventType.TA_DONE, CLEANUP_CONTAINER_TRANSITION)
@@ -260,6 +265,7 @@ public abstract class TaskAttemptImpl im
TaskAttemptState.SUCCESS_CONTAINER_CLEANUP,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_FAILMSG,
+ TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
TaskAttemptEventType.TA_TIMED_OUT,
TaskAttemptEventType.TA_CONTAINER_COMPLETED))
@@ -339,6 +345,7 @@ public abstract class TaskAttemptImpl im
TaskAttemptState.SUCCEEDED,
EnumSet.of(TaskAttemptEventType.TA_KILL,
TaskAttemptEventType.TA_FAILMSG,
+ TaskAttemptEventType.TA_DIAGNOSTICS_UPDATE,
TaskAttemptEventType.TA_CONTAINER_COMPLETED))
// Ignore-able events for FAILED state
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/launcher/ContainerLauncherImpl.java Thu Jun 16 07:39:18 2011
@@ -30,8 +30,10 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.app.AMConstants;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerAllocator;
@@ -40,6 +42,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -47,6 +50,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo;
@@ -66,6 +70,7 @@ public class ContainerLauncherImpl exten
private Thread eventHandlingThread;
private BlockingQueue<ContainerLauncherEvent> eventQueue =
new LinkedBlockingQueue<ContainerLauncherEvent>();
+ private RecordFactory recordFactory;
public ContainerLauncherImpl(AppContext context) {
super(ContainerLauncherImpl.class.getName());
@@ -80,6 +85,7 @@ public class ContainerLauncherImpl exten
myLocalConfig.setClass(
CommonConfigurationKeysPublic.HADOOP_SECURITY_INFO_CLASS_NAME,
ContainerManagerSecurityInfo.class, SecurityInfo.class);
+ this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
super.init(myLocalConfig);
}
@@ -172,6 +178,7 @@ public class ContainerLauncherImpl exten
case CONTAINER_REMOTE_LAUNCH:
ContainerRemoteLaunchEvent launchEv = (ContainerRemoteLaunchEvent) event;
+ TaskAttemptId taskAttemptID = launchEv.getTaskAttemptID();
try {
ContainerManager proxy =
@@ -184,19 +191,24 @@ public class ContainerLauncherImpl exten
// TODO: Make sure that child's mapred-local-dir is set correctly.
// Now launch the actual container
- StartContainerRequest startRequest = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(StartContainerRequest.class);
+ StartContainerRequest startRequest = recordFactory
+ .newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
proxy.startContainer(startRequest);
// after launching, send launched event to task attempt to move
// it from ASSIGNED to RUNNING state
context.getEventHandler().handle(
- new TaskAttemptEvent(launchEv.getTaskAttemptID(),
+ new TaskAttemptEvent(taskAttemptID,
TaskAttemptEventType.TA_CONTAINER_LAUNCHED));
} catch (Exception e) {
- LOG.error("Container launch failed", e);
+ String message = "Container launch failed for " + containerID
+ + " : " + StringUtils.stringifyException(e);
+ LOG.error(message);
context.getEventHandler().handle(
- new TaskAttemptEvent(launchEv.getTaskAttemptID(),
+ new TaskAttemptDiagnosticsUpdateEvent(taskAttemptID, message));
+ context.getEventHandler().handle(
+ new TaskAttemptEvent(taskAttemptID,
TaskAttemptEventType.TA_CONTAINER_LAUNCH_FAILED));
}
@@ -218,7 +230,8 @@ public class ContainerLauncherImpl exten
// TODO:check whether container is launched
// kill the remote container if already launched
- StopContainerRequest stopRequest = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(StopContainerRequest.class);
+ StopContainerRequest stopRequest = recordFactory
+ .newRecordInstance(StopContainerRequest.class);
stopRequest.setContainerId(event.getContainerID());
proxy.stopContainer(stopRequest);
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Thu Jun 16 07:39:18 2011
@@ -44,6 +44,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.yarn.api.records.Container;
@@ -412,6 +413,10 @@ public class RMContainerAllocator extend
//send the container completed event to Task attempt
eventHandler.handle(new TaskAttemptEvent(attemptID,
TaskAttemptEventType.TA_CONTAINER_COMPLETED));
+ // Send the diagnostics
+ String diagnostics = cont.getContainerStatus().getDiagnostics();
+ eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(
+ attemptID, diagnostics));
}
}
LOG.debug("Received Container :" + cont);
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java Thu Jun 16 07:39:18 2011
@@ -1,3 +1,21 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
package org.apache.hadoop.yarn.api.records;
import java.util.List;
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java Thu Jun 16 07:39:18 2011
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -38,6 +40,7 @@ import org.apache.hadoop.yarn.util.Proto
public class ContainerPBImpl extends ProtoBase<ContainerProto> implements Container {
+
ContainerProto proto = ContainerProto.getDefaultInstance();
ContainerProto.Builder builder = null;
boolean viaProto = false;
@@ -65,15 +68,26 @@ public class ContainerPBImpl extends Pro
}
private void mergeLocalToBuilder() {
- if (this.containerId != null && !((ContainerIdPBImpl)containerId).getProto().equals(builder.getId())) {
+ if (this.containerId != null
+ && !((ContainerIdPBImpl) containerId).getProto().equals(
+ builder.getId())) {
builder.setId(convertToProtoFormat(this.containerId));
}
- if (this.resource != null && !((ResourcePBImpl)this.resource).getProto().equals(builder.getResource())) {
+ if (this.resource != null
+ && !((ResourcePBImpl) this.resource).getProto().equals(
+ builder.getResource())) {
builder.setResource(convertToProtoFormat(this.resource));
}
- if (this.containerToken != null && !((ContainerTokenPBImpl)this.containerToken).getProto().equals(builder.getContainerToken())) {
+ if (this.containerToken != null
+ && !((ContainerTokenPBImpl) this.containerToken).getProto().equals(
+ builder.getContainerToken())) {
builder.setContainerToken(convertToProtoFormat(this.containerToken));
}
+ if (this.containerStatus != null
+ && !((ContainerStatusPBImpl) this.containerStatus).getProto().equals(
+ builder.getContainerStatus())) {
+ builder.setContainerStatus(convertToProtoFormat(this.containerStatus));
+ }
}
private void mergeLocalToProto() {
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/proto/yarn_protos.proto?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-api/src/main/proto/yarn_protos.proto Thu Jun 16 07:39:18 2011
@@ -219,7 +219,7 @@ message ContainerLaunchContextProto {
message ContainerStatusProto {
optional ContainerIdProto container_id = 1;
optional ContainerStateProto state = 2;
- optional string diagnostics = 3 [default = ""];
+ optional string diagnostics = 3 [default = "N/A"];
optional string exit_status = 4 [default = "N/A"];
}
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Thu Jun 16 07:39:18 2011
@@ -32,8 +32,10 @@ import org.apache.hadoop.fs.UnsupportedF
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -89,8 +91,10 @@ public class DefaultContainerExecutor ex
String userName, String appId, Path containerWorkDir)
throws IOException {
+ ContainerId containerId = container.getContainerID();
+
// create container dirs on all disks
- String containerIdStr = ConverterUtils.toString(container.getContainerID());
+ String containerIdStr = ConverterUtils.toString(containerId);
String appIdStr =
ConverterUtils.toString(container.getContainerID().getAppId());
String[] sLocalDirs =
@@ -128,7 +132,7 @@ public class DefaultContainerExecutor ex
LOG.info("launchContainer: " + Arrays.toString(command));
shExec = new ShellCommandExecutor(command,
new File(containerWorkDir.toUri().getPath()));
- launchCommandObjs.put(container.getLaunchContext().getContainerId(), shExec);
+ launchCommandObjs.put(containerId, shExec);
shExec.execute();
} catch (Exception e) {
if (null == shExec) {
@@ -136,10 +140,13 @@ public class DefaultContainerExecutor ex
}
int exitCode = shExec.getExitCode();
LOG.warn("Exit code from task is : " + exitCode);
- logOutput(shExec.getOutput());
+ String message = shExec.getOutput();
+ logOutput(message);
+ container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
+ message));
return exitCode;
} finally {
- launchCommandObjs.remove(container.getLaunchContext().getContainerId());
+ launchCommandObjs.remove(containerId);
}
return 0;
}
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Thu Jun 16 07:39:18 2011
@@ -29,9 +29,12 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -71,8 +74,8 @@ public class LinuxContainerExecutor exte
}
/**
- * Result codes returned from the C task-controller.
- * These must match the values in task-controller.h.
+ * Result codes returned from the C container-executor.
+ * These must match the values in container-executor.h.
*/
enum ResultCode {
OK(0),
@@ -137,7 +140,7 @@ public class LinuxContainerExecutor exte
}
} catch (ExitCodeException e) {
int exitCode = shExec.getExitCode();
- LOG.warn("Exit code from task is : " + exitCode);
+ LOG.warn("Exit code from container is : " + exitCode);
logOutput(shExec.getOutput());
throw new IOException("App initialization failed (" + exitCode + ")", e);
}
@@ -147,7 +150,9 @@ public class LinuxContainerExecutor exte
public int launchContainer(Container container,
Path nmPrivateCotainerScriptPath, Path nmPrivateTokensPath,
String user, String appId, Path containerWorkDir) throws IOException {
- String containerIdStr = ConverterUtils.toString(container.getContainerID());
+
+ ContainerId containerId = container.getContainerID();
+ String containerIdStr = ConverterUtils.toString(containerId);
List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe,
user,
@@ -159,34 +164,42 @@ public class LinuxContainerExecutor exte
nmPrivateTokensPath.toUri().getPath().toString()));
String[] commandArray = command.toArray(new String[command.size()]);
ShellCommandExecutor shExec = new ShellCommandExecutor(commandArray);
- launchCommandObjs.put(container.getLaunchContext().getContainerId(), shExec);
+ launchCommandObjs.put(containerId, shExec);
// DEBUG
LOG.info("launchContainer: " + Arrays.toString(commandArray));
if (LOG.isDebugEnabled()) {
LOG.debug("launchContainer: " + Arrays.toString(commandArray));
}
+ String output = shExec.getOutput();
try {
shExec.execute();
if (LOG.isDebugEnabled()) {
- logOutput(shExec.getOutput());
+ logOutput(output);
}
} catch (ExitCodeException e) {
int exitCode = shExec.getExitCode();
- LOG.warn("Exit code from task is : " + exitCode);
- // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the task was
+ LOG.warn("Exit code from container is : " + exitCode);
+ // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
// terminated/killed forcefully. In all other cases, log the
- // task-controller output
+ // container-executor's output
if (exitCode != 143 && exitCode != 137) {
- LOG.warn("Exception thrown while launching task JVM : ", e);
- logOutput(shExec.getOutput());
+ LOG.warn("Exception from container-launch : ", e);
+ logOutput(output);
+ String diagnostics = "Exception from container-launch: \n"
+ + StringUtils.stringifyException(e) + "\n" + output;
+ container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
+ diagnostics));
+ } else {
+ container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
+ "Container killed on request. Exit code is " + exitCode));
}
return exitCode;
} finally {
- launchCommandObjs.remove(container.getLaunchContext().getContainerId());
+ launchCommandObjs.remove(containerId);
}
if (LOG.isDebugEnabled()) {
- LOG.debug("Output from LinuxContainerExecutor's launchTask follows:");
- logOutput(shExec.getOutput());
+ LOG.debug("Output from LinuxContainerExecutor's launchContainer follows:");
+ logOutput(output);
}
return 0;
}
@@ -203,7 +216,7 @@ public class LinuxContainerExecutor exte
Integer.toString(signal.getValue()) };
ShellCommandExecutor shExec = new ShellCommandExecutor(command);
if (LOG.isDebugEnabled()) {
- LOG.debug("signalTask: " + Arrays.toString(command));
+ LOG.debug("signalContainer: " + Arrays.toString(command));
}
try {
shExec.execute();
@@ -248,7 +261,7 @@ public class LinuxContainerExecutor exte
}
} catch (IOException e) {
int exitCode = shExec.getExitCode();
- LOG.warn("Exit code from task is : " + exitCode);
+ LOG.warn("Exit code from container is : " + exitCode);
if (exitCode != 0) {
LOG.error("DeleteAsUser for " + dir.toUri().getPath()
+ " returned with non-zero exit code" + exitCode);
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Thu Jun 16 07:39:18 2011
@@ -189,11 +189,9 @@ public class NodeStatusUpdaterImpl exten
}
private NodeStatus getNodeStatus() {
- NodeStatus status = recordFactory.newRecordInstance(NodeStatus.class);
- status.setNodeId(this.nodeId);
- Map<String, List<org.apache.hadoop.yarn.api.records.Container>> activeContainers =
- status.getAllContainers();
+ NodeStatus nodeStatus = recordFactory.newRecordInstance(NodeStatus.class);
+ nodeStatus.setNodeId(this.nodeId);
int numActiveContainers = 0;
synchronized (this.context.getContainers()) {
@@ -204,12 +202,11 @@ public class NodeStatusUpdaterImpl exten
Container container = e.getValue();
String applicationId = String.valueOf(containerId.getAppId().getId()); // TODO: ID? Really?
- List<org.apache.hadoop.yarn.api.records.Container> applicationContainers = status.getContainers(applicationId);
- activeContainers.get(applicationId);
+ List<org.apache.hadoop.yarn.api.records.Container> applicationContainers = nodeStatus
+ .getContainers(applicationId);
if (applicationContainers == null) {
applicationContainers = new ArrayList<org.apache.hadoop.yarn.api.records.Container>();
- status.setContainers(applicationId, applicationContainers);
-// activeContainers.put(applicationId, applicationContainers);
+ nodeStatus.setContainers(applicationId, applicationContainers);
}
// Clone the container to send it to the RM
@@ -238,9 +235,9 @@ public class NodeStatusUpdaterImpl exten
}
LOG.debug("Node's health-status : " + nodeHealthStatus.getIsNodeHealthy()
+ ", " + nodeHealthStatus.getHealthReport());
- status.setNodeHealthStatus(nodeHealthStatus);
+ nodeStatus.setNodeHealthStatus(nodeHealthStatus);
- return status;
+ return nodeStatus;
}
@Override
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Thu Jun 16 07:39:18 2011
@@ -145,7 +145,7 @@ public class ContainerManagerImpl extend
addService(auxiluaryServices);
this.containersMonitor =
- new ContainersMonitorImpl(exec, dispatcher);
+ new ContainersMonitorImpl(exec, dispatcher, this.context);
addService(this.containersMonitor);
LogAggregationService logAggregationService =
@@ -228,15 +228,18 @@ public class ContainerManagerImpl extend
}
super.stop();
}
-
+
+ /**
+ * Start a container on this NodeManager.
+ */
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException {
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
LOG.info(" container is " + request);
-
- // parse credentials
+
+ // //////////// Parse credentials
ByteBuffer tokens = launchContext.getContainerTokens();
Credentials credentials = new Credentials();
if (tokens != null) {
@@ -255,6 +258,7 @@ public class ContainerManagerImpl extend
throw RPCUtil.getRemoteException(e);
}
}
+ // //////////// End of parsing credentials
Container container =
new ContainerImpl(this.dispatcher, launchContext, credentials, metrics);
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Thu Jun 16 07:39:18 2011
@@ -41,7 +41,6 @@ import org.apache.hadoop.fs.LocalDirAllo
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -87,6 +86,7 @@ public class ContainerLaunch implements
public Integer call() {
final ContainerLaunchContext launchContext = container.getLaunchContext();
final Map<Path,String> localResources = container.getLocalizedResources();
+ String containerIdStr = ConverterUtils.toString(container.getContainerID());
final String user = launchContext.getUser();
final Map<String,String> env = launchContext.getAllEnv();
final List<String> command = launchContext.getCommandList();
@@ -96,8 +96,6 @@ public class ContainerLaunch implements
// /////////////////////////// Variable expansion
// Before the container script gets written out.
List<String> newCmds = new ArrayList<String>(command.size());
- String containerIdStr =
- ConverterUtils.toString(container.getContainerID());
String appIdStr = app.toString();
Path containerLogDir =
this.logDirsSelector.getLocalPathForWrite(appIdStr + Path.SEPARATOR
@@ -172,7 +170,7 @@ public class ContainerLaunch implements
containerWorkDir, FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
writeLaunchEnv(containerScriptOutStream, env, localResources,
- command, appDirs);
+ launchContext.getCommandList(), appDirs);
// /////////// End of writing out container-script
// /////////// Write out the container-tokens in the nmPrivate space.
@@ -194,18 +192,6 @@ public class ContainerLaunch implements
ret =
exec.launchContainer(container, nmPrivateContainerScriptPath,
nmPrivateTokensPath, user, appIdStr, containerWorkDir);
- if (ret == ExitCode.KILLED.getExitCode()) {
- // If the process was killed, Send container_cleanedup_after_kill and
- // just break out of this method.
- dispatcher.getEventHandler().handle(
- new ContainerExitEvent(launchContext.getContainerId(),
- ContainerEventType.CONTAINER_KILLED_ON_REQUEST, ret));
- return ret;
- }
-
- if (ret != 0) {
- throw new ExitCodeException(ret, "Container failed");
- }
} catch (Throwable e) {
LOG.warn("Failed to launch container", e);
dispatcher.getEventHandler().handle(new ContainerExitEvent(
@@ -213,8 +199,25 @@ public class ContainerLaunch implements
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret));
return ret;
}
- LOG.info("Container " + container + " succeeded "
- + launchContext.getContainerId());
+
+ if (ret == ExitCode.KILLED.getExitCode()) {
+ // If the process was killed, Send container_cleanedup_after_kill and
+ // just break out of this method.
+ dispatcher.getEventHandler().handle(
+ new ContainerExitEvent(launchContext.getContainerId(),
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST, ret));
+ return ret;
+ }
+
+ if (ret != 0) {
+ LOG.warn("Container exited with a non-zero exit code " + ret);
+ this.dispatcher.getEventHandler().handle(new ContainerExitEvent(
+ launchContext.getContainerId(),
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret));
+ return ret;
+ }
+
+ LOG.info("Container " + containerIdStr + " succeeded ");
dispatcher.getEventHandler().handle(
new ContainerEvent(launchContext.getContainerId(),
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS));
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Thu Jun 16 07:39:18 2011
@@ -14,10 +14,13 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
@@ -46,6 +49,7 @@ public class ContainersMonitorImpl exten
final ContainerExecutor containerExecutor;
private final Dispatcher eventDispatcher;
+ private final Context context;
private ResourceCalculatorPlugin resourceCalculatorPlugin;
private long maxVmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
@@ -63,11 +67,12 @@ public class ContainersMonitorImpl exten
"limit : %d bytes; Physical %d bytes, limit %d bytes";
public ContainersMonitorImpl(ContainerExecutor exec,
- AsyncDispatcher dispatcher) {
+ AsyncDispatcher dispatcher, Context context) {
super("containers-monitor");
this.containerExecutor = exec;
this.eventDispatcher = dispatcher;
+ this.context = context;
this.containersToBeAdded = new HashMap<ContainerId, ProcessTreeInfo>();
this.containersToBeRemoved = new ArrayList<ContainerId>();
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Thu Jun 16 07:39:18 2011
@@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.uti
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -66,12 +67,18 @@ public class ContainerPage extends NMVie
protected void render(Block html) {
ContainerId containerID =
ConverterUtils.toContainerId(this.recordFactory, $(CONTAINER_ID));
- Container container = this.nmContext.getContainers().get(containerID);
- info("Container information")
- ._("ContainerID", $(CONTAINER_ID))
- ._("ContainerState", container.getContainerState())
- ._("logs", ujoin("containerlogs", $(CONTAINER_ID)), "Link to logs");
- html._(InfoBlock.class);
+ Container container = this.nmContext.getContainers().get(containerID);
+ ContainerStatus containerData = container.cloneAndGetContainerStatus();
+ info("Container information")
+ ._("ContainerID", $(CONTAINER_ID))
+ ._("ContainerState", container.getContainerState())
+ ._("ExitStatus", containerData.getExitStatus())
+ ._("Diagnostics", containerData.getDiagnostics())
+ ._("User", container.getUser())
+ ._("TotalMemoryNeeded",
+ container.getLaunchContext().getResource().getMemory())
+ ._("logs", ujoin("containerlogs", $(CONTAINER_ID)), "Link to logs");
+ html._(InfoBlock.class);
}
}
}
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Thu Jun 16 07:39:18 2011
@@ -120,7 +120,7 @@ public class TestContainersMonitor exten
// vmem limit
long limit = 700000;
- ContainersMonitorImpl test = new ContainersMonitorImpl(null, null);
+ ContainersMonitorImpl test = new ContainersMonitorImpl(null, null, null);
// create process trees
// tree rooted at 100 is over limit immediately, as it is
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java?rev=1136306&r1=1136305&r2=1136306&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java Thu Jun 16 07:39:18 2011
@@ -284,10 +284,6 @@ public class Application {
Resources.subtractFrom(currentConsumption, containerResource);
}
- synchronized public void completedContainers(List<Container> containers) {
- completedContainers.addAll(containers);
- }
-
/**
* Resources have been allocated to this application by the resource
* scheduler. Track them.