You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/09/11 19:26:07 UTC
svn commit: r1169484 [1/2] - in
/hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/...
Author: acmurthy
Date: Sun Sep 11 17:26:06 2011
New Revision: 1169484
URL: http://svn.apache.org/viewvc?rev=1169484&view=rev
Log:
MAPREDUCE-2933. Change allocate call to return ContainerStatus for completed containers rather than Container.
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeUpdateSchedulerEvent.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRM.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sun Sep 11 17:26:06 2011
@@ -264,6 +264,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2711. Update TestBlockPlacementPolicyRaid for the new namesystem
and block management APIs. (szetszwo)
+ MAPREDUCE-2933. Change allocate call to return ContainerStatus for
+ completed containers rather than Container. (acmurthy)
+
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Sun Sep 11 17:26:06 2011
@@ -53,6 +53,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -414,8 +415,8 @@ public class RMContainerAllocator extend
int headRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;//first time it would be null
AMResponse response = makeRemoteRequest();
int newHeadRoom = getAvailableResources() != null ? getAvailableResources().getMemory() : 0;
- List<Container> newContainers = response.getNewContainerList();
- List<Container> finishedContainers = response.getFinishedContainerList();
+ List<Container> newContainers = response.getAllocatedContainers();
+ List<ContainerStatus> finishedContainers = response.getCompletedContainersStatuses();
if (newContainers.size() + finishedContainers.size() > 0 || headRoom != newHeadRoom) {
//something changed
recalculateReduceSchedule = true;
@@ -426,12 +427,12 @@ public class RMContainerAllocator extend
allocatedContainers.add(cont);
LOG.debug("Received new Container :" + cont);
}
- for (Container cont : finishedContainers) {
+ for (ContainerStatus cont : finishedContainers) {
LOG.info("Received completed container " + cont);
- TaskAttemptId attemptID = assignedRequests.get(cont.getId());
+ TaskAttemptId attemptID = assignedRequests.get(cont.getContainerId());
if (attemptID == null) {
LOG.error("Container complete event for unknown container id "
- + cont.getId());
+ + cont.getContainerId());
} else {
assignedRequests.remove(attemptID);
if (attemptID.getTaskId().getTaskType().equals(TaskType.MAP)) {
@@ -443,7 +444,7 @@ public class RMContainerAllocator extend
eventHandler.handle(new TaskAttemptEvent(attemptID,
TaskAttemptEventType.TA_CONTAINER_COMPLETED));
// Send the diagnostics
- String diagnostics = cont.getContainerStatus().getDiagnostics();
+ String diagnostics = cont.getDiagnostics();
eventHandler.handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID,
diagnostics));
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java Sun Sep 11 17:26:06 2011
@@ -36,7 +36,6 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.AMResponse;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -123,10 +122,11 @@ public abstract class RMContainerRequest
availableResources = response.getAvailableResources();
LOG.info("getResources() for " + applicationId + ":" + " ask="
- + ask.size() + " release= " + release.size() + " newContainers="
- + response.getNewContainerCount() + " finishedContainers="
- + response.getFinishedContainerCount()
- + " resourcelimit=" + availableResources);
+ + ask.size() + " release= " + release.size() +
+ " newContainers=" + response.getAllocatedContainers().size() +
+ " finishedContainers=" +
+ response.getCompletedContainersStatuses().size() +
+ " resourcelimit=" + availableResources);
ask.clear();
release.clear();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMResponse.java Sun Sep 11 17:26:06 2011
@@ -86,32 +86,17 @@ public interface AMResponse {
*/
@Public
@Stable
- public List<Container> getNewContainerList();
+ public List<Container> getAllocatedContainers();
- @Private
- @Unstable
- public Container getNewContainer(int index);
-
- @Private
- @Unstable
- public int getNewContainerCount();
-
- @Private
- @Unstable
- public void addAllNewContainers(List<Container> containers);
-
- @Private
- @Unstable
- public void addNewContainer(Container container);
-
- @Private
- @Unstable
- public void removeNewContainer(int index);
+ /**
+ * Set the list of <em>newly allocated</em> <code>Container</code> by the
+ * <code>ResourceManager</code>.
+ * @param containers list of <em>newly allocated</em> <code>Container</code>
+ */
+ @Public
+ @Stable
+ public void setAllocatedContainers(List<Container> containers);
- @Private
- @Unstable
- public void clearNewContainers();
-
/**
* Get the <em>available headroom</em> for resources in the cluster for the
* application.
@@ -127,35 +112,18 @@ public interface AMResponse {
public void setAvailableResources(Resource limit);
/**
- * Get the list of <em>completed containers</em>.
- * @return the list of <em>completed containers</em>
+ * Get the list of <em>completed containers' statuses</em>.
+ * @return the list of <em>completed containers' statuses</em>
*/
@Public
@Stable
- public List<Container> getFinishedContainerList();
+ public List<ContainerStatus> getCompletedContainersStatuses();
- @Private
- @Unstable
- public Container getFinishedContainer(int index);
-
- @Private
- @Unstable
- public int getFinishedContainerCount();
-
-
- @Private
- @Unstable
- public void addAllFinishedContainers(List<Container> containers);
-
- @Private
- @Unstable
- public void addFinishedContainer(Container container);
-
- @Private
- @Unstable
- public void removeFinishedContainer(int index);
-
- @Private
- @Unstable
- public void clearFinishedContainers();
+ /**
+ * Set the list of list of <em>completed containers' statuses</em>.
+ * @param containers list of <em>completed containers' statuses</em>
+ */
+ @Public
+ @Stable
+ public void setCompletedContainersStatuses(List<ContainerStatus> containers);
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/AMResponsePBImpl.java Sun Sep 11 17:26:06 2011
@@ -25,11 +25,13 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.proto.YarnProtos.AMResponseProto;
import org.apache.hadoop.yarn.proto.YarnProtos.AMResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
@@ -41,8 +43,8 @@ public class AMResponsePBImpl extends Pr
Resource limit;
- private List<Container> newContainersList = null;
- private List<Container> finishedContainersList = null;
+ private List<Container> allocatedContainers = null;
+ private List<ContainerStatus> completedContainersStatuses = null;
// private boolean hasLocalContainerList = false;
@@ -63,15 +65,17 @@ public class AMResponsePBImpl extends Pr
}
private synchronized void mergeLocalToBuilder() {
- if (this.newContainersList != null) {
- builder.clearNewContainers();
- Iterable<ContainerProto> iterable = getProtoIterable(this.newContainersList);
- builder.addAllNewContainers(iterable);
+ if (this.allocatedContainers != null) {
+ builder.clearAllocatedContainers();
+ Iterable<ContainerProto> iterable =
+ getProtoIterable(this.allocatedContainers);
+ builder.addAllAllocatedContainers(iterable);
}
- if (this.finishedContainersList != null) {
- builder.clearFinishedContainers();
- Iterable<ContainerProto> iterable = getProtoIterable(this.finishedContainersList);
- builder.addAllFinishedContainers(iterable);
+ if (this.completedContainersStatuses != null) {
+ builder.clearCompletedContainerStatuses();
+ Iterable<ContainerStatusProto> iterable =
+ getContainerStatusProtoIterable(this.completedContainersStatuses);
+ builder.addAllCompletedContainerStatuses(iterable);
}
if (this.limit != null) {
builder.setLimit(convertToProtoFormat(this.limit));
@@ -139,42 +143,31 @@ public class AMResponsePBImpl extends Pr
}
@Override
- public synchronized List<Container> getNewContainerList() {
+ public synchronized List<Container> getAllocatedContainers() {
initLocalNewContainerList();
- return this.newContainersList;
- }
-
- @Override
- public synchronized Container getNewContainer(int index) {
- initLocalNewContainerList();
- return this.newContainersList.get(index);
- }
- @Override
- public synchronized int getNewContainerCount() {
- initLocalNewContainerList();
- return this.newContainersList.size();
+ return this.allocatedContainers;
}
//Once this is called. containerList will never be null - untill a getProto is called.
private synchronized void initLocalNewContainerList() {
- if (this.newContainersList != null) {
+ if (this.allocatedContainers != null) {
return;
}
AMResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerProto> list = p.getNewContainersList();
- newContainersList = new ArrayList<Container>();
+ List<ContainerProto> list = p.getAllocatedContainersList();
+ allocatedContainers = new ArrayList<Container>();
for (ContainerProto c : list) {
- newContainersList.add(convertFromProtoFormat(c));
+ allocatedContainers.add(convertFromProtoFormat(c));
}
}
@Override
- public synchronized void addAllNewContainers(final List<Container> containers) {
+ public synchronized void setAllocatedContainers(final List<Container> containers) {
if (containers == null)
return;
initLocalNewContainerList();
- newContainersList.addAll(containers);
+ allocatedContainers.addAll(containers);
}
private synchronized Iterable<ContainerProto> getProtoIterable(
@@ -207,86 +200,71 @@ public class AMResponsePBImpl extends Pr
}
};
}
-
- @Override
- public synchronized void addNewContainer(Container containers) {
- initLocalNewContainerList();
- if (containers == null)
- return;
- this.newContainersList.add(containers);
- }
-
- @Override
- public synchronized void removeNewContainer(int index) {
- initLocalNewContainerList();
- this.newContainersList.remove(index);
- }
- @Override
- public synchronized void clearNewContainers() {
- initLocalNewContainerList();
- this.newContainersList.clear();
+
+ private synchronized Iterable<ContainerStatusProto>
+ getContainerStatusProtoIterable(
+ final List<ContainerStatus> newContainersList) {
+ maybeInitBuilder();
+ return new Iterable<ContainerStatusProto>() {
+ @Override
+ public synchronized Iterator<ContainerStatusProto> iterator() {
+ return new Iterator<ContainerStatusProto>() {
+
+ Iterator<ContainerStatus> iter = newContainersList.iterator();
+
+ @Override
+ public synchronized boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public synchronized ContainerStatusProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public synchronized void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
}
//// Finished containers
@Override
- public synchronized List<Container> getFinishedContainerList() {
+ public synchronized List<ContainerStatus> getCompletedContainersStatuses() {
initLocalFinishedContainerList();
- return this.finishedContainersList;
- }
-
- @Override
- public synchronized Container getFinishedContainer(int index) {
- initLocalFinishedContainerList();
- return this.finishedContainersList.get(index);
- }
- @Override
- public synchronized int getFinishedContainerCount() {
- initLocalFinishedContainerList();
- return this.finishedContainersList.size();
+ return this.completedContainersStatuses;
}
//Once this is called. containerList will never be null - untill a getProto is called.
private synchronized void initLocalFinishedContainerList() {
- if (this.finishedContainersList != null) {
+ if (this.completedContainersStatuses != null) {
return;
}
AMResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerProto> list = p.getFinishedContainersList();
- finishedContainersList = new ArrayList<Container>();
+ List<ContainerStatusProto> list = p.getCompletedContainerStatusesList();
+ completedContainersStatuses = new ArrayList<ContainerStatus>();
- for (ContainerProto c : list) {
- finishedContainersList.add(convertFromProtoFormat(c));
+ for (ContainerStatusProto c : list) {
+ completedContainersStatuses.add(convertFromProtoFormat(c));
}
}
@Override
- public synchronized void addAllFinishedContainers(final List<Container> containers) {
+ public synchronized void setCompletedContainersStatuses(
+ final List<ContainerStatus> containers) {
if (containers == null)
return;
initLocalFinishedContainerList();
- finishedContainersList.addAll(containers);
+ completedContainersStatuses.addAll(containers);
}
- @Override
- public synchronized void addFinishedContainer(Container containers) {
- initLocalFinishedContainerList();
- if (containers == null)
- return;
- this.finishedContainersList.add(containers);
- }
-
- @Override
- public synchronized void removeFinishedContainer(int index) {
- initLocalFinishedContainerList();
- this.finishedContainersList.remove(index);
- }
- @Override
- public synchronized void clearFinishedContainers() {
- initLocalFinishedContainerList();
- this.finishedContainersList.clear();
- }
-
- private synchronized ContainerPBImpl convertFromProtoFormat(ContainerProto p) {
+ private synchronized ContainerPBImpl convertFromProtoFormat(
+ ContainerProto p) {
return new ContainerPBImpl(p);
}
@@ -294,6 +272,15 @@ public class AMResponsePBImpl extends Pr
return ((ContainerPBImpl)t).getProto();
}
+ private synchronized ContainerStatusPBImpl convertFromProtoFormat(
+ ContainerStatusProto p) {
+ return new ContainerStatusPBImpl(p);
+ }
+
+ private synchronized ContainerStatusProto convertToProtoFormat(ContainerStatus t) {
+ return ((ContainerStatusPBImpl)t).getProto();
+ }
+
private synchronized ResourcePBImpl convertFromProtoFormat(ResourceProto p) {
return new ResourcePBImpl(p);
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Sun Sep 11 17:26:06 2011
@@ -177,8 +177,8 @@ message ResourceRequestProto {
message AMResponseProto {
optional bool reboot = 1;
optional int32 response_id = 2;
- repeated ContainerProto new_containers = 3;
- repeated ContainerProto finished_containers = 4;
+ repeated ContainerProto allocated_containers = 3;
+ repeated ContainerStatusProto completed_container_statuses = 4;
optional ResourceProto limit = 5;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java Sun Sep 11 17:26:06 2011
@@ -18,10 +18,8 @@
package org.apache.hadoop.yarn.server.api.records;
import java.util.List;
-import java.util.Map;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -31,17 +29,13 @@ public interface NodeStatus {
public abstract NodeId getNodeId();
public abstract int getResponseId();
- public abstract Map<ApplicationId, List<Container>> getAllContainers();
- public abstract List<Container> getContainers(ApplicationId key);
+ public abstract List<ContainerStatus> getContainersStatuses();
+ public abstract void setContainersStatuses(
+ List<ContainerStatus> containersStatuses);
NodeHealthStatus getNodeHealthStatus();
void setNodeHealthStatus(NodeHealthStatus healthStatus);
public abstract void setNodeId(NodeId nodeId);
public abstract void setResponseId(int responseId);
-
- public abstract void addAllContainers(Map<ApplicationId, List<Container>> containers);
- public abstract void setContainers(ApplicationId key, List<Container> containers);
- public abstract void removeContainers(ApplicationId key);
- public abstract void clearContainers();
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java Sun Sep 11 17:26:06 2011
@@ -20,27 +20,19 @@ package org.apache.hadoop.yarn.server.ap
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ProtoBase;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeHealthStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ApplicationIdContainerListMapProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ContainerListProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
@@ -51,7 +43,7 @@ public class NodeStatusPBImpl extends Pr
boolean viaProto = false;
private NodeId nodeId = null;
- private Map<ApplicationIdProto, List<Container>> containers = null;
+ private List<ContainerStatus> containers = null;
private NodeHealthStatus nodeHealthStatus = null;
public NodeStatusPBImpl() {
@@ -99,7 +91,40 @@ public class NodeStatusPBImpl extends Pr
viaProto = false;
}
+ private void addContainersToProto() {
+ maybeInitBuilder();
+ builder.clearContainersStatuses();
+ if (containers == null)
+ return;
+ Iterable<ContainerStatusProto> iterable = new Iterable<ContainerStatusProto>() {
+ @Override
+ public Iterator<ContainerStatusProto> iterator() {
+ return new Iterator<ContainerStatusProto>() {
+
+ Iterator<ContainerStatus> iter = containers.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ContainerStatusProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+ }
+ };
+ builder.addAllContainersStatuses(iterable);
+ }
+
@Override
public int getResponseId() {
NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
@@ -133,24 +158,17 @@ public class NodeStatusPBImpl extends Pr
}
@Override
- public Map<ApplicationId, List<Container>> getAllContainers() {
+ public List<ContainerStatus> getContainersStatuses() {
initContainers();
- HashMap<ApplicationId, List<Container>> returnMap = new HashMap<ApplicationId, List<Container>>(
- this.containers.size());
- for (Entry<ApplicationIdProto, List<Container>> entry : this.containers.entrySet()) {
- returnMap.put(convertFromProtoFormat(entry.getKey()), entry.getValue());
- }
- return returnMap;
+ return this.containers;
}
@Override
- public List<Container> getContainers(ApplicationId applicationId) {
- initContainers();
- ApplicationIdProto applicationIdProto = convertToProtoFormat(applicationId);
- if (this.containers.get(applicationIdProto) == null) {
- this.containers.put(applicationIdProto, new ArrayList<Container>());
+ public void setContainersStatuses(List<ContainerStatus> containers) {
+ if (containers == null) {
+ builder.clearContainersStatuses();
}
- return this.containers.get(applicationIdProto);
+ this.containers = containers;
}
private void initContainers() {
@@ -158,60 +176,16 @@ public class NodeStatusPBImpl extends Pr
return;
}
NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
- List<ApplicationIdContainerListMapProto> list = p.getContainersList();
- this.containers = new HashMap<ApplicationIdProto, List<Container>>();
+ List<ContainerStatusProto> list = p.getContainersStatusesList();
+ this.containers = new ArrayList<ContainerStatus>();
- for (ApplicationIdContainerListMapProto c : list) {
- this.containers.put(c.getApplicationId(), convertFromProtoFormat(c.getValue()));
+ for (ContainerStatusProto c : list) {
+ this.containers.add(convertFromProtoFormat(c));
}
}
@Override
- public void addAllContainers(final Map<ApplicationId, List<Container>> containers) {
- if (containers == null)
- return;
- initContainers();
- for (Entry<ApplicationId, List<Container>> entry : containers.entrySet()) {
- this.containers.put(convertToProtoFormat(entry.getKey()), entry.getValue());
- }
- }
-
- private void addContainersToProto() {
- maybeInitBuilder();
- builder.clearContainers();
- viaProto = false;
- Iterable<ApplicationIdContainerListMapProto> iterable = new Iterable<ApplicationIdContainerListMapProto>() {
-
- @Override
- public Iterator<ApplicationIdContainerListMapProto> iterator() {
- return new Iterator<ApplicationIdContainerListMapProto>() {
-
- Iterator<ApplicationIdProto> keyIter = containers.keySet().iterator();
- @Override
- public boolean hasNext() {
- return keyIter.hasNext();
- }
-
- @Override
- public ApplicationIdContainerListMapProto next() {
- ApplicationIdProto applicationIdProto = keyIter.next();
- return ApplicationIdContainerListMapProto.newBuilder().setApplicationId(applicationIdProto).setValue(convertToProtoFormat(containers.get(applicationIdProto))).build();
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- };
- }
-
- };
- builder.addAllContainers(iterable);
- }
-
- @Override
public NodeHealthStatus getNodeHealthStatus() {
NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
if (nodeHealthStatus != null) {
@@ -233,66 +207,6 @@ public class NodeStatusPBImpl extends Pr
this.nodeHealthStatus = healthStatus;
}
- /*
- *
- * @Override
- public String getApplicationName() {
- ApplicationSubmissionContextProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasApplicationName()) {
- return null;
- }
- return (p.getApplicationName());
- }
-
- @Override
- public void setApplicationName(String applicationName) {
- maybeInitBuilder();
- if (applicationName == null) {
- builder.clearApplicationName();
- return;
- }
- builder.setApplicationName((applicationName));
- }
- */
-
- private ContainerListProto convertToProtoFormat(List<Container> src) {
- ContainerListProto.Builder ret = ContainerListProto.newBuilder();
- for (Container c : src) {
- ret.addContainer(((ContainerPBImpl)c).getProto());
- }
- return ret.build();
- }
-
- private List<Container> convertFromProtoFormat(ContainerListProto src) {
- List<Container> ret = new ArrayList<Container>();
- for (ContainerProto c : src.getContainerList()) {
- ret.add(convertFromProtoFormat(c));
- }
- return ret;
- }
-
- private Container convertFromProtoFormat(ContainerProto src) {
- return new ContainerPBImpl(src);
- }
-
- @Override
- public void setContainers(ApplicationId applicationId, List<Container> containers) {
- initContainers();
- this.containers.put(convertToProtoFormat(applicationId), containers);
- }
-
- @Override
- public void removeContainers(ApplicationId applicationId) {
- initContainers();
- this.containers.remove(convertToProtoFormat(applicationId));
- }
-
- @Override
- public void clearContainers() {
- initContainers();
- this.containers.clear();
- }
-
private NodeIdProto convertToProtoFormat(NodeId nodeId) {
return ((NodeIdPBImpl)nodeId).getProto();
}
@@ -301,14 +215,6 @@ public class NodeStatusPBImpl extends Pr
return new NodeIdPBImpl(proto);
}
- private ApplicationIdProto convertToProtoFormat(ApplicationId applicationId) {
- return ((ApplicationIdPBImpl)applicationId).getProto();
- }
-
- private ApplicationId convertFromProtoFormat(ApplicationIdProto proto) {
- return new ApplicationIdPBImpl(proto);
- }
-
private NodeHealthStatusProto convertToProtoFormat(
NodeHealthStatus healthStatus) {
return ((NodeHealthStatusPBImpl) healthStatus).getProto();
@@ -317,4 +223,12 @@ public class NodeStatusPBImpl extends Pr
private NodeHealthStatus convertFromProtoFormat(NodeHealthStatusProto proto) {
return new NodeHealthStatusPBImpl(proto);
}
+
+ private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto c) {
+ return new ContainerStatusPBImpl(c);
+ }
+
+ private ContainerStatusProto convertToProtoFormat(ContainerStatus c) {
+ return ((ContainerStatusPBImpl)c).getProto();
+ }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Sun Sep 11 17:26:06 2011
@@ -26,7 +26,7 @@ import "yarn_protos.proto";
message NodeStatusProto {
optional NodeIdProto node_id = 1;
optional int32 response_id = 2;
- repeated ApplicationIdContainerListMapProto containers = 3;
+ repeated ContainerStatusProto containersStatuses = 3;
optional NodeHealthStatusProto nodeHealthStatus = 4;
}
@@ -41,12 +41,3 @@ message HeartbeatResponseProto {
repeated ApplicationIdProto applications_to_cleanup = 4;
}
-message ContainerListProto {
- repeated ContainerProto container = 1;
-}
-
-message ApplicationIdContainerListMapProto {
- optional ApplicationIdProto application_id = 1;
- optional ContainerListProto value = 2;
-}
-
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Sun Sep 11 17:26:06 2011
@@ -30,13 +30,13 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.NodeHealthCheckerService;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityInfo;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -73,7 +73,6 @@ public class NodeStatusUpdaterImpl exten
private String rmAddress;
private Resource totalResource;
private String containerManagerBindAddress;
- private String nodeHttpAddress;
private String hostName;
private int containerManagerPort;
private int httpPort;
@@ -127,7 +126,6 @@ public class NodeStatusUpdaterImpl exten
this.httpPort = httpBindAddress.getPort();
this.containerManagerBindAddress =
this.hostName + ":" + this.containerManagerPort;
- this.nodeHttpAddress = this.hostName + ":" + this.httpPort;
LOG.info("Configured ContainerManager Address is "
+ this.containerManagerBindAddress);
// Registration has to be in start so that ContainerManager can get the
@@ -195,35 +193,28 @@ public class NodeStatusUpdaterImpl exten
nodeStatus.setNodeId(this.nodeId);
int numActiveContainers = 0;
+ List<ContainerStatus> containersStatuses = new ArrayList<ContainerStatus>();
for (Iterator<Entry<ContainerId, Container>> i =
this.context.getContainers().entrySet().iterator(); i.hasNext();) {
Entry<ContainerId, Container> e = i.next();
ContainerId containerId = e.getKey();
Container container = e.getValue();
- List<org.apache.hadoop.yarn.api.records.Container> applicationContainers = nodeStatus
- .getContainers(container.getContainerID().getAppId());
- if (applicationContainers == null) {
- applicationContainers = new ArrayList<org.apache.hadoop.yarn.api.records.Container>();
- nodeStatus.setContainers(container.getContainerID().getAppId(),
- applicationContainers);
- }
-
// Clone the container to send it to the RM
- org.apache.hadoop.yarn.api.records.Container c = container.cloneAndGetContainer();
- c.setNodeId(this.nodeId);
- c.setNodeHttpAddress(this.nodeHttpAddress); // TODO: don't set everytime.
- applicationContainers.add(c);
+ org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus =
+ container.cloneAndGetContainerStatus();
+ containersStatuses.add(containerStatus);
++numActiveContainers;
- LOG.info("Sending out status for container: " + c);
+ LOG.info("Sending out status for container: " + containerStatus);
- if (c.getState() == ContainerState.COMPLETE) {
+ if (containerStatus.getState() == ContainerState.COMPLETE) {
// Remove
i.remove();
LOG.info("Removed completed container " + containerId);
}
}
+ nodeStatus.setContainersStatuses(containersStatuses);
LOG.debug(this.containerManagerBindAddress + " sending out status for " + numActiveContainers
+ " containers");
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java Sun Sep 11 17:26:06 2011
@@ -40,8 +40,6 @@ public interface Container extends Event
Map<Path,String> getLocalizedResources();
- org.apache.hadoop.yarn.api.records.Container cloneAndGetContainer();
-
ContainerStatus cloneAndGetContainerStatus();
String toString();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Sun Sep 11 17:26:06 2011
@@ -327,24 +327,6 @@ public class ContainerImpl implements Co
}
@Override
- public
- org.apache.hadoop.yarn.api.records.Container cloneAndGetContainer() {
- this.readLock.lock();
- try {
- org.apache.hadoop.yarn.api.records.Container c =
- recordFactory.newRecordInstance(
- org.apache.hadoop.yarn.api.records.Container.class);
- c.setId(this.launchContext.getContainerId());
- c.setResource(this.launchContext.getResource());
- c.setState(getCurrentState());
- c.setContainerStatus(cloneAndGetContainerStatus());
- return c;
- } finally {
- this.readLock.unlock();
- }
- }
-
- @Override
public ContainerLaunchContext getLaunchContext() {
this.readLock.lock();
try {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Sun Sep 11 17:26:06 2011
@@ -21,6 +21,10 @@ package org.apache.hadoop.yarn.server.no
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
@@ -32,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -98,13 +103,30 @@ public class TestNodeStatusUpdater {
ContainerId firstContainerID = recordFactory.newRecordInstance(ContainerId.class);
ContainerId secondContainerID = recordFactory.newRecordInstance(ContainerId.class);
+ private Map<ApplicationId, List<ContainerStatus>> getAppToContainerStatusMap(
+ List<ContainerStatus> containers) {
+ Map<ApplicationId, List<ContainerStatus>> map =
+ new HashMap<ApplicationId, List<ContainerStatus>>();
+ for (ContainerStatus cs : containers) {
+ ApplicationId applicationId = cs.getContainerId().getAppId();
+ List<ContainerStatus> appContainers = map.get(applicationId);
+ if (appContainers == null) {
+ appContainers = new ArrayList<ContainerStatus>();
+ map.put(applicationId, appContainers);
+ }
+ appContainers.add(cs);
+ }
+ return map;
+ }
@Override
public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request) throws YarnRemoteException {
NodeStatus nodeStatus = request.getNodeStatus();
LOG.info("Got heartbeat number " + heartBeatID);
nodeStatus.setResponseId(heartBeatID++);
+ Map<ApplicationId, List<ContainerStatus>> appToContainers =
+ getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
if (heartBeatID == 1) {
- Assert.assertEquals(0, nodeStatus.getAllContainers().size());
+ Assert.assertEquals(0, nodeStatus.getContainersStatuses().size());
// Give a container to the NM.
applicationID.setId(heartBeatID);
@@ -121,11 +143,9 @@ public class TestNodeStatusUpdater {
} else if (heartBeatID == 2) {
// Checks on the RM end
Assert.assertEquals("Number of applications should only be one!", 1,
- nodeStatus.getAllContainers().size());
+ nodeStatus.getContainersStatuses().size());
Assert.assertEquals("Number of container for the app should be one!",
- 1, nodeStatus.getContainers(applicationID).size());
- Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
- .getResource().getMemory());
+ 1, appToContainers.get(applicationID).size());
// Checks on the NM end
ConcurrentMap<ContainerId, Container> activeContainers =
@@ -147,13 +167,9 @@ public class TestNodeStatusUpdater {
} else if (heartBeatID == 3) {
// Checks on the RM end
Assert.assertEquals("Number of applications should only be one!", 1,
- nodeStatus.getAllContainers().size());
+ appToContainers.size());
Assert.assertEquals("Number of container for the app should be two!",
- 2, nodeStatus.getContainers(applicationID).size());
- Assert.assertEquals(2, nodeStatus.getContainers(applicationID).get(0)
- .getResource().getMemory());
- Assert.assertEquals(3, nodeStatus.getContainers(applicationID).get(1)
- .getResource().getMemory());
+ 2, appToContainers.get(applicationID).size());
// Checks on the NM end
ConcurrentMap<ContainerId, Container> activeContainers =
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Sun Sep 11 17:26:06 2011
@@ -232,8 +232,8 @@ public class ApplicationMasterService ex
RMAppAttempt appAttempt = app.getRMAppAttempt(appAttemptId);
AMResponse response = recordFactory.newRecordInstance(AMResponse.class);
- response.addAllNewContainers(allocation.getContainers());
- response.addAllFinishedContainers(appAttempt
+ response.setAllocatedContainers(allocation.getContainers());
+ response.setCompletedContainersStatuses(appAttempt
.pullJustFinishedContainers());
response.setResponseId(lastResponse.getResponseId() + 1);
response.setAvailableResources(allocation.getResourceLimit());
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Sun Sep 11 17:26:06 2011
@@ -20,10 +20,6 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
import javax.crypto.SecretKey;
@@ -31,7 +27,6 @@ import org.apache.avro.ipc.Server;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.SecurityInfo;
@@ -252,7 +247,7 @@ public class ResourceTrackerService exte
// 4. Send status to RMNode, saving the latest response.
this.rmContext.getDispatcher().getEventHandler().handle(
new RMNodeStatusEvent(nodeId, remoteNodeStatus.getNodeHealthStatus(),
- remoteNodeStatus.getAllContainers(), latestResponse));
+ remoteNodeStatus.getContainersStatuses(), latestResponse));
nodeHeartBeatResponse.setHeartbeatResponse(latestResponse);
return nodeHeartBeatResponse;
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java Sun Sep 11 17:26:06 2011
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -47,9 +48,9 @@ public interface RMAppAttempt extends Ev
Set<NodeId> getRanNodes();
- List<Container> pullJustFinishedContainers();
+ List<ContainerStatus> pullJustFinishedContainers();
- List<Container> getJustFinishedContainers();
+ List<ContainerStatus> getJustFinishedContainers();
Container getMasterContainer();
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Sun Sep 11 17:26:06 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -97,8 +98,8 @@ public class RMAppAttemptImpl implements
//nodes on while this attempt's containers ran
private final Set<NodeId> ranNodes =
new HashSet<NodeId>();
- private final List<Container> justFinishedContainers =
- new ArrayList<Container>();
+ private final List<ContainerStatus> justFinishedContainers =
+ new ArrayList<ContainerStatus>();
private Container masterContainer;
private float progress = 0;
@@ -333,7 +334,7 @@ public class RMAppAttemptImpl implements
}
@Override
- public List<Container> getJustFinishedContainers() {
+ public List<ContainerStatus> getJustFinishedContainers() {
this.readLock.lock();
try {
return this.justFinishedContainers;
@@ -343,11 +344,11 @@ public class RMAppAttemptImpl implements
}
@Override
- public List<Container> pullJustFinishedContainers() {
+ public List<ContainerStatus> pullJustFinishedContainers() {
this.writeLock.lock();
try {
- List<Container> returnList = new ArrayList<Container>(
+ List<ContainerStatus> returnList = new ArrayList<ContainerStatus>(
this.justFinishedContainers.size());
returnList.addAll(this.justFinishedContainers);
this.justFinishedContainers.clear();
@@ -705,11 +706,13 @@ public class RMAppAttemptImpl implements
RMAppAttemptContainerFinishedEvent containerFinishedEvent
= (RMAppAttemptContainerFinishedEvent) event;
- Container container = containerFinishedEvent.getContainer();
+ ContainerStatus containerStatus =
+ containerFinishedEvent.getContainerStatus();
// Is this container the AmContainer? If the finished container is same as
// the AMContainer, AppAttempt fails
- if (appAttempt.masterContainer.getId().equals(container.getId())) {
+ if (appAttempt.masterContainer.getId().equals(
+ containerStatus.getContainerId())) {
new FinalTransition(RMAppAttemptState.FAILED).transition(
appAttempt, containerFinishedEvent);
return RMAppAttemptState.FAILED;
@@ -718,7 +721,7 @@ public class RMAppAttemptImpl implements
// Normal container.
// Put it in completedcontainers list
- appAttempt.justFinishedContainers.add(container);
+ appAttempt.justFinishedContainers.add(containerStatus);
return RMAppAttemptState.RUNNING;
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/event/RMAppAttemptContainerFinishedEvent.java Sun Sep 11 17:26:06 2011
@@ -19,22 +19,22 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
public class RMAppAttemptContainerFinishedEvent extends RMAppAttemptEvent {
- private final Container container;
+ private final ContainerStatus containerStatus;
public RMAppAttemptContainerFinishedEvent(ApplicationAttemptId appAttemptId,
- Container container) {
+ ContainerStatus containerStatus) {
super(appAttemptId, RMAppAttemptEventType.CONTAINER_FINISHED);
- this.container = container;
+ this.containerStatus = containerStatus;
}
- public Container getContainer() {
- return this.container;
+ public ContainerStatus getContainerStatus() {
+ return this.containerStatus;
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerFinishedEvent.java Sun Sep 11 17:26:06 2011
@@ -26,8 +26,8 @@ public class RMContainerFinishedEvent ex
private final ContainerStatus remoteContainerStatus;
public RMContainerFinishedEvent(ContainerId containerId,
- ContainerStatus containerStatus) {
- super(containerId, RMContainerEventType.FINISHED);
+ ContainerStatus containerStatus, RMContainerEventType event) {
+ super(containerId, event);
this.remoteContainerStatus = containerStatus;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Sun Sep 11 17:26:06 2011
@@ -92,7 +92,7 @@ public class RMContainerImpl implements
// Transitions from RUNNING state
.addTransition(RMContainerState.RUNNING, RMContainerState.COMPLETED,
- RMContainerEventType.FINISHED, new ContainerCompletedTransition())
+ RMContainerEventType.FINISHED, new FinishedTransition())
.addTransition(RMContainerState.RUNNING, RMContainerState.KILLED,
RMContainerEventType.KILL, new KillTransition())
@@ -273,10 +273,16 @@ public class RMContainerImpl implements
@Override
public void transition(RMContainerImpl container, RMContainerEvent event) {
+ RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
+
+ // Update container-status for diagnostics. Today we completely
+ // replace it on finish. We may just need to update diagnostics.
+ container.container.setContainerStatus(finishedEvent
+ .getRemoteContainerStatus());
// Inform AppAttempt
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
- container.appAttemptId, container.container));
+ container.appAttemptId, container.container.getContainerStatus()));
}
}
@@ -312,22 +318,4 @@ public class RMContainerImpl implements
}
}
- private static final class ContainerCompletedTransition extends
- FinishedTransition {
-
- @Override
- public void transition(RMContainerImpl container, RMContainerEvent event) {
-
- RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
-
- // Update container-status for diagnostics. Today we completely
- // replace it on finish. We may just need to update diagnostics.
- // ^TODO
- container.container.setContainerStatus(finishedEvent
- .getRemoteContainerStatus());
-
- // Inform appAttempt
- super.transition(container, event);
- }
- }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Sun Sep 11 17:26:06 2011
@@ -23,7 +23,6 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -36,9 +35,9 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -87,8 +86,8 @@ public class RMNodeImpl implements RMNod
.newRecordInstance(NodeHealthStatus.class);
/* set of containers that have just launched */
- private final Map<ContainerId, Container> justLaunchedContainers =
- new HashMap<ContainerId, Container>();
+ private final Map<ContainerId, ContainerStatus> justLaunchedContainers =
+ new HashMap<ContainerId, ContainerStatus>();
/* set of containers that need to be cleaned */
@@ -355,43 +354,29 @@ public class RMNodeImpl implements RMNod
// Filter the map to only obtain just launched containers and finished
// containers.
- Map<ApplicationId, List<Container>> remoteAppContainersMap = statusEvent
- .getContainersCollection();
- Map<ApplicationId, List<Container>> containersMapForScheduler = new HashMap<ApplicationId, List<Container>>(
- remoteAppContainersMap.size());
- for (Entry<ApplicationId, List<Container>> entrySet : remoteAppContainersMap
- .entrySet()) {
-
- ApplicationId appId = entrySet.getKey();
- List<Container> remoteContainerList = entrySet.getValue();
-
- if (!containersMapForScheduler.containsKey(appId)) {
- containersMapForScheduler.put(appId, new ArrayList<Container>(
- remoteContainerList.size()));
- }
- List<Container> entryForThisApp = containersMapForScheduler
- .get(appId);
-
- for (Container remoteContainer : remoteContainerList) {
-
- // Process running containers
- ContainerId containerId = remoteContainer.getId();
- if (remoteContainer.getState() == ContainerState.RUNNING) {
- if (!rmNode.justLaunchedContainers.containsKey(containerId)) {
- // Just launched container. RM knows about it the first time.
- rmNode.justLaunchedContainers.put(containerId, remoteContainer);
- entryForThisApp.add(remoteContainer);
- }
- } else {
- // A finished container
- rmNode.justLaunchedContainers.remove(containerId);
- entryForThisApp.add(remoteContainer);
+ List<ContainerStatus> newlyLaunchedContainers =
+ new ArrayList<ContainerStatus>();
+ List<ContainerStatus> completedContainers =
+ new ArrayList<ContainerStatus>();
+ for (ContainerStatus remoteContainer : statusEvent.getContainers()) {
+ // Process running containers
+ ContainerId containerId = remoteContainer.getContainerId();
+ if (remoteContainer.getState() == ContainerState.RUNNING) {
+ if (!rmNode.justLaunchedContainers.containsKey(containerId)) {
+ // Just launched container. RM knows about it the first time.
+ rmNode.justLaunchedContainers.put(containerId, remoteContainer);
+ newlyLaunchedContainers.add(remoteContainer);
}
+ } else {
+ // A finished container
+ rmNode.justLaunchedContainers.remove(containerId);
+ completedContainers.add(remoteContainer);
}
}
rmNode.context.getDispatcher().getEventHandler().handle(
- new NodeUpdateSchedulerEvent(rmNode, containersMapForScheduler));
+ new NodeUpdateSchedulerEvent(rmNode, newlyLaunchedContainers,
+ completedContainers));
return RMNodeState.RUNNING;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java Sun Sep 11 17:26:06 2011
@@ -19,10 +19,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
import java.util.List;
-import java.util.Map;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
@@ -30,11 +28,11 @@ import org.apache.hadoop.yarn.server.api
public class RMNodeStatusEvent extends RMNodeEvent {
private final NodeHealthStatus nodeHealthStatus;
- private Map<ApplicationId, List<Container>> containersCollection;
+ private List<ContainerStatus> containersCollection;
private final HeartbeatResponse latestResponse;
public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus,
- Map<ApplicationId, List<Container>> collection,
+ List<ContainerStatus> collection,
HeartbeatResponse latestResponse) {
super(nodeId, RMNodeEventType.STATUS_UPDATE);
this.nodeHealthStatus = nodeHealthStatus;
@@ -46,7 +44,7 @@ public class RMNodeStatusEvent extends R
return this.nodeHealthStatus;
}
- public Map<ApplicationId, List<Container>> getContainersCollection() {
+ public List<ContainerStatus> getContainers() {
return this.containersCollection;
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java Sun Sep 11 17:26:06 2011
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -161,26 +162,21 @@ public class SchedulerApp {
RMContainerEventType.LAUNCHED));
}
- public synchronized void killContainers(
- SchedulerApp application) {
- }
-
synchronized public void containerCompleted(RMContainer rmContainer,
- RMContainerEventType event) {
+ ContainerStatus containerStatus, RMContainerEventType event) {
Container container = rmContainer.getContainer();
ContainerId containerId = container.getId();
// Inform the container
- if (event.equals(RMContainerEventType.FINISHED)) {
- // Have to send diagnostics for finished containers.
- rmContainer.handle(new RMContainerFinishedEvent(containerId,
- container.getContainerStatus()));
- } else {
- rmContainer.handle(new RMContainerEvent(containerId, event));
- }
+ rmContainer.handle(
+ new RMContainerFinishedEvent(
+ containerId,
+ containerStatus,
+ event)
+ );
LOG.info("Completed container: " + rmContainer.getContainerId() +
- " in state: " + rmContainer.getState());
+ " in state: " + rmContainer.getState() + " event:" + event);
// Remove from the list of containers
liveContainers.remove(rmContainer.getContainerId());
Added: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1169484&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (added)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Sun Sep 11 17:26:06 2011
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+
+/**
+ * Utilities shared by schedulers.
+ */
+@Private
+@Unstable
+public class SchedulerUtils {
+
+ private static final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
+ public static final String RELEASED_CONTAINER =
+ "Container released by application";
+
+ public static final String LOST_CONTAINER =
+ "Container released on a *lost* node";
+
+ public static final String COMPLETED_APPLICATION =
+ "Container of a completed application";
+
+ public static final String EXPIRED_CONTAINER =
+ "Container expired since it unused";
+
+ public static final String UNRESERVED_CONTAINER =
+ "Container reservation no longer required.";
+
+ /**
+ * Utility to create a {@link ContainerStatus} during exceptional
+ * circumstances.
+ *
+ * @param containerId {@link ContainerId} of returned/released/lost container.
+ * @param diagnostics diagnostic message
+ * @return <code>ContainerStatus</code> for an returned/released/lost
+ * container
+ */
+ public static ContainerStatus createAbnormalContainerStatus(
+ ContainerId containerId, String diagnostics) {
+ ContainerStatus containerStatus =
+ recordFactory.newRecordInstance(ContainerStatus.class);
+ containerStatus.setContainerId(containerId);
+ containerStatus.setDiagnostics(diagnostics);
+ containerStatus.setExitStatus("ABORTED");
+ containerStatus.setState(ContainerState.COMPLETE);
+ return containerStatus;
+ }
+
+
+}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1169484&r1=1169483&r2=1169484&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java Sun Sep 11 17:26:06 2011
@@ -26,6 +26,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -165,11 +166,14 @@ extends org.apache.hadoop.yarn.server.re
* @param node node on which the container completed
* @param container completed container,
* <code>null</code> if it was just a reservation
+ * @param containerStatus <code>ContainerStatus</code> for the completed
+ * container
* @param event event to be sent to the container
*/
public void completedContainer(Resource clusterResource,
SchedulerApp application, SchedulerNode node,
- RMContainer container, RMContainerEventType event);
+ RMContainer container, ContainerStatus containerStatus,
+ RMContainerEventType event);
/**
* Get the number of applications in the queue.