You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by at...@apache.org on 2011/10/06 03:16:57 UTC
svn commit: r1179484 [5/6] - in
/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/
hadoop-mapreduce-client/hadoop-mapreduce-client-a...
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Thu Oct 6 01:16:48 2011
@@ -58,8 +58,10 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -128,10 +130,15 @@ implements ResourceScheduler, CapacitySc
public CapacityScheduler() {}
+ @Override
+ public QueueMetrics getRootQueueMetrics() {
+ return root.getMetrics();
+ }
+
public CSQueue getRootQueue() {
return root;
}
-
+
@Override
public CapacitySchedulerConfiguration getConfiguration() {
return conf;
@@ -152,14 +159,6 @@ implements ResourceScheduler, CapacitySc
return maximumAllocation;
}
- public synchronized Resource getUsedResource(NodeId nodeId) {
- return nodes.get(nodeId).getUsedResource();
- }
-
- public synchronized Resource getAvailableResource(NodeId nodeId) {
- return nodes.get(nodeId).getAvailableResource();
- }
-
public synchronized int getNumClusterNodes() {
return numNodeManagers;
}
@@ -401,7 +400,7 @@ implements ResourceScheduler, CapacitySc
}
// Release all reserved containers
- for (RMContainer rmContainer : application.getAllReservedContainers()) {
+ for (RMContainer rmContainer : application.getReservedContainers()) {
completedContainer(rmContainer,
SchedulerUtils.createAbnormalContainerStatus(
rmContainer.getContainerId(),
@@ -465,21 +464,25 @@ implements ResourceScheduler, CapacitySc
if (!ask.isEmpty()) {
- LOG.info("DEBUG --- allocate: pre-update" +
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("allocate: pre-update" +
" applicationAttemptId=" + applicationAttemptId +
" application=" + application);
+ }
application.showRequests();
// Update application requests
application.updateResourceRequests(ask);
- LOG.info("DEBUG --- allocate: post-update");
+ LOG.debug("allocate: post-update");
application.showRequests();
}
- LOG.info("DEBUG --- allocate:" +
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("allocate:" +
" applicationAttemptId=" + applicationAttemptId +
" #ask=" + ask.size());
+ }
return new Allocation(
application.pullNewlyAllocatedContainers(),
@@ -548,14 +551,16 @@ implements ResourceScheduler, CapacitySc
// Process completed containers
for (ContainerStatus completedContainer : completedContainers) {
ContainerId containerId = completedContainer.getContainerId();
- LOG.info("DEBUG --- Container FINISHED: " + containerId);
+ LOG.debug("Container FINISHED: " + containerId);
completedContainer(getRMContainer(containerId),
completedContainer, RMContainerEventType.FINISHED);
}
// Now node data structures are upto date and ready for scheduling.
- LOG.info("DEBUG -- Node being looked for scheduling " + nm
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Node being looked for scheduling " + nm
+ " availableResource: " + node.getAvailableResource());
+ }
// Assign new containers...
// 1. Check for reserved applications
@@ -733,6 +738,13 @@ implements ResourceScheduler, CapacitySc
return applications.get(applicationAttemptId);
}
+ @Override
+ public SchedulerAppReport getSchedulerAppInfo(
+ ApplicationAttemptId applicationAttemptId) {
+ SchedulerApp app = getApplication(applicationAttemptId);
+ return app == null ? null : new SchedulerAppReport(app);
+ }
+
@Lock(Lock.NoLock.class)
SchedulerNode getNode(NodeId nodeId) {
return nodes.get(nodeId);
@@ -764,8 +776,7 @@ implements ResourceScheduler, CapacitySc
@Override
public SchedulerNodeReport getNodeReport(NodeId nodeId) {
SchedulerNode node = getNode(nodeId);
- return new SchedulerNodeReport(
- node.getUsedResource(), node.getNumContainers());
+ return node == null ? null : new SchedulerNodeReport(node);
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Oct 6 01:16:48 2011
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
@@ -34,6 +35,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -41,6 +43,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -181,9 +184,10 @@ public class LeafQueue implements CSQueu
maxActiveApplications, maxActiveApplicationsPerUser,
state, acls);
- LOG.info("DEBUG --- LeafQueue:" +
- " name=" + queueName +
- ", fullname=" + getQueuePath());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("LeafQueue:" + " name=" + queueName
+ + ", fullname=" + getQueuePath());
+ }
this.pendingApplications =
new TreeSet<SchedulerApp>(applicationComparator);
@@ -670,9 +674,10 @@ public class LeafQueue implements CSQueu
public synchronized Resource
assignContainers(Resource clusterResource, SchedulerNode node) {
- LOG.info("DEBUG --- assignContainers:" +
- " node=" + node.getHostName() +
- " #applications=" + activeApplications.size());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("assignContainers: node=" + node.getHostName()
+ + " #applications=" + activeApplications.size());
+ }
// Check for reserved resources
RMContainer reservedContainer = node.getReservedContainer();
@@ -686,8 +691,10 @@ public class LeafQueue implements CSQueu
// Try to assign containers to applications in order
for (SchedulerApp application : activeApplications) {
- LOG.info("DEBUG --- pre-assignContainers for application "
- + application.getApplicationId());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("pre-assignContainers for application "
+ + application.getApplicationId());
+ }
application.showRequests();
synchronized (application) {
@@ -747,8 +754,10 @@ public class LeafQueue implements CSQueu
}
}
- LOG.info("DEBUG --- post-assignContainers for application "
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("post-assignContainers for application "
+ application.getApplicationId());
+ }
application.showRequests();
}
@@ -1052,6 +1061,7 @@ public class LeafQueue implements CSQueu
createContainer(application, node, capability, priority);
}
+
public Container createContainer(SchedulerApp application, SchedulerNode node,
Resource capability, Priority priority) {
Container container =
@@ -1065,9 +1075,9 @@ public class LeafQueue implements CSQueu
if (UserGroupInformation.isSecurityEnabled()) {
ContainerToken containerToken =
this.recordFactory.newRecordInstance(ContainerToken.class);
- ContainerTokenIdentifier tokenidentifier =
- new ContainerTokenIdentifier(container.getId(),
- container.getNodeId().toString(), container.getResource());
+ NodeId nodeId = container.getNodeId();
+ ContainerTokenIdentifier tokenidentifier = new ContainerTokenIdentifier(
+ container.getId(), nodeId.toString(), container.getResource());
containerToken.setIdentifier(
ByteBuffer.wrap(tokenidentifier.getBytes()));
containerToken.setKind(ContainerTokenIdentifier.KIND.toString());
@@ -1075,7 +1085,11 @@ public class LeafQueue implements CSQueu
ByteBuffer.wrap(
containerTokenSecretManager.createPassword(tokenidentifier))
);
- containerToken.setService(container.getNodeId().toString());
+ // RPC layer client expects ip:port as service for tokens
+ InetSocketAddress addr = NetUtils.createSocketAddr(nodeId.getHost(),
+ nodeId.getPort());
+ containerToken.setService(addr.getAddress().getHostAddress() + ":"
+ + addr.getPort());
container.setContainerToken(containerToken);
}
@@ -1086,11 +1100,10 @@ public class LeafQueue implements CSQueu
SchedulerApp application, Priority priority,
ResourceRequest request, NodeType type, RMContainer rmContainer) {
if (LOG.isDebugEnabled()) {
- LOG.info("DEBUG --- assignContainers:" +
- " node=" + node.getHostName() +
- " application=" + application.getApplicationId().getId() +
- " priority=" + priority.getPriority() +
- " request=" + request + " type=" + type);
+ LOG.debug("assignContainers: node=" + node.getHostName()
+ + " application=" + application.getApplicationId().getId()
+ + " priority=" + priority.getPriority()
+ + " request=" + request + " type=" + type);
}
Resource capability = request.getCapability();
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Thu Oct 6 01:16:48 2011
@@ -141,7 +141,7 @@ public class ParentQueue implements CSQu
maximumCapacity, absoluteMaxCapacity, state, acls);
this.queueComparator = comparator;
- this.childQueues = new TreeSet<CSQueue>(comparator);
+ this.childQueues = new TreeSet<CSQueue>(queueComparator);
LOG.info("Initialized parent-queue " + queueName +
" name=" + queueName +
@@ -197,7 +197,9 @@ public class ParentQueue implements CSQu
this.childQueues.clear();
this.childQueues.addAll(childQueues);
- LOG.info("DEBUG --- setChildQueues: " + getChildQueuesToPrint());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("setChildQueues: " + getChildQueuesToPrint());
+ }
}
@Override
@@ -497,8 +499,10 @@ public class ParentQueue implements CSQu
Resource assigned = Resources.createResource(0);
while (canAssign(node)) {
- LOG.info("DEBUG --- Trying to assign containers to child-queue of " +
- getQueueName());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to assign containers to child-queue of "
+ + getQueueName());
+ }
// Are we over maximum-capacity for this queue?
if (!assignToQueue(clusterResource)) {
@@ -527,11 +531,12 @@ public class ParentQueue implements CSQu
break;
}
- LOG.info("DEBUG ---" +
- " parentQ=" + getQueueName() +
- " assignedSoFarInThisIteration=" + assigned +
- " utilization=" + getUtilization());
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ParentQ=" + getQueueName()
+ + " assignedSoFarInThisIteration=" + assigned
+ + " utilization=" + getUtilization());
+ }
+
// Do not assign more than one container if this isn't the root queue
if (!rootQueue) {
break;
@@ -571,13 +576,15 @@ public class ParentQueue implements CSQu
// Try to assign to most 'under-served' sub-queue
for (Iterator<CSQueue> iter=childQueues.iterator(); iter.hasNext();) {
CSQueue childQueue = iter.next();
- LOG.info("DEBUG --- Trying to assign to" +
- " queue: " + childQueue.getQueuePath() +
- " stats: " + childQueue);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Trying to assign to queue: " + childQueue.getQueuePath()
+ + " stats: " + childQueue);
+ }
assigned = childQueue.assignContainers(cluster, node);
- LOG.info("DEBUG --- Assignedto" +
- " queue: " + childQueue.getQueuePath() +
- " stats: " + childQueue + " --> " + assigned.getMemory());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Assignedto queue: " + childQueue.getQueuePath()
+ + " stats: " + childQueue + " --> " + assigned.getMemory());
+ }
// If we do assign, remove the queue and re-insert in-order to re-sort
if (Resources.greaterThan(assigned, Resources.none())) {
@@ -602,8 +609,10 @@ public class ParentQueue implements CSQu
return sb.toString();
}
void printChildQueues() {
- LOG.info("DEBUG --- printChildQueues - queue: " + getQueuePath() +
- " child-queues: " + getChildQueuesToPrint());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("printChildQueues - queue: " + getQueuePath()
+ + " child-queues: " + getChildQueuesToPrint());
+ }
}
@Override
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Thu Oct 6 01:16:48 2011
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -35,6 +36,7 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.Lock;
@@ -47,6 +49,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -70,6 +73,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApp;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -82,7 +86,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import org.apache.hadoop.yarn.api.records.QueueState;
@LimitedPrivate("yarn")
@Evolving
@@ -170,14 +173,6 @@ public class FifoScheduler implements Re
}
};
- public synchronized Resource getUsedResource(NodeId nodeId) {
- return getNode(nodeId).getUsedResource();
- }
-
- public synchronized Resource getAvailableResource(NodeId nodeId) {
- return getNode(nodeId).getAvailableResource();
- }
-
@Override
public Resource getMinimumResourceCapability() {
return minimumAllocation;
@@ -285,6 +280,13 @@ public class FifoScheduler implements Re
return applications.get(applicationAttemptId);
}
+ @Override
+ public SchedulerAppReport getSchedulerAppInfo(
+ ApplicationAttemptId applicationAttemptId) {
+ SchedulerApp app = getApplication(applicationAttemptId);
+ return app == null ? null : new SchedulerAppReport(app);
+ }
+
private SchedulerNode getNode(NodeId nodeId) {
return nodes.get(nodeId);
}
@@ -535,16 +537,21 @@ public class FifoScheduler implements Re
if (UserGroupInformation.isSecurityEnabled()) {
ContainerToken containerToken =
recordFactory.newRecordInstance(ContainerToken.class);
+ NodeId nodeId = container.getNodeId();
ContainerTokenIdentifier tokenidentifier =
new ContainerTokenIdentifier(container.getId(),
- container.getNodeId().toString(), container.getResource());
+ nodeId.toString(), container.getResource());
containerToken.setIdentifier(
ByteBuffer.wrap(tokenidentifier.getBytes()));
containerToken.setKind(ContainerTokenIdentifier.KIND.toString());
containerToken.setPassword(
ByteBuffer.wrap(containerTokenSecretManager
.createPassword(tokenidentifier)));
- containerToken.setService(container.getNodeId().toString());
+ // RPC layer client expects ip:port as service for tokens
+ InetSocketAddress addr = NetUtils.createSocketAddr(
+ nodeId.getHost(), nodeId.getPort());
+ containerToken.setService(addr.getAddress().getHostAddress() + ":"
+ + addr.getPort());
container.setContainerToken(containerToken);
}
@@ -580,7 +587,7 @@ public class FifoScheduler implements Re
// Process completed containers
for (ContainerStatus completedContainer : completedContainers) {
ContainerId containerId = completedContainer.getContainerId();
- LOG.info("DEBUG --- Container FINISHED: " + containerId);
+ LOG.debug("Container FINISHED: " + containerId);
containerCompleted(getRMContainer(containerId),
completedContainer, RMContainerEventType.FINISHED);
}
@@ -703,6 +710,9 @@ public class FifoScheduler implements Re
// Inform the node
node.releaseContainer(container);
+
+ // Update total usage
+ Resources.subtractFrom(usedResource, container.getResource());
LOG.info("Application " + applicationAttemptId +
" released container " + container.getId() +
@@ -762,8 +772,7 @@ public class FifoScheduler implements Re
@Override
public synchronized SchedulerNodeReport getNodeReport(NodeId nodeId) {
SchedulerNode node = getNode(nodeId);
- return new SchedulerNodeReport(
- node.getUsedResource(), node.getNumContainers());
+ return node == null ? null : new SchedulerNodeReport(node);
}
private RMContainer getRMContainer(ContainerId containerId) {
@@ -772,4 +781,9 @@ public class FifoScheduler implements Re
return (application == null) ? null : application.getRMContainer(containerId);
}
+ @Override
+ public QueueMetrics getRootQueueMetrics() {
+ return DEFAULT_QUEUE.getMetrics();
+ }
+
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/tools/RMAdmin.java Thu Oct 6 01:16:48 2011
@@ -23,7 +23,6 @@ import java.security.PrivilegedAction;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityInfo;
@@ -149,11 +148,6 @@ public class RMAdmin extends Configured
YarnConfiguration.RM_ADMIN_ADDRESS);
final YarnRPC rpc = YarnRPC.create(conf);
- if (UserGroupInformation.isSecurityEnabled()) {
- conf.setClass(YarnConfiguration.YARN_SECURITY_INFO,
- AdminSecurityInfo.class, SecurityInfo.class);
- }
-
RMAdminProtocol adminProtocol =
getUGI(conf).doAs(new PrivilegedAction<RMAdminProtocol>() {
@Override
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java Thu Oct 6 01:16:48 2011
@@ -50,6 +50,7 @@ class AppsBlock extends HtmlBlock {
th(".name", "Name").
th(".queue", "Queue").
th(".state", "State").
+ th(".finalstatus", "FinalStatus").
th(".progress", "Progress").
th(".ui", "Tracking UI").
th(".note", "Note")._()._().
@@ -60,7 +61,7 @@ class AppsBlock extends HtmlBlock {
String trackingUrl = app.getTrackingUrl();
String ui = trackingUrl == null || trackingUrl.isEmpty() ? "UNASSIGNED" :
(app.getFinishTime() == 0 ?
- "ApplicationMaster URL" : "JobHistory URL");
+ "ApplicationMaster" : "History");
String percent = String.format("%.1f", app.getProgress() * 100);
tbody.
tr().
@@ -70,8 +71,8 @@ class AppsBlock extends HtmlBlock {
td(app.getUser().toString()).
td(app.getName().toString()).
td(app.getQueue().toString()).
- td(app.getState() == RMAppState.FINISHED ? app.getAMFinalState() :
- app.getState().toString()).
+ td(app.getState().toString()).
+ td(app.getFinalApplicationStatus().toString()).
td().
br().$title(percent)._(). // for sorting
div(_PROGRESSBAR).
@@ -79,7 +80,8 @@ class AppsBlock extends HtmlBlock {
div(_PROGRESSBAR_VALUE).
$style(join("width:", percent, '%'))._()._()._().
td().
- a(trackingUrl == null ? "#" : join("http://", trackingUrl), ui)._().
+ a(trackingUrl == null || trackingUrl.isEmpty() || "N/A".equalsIgnoreCase(trackingUrl) ?
+ "#" : join("http://", trackingUrl), ui)._().
td(app.getDiagnostics().toString())._();
if (list.rendering != Render.HTML && ++i >= 20) break;
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsList.java Thu Oct 6 01:16:48 2011
@@ -62,8 +62,8 @@ class AppsList implements ToJSON {
}
String appID = app.getApplicationId().toString();
String trackingUrl = app.getTrackingUrl();
- String ui = trackingUrl == null ? "UNASSIGNED" :
- (app.getFinishTime() == 0 ? "ApplicationMaster" : "JobHistory");
+ String ui = trackingUrl == null || trackingUrl.isEmpty() || "N/A".equalsIgnoreCase(trackingUrl) ?
+ "UNASSIGNED" : (app.getFinishTime() == 0 ? "ApplicationMaster" : "JobHistory");
out.append("[\"");
appendSortable(out, app.getApplicationId().getId());
appendLink(out, appID, rc.prefix(), "app", appID).append(_SEP).
@@ -73,7 +73,8 @@ class AppsList implements ToJSON {
append(app.getState().toString()).append(_SEP);
appendProgressBar(out, app.getProgress()).append(_SEP);
appendLink(out, ui, rc.prefix(),
- trackingUrl == null ? "#" : "http://", trackingUrl).
+ trackingUrl == null || trackingUrl.isEmpty() || "N/A".equalsIgnoreCase(trackingUrl) ?
+ "#" : "http://", trackingUrl).
append(_SEP).append(escapeJavaScript(escapeHtml(
app.getDiagnostics().toString()))).
append("\"]");
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java Thu Oct 6 01:16:48 2011
@@ -31,7 +31,6 @@ import org.apache.hadoop.yarn.webapp.ham
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
class CapacitySchedulerPage extends RmView {
static final String _Q = ".ui-state-default.ui-corner-all";
@@ -96,6 +95,7 @@ class CapacitySchedulerPage extends RmVi
@Override
public void render(Block html) {
+ html._(MetricsOverviewTable.class);
UL<DIV<DIV<Hamlet>>> ul = html.
div("#cs-wrapper.ui-widget").
div(".ui-widget-header.ui-corner-top").
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java Thu Oct 6 01:16:48 2011
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import com.google.inject.Inject;
-import com.google.inject.servlet.RequestScoped;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
@@ -35,7 +34,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
class DefaultSchedulerPage extends RmView {
static final String _Q = ".ui-state-default.ui-corner-all";
@@ -76,8 +74,9 @@ class DefaultSchedulerPage extends RmVie
int nodeContainers = 0;
for (RMNode ni : this.rmContext.getRMNodes().values()) {
- usedNodeMem += fs.getUsedResource(ni.getNodeID()).getMemory();
- availNodeMem += fs.getAvailableResource(ni.getNodeID()).getMemory();
+ SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
+ usedNodeMem += report.getUsedResource().getMemory();
+ availNodeMem += report.getAvailableResource().getMemory();
totNodeMem += ni.getTotalCapability().getMemory();
nodeContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
}
@@ -109,6 +108,7 @@ class DefaultSchedulerPage extends RmVie
@Override
public void render(Block html) {
+ html._(MetricsOverviewTable.class);
UL<DIV<DIV<Hamlet>>> ul = html.
div("#cs-wrapper.ui-widget").
div(".ui-widget-header.ui-corner-top").
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Thu Oct 6 01:16:48 2011
@@ -18,14 +18,21 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.util.Times;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
@@ -38,45 +45,75 @@ import com.google.inject.Inject;
class NodesPage extends RmView {
static class NodesBlock extends HtmlBlock {
+ private static final long BYTES_IN_MB = 1024 * 1024;
final RMContext rmContext;
+ final ResourceManager rm;
@Inject
- NodesBlock(RMContext context, ViewContext ctx) {
+ NodesBlock(RMContext context, ResourceManager rm, ViewContext ctx) {
super(ctx);
this.rmContext = context;
+ this.rm = rm;
}
@Override
protected void render(Block html) {
+ html._(MetricsOverviewTable.class);
+
+ ResourceScheduler sched = rm.getResourceScheduler();
+ String type = $(NODE_STATE);
TBODY<TABLE<Hamlet>> tbody = html.table("#nodes").
thead().
tr().
th(".rack", "Rack").
+ th(".state", "Node State").
th(".nodeaddress", "Node Address").
th(".nodehttpaddress", "Node HTTP Address").
th(".healthStatus", "Health-status").
th(".lastHealthUpdate", "Last health-update").
th(".healthReport", "Health-report").
th(".containers", "Containers").
-// th(".mem", "Mem Used (MB)").
-// th(".mem", "Mem Avail (MB)").
+ th(".mem", "Mem Used").
+ th(".mem", "Mem Avail").
_()._().
tbody();
+ RMNodeState stateFilter = null;
+ if(type != null && !type.isEmpty()) {
+ stateFilter = RMNodeState.valueOf(type.toUpperCase());
+ }
for (RMNode ni : this.rmContext.getRMNodes().values()) {
+ if(stateFilter != null) {
+ RMNodeState state = ni.getState();
+ if(!stateFilter.equals(state)) {
+ continue;
+ }
+ }
+ NodeId id = ni.getNodeID();
+ SchedulerNodeReport report = sched.getNodeReport(id);
+ int numContainers = 0;
+ int usedMemory = 0;
+ int availableMemory = 0;
+ if(report != null) {
+ numContainers = report.getNumContainers();
+ usedMemory = report.getUsedResource().getMemory();
+ availableMemory = report.getAvailableResource().getMemory();
+ }
+
NodeHealthStatus health = ni.getNodeHealthStatus();
tbody.tr().
td(ni.getRackName()).
+ td(String.valueOf(ni.getState())).
td(String.valueOf(ni.getNodeID().toString())).
td().a("http://" + ni.getHttpAddress(), ni.getHttpAddress())._().
td(health.getIsNodeHealthy() ? "Healthy" : "Unhealthy").
td(Times.format(health.getLastHealthReportTime())).
td(String.valueOf(health.getHealthReport())).
- // TODO: acm: refactor2 FIXME
- //td(String.valueOf(ni.getNumContainers())).
- // TODO: FIXME Vinodkv
-// td(String.valueOf(ni.getUsedResource().getMemory())).
-// td(String.valueOf(ni.getAvailableResource().getMemory())).
- td("n/a")._();
+ td(String.valueOf(numContainers)).
+ td().br().$title(String.valueOf(usedMemory))._().
+ _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
+ td().br().$title(String.valueOf(usedMemory))._().
+ _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
+ _();
}
tbody._()._();
}
@@ -84,7 +121,12 @@ class NodesPage extends RmView {
@Override protected void preHead(Page.HTML<_> html) {
commonPreHead(html);
- setTitle("Nodes of the cluster");
+ String type = $(NODE_STATE);
+ String title = "Nodes of the cluster";
+ if(type != null && !type.isEmpty()) {
+ title = title+" ("+type+")";
+ }
+ setTitle(title);
set(DATATABLES_ID, "nodes");
set(initID(DATATABLES, "nodes"), nodesTableInit());
setTableStyles(html, "nodes", ".healthStatus {width:10em}",
@@ -96,11 +138,10 @@ class NodesPage extends RmView {
}
private String nodesTableInit() {
- return tableInit().
- // rack, nodeid, host, healthStatus, health update ts, health report,
- // containers, memused, memavail
- append(", aoColumns:[null, null, null, null, null, null, ").
- append("{sType:'title-numeric', bSearchable:false}]}").
- toString();
+ StringBuilder b = tableInit().append(",aoColumnDefs:[");
+ b.append("{'bSearchable':false, 'aTargets': [7]} ,");
+ b.append("{'sType':'title-numeric', 'bSearchable':false, " +
+ "'aTargets': [ 8, 9] }]}");
+ return b.toString();
}
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java Thu Oct 6 01:16:48 2011
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.webapp.Web
public class RMWebApp extends WebApp {
static final String APP_ID = "app.id";
static final String QUEUE_NAME = "queue.name";
+ static final String NODE_STATE = "node.state";
private final ResourceManager rm;
@@ -44,9 +45,9 @@ public class RMWebApp extends WebApp {
bind(RMContext.class).toInstance(rm.getRMContext());
}
route("/", RmController.class);
- route("/nodes", RmController.class, "nodes");
+ route(pajoin("/nodes", NODE_STATE), RmController.class, "nodes");
route("/apps", RmController.class);
- route("/cluster", RmController.class, "info");
+ route("/cluster", RmController.class, "about");
route(pajoin("/app", APP_ID), RmController.class, "app");
route("/scheduler", RmController.class, "scheduler");
route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue");
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java Thu Oct 6 01:16:48 2011
@@ -22,8 +22,9 @@ import static org.apache.hadoop.yarn.ser
import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.QUEUE_NAME;
import static org.apache.hadoop.yarn.util.StringHelper.join;
+import javax.servlet.http.HttpServletResponse;
+
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -35,7 +36,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.webapp.Controller;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
@@ -50,25 +50,15 @@ public class RmController extends Contro
setTitle("Applications");
}
- public void info() {
+ public void about() {
setTitle("About the Cluster");
- long ts = ResourceManager.clusterTimeStamp;
- ResourceManager rm = getInstance(ResourceManager.class);
- info("Cluster overview").
- _("Cluster ID:", ts).
- _("ResourceManager state:", rm.getServiceState()).
- _("ResourceManager started on:", Times.format(ts)).
- _("ResourceManager version:", YarnVersionInfo.getBuildVersion() +
- " on " + YarnVersionInfo.getDate()).
- _("Hadoop version:", VersionInfo.getBuildVersion() +
- " on " + VersionInfo.getDate());
- render(InfoPage.class);
+ render(AboutPage.class);
}
public void app() {
String aid = $(APP_ID);
if (aid.isEmpty()) {
- setStatus(response().SC_BAD_REQUEST);
+ setStatus(HttpServletResponse.SC_BAD_REQUEST);
setTitle("Bad request: requires application ID");
return;
}
@@ -77,7 +67,7 @@ public class RmController extends Contro
RMApp app = context.getRMApps().get(appID);
if (app == null) {
// TODO: handle redirect to jobhistory server
- setStatus(response().SC_NOT_FOUND);
+ setStatus(HttpServletResponse.SC_NOT_FOUND);
setTitle("Application not found: "+ aid);
return;
}
@@ -89,9 +79,8 @@ public class RmController extends Contro
ResponseInfo info = info("Application Overview").
_("User:", app.getUser()).
_("Name:", app.getName()).
- _("State:", (app.getState() == RMAppState.FINISHED ?
- app.getAMFinalState() : app.getState().toString())
- ).
+ _("State:", app.getState().toString()).
+ _("FinalStatus:", app.getFinalApplicationStatus().toString()).
_("Started:", Times.format(app.getStartTime())).
_("Elapsed:", StringUtils.formatTime(
Times.elapsed(app.getStartTime(), app.getFinishTime()))).
@@ -108,7 +97,7 @@ public class RmController extends Contro
} else {
info._("AM container logs:", "AM not yet registered with RM");
}
- render(InfoPage.class);
+ render(AboutPage.class);
}
public void nodes() {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmView.java Thu Oct 6 01:16:48 2011
@@ -52,7 +52,7 @@ public class RmView extends TwoColumnLay
@Override
protected Class<? extends SubView> content() {
- return AppsBlock.class;
+ return AppsBlockWithMetrics.class;
}
private String appsTableInit() {
@@ -60,7 +60,7 @@ public class RmView extends TwoColumnLay
// id, user, name, queue, state, progress, ui, note
StringBuilder init = tableInit().
append(", aoColumns:[{sType:'title-numeric'}, null, null, null, null,").
- append("{sType:'title-numeric', bSearchable:false}, null, null]");
+ append("null,{sType:'title-numeric', bSearchable:false}, null, null]");
String rows = $("rowlimit");
int rowLimit = rows.isEmpty() ? MAX_DISPLAY_ROWS : Integer.parseInt(rows);
if (list.apps.size() < rowLimit) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/resources/capacity-scheduler.xml Thu Oct 6 01:16:48 2011
@@ -3,76 +3,92 @@
<property>
<name>yarn.scheduler.capacity.maximum-applications</name>
<value>10000</value>
- <description>Maximum number of applications that can be running.
+ <description>
+ Maximum number of applications that can be pending and running.
</description>
</property>
<property>
<name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
<value>0.1</value>
+ <description>
+ Maximum percent of resources in the cluster which can be used to run
+ application masters i.e. controls number of concurrent running
+ applications.
+ </description>
</property>
<property>
<name>yarn.scheduler.capacity.root.queues</name>
<value>default</value>
- <description>The queues at the this level (root is the root queue).
+ <description>
+ The queues at the this level (root is the root queue).
</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.capacity</name>
<value>100</value>
- <description>The total capacity as a percentage out of 100 for this queue.
- If it has child queues then this includes their capacity as well.
- The child queues capacity should add up to their parent queue's capacity
- or less.</description>
- </property>
-
- <property>
- <name>yarn.scheduler.capacity.root.acl_administer_queues</name>
- <value>*</value>
- <description>The ACL for who can administer this queue. i.e.
- change sub queue allocations.</description>
+ <description>
+ The total capacity as a percentage out of 100 for this queue.
+ If it has child queues then this includes their capacity as well.
+ The child queues capacity should add up to their parent queue's capacity
+ or less.
+ </description>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.capacity</name>
<value>100</value>
- <description>default queue target capacity.</description>
+ <description>Default queue target capacity.</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
<value>1</value>
- <description>default queue user limit a percantage from 0.0 to 1.0.
+ <description>
+ Default queue user limit a percentage from 0.0 to 1.0.
</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
<value>-1</value>
- <description>the maximum capacity of the default queue -1 disables.
+ <description>
+ The maximum capacity of the default queue. A value of -1 disables this.
</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.state</name>
<value>RUNNING</value>
- <description>The state of the default queue. can be RUNNING or STOPPED
+ <description>
+ The state of the default queue. State can be one of RUNNING or STOPPED.
</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.acl_submit_jobs</name>
<value>*</value>
- <description>The ACL of who can submit jobs to the default queue.
+ <description>
+ The ACL of who can submit jobs to the default queue.
</description>
</property>
<property>
<name>yarn.scheduler.capacity.root.default.acl_administer_jobs</name>
<value>*</value>
- <description>The ACL of who can administer jobs on the default queue.
+ <description>
+ The ACL of who can administer jobs on the default queue.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.scheduler.capacity.root.acl_administer_queues</name>
+ <value>*</value>
+ <description>
+ The ACL for who can administer this queue i.e. change sub-queue
+ allocations.
</description>
</property>
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Thu Oct 6 01:16:48 2011
@@ -164,9 +164,10 @@ public class Application {
if (requests == null) {
requests = new HashMap<String, ResourceRequest>();
this.requests.put(priority, requests);
- LOG.info("DEBUG --- Added" +
- " priority=" + priority +
- " application=" + applicationId);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("Added priority=" + priority + " application="
+ + applicationId);
+ }
}
final Resource capability = requestSpec.get(priority);
@@ -182,9 +183,10 @@ public class Application {
LOG.info("Added task " + task.getTaskId() + " to application " +
applicationId + " at priority " + priority);
- LOG.info("DEBUG --- addTask:" +
- " application=" + applicationId +
- " #asks=" + ask.size());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("addTask: application=" + applicationId
+ + " #asks=" + ask.size());
+ }
// Create resource requests
for (String host : task.getHosts()) {
@@ -245,23 +247,24 @@ public class Application {
org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest(
request)); // clone to ensure the RM doesn't manipulate the same obj
- LOG.info("DEBUG --- addResourceRequest:" +
- " applicationId=" + applicationId.getId() +
- " priority=" + priority.getPriority() +
- " resourceName=" + resourceName +
- " capability=" + capability +
- " numContainers=" + request.getNumContainers() +
- " #asks=" + ask.size());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("addResourceRequest: applicationId=" + applicationId.getId()
+ + " priority=" + priority.getPriority()
+ + " resourceName=" + resourceName + " capability=" + capability
+ + " numContainers=" + request.getNumContainers()
+ + " #asks=" + ask.size());
+ }
}
public synchronized List<Container> getResources() throws IOException {
- LOG.info("DEBUG --- getResources begin:" +
- " application=" + applicationId +
- " #ask=" + ask.size());
- for (ResourceRequest request : ask) {
- LOG.info("DEBUG --- getResources:" +
- " application=" + applicationId +
- " ask-request=" + request);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getResources begin:" + " application=" + applicationId
+ + " #ask=" + ask.size());
+
+ for (ResourceRequest request : ask) {
+ LOG.debug("getResources:" + " application=" + applicationId
+ + " ask-request=" + request);
+ }
}
// Get resources from the ResourceManager
@@ -280,9 +283,10 @@ public class Application {
// Clear state for next interaction with ResourceManager
ask.clear();
- LOG.info("DEBUG --- getResources() for " + applicationId + ":" +
- " ask=" + ask.size() +
- " recieved=" + containers.size());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("getResources() for " + applicationId + ":"
+ + " ask=" + ask.size() + " recieved=" + containers.size());
+ }
return containers;
}
@@ -353,22 +357,22 @@ public class Application {
NodeType type, Task task) {
if (type == NodeType.NODE_LOCAL) {
for (String host : task.getHosts()) {
- LOG.info("DEBUG --- updateResourceRequests:" +
- " application=" + applicationId +
- " type=" + type +
- " host=" + host +
- " request=" + ((requests == null) ? "null" : requests.get(host)));
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("updateResourceRequests:" + " application=" + applicationId
+ + " type=" + type + " host=" + host
+ + " request=" + ((requests == null) ? "null" : requests.get(host)));
+ }
updateResourceRequest(requests.get(host));
}
}
if (type == NodeType.NODE_LOCAL || type == NodeType.RACK_LOCAL) {
for (String rack : task.getRacks()) {
- LOG.info("DEBUG --- updateResourceRequests:" +
- " application=" + applicationId +
- " type=" + type +
- " rack=" + rack +
- " request=" + ((requests == null) ? "null" : requests.get(rack)));
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("updateResourceRequests:" + " application=" + applicationId
+ + " type=" + type + " rack=" + rack
+ + " request=" + ((requests == null) ? "null" : requests.get(rack)));
+ }
updateResourceRequest(requests.get(rack));
}
}
@@ -378,9 +382,10 @@ public class Application {
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode.ANY)
);
- LOG.info("DEBUG --- updateResourceRequests:" +
- " application=" + applicationId +
- " #asks=" + ask.size());
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("updateResourceRequests:" + " application=" + applicationId
+ + " #asks=" + ask.size());
+ }
}
private void updateResourceRequest(ResourceRequest request) {
@@ -392,9 +397,10 @@ public class Application {
org.apache.hadoop.yarn.util.BuilderUtils.newResourceRequest(
request)); // clone to ensure the RM doesn't manipulate the same obj
- LOG.info("DEBUG --- updateResourceRequest:" +
- " application=" + applicationId +
- " request=" + request);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("updateResourceRequest:" + " application=" + applicationId
+ + " request=" + request);
+ }
}
private ContainerLaunchContext createCLC(Container container) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Thu Oct 6 01:16:48 2011
@@ -30,7 +30,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -47,11 +47,11 @@ public class MockAM {
private final ApplicationAttemptId attemptId;
private final RMContext context;
private final AMRMProtocol amRMProtocol;
-
+
private final List<ResourceRequest> requests = new ArrayList<ResourceRequest>();
private final List<ContainerId> releases = new ArrayList<ContainerId>();
- MockAM(RMContext context, AMRMProtocol amRMProtocol,
+ MockAM(RMContext context, AMRMProtocol amRMProtocol,
ApplicationAttemptId attemptId) {
this.context = context;
this.amRMProtocol = amRMProtocol;
@@ -85,7 +85,7 @@ public class MockAM {
amRMProtocol.registerApplicationMaster(req);
}
- public void addRequests(String[] hosts, int memory, int priority,
+ public void addRequests(String[] hosts, int memory, int priority,
int containers) throws Exception {
requests.addAll(createReq(hosts, memory, priority, containers));
}
@@ -97,33 +97,33 @@ public class MockAM {
return response;
}
- public AMResponse allocate(
- String host, int memory, int numContainers,
+ public AMResponse allocate(
+ String host, int memory, int numContainers,
List<ContainerId> releases) throws Exception {
- List reqs = createReq(new String[]{host}, memory, 1, numContainers);
+ List<ResourceRequest> reqs = createReq(new String[]{host}, memory, 1, numContainers);
return allocate(reqs, releases);
}
- public List<ResourceRequest> createReq(String[] hosts, int memory, int priority,
+ public List<ResourceRequest> createReq(String[] hosts, int memory, int priority,
int containers) throws Exception {
List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
for (String host : hosts) {
- ResourceRequest hostReq = createResourceReq(host, memory, priority,
+ ResourceRequest hostReq = createResourceReq(host, memory, priority,
containers);
reqs.add(hostReq);
- ResourceRequest rackReq = createResourceReq("default-rack", memory,
+ ResourceRequest rackReq = createResourceReq("default-rack", memory,
priority, containers);
reqs.add(rackReq);
}
-
- ResourceRequest offRackReq = createResourceReq("*", memory, priority,
+
+ ResourceRequest offRackReq = createResourceReq("*", memory, priority,
containers);
reqs.add(offRackReq);
return reqs;
-
+
}
- public ResourceRequest createResourceReq(String resource, int memory, int priority,
+ public ResourceRequest createResourceReq(String resource, int memory, int priority,
int containers) throws Exception {
ResourceRequest req = Records.newRecord(ResourceRequest.class);
req.setHostName(resource);
@@ -138,7 +138,7 @@ public class MockAM {
}
public AMResponse allocate(
- List<ResourceRequest> resourceRequest, List<ContainerId> releases)
+ List<ResourceRequest> resourceRequest, List<ContainerId> releases)
throws Exception {
AllocateRequest req = BuilderUtils.newAllocateRequest(attemptId,
++responseId, 0F, resourceRequest, releases);
@@ -151,7 +151,7 @@ public class MockAM {
FinishApplicationMasterRequest req = Records.newRecord(FinishApplicationMasterRequest.class);
req.setAppAttemptId(attemptId);
req.setDiagnostics("");
- req.setFinalState("");
+ req.setFinishApplicationStatus(FinalApplicationStatus.SUCCEEDED);
req.setTrackingUrl("");
amRMProtocol.finishApplicationMaster(req);
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Thu Oct 6 01:16:48 2011
@@ -194,12 +194,11 @@ public class NodeManager implements Cont
Resources.subtractFrom(available, containerLaunchContext.getResource());
Resources.addTo(used, containerLaunchContext.getResource());
- LOG.info("DEBUG --- startContainer:" +
- " node=" + containerManagerAddress +
- " application=" + applicationId +
- " container=" + container +
- " available=" + available +
- " used=" + used);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("startContainer:" + " node=" + containerManagerAddress
+ + " application=" + applicationId + " container=" + container
+ + " available=" + available + " used=" + used);
+ }
StartContainerResponse response = recordFactory.newRecordInstance(StartContainerResponse.class);
return response;
@@ -254,12 +253,11 @@ public class NodeManager implements Cont
Resources.addTo(available, container.getResource());
Resources.subtractFrom(used, container.getResource());
- LOG.info("DEBUG --- stopContainer:" +
- " node=" + containerManagerAddress +
- " application=" + applicationId +
- " container=" + containerID +
- " available=" + available +
- " used=" + used);
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("stopContainer:" + " node=" + containerManagerAddress
+ + " application=" + applicationId + " container=" + containerID
+ + " available=" + available + " used=" + used);
+ }
StopContainerResponse response = recordFactory.newRecordInstance(StopContainerResponse.class);
return response;
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Thu Oct 6 01:16:48 2011
@@ -28,10 +28,12 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.yarn.api.records.AMResponse;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
@@ -71,8 +73,9 @@ public class TestFifoScheduler {
RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
am1.registerAppAttempt();
- Assert.assertEquals(2 * GB, rm.getResourceScheduler().getUsedResource(
- nm1.getNodeId()).getMemory());
+ SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
+ nm1.getNodeId());
+ Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemory());
RMApp app2 = rm.submitApp(2048);
// kick the scheduling, 2GB given to AM, remaining 2 GB on nm2
@@ -80,8 +83,9 @@ public class TestFifoScheduler {
RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId());
am2.registerAppAttempt();
- Assert.assertEquals(2 * GB, rm.getResourceScheduler().getUsedResource(
- nm2.getNodeId()).getMemory());
+ SchedulerNodeReport report_nm2 = rm.getResourceScheduler().getNodeReport(
+ nm2.getNodeId());
+ Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory());
// add request for containers
am1.addRequests(new String[] { "h1", "h2" }, GB, 1, 1);
@@ -114,16 +118,14 @@ public class TestFifoScheduler {
Assert.assertEquals(1, allocated2.size());
Assert.assertEquals(3 * GB, allocated2.get(0).getResource().getMemory());
Assert.assertEquals(nm1.getNodeId(), allocated2.get(0).getNodeId());
+
+ report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
+ report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
+ Assert.assertEquals(0, report_nm1.getAvailableResource().getMemory());
+ Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemory());
- Assert.assertEquals(0, rm.getResourceScheduler().getAvailableResource(
- nm1.getNodeId()).getMemory());
- Assert.assertEquals(2 * GB, rm.getResourceScheduler()
- .getAvailableResource(nm2.getNodeId()).getMemory());
-
- Assert.assertEquals(6 * GB, rm.getResourceScheduler().getUsedResource(
- nm1.getNodeId()).getMemory());
- Assert.assertEquals(2 * GB, rm.getResourceScheduler().getUsedResource(
- nm2.getNodeId()).getMemory());
+ Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemory());
+ Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemory());
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemory());
@@ -138,8 +140,8 @@ public class TestFifoScheduler {
}
Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
- Assert.assertEquals(5 * GB, rm.getResourceScheduler().getUsedResource(
- nm1.getNodeId()).getMemory());
+ report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
+ Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemory());
rm.stop();
}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java Thu Oct 6 01:16:48 2011
@@ -22,10 +22,11 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationMaster;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationStatus;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -69,7 +70,7 @@ public abstract class MockAsm extends Mo
}
@Override
- public ApplicationState getState() {
+ public YarnApplicationState getState() {
throw new UnsupportedOperationException("Not supported yet.");
}
@@ -119,7 +120,7 @@ public abstract class MockAsm extends Mo
}
@Override
- public void setState(ApplicationState state) {
+ public void setState(YarnApplicationState state) {
throw new UnsupportedOperationException("Not supported yet.");
}
@@ -207,11 +208,11 @@ public abstract class MockAsm extends Mo
}
@Override
public void handle(RMAppEvent event) {
- throw new UnsupportedOperationException("Not supported yet.");
+ throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public String getAMFinalState() {
+ public FinalApplicationStatus getFinalApplicationStatus() {
throw new UnsupportedOperationException("Not supported yet.");
}
}
@@ -274,9 +275,14 @@ public abstract class MockAsm extends Mo
public float getProgress() {
return (float)Math.random();
}
+ @Override
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return FinalApplicationStatus.UNDEFINED;
+ }
+
};
}
-
+
public static List<RMApp> newApplications(int n) {
List<RMApp> list = Lists.newArrayList();
for (int i = 0; i < n; ++i) {
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java Thu Oct 6 01:16:48 2011
@@ -32,7 +32,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationMaster;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
@@ -65,7 +65,7 @@ public class TestAMLaunchFailure {
// private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
// ApplicationsManagerImpl asmImpl;
// YarnScheduler scheduler = new DummyYarnScheduler();
-// ApplicationTokenSecretManager applicationTokenSecretManager =
+// ApplicationTokenSecretManager applicationTokenSecretManager =
// new ApplicationTokenSecretManager();
// private ClientRMService clientService;
//
@@ -98,7 +98,7 @@ public class TestAMLaunchFailure {
// , ApplicationStore appStore)
// throws IOException {
// // TODO Auto-generated method stub
-//
+//
// }
//
// @Override
@@ -199,7 +199,7 @@ public class TestAMLaunchFailure {
// conf.setLong(YarnConfiguration.AM_EXPIRY_INTERVAL, 3000L);
// conf.setInt(RMConfig.AM_MAX_RETRIES, 1);
// asmImpl.init(conf);
-// asmImpl.start();
+// asmImpl.start();
// }
//
// @After
@@ -221,7 +221,7 @@ public class TestAMLaunchFailure {
// .newRecordInstance(SubmitApplicationRequest.class);
// request.setApplicationSubmissionContext(submissionContext);
// clientService.submitApplication(request);
-// AppAttempt application = context.getApplications().get(appID);
+// AppAttempt application = context.getApplications().get(appID);
//
// while (application.getState() != ApplicationState.FAILED) {
// LOG.info("Waiting for application to go to FAILED state."
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Thu Oct 6 01:16:48 2011
@@ -33,7 +33,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationMaster;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -75,7 +75,7 @@ public class TestAMRestart {
// private static final Log LOG = LogFactory.getLog(TestAMRestart.class);
// ApplicationsManagerImpl appImpl;
// RMContext asmContext = new RMContextImpl(new MemStore());
-// ApplicationTokenSecretManager appTokenSecretManager =
+// ApplicationTokenSecretManager appTokenSecretManager =
// new ApplicationTokenSecretManager();
// DummyResourceScheduler scheduler;
// private ClientRMService clientRMService;
@@ -90,7 +90,7 @@ public class TestAMRestart {
// int launcherLaunchCalled = 0;
// int launcherCleanupCalled = 0;
// private final static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-//
+//
// private class ExtApplicationsManagerImpl extends ApplicationsManagerImpl {
// public ExtApplicationsManagerImpl(
// ApplicationTokenSecretManager applicationTokenSecretManager,
@@ -115,7 +115,7 @@ public class TestAMRestart {
// LOG.info("DEBUG -- waiting for launch");
// synchronized(launchNotify) {
// while (launchNotify.get() == 0) {
-// try {
+// try {
// launchNotify.wait();
// } catch (InterruptedException e) {
// }
@@ -151,11 +151,11 @@ public class TestAMRestart {
// }
//
// private class DummyResourceScheduler implements ResourceScheduler {
-//
+//
// @Override
// public void removeNode(RMNode node) {
// }
-//
+//
// @Override
// public Allocation allocate(ApplicationId applicationId,
// List<ResourceRequest> ask, List<Container> release) throws IOException {
@@ -222,7 +222,7 @@ public class TestAMRestart {
//
// @Override
// public void nodeUpdate(RMNode nodeInfo,
-// Map<String, List<Container>> containers) {
+// Map<String, List<Container>> containers) {
// }
//
// @Override
@@ -253,7 +253,7 @@ public class TestAMRestart {
// asmContext.getDispatcher().start();
// asmContext.getDispatcher().register(ApplicationTrackerEventType.class, scheduler);
// appImpl = new ExtApplicationsManagerImpl(appTokenSecretManager, scheduler, asmContext);
-//
+//
// conf.setLong(YarnConfiguration.AM_EXPIRY_INTERVAL, 1000L);
// conf.setInt(RMConfig.AM_MAX_RETRIES, maxFailures);
// appImpl.init(conf);
@@ -261,7 +261,7 @@ public class TestAMRestart {
//
// this.clientRMService = new ClientRMService(asmContext, appImpl
// .getAmLivelinessMonitor(), appImpl.getClientToAMSecretManager(),
-// scheduler);
+// scheduler);
// this.clientRMService.init(conf);
// }
//
@@ -269,7 +269,7 @@ public class TestAMRestart {
// public void tearDown() {
// }
//
-// private void waitForFailed(AppAttempt application, ApplicationState
+// private void waitForFailed(AppAttempt application, ApplicationState
// finalState) throws Exception {
// int count = 0;
// while(application.getState() != finalState && count < 10) {
@@ -292,7 +292,7 @@ public class TestAMRestart {
// .newRecordInstance(SubmitApplicationRequest.class);
// request.setApplicationSubmissionContext(subContext);
// clientRMService.submitApplication(request);
-// AppAttempt application = asmContext.getApplications().get(appID);
+// AppAttempt application = asmContext.getApplications().get(appID);
// synchronized (schedulerNotify) {
// while(schedulerNotify.get() == 0) {
// schedulerNotify.wait();
@@ -306,4 +306,4 @@ public class TestAMRestart {
// waitForFailed(application, ApplicationState.FAILED);
// stop = true;
// }
-}
\ No newline at end of file
+}
Modified: hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java?rev=1179484&r1=1179483&r2=1179484&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java (original)
+++ hadoop/common/branches/HDFS-1623/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestASMStateMachine.java Thu Oct 6 01:16:48 2011
@@ -26,7 +26,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.ApplicationStatus;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -152,7 +152,7 @@ public class TestASMStateMachine {
// }
// }
//
-// private void waitForState( ApplicationState
+// private void waitForState( ApplicationState
// finalState, AppAttemptImpl masterInfo) throws Exception {
// int count = 0;
// while(masterInfo.getState() != finalState && count < 10) {
@@ -160,10 +160,10 @@ public class TestASMStateMachine {
// count++;
// }
// Assert.assertEquals(finalState, masterInfo.getState());
-// }
-//
-// /* Test the state machine.
-// *
+// }
+//
+// /* Test the state machine.
+// *
// */
// @Test
// public void testStateMachine() throws Exception {
@@ -211,6 +211,6 @@ public class TestASMStateMachine {
// /* check if expiry doesnt make it failed */
// handler.handle(new ApplicationEvent(ApplicationEventType.EXPIRE,
// masterInfo.getApplicationID()));
-// Assert.assertEquals(ApplicationState.COMPLETED, masterInfo.getState());
+// Assert.assertEquals(ApplicationState.COMPLETED, masterInfo.getState());
// }
}