You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/04/17 07:19:07 UTC
svn commit: r1094103 [2/2] - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/
mr-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
mr-client/hadoop-ma...
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java Sun Apr 17 05:19:05 2011
@@ -21,8 +21,10 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@@ -31,9 +33,13 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager;
@@ -66,6 +72,13 @@ public class ParentQueue implements Queu
private volatile int numApplications;
private volatile int numContainers;
+ private QueueInfo queueInfo;
+ private Map<ApplicationId, org.apache.hadoop.yarn.api.records.Application>
+ applicationInfos;
+
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
public ParentQueue(CapacitySchedulerContext cs,
String queueName, Comparator<Queue> comparator, Queue parent) {
minimumAllocation = cs.getMinimumAllocation();
@@ -91,6 +104,16 @@ public class ParentQueue implements Queu
this.childQueues = new TreeSet<Queue>(comparator);
+ this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
+ this.queueInfo.setCapacity(capacity);
+ this.queueInfo.setMaximumCapacity(maximumCapacity);
+ this.queueInfo.setQueueName(queueName);
+ this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
+
+ this.applicationInfos =
+ new HashMap<ApplicationId,
+ org.apache.hadoop.yarn.api.records.Application>();
+
LOG.info("Initialized parent-queue " + queueName +
" name=" + queueName +
", fullname=" + getQueuePath() +
@@ -188,6 +211,33 @@ public class ParentQueue implements Queu
return numApplications;
}
+ @Override
+ public synchronized QueueInfo getQueueInfo(boolean includeApplications,
+ boolean includeChildQueues, boolean recursive) {
+ queueInfo.setCurrentCapacity(usedCapacity);
+
+ if (includeApplications) {
+ queueInfo.setApplications(
+ new ArrayList<org.apache.hadoop.yarn.api.records.Application>(
+ applicationInfos.values()));
+ } else {
+ queueInfo.setApplications(
+ new ArrayList<org.apache.hadoop.yarn.api.records.Application>());
+ }
+
+ List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
+ if (includeChildQueues) {
+ for (Queue child : childQueues) {
+ // Get queue information recursively?
+ childQueuesInfo.add(
+ child.getQueueInfo(includeApplications, recursive, recursive));
+ }
+ }
+ queueInfo.setChildQueues(childQueuesInfo);
+
+ return queueInfo;
+}
+
public String toString() {
return queueName + ":" + capacity + ":" + absoluteCapacity + ":" +
getUsedCapacity() + ":" + getUtilization() + ":" +
@@ -207,6 +257,9 @@ public class ParentQueue implements Queu
++numApplications;
+ applicationInfos.put(application.getApplicationId(),
+ application.getApplicationInfo());
+
LOG.info("Application submission -" +
" appId: " + application.getApplicationId() +
" user: " + user +
@@ -229,7 +282,8 @@ public class ParentQueue implements Queu
}
--numApplications;
-
+ applicationInfos.remove(application.getApplicationId());
+
LOG.info("Application completion -" +
" appId: " + application.getApplicationId() +
" user: " + application.getUser() +
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Sun Apr 17 05:19:05 2011
@@ -32,17 +32,16 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.api.records.NodeId;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
@@ -76,13 +75,32 @@ public class FifoScheduler implements Re
new TreeMap<ApplicationId, Application>(
new org.apache.hadoop.yarn.server.resourcemanager.resource.ApplicationID.Comparator());
- private static final Queue DEFAULT_QUEUE = new Queue() {
+ private final Queue DEFAULT_QUEUE = new Queue() {
+ QueueInfo queueInfo;
+
@Override
public String getQueueName() {
return "default";
}
+
+ @Override
+ public QueueInfo getQueueInfo(boolean includeApplications,
+ boolean includeChildQueues, boolean recursive) {
+ queueInfo.setQueueName(DEFAULT_QUEUE.getQueueName());
+ queueInfo.setCapacity(100.0f);
+ queueInfo.setMaximumCapacity(100.0f);
+ queueInfo.setChildQueues(new ArrayList<QueueInfo>());
+
+ if (includeApplications) {
+ queueInfo.setApplications(getApplications());
+ } else {
+ queueInfo.setApplications(
+ new ArrayList<org.apache.hadoop.yarn.api.records.Application>());
+ }
+ return queueInfo;
+ }
};
-
+
public FifoScheduler() {}
public FifoScheduler(Configuration conf,
@@ -160,6 +178,7 @@ public class FifoScheduler implements Re
return applications.get(applicationId);
}
+ @Override
public synchronized void addApplication(ApplicationId applicationId,
String user, String unusedQueue, Priority unusedPriority)
throws IOException {
@@ -169,6 +188,7 @@ public class FifoScheduler implements Re
", currently active: " + applications.size());
}
+ @Override
public synchronized void removeApplication(ApplicationId applicationId)
throws IOException {
Application application = getApplication(applicationId);
@@ -462,6 +482,25 @@ public class FifoScheduler implements Re
nodes.remove(nodeInfo.getHostName());
}
+
+ @Override
+ public QueueInfo getQueueInfo(String queueName, boolean includeApplications,
+ boolean includeChildQueues, boolean recursive) {
+ QueueInfo queueInfo =
+ DEFAULT_QUEUE.getQueueInfo(includeApplications, false, false);
+ return queueInfo;
+ }
+
+ private synchronized List<org.apache.hadoop.yarn.api.records.Application>
+ getApplications() {
+ List<org.apache.hadoop.yarn.api.records.Application> applications =
+ new ArrayList<org.apache.hadoop.yarn.api.records.Application>();
+ for (Application application : this.applications.values()) {
+ applications.add(application.getApplicationInfo());
+ }
+ return applications;
+ }
+
public synchronized boolean isTracked(NodeInfo nodeInfo) {
NodeManager node = nodes.get(nodeInfo.getHostName());
return (node == null? false: true);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMLaunchFailure.java Sun Apr 17 05:19:05 2011
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -70,6 +71,23 @@ public class TestAMLaunchFailure extends
List<ResourceRequest> ask, List<Container> release) throws IOException {
return Arrays.asList(container);
}
+
+ @Override
+ public void addApplication(ApplicationId applicationId, String user,
+ String queue, Priority priority) throws IOException {
+ }
+
+ @Override
+ public void removeApplication(ApplicationId applicationId)
+ throws IOException {
+ }
+
+ @Override
+ public QueueInfo getQueueInfo(String queueName,
+ boolean includeApplications, boolean includeChildQueues,
+ boolean recursive) throws IOException {
+ return null;
+ }
}
private class DummyApplicationTracker implements EventHandler<ASMEvent<ApplicationTrackerEventType>> {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCResponseId.java Sun Apr 17 05:19:05 2011
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -80,6 +81,23 @@ public class TestAMRMRPCResponseId exten
List<ResourceRequest> ask, List<Container> release) throws IOException {
return null;
}
+
+ @Override
+ public void addApplication(ApplicationId applicationId, String user,
+ String queue, Priority priority) throws IOException {
+ }
+
+ @Override
+ public void removeApplication(ApplicationId applicationId)
+ throws IOException {
+ }
+
+ @Override
+ public QueueInfo getQueueInfo(String queueName,
+ boolean includeApplications, boolean includeChildQueues,
+ boolean recursive) throws IOException {
+ return null;
+ }
}
@Before
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Sun Apr 17 05:19:05 2011
@@ -21,6 +21,8 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -179,6 +181,20 @@ public class TestAMRestart extends TestC
public void reinitialize(Configuration conf,
ContainerTokenSecretManager secretManager) {
}
+ @Override
+ public void addApplication(ApplicationId applicationId, String user,
+ String queue, Priority priority) throws IOException {
+ }
+ @Override
+ public void removeApplication(ApplicationId applicationId)
+ throws IOException {
+ }
+ @Override
+ public QueueInfo getQueueInfo(String queueName,
+ boolean includeApplications, boolean includeChildQueues,
+ boolean recursive) throws IOException {
+ return null;
+ }
}
@Before
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java?rev=1094103&r1=1094102&r2=1094103&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java Sun Apr 17 05:19:05 2011
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -95,6 +96,20 @@ public class TestSchedulerNegotiator ext
@Override
public void handle(ASMEvent<ApplicationTrackerEventType> event) {
}
+ @Override
+ public void addApplication(ApplicationId applicationId, String user,
+ String queue, Priority priority) throws IOException {
+ }
+ @Override
+ public void removeApplication(ApplicationId applicationId)
+ throws IOException {
+ }
+ @Override
+ public QueueInfo getQueueInfo(String queueName,
+ boolean includeApplications, boolean includeChildQueues,
+ boolean recursive) throws IOException {
+ return null;
+ }
}
@Before