You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sa...@apache.org on 2013/08/09 02:37:29 UTC
svn commit: r1512115 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/
hadoop-yarn/hadoop-yarn-server/hado...
Author: sandy
Date: Fri Aug 9 00:37:29 2013
New Revision: 1512115
URL: http://svn.apache.org/r1512115
Log:
YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1512115&r1=1512114&r2=1512115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Aug 9 00:37:29 2013
@@ -23,6 +23,8 @@ Release 2.1.1-beta - UNRELEASED
IMPROVEMENTS
+ YARN-589. Expose a REST API for monitoring the fair scheduler (Sandy Ryza).
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java?rev=1512115&r1=1512114&r2=1512115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java Fri Aug 9 00:37:29 2013
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
@@ -60,6 +61,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
@@ -145,6 +147,9 @@ public class RMWebServices {
CapacityScheduler cs = (CapacityScheduler) rs;
CSQueue root = cs.getRootQueue();
sinfo = new CapacitySchedulerInfo(root);
+ } else if (rs instanceof FairScheduler) {
+ FairScheduler fs = (FairScheduler) rs;
+ sinfo = new FairSchedulerInfo(fs);
} else if (rs instanceof FifoScheduler) {
sinfo = new FifoSchedulerInfo(this.rm);
} else {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java?rev=1512115&r1=1512114&r2=1512115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java Fri Aug 9 00:37:29 2013
@@ -18,14 +18,31 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.XmlType;
+
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
-public class FairSchedulerInfo {
+@XmlRootElement(name = "fairScheduler")
+@XmlType(name = "fairScheduler")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class FairSchedulerInfo extends SchedulerInfo {
+ private FairSchedulerQueueInfo rootQueue;
+
+ @XmlTransient
private FairScheduler scheduler;
+ public FairSchedulerInfo() {
+ } // JAXB needs this
+
public FairSchedulerInfo(FairScheduler fs) {
scheduler = fs;
+ rootQueue = new FairSchedulerQueueInfo(scheduler.getQueueManager().
+ getRootQueue(), scheduler);
}
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
@@ -34,7 +51,6 @@ public class FairSchedulerInfo {
}
public FairSchedulerQueueInfo getRootQueueInfo() {
- return new FairSchedulerQueueInfo(scheduler.getQueueManager().
- getRootQueue(), scheduler);
+ return rootQueue;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java?rev=1512115&r1=1512114&r2=1512115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java Fri Aug 9 00:37:29 2013
@@ -20,14 +20,23 @@ package org.apache.hadoop.yarn.server.re
import java.util.Collection;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
public class FairSchedulerLeafQueueInfo extends FairSchedulerQueueInfo {
private int numPendingApps;
private int numActiveApps;
+ public FairSchedulerLeafQueueInfo() {
+ }
+
public FairSchedulerLeafQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
super(queue, scheduler);
Collection<AppSchedulable> apps = queue.getAppSchedulables();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java?rev=1512115&r1=1512114&r2=1512115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java Fri Aug 9 00:37:29 2013
@@ -22,6 +22,12 @@ package org.apache.hadoop.yarn.server.re
import java.util.ArrayList;
import java.util.Collection;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
+import javax.xml.bind.annotation.XmlTransient;
+
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -29,39 +35,53 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
import org.apache.hadoop.yarn.util.resource.Resources;
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+@XmlSeeAlso({FairSchedulerLeafQueueInfo.class})
public class FairSchedulerQueueInfo {
private int maxApps;
+ @XmlTransient
private float fractionMemUsed;
+ @XmlTransient
private float fractionMemFairShare;
+ @XmlTransient
private float fractionMemMinShare;
+ @XmlTransient
private float fractionMemMaxShare;
- private Resource minResources;
- private Resource maxResources;
- private Resource usedResources;
- private Resource fairResources;
- private Resource clusterResources;
+ private ResourceInfo minResources;
+ private ResourceInfo maxResources;
+ private ResourceInfo usedResources;
+ private ResourceInfo fairResources;
+ private ResourceInfo clusterResources;
private String queueName;
+ private String schedulingPolicy;
+
+ private Collection<FairSchedulerQueueInfo> childQueues;
- private Collection<FairSchedulerQueueInfo> childInfos;
+ public FairSchedulerQueueInfo() {
+ }
public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
QueueManager manager = scheduler.getQueueManager();
queueName = queue.getName();
-
- clusterResources = scheduler.getClusterCapacity();
+ schedulingPolicy = queue.getPolicy().getName();
+
+ clusterResources = new ResourceInfo(scheduler.getClusterCapacity());
- usedResources = queue.getResourceUsage();
+ usedResources = new ResourceInfo(queue.getResourceUsage());
fractionMemUsed = (float)usedResources.getMemory() /
clusterResources.getMemory();
- fairResources = queue.getFairShare();
- minResources = queue.getMinShare();
- maxResources = queue.getMaxShare();
- maxResources = Resources.componentwiseMin(maxResources, clusterResources);
+ fairResources = new ResourceInfo(queue.getFairShare());
+ minResources = new ResourceInfo(queue.getMinShare());
+ maxResources = new ResourceInfo(queue.getMaxShare());
+ maxResources = new ResourceInfo(
+ Resources.componentwiseMin(queue.getMaxShare(),
+ scheduler.getClusterCapacity()));
fractionMemFairShare = (float)fairResources.getMemory() / clusterResources.getMemory();
fractionMemMinShare = (float)minResources.getMemory() / clusterResources.getMemory();
@@ -69,13 +89,13 @@ public class FairSchedulerQueueInfo {
maxApps = manager.getQueueMaxApps(queueName);
- Collection<FSQueue> childQueues = queue.getChildQueues();
- childInfos = new ArrayList<FairSchedulerQueueInfo>();
- for (FSQueue child : childQueues) {
+ Collection<FSQueue> children = queue.getChildQueues();
+ childQueues = new ArrayList<FairSchedulerQueueInfo>();
+ for (FSQueue child : children) {
if (child instanceof FSLeafQueue) {
- childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
+ childQueues.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
} else {
- childInfos.add(new FairSchedulerQueueInfo(child, scheduler));
+ childQueues.add(new FairSchedulerQueueInfo(child, scheduler));
}
}
}
@@ -90,15 +110,15 @@ public class FairSchedulerQueueInfo {
/**
* Returns the fair share of this queue in megabytes.
*/
- public Resource getFairShare() {
+ public ResourceInfo getFairShare() {
return fairResources;
}
- public Resource getMinResources() {
+ public ResourceInfo getMinResources() {
return minResources;
}
- public Resource getMaxResources() {
+ public ResourceInfo getMaxResources() {
return maxResources;
}
@@ -110,7 +130,7 @@ public class FairSchedulerQueueInfo {
return queueName;
}
- public Resource getUsedResources() {
+ public ResourceInfo getUsedResources() {
return usedResources;
}
@@ -138,7 +158,14 @@ public class FairSchedulerQueueInfo {
return fractionMemMaxShare;
}
+ /**
+ * Returns the name of the scheduling policy used by this queue.
+ */
+ public String getSchedulingPolicy() {
+ return schedulingPolicy;
+ }
+
public Collection<FairSchedulerQueueInfo> getChildQueues() {
- return childInfos;
+ return childQueues;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java?rev=1512115&r1=1512114&r2=1512115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java Fri Aug 9 00:37:29 2013
@@ -45,4 +45,9 @@ public class ResourceInfo {
public int getvCores() {
return vCores;
}
+
+ @Override
+ public String toString() {
+ return "<memory:" + memory + ", vCores:" + vCores + ">";
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java?rev=1512115&r1=1512114&r2=1512115&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java Fri Aug 9 00:37:29 2013
@@ -22,7 +22,8 @@ import javax.xml.bind.annotation.XmlRoot
import javax.xml.bind.annotation.XmlSeeAlso;
@XmlRootElement
-@XmlSeeAlso({ CapacitySchedulerInfo.class, FifoSchedulerInfo.class })
+@XmlSeeAlso({ CapacitySchedulerInfo.class, FairSchedulerInfo.class,
+ FifoSchedulerInfo.class })
public class SchedulerInfo {
public SchedulerInfo() {