You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by tu...@apache.org on 2012/12/19 04:06:50 UTC
svn commit: r1423743 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/
hadoop-yarn/hadoop-yarn-server/hado...
Author: tucu
Date: Wed Dec 19 03:06:49 2012
New Revision: 1423743
URL: http://svn.apache.org/viewvc?rev=1423743&view=rev
Log:
YARN-254. Update fair scheduler web UI for hierarchical queues. (sandyr via tucu)
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
- copied unchanged from r1423742, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerLeafQueueInfo.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1423743&r1=1423742&r2=1423743&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Wed Dec 19 03:06:49 2012
@@ -56,6 +56,9 @@ Release 2.0.3-alpha - Unreleased
YARN-129. Simplify classpath construction for mini YARN tests. (tomwhite)
+ YARN-254. Update fair scheduler web UI for hierarchical queues.
+ (sandyr via tucu)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java?rev=1423743&r1=1423742&r2=1423743&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java Wed Dec 19 03:06:49 2012
@@ -20,21 +20,21 @@ package org.apache.hadoop.yarn.server.re
import static org.apache.hadoop.yarn.util.StringHelper.join;
-import java.util.List;
+import java.util.Collection;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerLeafQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerQueueInfo;
import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
-import org.apache.hadoop.yarn.webapp.view.HtmlPage.Page;
-import org.apache.hadoop.yarn.webapp.view.HtmlPage._;
import com.google.inject.Inject;
import com.google.inject.servlet.RequestScoped;
@@ -50,16 +50,15 @@ public class FairSchedulerPage extends R
@RequestScoped
static class FSQInfo {
- FairSchedulerInfo fsinfo;
FairSchedulerQueueInfo qinfo;
}
- static class QueueInfoBlock extends HtmlBlock {
- final FairSchedulerQueueInfo qinfo;
+ static class LeafQueueBlock extends HtmlBlock {
+ final FairSchedulerLeafQueueInfo qinfo;
- @Inject QueueInfoBlock(ViewContext ctx, FSQInfo info) {
+ @Inject LeafQueueBlock(ViewContext ctx, FSQInfo info) {
super(ctx);
- qinfo = (FairSchedulerQueueInfo) info.qinfo;
+ qinfo = (FairSchedulerLeafQueueInfo)info.qinfo;
}
@Override
@@ -83,6 +82,47 @@ public class FairSchedulerPage extends R
}
}
+ static class QueueBlock extends HtmlBlock {
+ final FSQInfo fsqinfo;
+
+ @Inject QueueBlock(FSQInfo info) {
+ fsqinfo = info;
+ }
+
+ @Override
+ public void render(Block html) {
+ Collection<FairSchedulerQueueInfo> subQueues = fsqinfo.qinfo.getChildQueues();
+ UL<Hamlet> ul = html.ul("#pq");
+ for (FairSchedulerQueueInfo info : subQueues) {
+ float capacity = info.getMaxResourcesFraction();
+ float fairShare = info.getFairShareFraction();
+ float used = info.getUsedFraction();
+ LI<UL<Hamlet>> li = ul.
+ li().
+ a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
+ $title(join("Fair Share:", percent(fairShare))).
+ span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
+ _('.')._().
+ span().$style(join(width(used/capacity),
+ ";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
+ _('.')._().
+ span(".q", info.getQueueName())._().
+ span().$class("qstats").$style(left(Q_STATS_POS)).
+ _(join(percent(used), " used"))._();
+
+ fsqinfo.qinfo = info;
+ if (info instanceof FairSchedulerLeafQueueInfo) {
+ li.ul("#lq").li()._(LeafQueueBlock.class)._()._();
+ } else {
+ li._(QueueBlock.class);
+ }
+ li._();
+ }
+
+ ul._();
+ }
+ }
+
static class QueuesBlock extends HtmlBlock {
final FairScheduler fs;
final FSQInfo fsqinfo;
@@ -91,8 +131,9 @@ public class FairSchedulerPage extends R
fs = (FairScheduler)rm.getResourceScheduler();
fsqinfo = info;
}
-
- @Override public void render(Block html) {
+
+ @Override
+ public void render(Block html) {
html._(MetricsOverviewTable.class);
UL<DIV<DIV<Hamlet>>> ul = html.
div("#cs-wrapper.ui-widget").
@@ -108,8 +149,8 @@ public class FairSchedulerPage extends R
span(".q", "default")._()._();
} else {
FairSchedulerInfo sinfo = new FairSchedulerInfo(fs);
- fsqinfo.fsinfo = sinfo;
- fsqinfo.qinfo = null;
+ fsqinfo.qinfo = sinfo.getRootQueueInfo();
+ float used = fsqinfo.qinfo.getUsedFraction();
ul.
li().$style("margin-bottom: 1em").
@@ -122,29 +163,15 @@ public class FairSchedulerPage extends R
_("Used (over fair share)")._().
span().$class("qlegend ui-corner-all ui-state-default").
_("Max Capacity")._().
- _();
-
- List<FairSchedulerQueueInfo> subQueues = fsqinfo.fsinfo.getQueueInfos();
- for (FairSchedulerQueueInfo info : subQueues) {
- fsqinfo.qinfo = info;
- float capacity = info.getMaxResourcesFraction();
- float fairShare = info.getFairShareFraction();
- float used = info.getUsedFraction();
- ul.
- li().
- a(_Q).$style(width(capacity * Q_MAX_WIDTH)).
- $title(join("Fair Share:", percent(fairShare))).
- span().$style(join(Q_GIVEN, ";font-size:1px;", width(fairShare/capacity))).
- _('.')._().
- span().$style(join(width(used/capacity),
- ";font-size:1px;left:0%;", used > fairShare ? Q_OVER : Q_UNDER)).
- _('.')._().
- span(".q", info.getQueueName())._().
- span().$class("qstats").$style(left(Q_STATS_POS)).
- _(join(percent(used), " used"))._().
- ul("#lq").li()._(QueueInfoBlock.class)._()._().
- _();
- }
+ _().
+ li().
+ a(_Q).$style(width(Q_MAX_WIDTH)).
+ span().$style(join(width(used), ";left:0%;",
+ used > 1 ? Q_OVER : Q_UNDER))._(".")._().
+ span(".q", "root")._().
+ span().$class("qstats").$style(left(Q_STATS_POS)).
+ _(join(percent(used), " used"))._().
+ _(QueueBlock.class)._();
}
ul._()._().
script().$type("text/javascript").
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java?rev=1423743&r1=1423742&r2=1423743&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerInfo.java Wed Dec 19 03:06:49 2012
@@ -18,33 +18,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
public class FairSchedulerInfo {
- private List<FairSchedulerQueueInfo> queueInfos;
private FairScheduler scheduler;
public FairSchedulerInfo(FairScheduler fs) {
scheduler = fs;
- Collection<FSLeafQueue> queues = fs.getQueueManager().getLeafQueues();
- queueInfos = new ArrayList<FairSchedulerQueueInfo>();
- for (FSLeafQueue queue : queues) {
- queueInfos.add(new FairSchedulerQueueInfo(queue, fs));
- }
- }
-
- public List<FairSchedulerQueueInfo> getQueueInfos() {
- return queueInfos;
}
public int getAppFairShare(ApplicationAttemptId appAttemptId) {
return scheduler.getSchedulerApp(appAttemptId).
getAppSchedulable().getFairShare().getMemory();
}
+
+ public FairSchedulerQueueInfo getRootQueueInfo() {
+ return new FairSchedulerQueueInfo(scheduler.getQueueManager().
+ getRootQueue(), scheduler);
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java?rev=1423743&r1=1423742&r2=1423743&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java Wed Dec 19 03:06:49 2012
@@ -18,19 +18,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import java.util.ArrayList;
import java.util.Collection;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.AppSchedulable;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueueManager;
-public class FairSchedulerQueueInfo {
- private int numPendingApps;
- private int numActiveApps;
-
+public class FairSchedulerQueueInfo {
private int fairShare;
private int minShare;
private int maxShare;
@@ -48,16 +47,9 @@ public class FairSchedulerQueueInfo {
private String queueName;
- public FairSchedulerQueueInfo(FSLeafQueue queue, FairScheduler scheduler) {
- Collection<AppSchedulable> apps = queue.getAppSchedulables();
- for (AppSchedulable app : apps) {
- if (app.getApp().isPending()) {
- numPendingApps++;
- } else {
- numActiveApps++;
- }
- }
-
+ private Collection<FairSchedulerQueueInfo> childInfos;
+
+ public FairSchedulerQueueInfo(FSQueue queue, FairScheduler scheduler) {
QueueManager manager = scheduler.getQueueManager();
queueName = queue.getName();
@@ -81,6 +73,16 @@ public class FairSchedulerQueueInfo {
fractionMinShare = (float)minShare / clusterMaxMem;
maxApps = manager.getQueueMaxApps(queueName);
+
+ Collection<FSQueue> childQueues = queue.getChildQueues();
+ childInfos = new ArrayList<FairSchedulerQueueInfo>();
+ for (FSQueue child : childQueues) {
+ if (child instanceof FSLeafQueue) {
+ childInfos.add(new FairSchedulerLeafQueueInfo((FSLeafQueue)child, scheduler));
+ } else {
+ childInfos.add(new FairSchedulerQueueInfo(child, scheduler));
+ }
+ }
}
/**
@@ -96,15 +98,7 @@ public class FairSchedulerQueueInfo {
public int getFairShare() {
return fairShare;
}
-
- public int getNumActiveApplications() {
- return numPendingApps;
- }
-
- public int getNumPendingApplications() {
- return numActiveApps;
- }
-
+
public Resource getMinResources() {
return minResources;
}
@@ -148,4 +142,8 @@ public class FairSchedulerQueueInfo {
public float getMaxResourcesFraction() {
return (float)maxShare / clusterMaxMem;
}
+
+ public Collection<FairSchedulerQueueInfo> getChildQueues() {
+ return childInfos;
+ }
}