You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/09/09 04:05:56 UTC
svn commit: r1166959 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/
hadoop-yarn/hadoop-...
Author: acmurthy
Date: Fri Sep 9 02:05:55 2011
New Revision: 1166959
URL: http://svn.apache.org/viewvc?rev=1166959&view=rev
Log:
fMerge -r 1166957:1166958 from trunk to branch-0.23 to fix MAPREDUCE-2690.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1166959&r1=1166958&r2=1166959&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri Sep 9 02:05:55 2011
@@ -245,6 +245,8 @@ Release 0.23.0 - Unreleased
MAPREDUCE-2864. Normalize configuration variable names for YARN. (Robert
Evans via acmurthy)
+ MAPREDUCE-2690. Web-page for FifoScheduler. (Eric Payne via acmurthy)
+
OPTIMIZATIONS
MAPREDUCE-2026. Make JobTracker.getJobCounters() and
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1166959&r1=1166958&r2=1166959&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Sep 9 02:05:55 2011
@@ -82,6 +82,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.api.records.QueueState;
@LimitedPrivate("yarn")
@Evolving
@@ -144,6 +145,7 @@ public class FifoScheduler implements Re
queueInfo.setCapacity(100.0f);
queueInfo.setMaximumCapacity(100.0f);
queueInfo.setChildQueues(new ArrayList<QueueInfo>());
+ queueInfo.setQueueState(QueueState.RUNNING);
return queueInfo;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java?rev=1166959&r1=1166958&r2=1166959&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/DefaultSchedulerPage.java Fri Sep 9 02:05:55 2011
@@ -18,20 +18,177 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import com.google.inject.Inject;
+import com.google.inject.servlet.RequestScoped;
+
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+import static org.apache.hadoop.yarn.util.StringHelper.*;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
class DefaultSchedulerPage extends RmView {
+ static final String _Q = ".ui-state-default.ui-corner-all";
+ static final float WIDTH_F = 0.8f;
+ static final String Q_END = "left:101%";
+ static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)";
+ static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)";
+ static final float EPSILON = 1e-8f;
+
+ static class QueueInfoBlock extends HtmlBlock {
+ final RMContext rmContext;
+ final FifoScheduler fs;
+ final String qName;
+ final QueueInfo qInfo;
+
+ @Inject QueueInfoBlock(RMContext context, ViewContext ctx, ResourceManager rm) {
+ super(ctx);
+ this.rmContext = context;
+
+ fs = (FifoScheduler) rm.getResourceScheduler();
+ qName = fs.getQueueInfo("",false,false).getQueueName();
+ qInfo = fs.getQueueInfo(qName,true,true);
+ }
- static class QueueBlock extends HtmlBlock {
@Override public void render(Block html) {
- html.h2("Under construction");
+ String minmemoryresource =
+ Integer.toString(fs.getMinimumResourceCapability().getMemory());
+ String maxmemoryresource =
+ Integer.toString(fs.getMaximumResourceCapability().getMemory());
+ String qstate = (qInfo.getQueueState() == QueueState.RUNNING) ?
+ "Running" :
+ (qInfo.getQueueState() == QueueState.STOPPED) ?
+ "Stopped" : "Unknown";
+
+ int usedNodeMem = 0;
+ int availNodeMem = 0;
+ int totNodeMem = 0;
+ int nodeContainers = 0;
+
+ for (RMNode ni : this.rmContext.getRMNodes().values()) {
+ usedNodeMem += fs.getUsedResource(ni.getNodeID()).getMemory();
+ availNodeMem += fs.getAvailableResource(ni.getNodeID()).getMemory();
+ totNodeMem += ni.getTotalCapability().getMemory();
+ nodeContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
+ }
+
+ info("\'" + qName + "\' Queue Status").
+ _("Queue State:" , qstate).
+ _("Minimum Queue Memory Capacity:" , minmemoryresource).
+ _("Maximum Queue Memory Capacity:" , maxmemoryresource).
+ _("Number of Nodes:" , Integer.toString(this.rmContext.getRMNodes().size())).
+ _("Used Node Capacity:" , Integer.toString(usedNodeMem)).
+ _("Available Node Capacity:" , Integer.toString(availNodeMem)).
+ _("Total Node Capacity:" , Integer.toString(totNodeMem)).
+ _("Number of Node Containers:" , Integer.toString(nodeContainers));
+
+ html._(InfoBlock.class);
}
}
+ static class QueuesBlock extends HtmlBlock {
+ final FifoScheduler fs;
+ final String qName;
+ final QueueInfo qInfo;
+
+ @Inject QueuesBlock(ResourceManager rm) {
+ fs = (FifoScheduler) rm.getResourceScheduler();
+ qName = fs.getQueueInfo("",false,false).getQueueName();
+ qInfo = fs.getQueueInfo(qName,false,false);
+ }
+
+ @Override
+ public void render(Block html) {
+ UL<DIV<DIV<Hamlet>>> ul = html.
+ div("#cs-wrapper.ui-widget").
+ div(".ui-widget-header.ui-corner-top").
+ _("FifoScheduler Queue")._().
+ div("#cs.ui-widget-content.ui-corner-bottom").
+ ul();
+
+ if (fs == null) {
+ ul.
+ li().
+ a(_Q).$style(width(WIDTH_F)).
+ span().$style(Q_END)._("100% ")._().
+ span(".q", "default")._()._();
+ } else {
+ float used = qInfo.getCurrentCapacity() / 100.0f;
+ float set = qInfo.getCapacity() / 100.0f;
+ float delta = Math.abs(set - used) + 0.001f;
+ ul.
+ li().
+ a(_Q).$style(width(WIDTH_F)).
+ $title(join("used:", percent(used))).
+ span().$style(Q_END)._("100%")._().
+ span().$style(join(width(delta), ';', used > set ? OVER : UNDER,
+ ';', used > set ? left(set) : left(used)))._(".")._().
+ span(".q", qName)._().
+ _(QueueInfoBlock.class)._();
+ }
+
+ ul._()._().
+ script().$type("text/javascript").
+ _("$('#cs').hide();")._()._().
+ _(AppsBlock.class);
+ }
+ }
+
+
+ @Override protected void postHead(Page.HTML<_> html) {
+ html.
+ style().$type("text/css").
+ _("#cs { padding: 0.5em 0 1em 0; margin-bottom: 1em; position: relative }",
+ "#cs ul { list-style: none }",
+ "#cs a { font-weight: normal; margin: 2px; position: relative }",
+ "#cs a span { font-weight: normal; font-size: 80% }",
+ "#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }",
+ "table.info tr th {width: 50%}")._(). // to center info table
+ script("/static/jt/jquery.jstree.js").
+ script().$type("text/javascript").
+ _("$(function() {",
+ " $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');",
+ " $('#cs').bind('loaded.jstree', function (e, data) {",
+ " data.inst.open_all(); }).",
+ " jstree({",
+ " core: { animation: 188, html_titles: true },",
+ " plugins: ['themeroller', 'html_data', 'ui'],",
+ " themeroller: { item_open: 'ui-icon-minus',",
+ " item_clsd: 'ui-icon-plus', item_leaf: 'ui-icon-gear'",
+ " }",
+ " });",
+ " $('#cs').bind('select_node.jstree', function(e, data) {",
+ " var q = $('.q', data.rslt.obj).first().text();",
+ " if (q == 'root') q = '';",
+ " $('#apps').dataTable().fnFilter(q, 3);",
+ " });",
+ " $('#cs').show();",
+ "});")._();
+ }
+
@Override protected Class<? extends SubView> content() {
- return QueueBlock.class;
+ return QueuesBlock.class;
+ }
+
+ static String percent(float f) {
+ return String.format("%.1f%%", f * 100);
+ }
+
+ static String width(float f) {
+ return String.format("width:%.1f%%", f * 100);
+ }
+
+ static String left(float f) {
+ return String.format("left:%.1f%%", f * 100);
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1166959&r1=1166958&r2=1166959&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Fri Sep 9 02:05:55 2011
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
import org.junit.Test;
@@ -168,9 +169,38 @@ public class TestRMWebApp {
conf.setCapacity(C13, 40);
}
+ public static ResourceManager mockFifoRm(int apps, int racks, int nodes,
+ int mbsPerNode)
+ throws Exception {
+ ResourceManager rm = mock(ResourceManager.class);
+ RMContext rmContext = mockRMContext(apps, racks, nodes,
+ mbsPerNode);
+ ResourceScheduler rs = mockFifoScheduler();
+ when(rm.getResourceScheduler()).thenReturn(rs);
+ when(rm.getRMContext()).thenReturn(rmContext);
+ return rm;
+ }
+
+ public static FifoScheduler mockFifoScheduler() throws Exception {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ setupFifoQueueConfiguration(conf);
+
+ FifoScheduler fs = new FifoScheduler();
+ fs.reinitialize(conf, null, null);
+ return fs;
+ }
+
+ static void setupFifoQueueConfiguration(CapacitySchedulerConfiguration conf) {
+ // Define default queue
+ conf.setQueues("default", new String[] {"default"});
+ conf.setCapacity("default", 100);
+ }
+
public static void main(String[] args) throws Exception {
// For manual testing
WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
start(new RMWebApp(mockRm(101, 8, 8, 8*GiB))).joinThread();
+ WebApps.$for("yarn", new TestRMWebApp()).at(8888).inDevMode().
+ start(new RMWebApp(mockFifoRm(10, 1, 4, 8*GiB))).joinThread();
}
}