You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by tg...@apache.org on 2014/06/24 21:34:35 UTC
svn commit: r1605162 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/...
Author: tgraves
Date: Tue Jun 24 19:34:34 2014
New Revision: 1605162
URL: http://svn.apache.org/r1605162
Log:
YARN-2072. RM/NM UIs and webservices are missing vcore information. (Nathan Roberts via tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Tue Jun 24 19:34:34 2014
@@ -183,6 +183,9 @@ Release 2.5.0 - UNRELEASED
YARN-2109. Fix TestRM to work with both schedulers. (Anubhav Dhoot via kasha)
+ YARN-2072. RM/NM UIs and webservices are missing vcore information.
+ (Nathan Roberts via tgraves)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java Tue Jun 24 19:34:34 2014
@@ -27,4 +27,6 @@ public interface ResourceView {
long getPmemAllocatedForContainers();
boolean isPmemCheckEnabled();
+
+ long getVCoresAllocatedForContainers();
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Tue Jun 24 19:34:34 2014
@@ -70,6 +70,8 @@ public class ContainersMonitorImpl exten
private boolean pmemCheckEnabled;
private boolean vmemCheckEnabled;
+ private long maxVCoresAllottedForContainers;
+
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
public ContainersMonitorImpl(ContainerExecutor exec,
@@ -108,10 +110,16 @@ public class ContainersMonitorImpl exten
YarnConfiguration.NM_PMEM_MB,
YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l;
+ long configuredVCoresForContainers = conf.getLong(
+ YarnConfiguration.NM_VCORES,
+ YarnConfiguration.DEFAULT_NM_VCORES);
+
+
// Setting these irrespective of whether checks are enabled. Required in
// the UI.
// ///////// Physical memory configuration //////
this.maxPmemAllottedForContainers = configuredPMemForContainers;
+ this.maxVCoresAllottedForContainers = configuredVCoresForContainers;
// ///////// Virtual memory configuration //////
float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO,
@@ -518,6 +526,11 @@ public class ContainersMonitorImpl exten
return this.maxPmemAllottedForContainers;
}
+ @Override
+ public long getVCoresAllocatedForContainers() {
+ return this.maxVCoresAllottedForContainers;
+ }
+
/**
* Is the total virtual memory check enabled?
*
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/metrics/NodeManagerMetrics.java Tue Jun 24 19:34:34 2014
@@ -40,6 +40,9 @@ public class NodeManagerMetrics {
@Metric("Current # of allocated containers")
MutableGaugeInt allocatedContainers;
@Metric MutableGaugeInt availableGB;
+ @Metric("Current allocated Virtual Cores")
+ MutableGaugeInt allocatedVCores;
+ @Metric MutableGaugeInt availableVCores;
public static NodeManagerMetrics create() {
return create(DefaultMetricsSystem.instance());
@@ -88,16 +91,21 @@ public class NodeManagerMetrics {
allocatedContainers.incr();
allocatedGB.incr(res.getMemory() / 1024);
availableGB.decr(res.getMemory() / 1024);
+ allocatedVCores.incr(res.getVirtualCores());
+ availableVCores.decr(res.getVirtualCores());
}
public void releaseContainer(Resource res) {
allocatedContainers.decr();
allocatedGB.decr(res.getMemory() / 1024);
availableGB.incr(res.getMemory() / 1024);
+ allocatedVCores.decr(res.getVirtualCores());
+ availableVCores.incr(res.getVirtualCores());
}
public void addResource(Resource res) {
availableGB.incr(res.getMemory() / 1024);
+ availableVCores.incr(res.getVirtualCores());
}
public int getRunningContainers() {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Tue Jun 24 19:34:34 2014
@@ -85,6 +85,7 @@ public class ContainerPage extends NMVie
._("Diagnostics", info.getDiagnostics())
._("User", info.getUser())
._("TotalMemoryNeeded", info.getMemoryNeeded())
+ ._("TotalVCoresNeeded", info.getVCoresNeeded())
._("logs", info.getShortLogLink(), "Link to logs");
html._(InfoBlock.class);
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java Tue Jun 24 19:34:34 2014
@@ -73,6 +73,8 @@ public class NodePage extends NMView {
StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB))
._("Pmem enforcement enabled",
info.isVmemCheckEnabled())
+ ._("Total VCores allocated for Containers",
+ String.valueOf(info.getTotalVCoresAllocated()))
._("NodeHealthyStatus",
info.getHealthStatus())
._("LastNodeHealthTime", new Date(
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java Tue Jun 24 19:34:34 2014
@@ -42,6 +42,7 @@ public class ContainerInfo {
protected String diagnostics;
protected String user;
protected long totalMemoryNeededMB;
+ protected long totalVCoresNeeded;
protected String containerLogsLink;
protected String nodeId;
@XmlTransient
@@ -76,6 +77,7 @@ public class ContainerInfo {
Resource res = container.getResource();
if (res != null) {
this.totalMemoryNeededMB = res.getMemory();
+ this.totalVCoresNeeded = res.getVirtualCores();
}
this.containerLogsShortLink = ujoin("containerlogs", this.id,
container.getUser());
@@ -130,4 +132,8 @@ public class ContainerInfo {
return this.totalMemoryNeededMB;
}
+ public long getVCoresNeeded() {
+ return this.totalVCoresNeeded;
+ }
+
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java Tue Jun 24 19:34:34 2014
@@ -36,6 +36,7 @@ public class NodeInfo {
protected String healthReport;
protected long totalVmemAllocatedContainersMB;
protected long totalPmemAllocatedContainersMB;
+ protected long totalVCoresAllocatedContainers;
protected boolean vmemCheckEnabled;
protected boolean pmemCheckEnabled;
protected long lastNodeUpdateTime;
@@ -62,6 +63,8 @@ public class NodeInfo {
this.totalPmemAllocatedContainersMB = resourceView
.getPmemAllocatedForContainers() / BYTES_IN_MB;
this.pmemCheckEnabled = resourceView.isPmemCheckEnabled();
+ this.totalVCoresAllocatedContainers = resourceView
+ .getVCoresAllocatedForContainers();
this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy();
this.lastNodeUpdateTime = context.getNodeHealthStatus()
.getLastHealthReportTime();
@@ -124,6 +127,10 @@ public class NodeInfo {
return this.totalVmemAllocatedContainersMB;
}
+ public long getTotalVCoresAllocated() {
+ return this.totalVCoresAllocatedContainers;
+ }
+
public boolean isVmemCheckEnabled() {
return this.vmemCheckEnabled;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/metrics/TestNodeManagerMetrics.java Tue Jun 24 19:34:34 2014
@@ -31,8 +31,11 @@ public class TestNodeManagerMetrics {
NodeManagerMetrics metrics = NodeManagerMetrics.create();
Resource total = Records.newRecord(Resource.class);
total.setMemory(8*GiB);
+ total.setVirtualCores(16);
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(1*GiB);
+ resource.setVirtualCores(2);
+
metrics.addResource(total);
@@ -57,12 +60,12 @@ public class TestNodeManagerMetrics {
metrics.initingContainer();
metrics.runningContainer();
- checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6);
+ checkMetrics(5, 1, 1, 1, 1, 1, 2, 2, 6, 4, 12);
}
private void checkMetrics(int launched, int completed, int failed, int killed,
int initing, int running, int allocatedGB,
- int allocatedContainers, int availableGB) {
+ int allocatedContainers, int availableGB, int allocatedVCores, int availableVCores) {
MetricsRecordBuilder rb = getMetrics("NodeManagerMetrics");
assertCounter("ContainersLaunched", launched, rb);
assertCounter("ContainersCompleted", completed, rb);
@@ -71,7 +74,10 @@ public class TestNodeManagerMetrics {
assertGauge("ContainersIniting", initing, rb);
assertGauge("ContainersRunning", running, rb);
assertGauge("AllocatedGB", allocatedGB, rb);
+ assertGauge("AllocatedVCores", allocatedVCores, rb);
assertGauge("AllocatedContainers", allocatedContainers, rb);
assertGauge("AvailableGB", availableGB, rb);
+ assertGauge("AvailableVCores",availableVCores, rb);
+
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Tue Jun 24 19:34:34 2014
@@ -91,6 +91,10 @@ public class TestNMWebServer {
return 0;
}
@Override
+ public long getVCoresAllocatedForContainers() {
+ return 0;
+ }
+ @Override
public boolean isVmemCheckEnabled() {
return true;
}
@@ -150,6 +154,10 @@ public class TestNMWebServer {
return 0;
}
@Override
+ public long getVCoresAllocatedForContainers() {
+ return 0;
+ }
+ @Override
public boolean isVmemCheckEnabled() {
return true;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java Tue Jun 24 19:34:34 2014
@@ -124,6 +124,10 @@ public class TestNMWebServices extends J
return new Long("17179869184");
}
@Override
+ public long getVCoresAllocatedForContainers() {
+ return new Long("4000");
+ }
+ @Override
public boolean isVmemCheckEnabled() {
return true;
}
@@ -375,6 +379,8 @@ public class TestNMWebServices extends J
"totalVmemAllocatedContainersMB"),
WebServicesTestUtils.getXmlLong(element,
"totalPmemAllocatedContainersMB"),
+ WebServicesTestUtils.getXmlLong(element,
+ "totalVCoresAllocatedContainers"),
WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"),
WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"),
WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"),
@@ -393,10 +399,11 @@ public class TestNMWebServices extends J
public void verifyNodeInfo(JSONObject json) throws JSONException, Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("nodeInfo");
- assertEquals("incorrect number of elements", 15, info.length());
+ assertEquals("incorrect number of elements", 16, info.length());
verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"),
info.getLong("totalVmemAllocatedContainersMB"),
info.getLong("totalPmemAllocatedContainersMB"),
+ info.getLong("totalVCoresAllocatedContainers"),
info.getBoolean("vmemCheckEnabled"),
info.getBoolean("pmemCheckEnabled"),
info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"),
@@ -410,6 +417,7 @@ public class TestNMWebServices extends J
public void verifyNodeInfoGeneric(String id, String healthReport,
long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB,
+ long totalVCoresAllocatedContainers,
boolean vmemCheckEnabled, boolean pmemCheckEnabled,
long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName,
String hadoopVersionBuiltOn, String hadoopBuildVersion,
@@ -423,6 +431,8 @@ public class TestNMWebServices extends J
totalVmemAllocatedContainersMB);
assertEquals("totalPmemAllocatedContainersMB incorrect", 16384,
totalPmemAllocatedContainersMB);
+ assertEquals("totalVCoresAllocatedContainers incorrect", 4000,
+ totalVCoresAllocatedContainers);
assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled);
assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled);
assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java Tue Jun 24 19:34:34 2014
@@ -117,6 +117,12 @@ public class TestNMWebServicesApps exten
}
@Override
+ public long getVCoresAllocatedForContainers() {
+ return new Long("4000");
+ }
+
+
+ @Override
public boolean isVmemCheckEnabled() {
return true;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java Tue Jun 24 19:34:34 2014
@@ -107,6 +107,11 @@ public class TestNMWebServicesContainers
}
@Override
+ public long getVCoresAllocatedForContainers() {
+ return new Long("4000");
+ }
+
+ @Override
public boolean isVmemCheckEnabled() {
return true;
}
@@ -461,24 +466,27 @@ public class TestNMWebServicesContainers
WebServicesTestUtils.getXmlString(element, "diagnostics"),
WebServicesTestUtils.getXmlString(element, "nodeId"),
WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
+ WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
}
}
public void verifyNodeContainerInfo(JSONObject info, Container cont)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 8, info.length());
+ assertEquals("incorrect number of elements", 9, info.length());
verifyNodeContainerInfoGeneric(cont, info.getString("id"),
info.getString("state"), info.getString("user"),
info.getInt("exitCode"), info.getString("diagnostics"),
info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
+ info.getInt("totalVCoresNeeded"),
info.getString("containerLogsLink"));
}
public void verifyNodeContainerInfoGeneric(Container cont, String id,
String state, String user, int exitCode, String diagnostics,
- String nodeId, int totalMemoryNeededMB, String logsLink)
+ String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded,
+ String logsLink)
throws JSONException, Exception {
WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
.toString(), id);
@@ -495,6 +503,9 @@ public class TestNMWebServicesContainers
assertEquals("totalMemoryNeededMB wrong",
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
totalMemoryNeededMB);
+ assertEquals("totalVCoresNeeded wrong",
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ totalVCoresNeeded);
String shortLink =
ujoin("containerlogs", cont.getContainerId().toString(),
cont.getUser());
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java Tue Jun 24 19:34:34 2014
@@ -72,6 +72,9 @@ public class MetricsOverviewTable extend
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Total")._().
th().$class("ui-state-default")._("Memory Reserved")._().
+ th().$class("ui-state-default")._("VCores Used")._().
+ th().$class("ui-state-default")._("VCores Total")._().
+ th().$class("ui-state-default")._("VCores Reserved")._().
th().$class("ui-state-default")._("Active Nodes")._().
th().$class("ui-state-default")._("Decommissioned Nodes")._().
th().$class("ui-state-default")._("Lost Nodes")._().
@@ -94,6 +97,9 @@ public class MetricsOverviewTable extend
td(StringUtils.byteDesc(clusterMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getTotalMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(clusterMetrics.getReservedMB() * BYTES_IN_MB)).
+ td(String.valueOf(clusterMetrics.getAllocatedVirtualCores())).
+ td(String.valueOf(clusterMetrics.getTotalVirtualCores())).
+ td(String.valueOf(clusterMetrics.getReservedVirtualCores())).
td().a(url("nodes"),String.valueOf(clusterMetrics.getActiveNodes()))._().
td().a(url("nodes/decommissioned"),String.valueOf(clusterMetrics.getDecommissionedNodes()))._().
td().a(url("nodes/lost"),String.valueOf(clusterMetrics.getLostNodes()))._().
@@ -120,6 +126,9 @@ public class MetricsOverviewTable extend
th().$class("ui-state-default")._("Memory Used")._().
th().$class("ui-state-default")._("Memory Pending")._().
th().$class("ui-state-default")._("Memory Reserved")._().
+ th().$class("ui-state-default")._("VCores Used")._().
+ th().$class("ui-state-default")._("VCores Pending")._().
+ th().$class("ui-state-default")._("VCores Reserved")._().
_().
_().
tbody().$class("ui-widget-content").
@@ -139,6 +148,9 @@ public class MetricsOverviewTable extend
td(StringUtils.byteDesc(userMetrics.getAllocatedMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getPendingMB() * BYTES_IN_MB)).
td(StringUtils.byteDesc(userMetrics.getReservedMB() * BYTES_IN_MB)).
+ td(String.valueOf(userMetrics.getAllocatedVirtualCores())).
+ td(String.valueOf(userMetrics.getPendingVirtualCores())).
+ td(String.valueOf(userMetrics.getReservedVirtualCores())).
_().
_()._();
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Tue Jun 24 19:34:34 2014
@@ -75,6 +75,8 @@ class NodesPage extends RmView {
th(".containers", "Containers").
th(".mem", "Mem Used").
th(".mem", "Mem Avail").
+ th(".vcores", "VCores Used").
+ th(".vcores", "VCores Avail").
th(".nodeManagerVersion", "Version").
_()._().
tbody();
@@ -127,8 +129,10 @@ class NodesPage extends RmView {
td(String.valueOf(info.getNumContainers())).
td().br().$title(String.valueOf(usedMemory))._().
_(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().
- td().br().$title(String.valueOf(usedMemory))._().
+ td().br().$title(String.valueOf(availableMemory))._().
_(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._().
+ td(String.valueOf(info.getUsedVirtualCores())).
+ td(String.valueOf(info.getAvailableVirtualCores())).
td(ni.getNodeManagerVersion()).
_();
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java Tue Jun 24 19:34:34 2014
@@ -37,16 +37,21 @@ public class ClusterMetricsInfo {
protected int appsRunning;
protected int appsFailed;
protected int appsKilled;
-
+
protected long reservedMB;
protected long availableMB;
protected long allocatedMB;
-
+
+ protected long reservedVirtualCores;
+ protected long availableVirtualCores;
+ protected long allocatedVirtualCores;
+
protected int containersAllocated;
protected int containersReserved;
protected int containersPending;
-
+
protected long totalMB;
+ protected long totalVirtualCores;
protected int totalNodes;
protected int lostNodes;
protected int unhealthyNodes;
@@ -68,16 +73,21 @@ public class ClusterMetricsInfo {
this.appsRunning = metrics.getAppsRunning();
this.appsFailed = metrics.getAppsFailed();
this.appsKilled = metrics.getAppsKilled();
-
+
this.reservedMB = metrics.getReservedMB();
this.availableMB = metrics.getAvailableMB();
this.allocatedMB = metrics.getAllocatedMB();
-
+
+ this.reservedVirtualCores = metrics.getReservedVirtualCores();
+ this.availableVirtualCores = metrics.getAvailableVirtualCores();
+ this.allocatedVirtualCores = metrics.getAllocatedVirtualCores();
+
this.containersAllocated = metrics.getAllocatedContainers();
this.containersPending = metrics.getPendingContainers();
this.containersReserved = metrics.getReservedContainers();
-
+
this.totalMB = availableMB + allocatedMB;
+ this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores;
this.activeNodes = clusterMetrics.getNumActiveNMs();
this.lostNodes = clusterMetrics.getNumLostNMs();
this.unhealthyNodes = clusterMetrics.getUnhealthyNMs();
@@ -123,6 +133,18 @@ public class ClusterMetricsInfo {
return this.allocatedMB;
}
+ public long getReservedVirtualCores() {
+ return this.reservedVirtualCores;
+ }
+
+ public long getAvailableVirtualCores() {
+ return this.availableVirtualCores;
+ }
+
+ public long getAllocatedVirtualCores() {
+ return this.allocatedVirtualCores;
+ }
+
public int getContainersAllocated() {
return this.containersAllocated;
}
@@ -134,15 +156,19 @@ public class ClusterMetricsInfo {
public int getPendingContainers() {
return this.containersPending;
}
-
+
public long getTotalMB() {
return this.totalMB;
}
+ public long getTotalVirtualCores() {
+ return this.totalVirtualCores;
+ }
+
public int getTotalNodes() {
return this.totalNodes;
}
-
+
public int getActiveNodes() {
return this.activeNodes;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java Tue Jun 24 19:34:34 2014
@@ -43,6 +43,8 @@ public class NodeInfo {
protected int numContainers;
protected long usedMemoryMB;
protected long availMemoryMB;
+ protected long usedVirtualCores;
+ protected long availableVirtualCores;
public NodeInfo() {
} // JAXB needs this
@@ -57,6 +59,8 @@ public class NodeInfo {
this.numContainers = report.getNumContainers();
this.usedMemoryMB = report.getUsedResource().getMemory();
this.availMemoryMB = report.getAvailableResource().getMemory();
+ this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+ this.availableVirtualCores = report.getAvailableResource().getVirtualCores();
}
this.id = id.toString();
this.rack = ni.getRackName();
@@ -83,7 +87,7 @@ public class NodeInfo {
public String getNodeHTTPAddress() {
return this.nodeHTTPAddress;
}
-
+
public void setNodeHTTPAddress(String nodeHTTPAddress) {
this.nodeHTTPAddress = nodeHTTPAddress;
}
@@ -112,4 +116,12 @@ public class NodeInfo {
return this.availMemoryMB;
}
+ public long getUsedVirtualCores() {
+ return this.usedVirtualCores;
+ }
+
+ public long getAvailableVirtualCores() {
+ return this.availableVirtualCores;
+ }
+
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java Tue Jun 24 19:34:34 2014
@@ -43,6 +43,9 @@ public class UserMetricsInfo {
protected long reservedMB;
protected long pendingMB;
protected long allocatedMB;
+ protected long reservedVirtualCores;
+ protected long pendingVirtualCores;
+ protected long allocatedVirtualCores;
@XmlTransient
protected boolean userMetricsAvailable;
@@ -59,7 +62,7 @@ public class UserMetricsInfo {
if (userMetrics != null) {
this.userMetricsAvailable = true;
-
+
this.appsSubmitted = userMetrics.getAppsSubmitted();
this.appsCompleted = metrics.getAppsCompleted();
this.appsPending = metrics.getAppsPending();
@@ -70,10 +73,14 @@ public class UserMetricsInfo {
this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers();
-
+
this.reservedMB = userMetrics.getReservedMB();
this.pendingMB = userMetrics.getPendingMB();
this.allocatedMB = userMetrics.getAllocatedMB();
+
+ this.reservedVirtualCores = userMetrics.getReservedVirtualCores();
+ this.pendingVirtualCores = userMetrics.getPendingVirtualCores();
+ this.allocatedVirtualCores = userMetrics.getAllocatedVirtualCores();
}
}
@@ -117,6 +124,18 @@ public class UserMetricsInfo {
return this.pendingMB;
}
+ public long getReservedVirtualCores() {
+ return this.reservedVirtualCores;
+ }
+
+ public long getAllocatedVirtualCores() {
+ return this.allocatedVirtualCores;
+ }
+
+ public long getPendingVirtualCores() {
+ return this.pendingVirtualCores;
+ }
+
public int getReservedContainers() {
return this.reservedContainers;
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java Tue Jun 24 19:34:34 2014
@@ -48,8 +48,8 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
- final int numberOfThInMetricsTable = 13;
- final int numberOfActualTableHeaders = 10;
+ final int numberOfThInMetricsTable = 16;
+ final int numberOfActualTableHeaders = 12;
private Injector injector;
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Tue Jun 24 19:34:34 2014
@@ -389,6 +389,10 @@ public class TestRMWebServices extends J
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
WebServicesTestUtils.getXmlInt(element, "availableMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
+ WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "containersAllocated"),
WebServicesTestUtils.getXmlInt(element, "totalMB"),
WebServicesTestUtils.getXmlInt(element, "totalNodes"),
@@ -404,11 +408,13 @@ public class TestRMWebServices extends J
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
- assertEquals("incorrect number of elements", 19, clusterinfo.length());
+ assertEquals("incorrect number of elements", 23, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
clusterinfo.getInt("allocatedMB"),
+ clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"),
+ clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"),
clusterinfo.getInt("containersAllocated"),
clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"),
clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"),
@@ -418,7 +424,9 @@ public class TestRMWebServices extends J
public void verifyClusterMetrics(int submittedApps, int completedApps,
int reservedMB, int availableMB,
- int allocMB, int containersAlloc, int totalMB, int totalNodes,
+ int allocMB, int reservedVirtualCores, int availableVirtualCores,
+ int allocVirtualCores, int totalVirtualCores,
+ int containersAlloc, int totalMB, int totalNodes,
int lostNodes, int unhealthyNodes, int decommissionedNodes,
int rebootedNodes, int activeNodes) throws JSONException, Exception {
@@ -428,7 +436,8 @@ public class TestRMWebServices extends J
long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
-
+ long totalVirtualCoresExpect =
+ metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
@@ -439,6 +448,12 @@ public class TestRMWebServices extends J
metrics.getAvailableMB(), availableMB);
assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
+ assertEquals("reservedVirtualCores doesn't match",
+ metrics.getReservedVirtualCores(), reservedVirtualCores);
+ assertEquals("availableVirtualCores doesn't match",
+ metrics.getAvailableVirtualCores(), availableVirtualCores);
+ assertEquals("allocatedVirtualCores doesn't match",
+ totalVirtualCoresExpect, allocVirtualCores);
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
assertEquals(
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Tue Jun 24 19:34:34 2014
@@ -656,13 +656,15 @@ public class TestRMWebServicesNodes exte
WebServicesTestUtils.getXmlInt(element, "numContainers"),
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
+ WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
+ WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
WebServicesTestUtils.getXmlString(element, "version"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 11, nodeInfo.length());
+ assertEquals("incorrect number of elements", 13, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
@@ -671,6 +673,7 @@ public class TestRMWebServicesNodes exte
nodeInfo.getLong("lastHealthUpdate"),
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
+ nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
nodeInfo.getString("version"));
}
@@ -678,7 +681,8 @@ public class TestRMWebServicesNodes exte
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
- int numContainers, long usedMemoryMB, long availMemoryMB, String version)
+ int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
+ long availVirtualCores, String version)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@@ -712,6 +716,10 @@ public class TestRMWebServicesNodes exte
.getUsedResource().getMemory(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
.getAvailableResource().getMemory(), availMemoryMB);
+ assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
+ .getUsedResource().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
+ .getAvailableResource().getVirtualCores(), availVirtualCores);
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/NodeManagerRest.apt.vm Tue Jun 24 19:34:34 2014
@@ -62,6 +62,8 @@ NodeManager REST API's.
*---------------+--------------+-------------------------------+
| totalVmemAllocatedContainersMB | long | The amount of virtual memory allocated for use by containers in MB |
*---------------+--------------+-------------------------------+
+| totalVCoresAllocatedContainers | long | The number of virtual cores allocated for use by containers |
+*---------------+--------------+-------------------------------+
| lastNodeUpdateTime | long | The last timestamp at which the health report was received (in ms since epoch)|
*---------------+--------------+-------------------------------+
| healthReport | string | The diagnostic health report of the node |
@@ -109,6 +111,7 @@ NodeManager REST API's.
"nodeManagerBuildVersion" : "0.23.1-SNAPSHOT from 1228355 by user1 source checksum 20647f76c36430e888cc7204826a445c",
"lastNodeUpdateTime" : 1326222266126,
"totalVmemAllocatedContainersMB" : 17203,
+ "totalVCoresAllocatedContainers" : 8,
"nodeHealthy" : true,
"healthReport" : "",
"totalPmemAllocatedContainersMB" : 8192,
@@ -148,6 +151,7 @@ NodeManager REST API's.
<healthReport/>
<totalVmemAllocatedContainersMB>17203</totalVmemAllocatedContainersMB>
<totalPmemAllocatedContainersMB>8192</totalPmemAllocatedContainersMB>
+ <totalVCoresAllocatedContainers>8</totalVCoresAllocatedContainers>
<lastNodeUpdateTime>1326222386134</lastNodeUpdateTime>
<nodeHealthy>true</nodeHealthy>
<nodeManagerVersion>0.23.1-SNAPSHOT</nodeManagerVersion>
@@ -446,6 +450,7 @@ NodeManager REST API's.
{
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
+ "totalVCoresNeeded" : 1,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1",
@@ -456,6 +461,7 @@ NodeManager REST API's.
{
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
+ "totalVCoresNeeded" : 2,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1",
@@ -498,6 +504,7 @@ NodeManager REST API's.
<diagnostics/>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
+ <totalVCoresNeeded>1</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000001/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>
@@ -508,6 +515,7 @@ NodeManager REST API's.
<diagnostics>Container killed by the ApplicationMaster.</diagnostics>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
+ <totalVCoresNeeded>2</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0006_01_000003/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>
@@ -560,6 +568,8 @@ NodeManager REST API's.
*---------------+--------------+-------------------------------+
| totalMemoryNeededMB | long | Total amout of memory needed by the container (in MB) |
*---------------+--------------+-------------------------------+
+| totalVCoresNeeded | long | Total number of virtual cores needed by the container |
+*---------------+--------------+-------------------------------+
** Response Examples
@@ -587,6 +597,7 @@ NodeManager REST API's.
"container" : {
"nodeId" : "host.domain.com:8041",
"totalMemoryNeededMB" : 2048,
+ "totalVCoresNeeded" : 1,
"state" : "RUNNING",
"diagnostics" : "",
"containerLogsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1",
@@ -626,6 +637,7 @@ NodeManager REST API's.
<diagnostics/>
<user>user1</user>
<totalMemoryNeededMB>2048</totalMemoryNeededMB>
+ <totalVCoresNeeded>1</totalVCoresNeeded>
<containerLogsLink>http://host.domain.com:8042/node/containerlogs/container_1326121700862_0007_01_000001/user1</containerLogsLink>
<nodeId>host.domain.com:8041</nodeId>
</container>
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm?rev=1605162&r1=1605161&r2=1605162&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm Tue Jun 24 19:34:34 2014
@@ -195,6 +195,14 @@ ResourceManager REST API's.
*---------------+--------------+-------------------------------+
| totalMB | long | The amount of total memory in MB |
*---------------+--------------+-------------------------------+
+| reservedVirtualCores | long | The number of reserved virtual cores |
+*---------------+--------------+-------------------------------+
+| availableVirtualCores | long | The number of available virtual cores |
+*---------------+--------------+-------------------------------+
+| allocatedVirtualCores | long | The number of allocated virtual cores |
+*---------------+--------------+-------------------------------+
+| totalVirtualCores | long | The total number of virtual cores |
+*---------------+--------------+-------------------------------+
| containersAllocated | int | The number of containers allocated |
*---------------+--------------+-------------------------------+
| containersReserved | int | The number of containers reserved |
@@ -247,10 +255,14 @@ ResourceManager REST API's.
"reservedMB":0,
"availableMB":17408,
"allocatedMB":0,
+ "reservedVirtualCores":0,
+ "availableVirtualCores":7,
+ "allocatedVirtualCores":1,
"containersAllocated":0,
"containersReserved":0,
"containersPending":0,
"totalMB":17408,
+ "totalVirtualCores":8,
"totalNodes":1,
"lostNodes":0,
"unhealthyNodes":0,
@@ -293,10 +305,14 @@ ResourceManager REST API's.
<reservedMB>0</reservedMB>
<availableMB>17408</availableMB>
<allocatedMB>0</allocatedMB>
+ <reservedVirtualCores>0</reservedVirtualCores>
+ <availableVirtualCores>7</availableVirtualCores>
+ <allocatedVirtualCores>1</allocatedVirtualCores>
<containersAllocated>0</containersAllocated>
<containersReserved>0</containersReserved>
<containersPending>0</containersPending>
<totalMB>17408</totalMB>
+ <totalVirtualCores>8</totalVirtualCores>
<totalNodes>1</totalNodes>
<lostNodes>0</lostNodes>
<unhealthyNodes>0</unhealthyNodes>
@@ -2040,8 +2056,10 @@ Server: Jetty(6.1.26)
"lastHealthUpdate":1324056895432,
"healthReport":"Healthy",
"numContainers":0,
- "usedMemoryMB":0
- "availMemoryMB":8192
+ "usedMemoryMB":0,
+ "availMemoryMB":8192,
+ "usedVirtualCores":0,
+ "availableVirtualCores":8
},
{
"rack":"\/default-rack",
@@ -2054,7 +2072,9 @@ Server: Jetty(6.1.26)
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0,
- "availMemoryMB":8192
+ "availMemoryMB":8192,
+ "usedVirtualCores":0,
+ "availableVirtualCores":8
}
]
}
@@ -2096,6 +2116,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
+ <usedVirtualCores>0</usedVirtualCores>
+ <availableVirtualCores>8</availableVirtualCores>
</node>
<node>
<rack>/default-rack</rack>
@@ -2109,6 +2131,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
+ <usedVirtualCores>0</usedVirtualCores>
+ <availableVirtualCores>8</availableVirtualCores>
</node>
</nodes>
+---+
@@ -2159,10 +2183,14 @@ Server: Jetty(6.1.26)
*---------------+--------------+-------------------------------+
| lastHealthUpdate | long | The last time the node reported its health (in ms since epoch)|
*---------------+--------------+-------------------------------+
-| usedMemoryMB | long | The total about of memory currently used on the node (in MB)|
+| usedMemoryMB | long | The total amount of memory currently used on the node (in MB)|
*---------------+--------------+-------------------------------+
| availMemoryMB | long | The total amount of memory currently available on the node (in MB)|
*---------------+--------------+-------------------------------+
+| usedVirtualCores | long | The total number of vCores currently used on the node |
+*---------------+--------------+-------------------------------+
+| availableVirtualCores | long | The total number of vCores available on the node |
+*---------------+--------------+-------------------------------+
| numContainers | int | The total number of containers currently running on the node|
*---------------+--------------+-------------------------------+
@@ -2201,7 +2229,9 @@ Server: Jetty(6.1.26)
"healthReport":"Healthy",
"numContainers":0,
"usedMemoryMB":0,
- "availMemoryMB":5120
+ "availMemoryMB":5120,
+ "usedVirtualCores":0,
+ "availableVirtualCores":8
}
}
+---+
@@ -2240,6 +2270,8 @@ Server: Jetty(6.1.26)
<numContainers>0</numContainers>
<usedMemoryMB>0</usedMemoryMB>
<availMemoryMB>5120</availMemoryMB>
+ <usedVirtualCores>0</usedVirtualCores>
+ <availableVirtualCores>5120</availableVirtualCores>
</node>
+---+