You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/01/29 05:11:01 UTC
[43/50] [abbrv] hadoop git commit: YARN-7337. Expose per-node
over-allocation info in Node Report Contributed by Haibo Chen.
YARN-7337. Expose per-node over-allocation info in Node Report Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8390588a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8390588a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8390588a
Branch: refs/heads/YARN-1011
Commit: 8390588a225a180cbf7ebcde02b64b20adcb0278
Parents: 7dcead0
Author: Miklos Szegedi <mi...@cloudera.com>
Authored: Wed Nov 22 08:39:03 2017 -0800
Committer: Haibo Chen <ha...@apache.org>
Committed: Sun Jan 28 18:17:45 2018 -0800
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/TestRMNMInfo.java | 14 ++-
.../hadoop/yarn/api/records/NodeReport.java | 98 +++++++++++++++-----
.../src/main/proto/yarn_protos.proto | 7 +-
.../applications/distributedshell/Client.java | 2 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 49 +++++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 43 ++++++---
.../api/records/impl/pb/NodeReportPBImpl.java | 94 +++++++++++++++----
.../hadoop/yarn/server/utils/BuilderUtils.java | 37 ++++----
.../server/resourcemanager/ClientRMService.java | 21 +++--
.../resourcemanager/DefaultAMSProcessor.java | 22 +++--
.../yarn/server/resourcemanager/RMNMInfo.java | 45 +++++----
.../resourcemanager/rmnode/RMNodeImpl.java | 2 +-
.../scheduler/SchedulerNodeReport.java | 50 +++++++---
.../webapp/dao/FifoSchedulerInfo.java | 9 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 17 ++--
.../server/resourcemanager/NodeManager.java | 4 +-
.../capacity/TestApplicationPriority.java | 66 ++++++++-----
.../capacity/TestCapacityScheduler.java | 32 ++++---
.../TestNodeLabelContainerAllocation.java | 40 ++++----
.../scheduler/fifo/TestFifoScheduler.java | 50 ++++++----
.../webapp/TestRMWebServicesNodes.java | 13 +--
21 files changed, 491 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
index efea709..76b5493 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
@@ -124,11 +124,17 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected number of used containers",
- 0, n.get("NumContainers").asInt());
+ n.get("NodeId") + ": Unexpected number of guaranteed containers used",
+ 0, n.get("NumContainers").asInt());
+ Assert.assertEquals(n.get("NodeId") +
+ ": Unexpected number of opportunistic containers used",
+ 0, n.get("NumOpportunisticContainers").asInt());
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected amount of used memory",
+ n.get("NodeId") + ": Unexpected amount of guaranteed memory used",
0, n.get("UsedMemoryMB").asInt());
+ Assert.assertEquals(
+ n.get("NodeId") + ": Unexpected amount of used opportunistic memory",
+ 0, n.get("UsedOpportunisticMemoryMB").asInt());
Assert.assertNotNull(n.get("AvailableMemoryMB"));
}
}
@@ -161,6 +167,8 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
+ Assert.assertNull(n.get("NumOpportunisticContainers"));
+ Assert.assertNull(n.get("UsedOpportunisticMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index 3a80641..53e9f92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -36,9 +36,12 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@link NodeId} of the node.</li>
* <li>HTTP Tracking URL of the node.</li>
* <li>Rack name for the node.</li>
- * <li>Used {@link Resource} on the node.</li>
+ * <li>Used guaranteed {@link Resource} on the node.</li>
+ * <li>Used opportunistic {@link Resource} on the node.</li>
* <li>Total available {@link Resource} of the node.</li>
- * <li>Number of running containers on the node.</li>
+ * <li>Number of total running containers on the node.</li>
+ * <li>Number of running guaranteed containers on the node.</li>
+ * <li>Number of running opportunistic containers on the node.</li>
* </ul>
*
* @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
@@ -50,33 +53,37 @@ public abstract class NodeReport {
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newInstance(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime) {
+ return newInstance(nodeId, nodeState, httpAddress, rackName, guaranteedUsed,
+ capability, numGuaranteedContainers, healthReport, lastHealthReportTime,
+ null, null, null, null, 0);
}
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType,
+ Resource opportunisticUsed, int numOpportunisticContainers) {
NodeReport nodeReport = Records.newRecord(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setGuaranteedResourceUsed(guaranteedUsed);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
nodeReport.setDecommissioningTimeout(decommissioningTimeout);
nodeReport.setNodeUpdateType(nodeUpdateType);
+ nodeReport.setOpportunisticResourceUsed(opportunisticUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
return nodeReport;
}
@@ -129,17 +136,43 @@ public abstract class NodeReport {
public abstract void setRackName(String rackName);
/**
- * Get <em>used</em> <code>Resource</code> on the node.
- * @return <em>used</em> <code>Resource</code> on the node
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsed();
@Private
@Unstable
+ @Deprecated
public abstract void setUsed(Resource used);
-
+
+ /**
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourceUsed(Resource guaranteed);
+
+ /**
+ * Get <em>opportunistic</em> <code>Resource</code> used on the node.
+ * @return <em>opportunistic</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceUsed(Resource opportunistic);
+
/**
* Get the <em>total</em> <code>Resource</code> on the node.
* @return <em>total</em> <code>Resource</code> on the node
@@ -153,19 +186,40 @@ public abstract class NodeReport {
public abstract void setCapability(Resource capability);
/**
- * Get the <em>number of allocated containers</em> on the node.
- * @return <em>number of allocated containers</em> on the node
+ * Get the <em>number of guaranteed containers</em> allocated on the node.
+ * @return <em>number of guaranteed containers</em> allocated on the node
*/
@Private
@Unstable
- public abstract int getNumContainers();
+ public abstract int getNumGuaranteedContainers();
@Private
@Unstable
- public abstract void setNumContainers(int numContainers);
-
+ public abstract void setNumGuaranteedContainers(int numContainers);
- /**
+ /**
+ * Get the <em>number of opportunistic containers</em> allocated on the node.
+ * @return <em>number of opportunistic containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public abstract int getNumOpportunisticContainers();
+
+ @Private
+ @Unstable
+ public abstract void setNumOpportunisticContainers(int numContainers);
+
+ /**
+ * Get the <em>number of containers</em> allocated on the node.
+ * @return <em>number of containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public int getNumTotalContainers() {
+ return getNumGuaranteedContainers() + getNumOpportunisticContainers();
+ }
+
+ /**
* Get the <em>diagnostic health report</em> of the node.
* @return <em>diagnostic health report</em> of the node
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index b6ea5f9..44e1b88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -342,9 +342,9 @@ message NodeReportProto {
optional NodeIdProto nodeId = 1;
optional string httpAddress = 2;
optional string rackName = 3;
- optional ResourceProto used = 4;
+ optional ResourceProto guaranteedResourceUsed = 4;
optional ResourceProto capability = 5;
- optional int32 numContainers = 6;
+ optional int32 numGuaranteedContainers = 6;
optional NodeStateProto node_state = 7;
optional string health_report = 8;
optional int64 last_health_report_time = 9;
@@ -353,6 +353,9 @@ message NodeReportProto {
optional ResourceUtilizationProto node_utilization = 12;
optional uint32 decommissioning_timeout = 13;
optional NodeUpdateTypeProto node_update_type = 14;
+ optional int32 numOpportunisticContainers = 15;
+ optional ResourceProto opportunisticResourceUsed = 16;
+ optional int32 numTotalContainers = 17;
}
message NodeIdToLabelsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index ef635d3..7520d30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -600,7 +600,7 @@ public class Client {
+ ", nodeId=" + node.getNodeId()
+ ", nodeAddress=" + node.getHttpAddress()
+ ", nodeRackName=" + node.getRackName()
- + ", nodeNumContainers=" + node.getNumContainers());
+ + ", nodeNumContainers=" + node.getNumGuaranteedContainers());
}
QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index 288a5d2..be466fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
@Private
@Unstable
@@ -185,7 +185,7 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
.getNodeState(), nodeReport.getHttpAddress(), nodeReport
- .getNumContainers());
+ .getNumGuaranteedContainers());
}
writer.flush();
}
@@ -210,13 +210,18 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
nodeReport.getNodeState(), nodeReport.getHttpAddress(),
- nodeReport.getNumContainers());
+ nodeReport.getNumGuaranteedContainers());
writer.println("Detailed Node Information :");
writer.print("\tConfigured Resources : ");
writer.println(nodeReport.getCapability());
- writer.print("\tAllocated Resources : ");
- if (nodeReport.getUsed() != null) {
- writer.print(nodeReport.getUsed());
+ writer.print("\tAllocated Guaranteed Resources : ");
+ if (nodeReport.getGuaranteedResourceUsed() != null) {
+ writer.print(nodeReport.getGuaranteedResourceUsed());
+ }
+ writer.println();
+ writer.print("\tAllocated Opportunistic Resources : ");
+ if (nodeReport.getOpportunisticResourceUsed() != null) {
+ writer.print(nodeReport.getOpportunisticResourceUsed());
}
writer.println();
@@ -287,16 +292,32 @@ public class NodeCLI extends YarnCLI {
nodeReportStr.print("\tHealth-Report : ");
nodeReportStr
.println(nodeReport.getHealthReport());
- nodeReportStr.print("\tContainers : ");
- nodeReportStr.println(nodeReport.getNumContainers());
- nodeReportStr.print("\tMemory-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB"
- : (nodeReport.getUsed().getMemorySize() + "MB"));
+ nodeReportStr.print("\tGuaranteed Containers : ");
+ nodeReportStr.println(nodeReport.getNumGuaranteedContainers());
+ nodeReportStr.print("\tOpportunistic Containers : ");
+ nodeReportStr.println(nodeReport.getNumOpportunisticContainers());
+ Resource guaranteedResourceUsed =
+ nodeReport.getGuaranteedResourceUsed();
+ Resource opportunisticResourceUsed =
+ nodeReport.getOpportunisticResourceUsed();
+ nodeReportStr.print("\tGuaranteed-Memory-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0MB" :
+ (guaranteedResourceUsed.getMemorySize() + "MB"));
+ nodeReportStr.print("\tOpportunistic-Memory-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0MB" :
+ (opportunisticResourceUsed.getMemorySize() + "MB"));
nodeReportStr.print("\tMemory-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB");
- nodeReportStr.print("\tCPU-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores"
- : (nodeReport.getUsed().getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tGuaranteed-CPU-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0 vcores" :
+ (guaranteedResourceUsed.getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tOpportunistic-CPU-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0 vcores" :
+ (opportunisticResourceUsed.getVirtualCores() + " vcores"));
nodeReportStr.print("\tCPU-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
nodeReportStr.print("\tNode-Labels : ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index fdd3fc8..e172acd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1352,7 +1352,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1360,7 +1361,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1561,10 +1563,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
@@ -1597,10 +1602,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : ");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
@@ -1633,10 +1641,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tResource Utilization by Node : ");
@@ -1989,10 +2000,12 @@ public class TestYarnCLI {
// ordered
nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
}
- NodeReport nodeReport = NodeReport.newInstance(NodeId
- .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
- "rack1", Records.newRecord(Resource.class), Records
- .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null);
+ NodeReport nodeReport = NodeReport.newInstance(
+ NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
+ "rack1", Records.newRecord(Resource.class),
+ Records.newRecord(Resource.class), 0, "",
+ 0, nodeLabels, null, null,
+ Records.newRecord(Resource.class), 0);
if (!emptyResourceUtilization) {
ResourceUtilization containersUtilization = ResourceUtilization
.newInstance(1024, 2048, 4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index ced588d..0960eb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -45,7 +45,8 @@ public class NodeReportPBImpl extends NodeReport {
private NodeReportProto.Builder builder = null;
private boolean viaProto = false;
private NodeId nodeId;
- private Resource used;
+ private Resource guaranteedResourceUsed;
+ private Resource opportunisticResourceUsed;
private Resource capability;
private ResourceUtilization containersUtilization = null;
private ResourceUtilization nodeUtilization = null;
@@ -109,9 +110,17 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public int getNumContainers() {
+ public int getNumGuaranteedContainers() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- return (p.hasNumContainers()) ? p.getNumContainers() : 0;
+ return (p.hasNumGuaranteedContainers()) ?
+ p.getNumGuaranteedContainers() : 0;
+ }
+
+ @Override
+ public int getNumOpportunisticContainers() {
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasNumOpportunisticContainers()) ?
+ p.getNumOpportunisticContainers() : 0;
}
@Override
@@ -120,18 +129,25 @@ public class NodeReportPBImpl extends NodeReport {
return (p.hasRackName()) ? p.getRackName() : null;
}
+ @Deprecated
@Override
public Resource getUsed() {
- if (this.used != null) {
- return this.used;
+ return getGuaranteedResourceUsed();
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsed() {
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
}
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasUsed()) {
+ if (!p.hasOpportunisticResourceUsed()) {
return null;
}
- this.used = convertFromProtoFormat(p.getUsed());
- return this.used;
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getOpportunisticResourceUsed());
+ return this.opportunisticResourceUsed;
}
@Override
@@ -196,13 +212,24 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public void setNumContainers(int numContainers) {
+ public void setNumGuaranteedContainers(int numContainers) {
+ maybeInitBuilder();
+ if (numContainers == 0) {
+ builder.clearNumGuaranteedContainers();
+ return;
+ }
+ builder.setNumGuaranteedContainers(numContainers);
+ }
+
+ @Override
+ public void setNumOpportunisticContainers(int numContainers) {
maybeInitBuilder();
if (numContainers == 0) {
- builder.clearNumContainers();
+ builder.clearNumOpportunisticContainers();
return;
}
- builder.setNumContainers(numContainers);
+ builder.setNumOpportunisticContainers(numContainers);
+
}
@Override
@@ -215,13 +242,43 @@ public class NodeReportPBImpl extends NodeReport {
builder.setRackName(rackName);
}
+ @Deprecated
@Override
public void setUsed(Resource used) {
+ setGuaranteedResourceUsed(used);
+ }
+
+ @Override
+ public Resource getGuaranteedResourceUsed() {
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
+ }
+
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasGuaranteedResourceUsed()) {
+ return null;
+ }
+ this.guaranteedResourceUsed =
+ convertFromProtoFormat(p.getGuaranteedResourceUsed());
+ return this.guaranteedResourceUsed;
+ }
+
+ @Override
+ public void setGuaranteedResourceUsed(Resource guaranteed) {
+ maybeInitBuilder();
+ if (guaranteedResourceUsed == null) {
+ builder.clearGuaranteedResourceUsed();
+ }
+ this.guaranteedResourceUsed = guaranteed;
+ }
+
+ @Override
+ public void setOpportunisticResourceUsed(Resource opportunisticUsed) {
maybeInitBuilder();
- if (used == null) {
- builder.clearUsed();
+ if (opportunisticUsed == null) {
+ builder.clearOpportunisticResourceUsed();
}
- this.used = used;
+ this.opportunisticResourceUsed = opportunisticUsed;
}
public NodeReportProto getProto() {
@@ -258,8 +315,13 @@ public class NodeReportPBImpl extends NodeReport {
builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.used != null) {
- builder.setUsed(convertToProtoFormat(this.used));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setGuaranteedResourceUsed(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setOpportunisticResourceUsed(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.capability != null) {
builder.setCapability(convertToProtoFormat(this.capability));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 83f912f..63333ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -185,26 +185,23 @@ public class BuilderUtils {
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainer,
+ String healthReport, long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) {
+ return newNodeReport(nodeId, nodeState, httpAddress, rackName,
+ guaranteedResourceUsed, capability, numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainer, healthReport,
+ lastHealthReportTime, nodeLabels, null, null,
+ decommissioningTimeout, nodeUpdateType);
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- nodeLabels, null, null, decommissioningTimeout, nodeUpdateType);
- }
-
- public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainers,
+ String healthReport, long lastHealthReportTime,
Set<String> nodeLabels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Integer decommissioningTimeout,
NodeUpdateType nodeUpdateType) {
@@ -213,9 +210,11 @@ public class BuilderUtils {
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
+ nodeReport.setGuaranteedResourceUsed(guaranteedResourceUsed);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
+ nodeReport.setOpportunisticResourceUsed(opportunisticResourceUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 55a3f0b..7b560fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1027,17 +1027,26 @@ public class ClientRMService extends AbstractService implements
private NodeReport createNodeReports(RMNode rmNode) {
SchedulerNodeReport schedulerNodeReport =
scheduler.getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ guaranteedResourceUsed = schedulerNodeReport.getGuaranteedResourceUsed();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeReport report =
BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed,
+ rmNode.getTotalCapability(), numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 713947f..cea00c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -340,17 +340,27 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
RMNode rmNode = rmNodeEntry.getKey();
SchedulerNodeReport schedulerNodeReport =
getScheduler().getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ guaranteedResourceUsed =
+ schedulerNodeReport.getGuaranteedResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeId nodeId = rmNode.getNodeID();
NodeReport report =
BuilderUtils.newNodeReport(nodeId, rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed, rmNode.getTotalCapability(),
+ numGuaranteedContainers, opportunisticResourceUsed,
+ numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(),
rmNodeEntry.getValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 1b7ddd3..849db5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -79,27 +79,32 @@ public class RMNMInfo implements RMNMInfoBeans {
List<InfoMap> nodesInfo = new ArrayList<InfoMap>();
for (final RMNode ni : nodes) {
- SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
- InfoMap info = new InfoMap();
- info.put("HostName", ni.getHostName());
- info.put("Rack", ni.getRackName());
- info.put("State", ni.getState().toString());
- info.put("NodeId", ni.getNodeID());
- info.put("NodeHTTPAddress", ni.getHttpAddress());
- info.put("LastHealthUpdate",
- ni.getLastHealthReportTime());
- info.put("HealthReport",
- ni.getHealthReport());
- info.put("NodeManagerVersion",
- ni.getNodeManagerVersion());
- if(report != null) {
- info.put("NumContainers", report.getNumContainers());
- info.put("UsedMemoryMB", report.getUsedResource().getMemorySize());
- info.put("AvailableMemoryMB",
- report.getAvailableResource().getMemorySize());
- }
+ SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
+ InfoMap info = new InfoMap();
+ info.put("HostName", ni.getHostName());
+ info.put("Rack", ni.getRackName());
+ info.put("State", ni.getState().toString());
+ info.put("NodeId", ni.getNodeID());
+ info.put("NodeHTTPAddress", ni.getHttpAddress());
+ info.put("LastHealthUpdate",
+ ni.getLastHealthReportTime());
+ info.put("HealthReport",
+ ni.getHealthReport());
+ info.put("NodeManagerVersion",
+ ni.getNodeManagerVersion());
+ if(report != null) {
+ info.put("NumContainers", report.getNumGuaranteedContainers());
+ info.put("NumOpportunisticContainers",
+ report.getNumOpportunisticContainers());
+ info.put("UsedMemoryMB",
+ report.getGuaranteedResourceUsed().getMemorySize());
+ info.put("UsedOpportunisticMemoryMB",
+ report.getOpportunisticResourceUsed().getMemorySize());
+ info.put("AvailableMemoryMB",
+ report.getAvailableGuaranteedResource().getMemorySize());
+ }
- nodesInfo.add(info);
+ nodesInfo.add(info);
}
return JSON.toString(nodesInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 3cac38f..9e2781f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -139,7 +139,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
/** Physical resources in the node. */
private volatile Resource physicalResource;
- /* Container Queue Information for the node.. Used by Distributed Scheduler */
+ /* Container Queue Information for the node..*/
private OpportunisticContainersStatus opportunisticContainersStatus;
private final ContainerAllocationExpirer containerAllocationExpirer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index ea30d78..25621a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -28,34 +28,54 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Private
@Stable
public class SchedulerNodeReport {
- private final Resource used;
- private final Resource avail;
- private final int num;
+ private final Resource guaranteedResourceUsage;
+ private final Resource opportunisticResourceUsage;
+ private final Resource guaranteedResourceAvail;
+ private final int numOfGuaranteedContainers;
+ private final int numOfOpportunisticContainers;
public SchedulerNodeReport(SchedulerNode node) {
- this.used = node.getAllocatedResource();
- this.avail = node.getUnallocatedResource();
- this.num = node.getNumGuaranteedContainers();
+ this.guaranteedResourceUsage = node.getAllocatedResource();
+ this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated();
+ this.guaranteedResourceAvail = node.getUnallocatedResource();
+ this.numOfGuaranteedContainers = node.getNumGuaranteedContainers();
+ this.numOfOpportunisticContainers = node.getNumOpportunisticContainers();
}
/**
- * @return the amount of resources currently used by the node.
+ * @return the amount of guaranteed resources currently used by the node.
*/
- public Resource getUsedResource() {
- return used;
+ public Resource getGuaranteedResourceUsed() {
+ return guaranteedResourceUsage;
}
/**
- * @return the amount of resources currently available on the node
+ * @return the amount of opportunistic resources currently used by the node.
*/
- public Resource getAvailableResource() {
- return avail;
+ public Resource getOpportunisticResourceUsed() {
+ return opportunisticResourceUsage;
}
/**
- * @return the number of containers currently running on this node.
+ * @return the amount of guaranteed resources currently available on the node
*/
- public int getNumContainers() {
- return num;
+ public Resource getAvailableGuaranteedResource() {
+ return guaranteedResourceAvail;
+ }
+
+ /**
+ * @return the number of guaranteed containers currently running on
+ * this node.
+ */
+ public int getNumGuaranteedContainers() {
+ return numOfGuaranteedContainers;
+ }
+
+ /**
+ * @return the number of opportunistic containers currently running on
+ * this node.
+ */
+ public int getNumOpportunisticContainers() {
+ return numOfOpportunisticContainers;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
index 1752546..f5d8d07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
@@ -79,10 +79,13 @@ public class FifoSchedulerInfo extends SchedulerInfo {
for (RMNode ni : rmContext.getRMNodes().values()) {
SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
- this.usedNodeCapacity += report.getUsedResource().getMemorySize();
- this.availNodeCapacity += report.getAvailableResource().getMemorySize();
+ this.usedNodeCapacity +=
+ report.getGuaranteedResourceUsed().getMemorySize();
+ this.availNodeCapacity +=
+ report.getAvailableGuaranteedResource().getMemorySize();
this.totalNodeCapacity += ni.getTotalCapability().getMemorySize();
- this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
+ this.numContainers +=
+ fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 3cec215..2c67dfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -71,14 +71,17 @@ public class NodeInfo {
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
if (report != null) {
- this.numContainers = report.getNumContainers();
- this.usedMemoryMB = report.getUsedResource().getMemorySize();
- this.availMemoryMB = report.getAvailableResource().getMemorySize();
- this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+ this.numContainers = report.getNumGuaranteedContainers();
+ this.usedMemoryMB = report.getGuaranteedResourceUsed().getMemorySize();
+ this.availMemoryMB =
+ report.getAvailableGuaranteedResource().getMemorySize();
+ this.usedVirtualCores =
+ report.getGuaranteedResourceUsed().getVirtualCores();
this.availableVirtualCores =
- report.getAvailableResource().getVirtualCores();
- this.usedResource = new ResourceInfo(report.getUsedResource());
- this.availableResource = new ResourceInfo(report.getAvailableResource());
+ report.getAvailableGuaranteedResource().getVirtualCores();
+ this.usedResource = new ResourceInfo(report.getGuaranteedResourceUsed());
+ this.availableResource =
+ new ResourceInfo(report.getAvailableGuaranteedResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index ee974e3..96613fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -225,10 +225,10 @@ public class NodeManager implements ContainerManagementProtocol {
LOG.info("Checking resource usage for " + containerManagerAddress);
Assert.assertEquals(available.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getAvailableResource().getMemorySize());
+ this.nodeId).getAvailableGuaranteedResource().getMemorySize());
Assert.assertEquals(used.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getUsedResource().getMemorySize());
+ this.nodeId).getGuaranteedResourceUsed().getMemorySize());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index cad0151..9c317e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -176,8 +176,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -189,8 +191,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -210,8 +214,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// send updated request for App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -226,8 +232,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -261,8 +269,10 @@ public class TestApplicationPriority {
// check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(8 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 7
Priority appPriority2 = Priority.newInstance(7);
@@ -288,8 +298,10 @@ public class TestApplicationPriority {
// check node report, 1 GB used and 7 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(7 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -486,8 +498,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -499,8 +513,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -522,8 +538,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -535,8 +553,10 @@ public class TestApplicationPriority {
Assert.assertEquals(2, allocated2.size());
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// kill 1 more
counter = 0;
@@ -552,8 +572,10 @@ public class TestApplicationPriority {
// check node report, 14 GB used and 2 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(14 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Change the priority of App1 to 3 (lowest)
Priority appPriority3 = Priority.newInstance(3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 8ab63c1..849a2c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -1267,8 +1267,10 @@ public class TestCapacityScheduler {
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1289,8 +1291,10 @@ public class TestCapacityScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1309,7 +1313,7 @@ public class TestCapacityScheduler {
waitCount = 0;
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- if (report_nm1.getAvailableResource().getMemorySize() != 0) {
+ if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1318,8 +1322,10 @@ public class TestCapacityScheduler {
}
// Now, the used resource is still 4 GB, and available resource is minus value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
@@ -1335,9 +1341,11 @@ public class TestCapacityScheduler {
Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Verify no NPE is trigger in schedule after resource is updated.
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
@@ -2837,8 +2845,10 @@ public class TestCapacityScheduler {
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(9 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 02c0cc5..8023457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -1947,15 +1947,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(5 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(5 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB());
@@ -2047,15 +2049,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(3 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(7 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(9 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
double delta = 0.0001;
@@ -2177,9 +2181,10 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(14 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Try to launch app2 in a2, asked 2GB, should success
// app2 -> a2
@@ -2196,9 +2201,10 @@ public class TestNodeLabelContainerAllocation {
reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a2
cs.killAllAppsInQueue("a2");
@@ -2217,9 +2223,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
cs.getApplicationAttempt(am3.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a1
cs.killAllAppsInQueue("a1");
@@ -2240,9 +2247,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
cs.getApplicationAttempt(am4.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
rm1.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 63da0c3..0640004 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -697,7 +697,8 @@ public class TestFifoScheduler {
am1.registerAppAttempt();
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
RMApp app2 = rm.submitApp(2048);
// kick the scheduling, 2GB given to AM, remaining 2 GB on nm2
@@ -707,7 +708,8 @@ public class TestFifoScheduler {
am2.registerAppAttempt();
SchedulerNodeReport report_nm2 =
rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1);
@@ -743,11 +745,15 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getAvailableGuaranteedResource().getMemorySize());
- Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemorySize());
@@ -765,7 +771,8 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ 5 * GB, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -822,7 +829,8 @@ public class TestFifoScheduler {
int checkAlloc =
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ checkAlloc, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -1113,8 +1121,10 @@ public class TestFifoScheduler {
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1135,8 +1145,10 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1155,7 +1167,7 @@ public class TestFifoScheduler {
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
if (null != report_nm1 &&
- report_nm1.getAvailableResource().getMemorySize() != 0) {
+ report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1165,8 +1177,10 @@ public class TestFifoScheduler {
// Now, the used resource is still 4 GB, and available resource is minus
// value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource
// over-commitment.
@@ -1184,9 +1198,11 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8390588a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index fb597fc..f972681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -811,15 +811,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
if (report != null) {
assertEquals("numContainers doesn't match: " + numContainers,
- report.getNumContainers(), numContainers);
+ report.getNumGuaranteedContainers(), numContainers);
assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report
- .getUsedResource().getMemorySize(), usedMemoryMB);
+ .getGuaranteedResourceUsed().getMemorySize(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
- .getAvailableResource().getMemorySize(), availMemoryMB);
+ .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB);
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
- .getUsedResource().getVirtualCores(), usedVirtualCores);
- assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
- .getAvailableResource().getVirtualCores(), availVirtualCores);
+ .getGuaranteedResourceUsed().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores,
+ report.getAvailableGuaranteedResource().getVirtualCores(),
+ availVirtualCores);
}
if (opportunisticStatus != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org