You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2018/04/02 22:36:13 UTC
[01/10] hadoop git commit: YARN-7460. Exclude findbugs warnings on
SchedulerNode.numGuaranteedContainers and numOpportunisticContainers.
Contributed by Haibo Chen. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-1011 8ea3f763d -> dece162b0 (forced update)
YARN-7460. Exclude findbugs warnings on SchedulerNode.numGuaranteedContainers and numOpportunisticContainers. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38b6d75f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38b6d75f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38b6d75f
Branch: refs/heads/YARN-1011
Commit: 38b6d75fe8585abd69511680f7c066f1aba9c9a2
Parents: bdf3e66
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Nov 20 13:27:32 2017 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:33:51 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 10 ++++++++++
1 file changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38b6d75f/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 6a10312..26ba5b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -76,6 +76,16 @@
<Bug pattern="VO_VOLATILE_INCREMENT" />
</Match>
<Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numOpportunisticContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode" />
+ <Field name="numGuaranteedContainers" />
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
+ <Match>
<Class name="org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl$AppRejectedTransition" />
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/10] hadoop git commit: YARN-6750. Add a configuration to cap how
much a NM can be overallocated. Contributed by Haibo Chen.
Posted by sz...@apache.org.
YARN-6750. Add a configuration to cap how much a NM can be overallocated. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2109c367
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2109c367
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2109c367
Branch: refs/heads/YARN-1011
Commit: 2109c3671065fea0eaae6644d576feb64bb7b981
Parents: e48809c
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 09:17:56 2017 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:34:45 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 10 ++
.../scheduler/AbstractYarnScheduler.java | 4 +
.../scheduler/SchedulerNode.java | 41 +++++--
.../scheduler/fair/FSSchedulerNode.java | 5 +
.../scheduler/fair/FairScheduler.java | 2 +-
.../scheduler/fair/TestFairScheduler.java | 107 +++++++++++++++++++
7 files changed, 167 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2109c367/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 15f6480..8d55737 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -384,6 +384,11 @@ public class YarnConfiguration extends Configuration {
/** ACL used in case none is found. Allows nothing. */
public static final String DEFAULT_YARN_APP_ACL = " ";
+ /** The global max overallocation per node in terms of their capacity. */
+ public static final String PER_NODE_MAX_OVERALLOCATION_RATIO =
+ RM_PREFIX + "overallocation.per-node-max-ratio";
+ public static final float DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO = 4.0f;
+
/** Setting that controls whether opportunistic container allocation
* is enabled or not. */
@Unstable
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2109c367/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 7962dae..c184226 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3182,6 +3182,16 @@
<property>
<description>
+ The maximum amount of resources, specified as a ratio to node capacity,
+ that can be allocated to opportunistic containers on any given node in
+ the cluster.
+ </description>
+ <name>yarn.resourcemanager.overallocation.per-node-max-ratio</name>
+ <value>4.0</value>
+ </property>
+
+ <property>
+ <description>
Frequency for computing least loaded NMs.
</description>
<name>yarn.resourcemanager.nm-container-queuing.sorting-nodes-interval-ms</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2109c367/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ca66055..a755a4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -118,6 +118,7 @@ public abstract class AbstractYarnScheduler
new ClusterNodeTracker<>();
protected Resource minimumAllocation;
+ protected float maxOverAllocationRatioPerNode;
protected volatile RMContext rmContext;
@@ -200,6 +201,9 @@ public abstract class AbstractYarnScheduler
nodeTracker.setConfiguredMaxAllocationWaitTime(
configuredMaximumAllocationWaitTime);
maxClusterLevelAppPriority = getMaxPriorityFromConf(conf);
+ maxOverAllocationRatioPerNode = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
createReleaseCache();
autoUpdateContainers =
conf.getBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2109c367/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index e942981..b10cac5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -90,10 +90,14 @@ public abstract class SchedulerNode {
protected Resource resourceAllocatedPendingLaunch =
Resource.newInstance(0, 0);
+ // The max amount of resources that can be allocated to opportunistic
+ // containers on the node, specified as a ratio to its capacity
+ private final float maxOverAllocationRatio;
+
private volatile Set<String> labels = null;
public SchedulerNode(RMNode node, boolean usePortForNodeName,
- Set<String> labels) {
+ Set<String> labels, float maxOverAllocationRatio) {
this.rmNode = node;
this.unallocatedResource = Resources.clone(node.getTotalCapability());
this.capacity = Resources.clone(node.getTotalCapability());
@@ -103,10 +107,24 @@ public abstract class SchedulerNode {
nodeName = rmNode.getHostName();
}
this.labels = ImmutableSet.copyOf(labels);
+ this.maxOverAllocationRatio = maxOverAllocationRatio;
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> labels) {
+ this(node, usePortForNodeName, labels,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverAllocationRatio) {
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ maxOverAllocationRatio);
}
public SchedulerNode(RMNode node, boolean usePortForNodeName) {
- this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
}
public RMNode getRMNode() {
@@ -604,9 +622,11 @@ public abstract class SchedulerNode {
/**
* Get the amount of resources that can be allocated to opportunistic
- * containers in the case of overallocation. It is calculated as
+ * containers in the case of overallocation, calculated as
* node capacity - (node utilization + resources of allocated-yet-not-started
- * containers).
+ * containers), subject to the maximum amount of resources that can be
+ * allocated to opportunistic containers on the node specified as a ratio to
+ * its capacity.
* @return the amount of resources that are available to be allocated to
* opportunistic containers
*/
@@ -639,11 +659,20 @@ public abstract class SchedulerNode {
Resource resourceAllowedForOpportunisticContainers =
Resources.createResource(allowedMemory, allowedCpu);
- // TODO cap the resources allocated to OPPORTUNISTIC containers on a node
- // in terms of its capacity. i.e. return min(max_ratio * capacity, allowed)
+ // cap the total amount of resources allocated to OPPORTUNISTIC containers
+ Resource maxOverallocation = getMaxOverallocationAllowed();
+ Resources.subtractFrom(maxOverallocation, allocatedResourceOpportunistic);
+ resourceAllowedForOpportunisticContainers = Resources.componentwiseMin(
+ maxOverallocation, resourceAllowedForOpportunisticContainers);
+
return resourceAllowedForOpportunisticContainers;
}
+ private Resource getMaxOverallocationAllowed() {
+ long maxMemory = (long) (capacity.getMemorySize() * maxOverAllocationRatio);
+ int maxVcore = (int) (capacity.getVirtualCores() * maxOverAllocationRatio);
+ return Resource.newInstance(maxMemory, maxVcore);
+ }
private static class ContainerInfo {
private final RMContainer container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2109c367/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 95490f5..a53dda4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -66,10 +66,15 @@ public class FSSchedulerNode extends SchedulerNode {
// slated for preemption
private Resource totalResourcesPreempted = Resource.newInstance(0, 0);
+ @VisibleForTesting
public FSSchedulerNode(RMNode node, boolean usePortForNodeName) {
super(node, usePortForNodeName);
}
+ public FSSchedulerNode(RMNode node, boolean usePortForNodeName,
+ float maxOverallocationRatio) {
+ super(node, usePortForNodeName, maxOverallocationRatio);
+ }
/**
* Total amount of reserved resources including reservations and preempted
* containers.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2109c367/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index a8e348c..3929f59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -751,7 +751,7 @@ public class FairScheduler extends
try {
writeLock.lock();
FSSchedulerNode schedulerNode = new FSSchedulerNode(node,
- usePortForNodeName);
+ usePortForNodeName, maxOverAllocationRatioPerNode);
nodeTracker.addNode(schedulerNode);
triggerUpdate();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2109c367/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 1f89ace..1870ea5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -3137,6 +3137,113 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ /**
+ * Test that max overallocation per node is enforced by Fair Scheduler.
+ * @throws Exception
+ */
+ @Test
+ public void testMaxOverallocationPerNode() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+ float maxOverallocationRatio = conf.getFloat(
+ YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ YarnConfiguration.DEFAULT_PER_NODE_MAX_OVERALLOCATION_RATIO);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO, 1.5f);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 1G of memory and 1 vcores and an overallocation
+ // threshold of 1.0f and 1.0f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(1f, 1f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(1024, 1), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that takes up the whole node
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(1024, "queue1", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is zero after the container runs
+ ContainerStatus containerStatus1 = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus1),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create a scheduling request that should get allocated an OPPORTUNISTIC
+ // container because the node utilization is zero
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1024, "queue2", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.OPPORTUNISTIC,
+ allocatedContainers2.get(0).getExecutionType());
+ assertEquals(1024, scheduler.getQueueManager().getQueue("queue2").
+ getOpportunisticResourceUsage().getMemorySize());
+
+ // node utilization is still zero after the container runs
+ ContainerStatus containerStatus2 = ContainerStatus.newInstance(
+ allocatedContainers2.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus2),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(0, 0, 0.0f));
+
+ // create another scheduling request that should not get any allocation
+ // because of the max overallocation on the node will be exceeded.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1024, "queue3", "user1", 1);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3").
+ getOpportunisticResourceUsage().getMemorySize());
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ conf.setFloat(YarnConfiguration.PER_NODE_MAX_OVERALLOCATION_RATIO,
+ maxOverallocationRatio);
+ }
+ }
+
@Test
public void testAclSubmitApplication() throws Exception {
// Set acl's
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/10] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by sz...@apache.org.
YARN-5473. Expose per-application over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29de8de8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29de8de8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29de8de8
Branch: refs/heads/YARN-1011
Commit: 29de8de89832936a60de9f143c55aaf9e65152f7
Parents: 83f03dc
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Jan 23 22:34:49 2018 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:35:06 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapreduce/TypeConverter.java | 4 +-
.../hadoop/mapreduce/TestTypeConverter.java | 4 +-
.../hadoop/mapred/TestResourceMgrDelegate.java | 2 +-
.../records/ApplicationResourceUsageReport.java | 158 +++++++++++---
.../src/main/proto/yarn_protos.proto | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +-
.../apache/hadoop/yarn/client/cli/TopCLI.java | 18 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 23 ++-
.../ApplicationResourceUsageReportPBImpl.java | 205 +++++++++++++++----
.../hadoop/yarn/util/resource/Resources.java | 20 ++
...pplicationHistoryManagerOnTimelineStore.java | 83 +++++---
.../TestApplicationHistoryClientService.java | 8 +-
...pplicationHistoryManagerOnTimelineStore.java | 18 +-
.../metrics/ApplicationMetricsConstants.java | 10 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +-
.../hadoop/yarn/server/webapp/dao/AppInfo.java | 39 +++-
.../server/resourcemanager/RMAppManager.java | 18 +-
.../server/resourcemanager/RMServerUtils.java | 3 +-
.../metrics/TimelineServiceV1Publisher.java | 14 +-
.../metrics/TimelineServiceV2Publisher.java | 14 +-
.../resourcemanager/recovery/RMStateStore.java | 5 +-
.../records/ApplicationAttemptStateData.java | 144 +++++++++++--
.../pb/ApplicationAttemptStateDataPBImpl.java | 110 ++++++++--
.../server/resourcemanager/rmapp/RMAppImpl.java | 38 ++--
.../resourcemanager/rmapp/RMAppMetrics.java | 38 ++--
.../attempt/AggregateAppResourceUsage.java | 51 +++--
.../rmapp/attempt/RMAppAttemptImpl.java | 21 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 47 +++--
.../ContainerResourceUsageReport.java | 46 +++++
.../rmcontainer/RMContainer.java | 7 +-
.../rmcontainer/RMContainerImpl.java | 119 ++++++++---
.../scheduler/AbstractYarnScheduler.java | 2 +-
.../scheduler/SchedulerApplicationAttempt.java | 63 +++---
.../scheduler/YarnScheduler.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
.../scheduler/fair/FSAppAttempt.java | 9 +-
.../webapp/FairSchedulerAppsBlock.java | 8 +
.../resourcemanager/webapp/RMAppBlock.java | 9 +-
.../resourcemanager/webapp/RMAppsBlock.java | 10 +
.../resourcemanager/webapp/dao/AppInfo.java | 78 +++++--
.../yarn_server_resourcemanager_recovery.proto | 1 +
.../server/resourcemanager/TestAppManager.java | 39 ++--
.../resourcemanager/TestApplicationACLs.java | 4 +-
.../resourcemanager/TestClientRMService.java | 45 ++--
.../TestContainerResourceUsage.java | 184 ++++++++++-------
.../applicationsmanager/MockAsm.java | 4 +-
.../TestCombinedSystemMetricsPublisher.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 23 ++-
.../TestSystemMetricsPublisherForV2.java | 12 +-
.../recovery/RMStateStoreTestBase.java | 12 +-
.../recovery/TestZKRMStateStore.java | 40 ++--
.../rmapp/TestRMAppTransitions.java | 6 +-
.../attempt/TestRMAppAttemptTransitions.java | 32 +--
.../capacity/TestCapacityScheduler.java | 5 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 20 +-
.../resourcemanager/webapp/TestAppPage.java | 2 +-
.../resourcemanager/webapp/TestRMWebApp.java | 3 +-
.../webapp/TestRMWebAppFairScheduler.java | 2 +-
.../webapp/TestRMWebServicesApps.java | 2 +-
.../router/webapp/RouterWebServiceUtil.java | 14 +-
.../router/webapp/TestRouterWebServiceUtil.java | 14 +-
61 files changed, 1427 insertions(+), 517 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index aea931a..23409fc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -527,8 +527,8 @@ public class TypeConverter {
jobStatus.setNumUsedSlots(resourceUsageReport.getNumUsedContainers());
jobStatus.setReservedMem(
(int)resourceUsageReport.getReservedResources().getMemorySize());
- jobStatus.setUsedMem(
- (int) resourceUsageReport.getUsedResources().getMemorySize());
+ jobStatus.setUsedMem((int) resourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
}
return jobStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
index 629a246..acd9c35 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/TestTypeConverter.java
@@ -94,7 +94,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
applicationReport.setApplicationResourceUsageReport(appUsageRpt);
JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
Assert.assertEquals(appStartTime, jobStatus.getStartTime());
@@ -133,7 +133,7 @@ public class TestTypeConverter {
appUsageRpt.setNumReservedContainers(1);
appUsageRpt.setNumUsedContainers(3);
appUsageRpt.setReservedResources(r);
- appUsageRpt.setUsedResources(r);
+ appUsageRpt.setGuaranteedResourcesUsed(r);
when(mockReport.getApplicationResourceUsageReport()).thenReturn(appUsageRpt);
JobStatus status = TypeConverter.fromYarn(mockReport, jobFile);
Assert.assertNotNull("fromYarn returned null status", status);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
index d6754f0..0506b27 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestResourceMgrDelegate.java
@@ -137,7 +137,7 @@ public class TestResourceMgrDelegate {
Records.newRecord(Resource.class));
Mockito.when(appResources.getReservedResources()).thenReturn(
Records.newRecord(Resource.class));
- Mockito.when(appResources.getUsedResources()).thenReturn(
+ Mockito.when(appResources.getGuaranteedResourcesUsed()).thenReturn(
Records.newRecord(Resource.class));
Mockito.when(appReport.getApplicationResourceUsageReport()).thenReturn(
appResources);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
index d2e33ff..4517db9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationResourceUsageReport.java
@@ -36,21 +36,27 @@ public abstract class ApplicationResourceUsageReport {
@Private
@Unstable
public static ApplicationResourceUsageReport newInstance(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap, float queueUsagePerc,
- float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap, float queueUsagePerc,
+ float clusterUsagePerc, Map<String, Long> preemtedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
+
ApplicationResourceUsageReport report =
Records.newRecord(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setQueueUsagePercentage(queueUsagePerc);
report.setClusterUsagePercentage(clusterUsagePerc);
report.setPreemptedResourceSecondsMap(preemtedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
@@ -87,16 +93,40 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNumReservedContainers(int num_reserved_containers);
/**
- * Get the used <code>Resource</code>. -1 for invalid/inaccessible reports.
- * @return the used <code>Resource</code>
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsedResources();
+ /**
+ * Get the guaranteed <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the guaranteed <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourcesUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourcesUsed(Resource resources);
+
+ /**
+ * Get the opportunistic <code>Resource</code> used.
+ * -1 for invalid/inaccessible reports.
+ * @return the opportunistic <code>Resource</code> used
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourcesUsed();
+
@Private
@Unstable
- public abstract void setUsedResources(Resource resources);
+ public abstract void setOpportunisticResourcesUsed(Resource resources);
/**
* Get the reserved <code>Resource</code>. -1 for invalid/inaccessible reports.
@@ -123,42 +153,84 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setNeededResources(Resource needed_resources);
/**
- * Set the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @param memory_seconds the aggregated amount of memory seconds
+ * Set the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @param memorySeconds the aggregated amount of guaranteed memory seconds
*/
@Private
@Unstable
- public abstract void setMemorySeconds(long memory_seconds);
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
/**
- * Get the aggregated amount of memory (in megabytes) the application has
- * allocated times the number of seconds the application has been running.
- * @return the aggregated amount of memory seconds
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ /**
+ * Get the aggregated amount of guaranteed memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of guaranteed memory seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
/**
- * Set the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @param vcore_seconds the aggregated number of vcore seconds
+ * Set the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @param vcoreSeconds the aggregated number of guaranteed vcore seconds
*/
@Private
@Unstable
- public abstract void setVcoreSeconds(long vcore_seconds);
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of vcores that the application has allocated
- * times the number of seconds the application has been running.
- * @return the aggregated number of vcore seconds
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ /**
+ * Get the aggregated number of guaranteed vcores that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return the aggregated number of guaranteed vcore seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
/**
+ * Get the aggregated amount of opportunistic memory (in megabytes) the
+ * application has allocated times the number of seconds the application
+ * has been running.
+ * @return the aggregated amount of opportunistic memory seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the aggregated number of opportunistic vcores that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return the aggregated number of opportunistic vcore seconds
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the percentage of resources of the queue that the app is using.
* @return the percentage of resources of the queue that the app is using.
*/
@@ -231,23 +303,35 @@ public abstract class ApplicationResourceUsageReport {
public abstract long getPreemptedVcoreSeconds();
/**
- * Get the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
- * @return map containing the resource name and aggregated resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Private
@Unstable
- public abstract void setResourceSecondsMap(
+ public abstract void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
@@ -272,4 +356,24 @@ public abstract class ApplicationResourceUsageReport {
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ * @param opportunisticResourceSecondsMap map containing the resource name
+ * and aggregated opportunistic resource-seconds
+ */
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 44e1b88..c9f0a69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -251,6 +251,8 @@ message ApplicationResourceUsageReportProto {
optional int64 preempted_vcore_seconds = 11;
repeated StringLongMapProto application_resource_usage_map = 12;
repeated StringLongMapProto application_preempted_resource_usage_map = 13;
+ optional ResourceProto used_opportunistic_resources = 14;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 15;
}
message ApplicationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index daca296..2b0f96a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -990,15 +990,20 @@ public class ApplicationCLI extends YarnCLI {
private void printResourceUsage(PrintWriter appReportStr,
ApplicationResourceUsageReport usageReport) {
- appReportStr.print("\tAggregate Resource Allocation : ");
+ appReportStr.print("\tAggregate Guaranteed Resource Allocation : ");
if (usageReport != null) {
- appReportStr.println(
- getResourceSecondsString(usageReport.getResourceSecondsMap()));
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getGuaranteedResourceSecondsMap()));
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println(getResourceSecondsString(
+ usageReport.getOpportunisticResourceSecondsMap()));
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println(getResourceSecondsString(
usageReport.getPreemptedResourceSecondsMap()));
} else {
appReportStr.println("N/A");
+ appReportStr.print("\tAggregate Opportunistic Resource Allocation : ");
+ appReportStr.println("N/A");
appReportStr.print("\tAggregate Resource Preempted : ");
appReportStr.println("N/A");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index d6c33f4..ba6eb3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -170,13 +170,11 @@ public class TopCLI extends YarnCLI {
appReport.getApplicationResourceUsageReport()
.getNumReservedContainers();
displayStringsMap.put(Columns.RCONT, String.valueOf(reservedContainers));
- usedVirtualCores =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getVirtualCores();
+ usedVirtualCores = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getVirtualCores();
displayStringsMap.put(Columns.VCORES, String.valueOf(usedVirtualCores));
- usedMemory =
- appReport.getApplicationResourceUsageReport().getUsedResources()
- .getMemorySize() / 1024;
+ usedMemory = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedResourcesUsed().getMemorySize() / 1024;
displayStringsMap.put(Columns.MEM, String.valueOf(usedMemory) + "G");
reservedVirtualCores =
appReport.getApplicationResourceUsageReport().getReservedResources()
@@ -195,11 +193,11 @@ public class TopCLI extends YarnCLI {
progress = appReport.getProgress() * 100;
displayStringsMap.put(Columns.PROGRESS, String.format("%.2f", progress));
// store in GBSeconds
- memorySeconds =
- appReport.getApplicationResourceUsageReport().getMemorySeconds() / 1024;
+ memorySeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedMemorySeconds() / 1024;
displayStringsMap.put(Columns.MEMSECS, String.valueOf(memorySeconds));
- vcoreSeconds =
- appReport.getApplicationResourceUsageReport().getVcoreSeconds();
+ vcoreSeconds = appReport.getApplicationResourceUsageReport()
+ .getGuaranteedVcoreSeconds();
displayStringsMap.put(Columns.VCORESECS, String.valueOf(vcoreSeconds));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index e172acd..c2e3d9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -121,18 +121,27 @@ public class TestYarnCLI {
for (int i = 0; i < 2; ++i) {
ApplicationCLI cli = createAndGetAppCLI();
ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 123456L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 4567L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 123456L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 4567L);
preemptedResoureSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), 1111L);
preemptedResoureSecondsMap
.put(ResourceInformation.VCORES.getName(), 2222L);
ApplicationResourceUsageReport usageReport = i == 0 ? null :
ApplicationResourceUsageReport
- .newInstance(2, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(2, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
ApplicationReport newApplicationReport = ApplicationReport.newInstance(
applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
"user", "queue", "appname", "host", 124, null,
@@ -168,7 +177,9 @@ public class TestYarnCLI {
pw.println("\tTracking-URL : N/A");
pw.println("\tRPC Port : 124");
pw.println("\tAM Host : host");
- pw.println("\tAggregate Resource Allocation : " +
+ pw.println("\tAggregate Guaranteed Resource Allocation : " +
+ (i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
+ pw.println("\tAggregate Opportunistic Resource Allocation : " +
(i == 0 ? "N/A" : "123456 MB-seconds, 4567 vcore-seconds"));
pw.println("\tAggregate Resource Preempted : " +
(i == 0 ? "N/A" : "1111 MB-seconds, 2222 vcore-seconds"));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
index 14ede5d..8ff1631 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationResourceUsageReportPBImpl.java
@@ -40,12 +40,14 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto.Builder builder = null;
boolean viaProto = false;
- Resource usedResources;
- Resource reservedResources;
- Resource neededResources;
+ private Resource guaranteedResourceUsed;
+ private Resource reservedResources;
+ private Resource neededResources;
+ private Resource opportunisticResourceUsed;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationResourceUsageReportPBImpl() {
builder = ApplicationResourceUsageReportProto.newBuilder();
@@ -55,8 +57,9 @@ extends ApplicationResourceUsageReport {
ApplicationResourceUsageReportProto proto) {
this.proto = proto;
viaProto = true;
- getResourceSecondsMap();
+ getGuaranteedResourceSecondsMap();
getPreemptedResourceSecondsMap();
+ getOpportunisticResourceSecondsMap();
}
public synchronized ApplicationResourceUsageReportProto getProto() {
@@ -87,8 +90,13 @@ extends ApplicationResourceUsageReport {
}
private void mergeLocalToBuilder() {
- if (this.usedResources != null) {
- builder.setUsedResources(convertToProtoFormat(this.usedResources));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setUsedResources(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setUsedOpportunisticResources(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.reservedResources != null) {
builder.setReservedResources(
@@ -99,19 +107,28 @@ extends ApplicationResourceUsageReport {
}
builder.clearApplicationResourceUsageMap();
builder.clearApplicationPreemptedResourceUsageMap();
+ builder.clearApplicationOpportunisticResourceUsageMap();
if (preemptedResourceSecondsMap != null && !preemptedResourceSecondsMap
.isEmpty()) {
builder.addAllApplicationPreemptedResourceUsageMap(ProtoUtils
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
- if (resourceSecondsMap != null && !resourceSecondsMap.isEmpty()) {
+ if (guaranteedResourceSecondsMap != null &&
+ !guaranteedResourceSecondsMap.isEmpty()) {
builder.addAllApplicationResourceUsageMap(
- ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ guaranteedResourceSecondsMap));
+ }
+ if (opportunisticResourceSecondsMap != null &&
+ !opportunisticResourceSecondsMap.isEmpty()) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(
+ opportunisticResourceSecondsMap));
}
- builder.setMemorySeconds(this.getMemorySeconds());
- builder.setVcoreSeconds(this.getVcoreSeconds());
+ builder.setMemorySeconds(this.getGuaranteedMemorySeconds());
+ builder.setVcoreSeconds(this.getGuaranteedVcoreSeconds());
builder.setPreemptedMemorySeconds(this.getPreemptedMemorySeconds());
builder.setPreemptedVcoreSeconds(this.getPreemptedVcoreSeconds());
}
@@ -157,24 +174,53 @@ extends ApplicationResourceUsageReport {
}
@Override
+ @Deprecated
public synchronized Resource getUsedResources() {
+ return getGuaranteedResourcesUsed();
+ }
+
+ @Override
+ public synchronized Resource getGuaranteedResourcesUsed() {
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- if (this.usedResources != null) {
- return this.usedResources;
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
}
if (!p.hasUsedResources()) {
return null;
}
- this.usedResources = convertFromProtoFormat(p.getUsedResources());
- return this.usedResources;
+ this.guaranteedResourceUsed = convertFromProtoFormat(p.getUsedResources());
+ return this.guaranteedResourceUsed;
}
@Override
- public synchronized void setUsedResources(Resource resources) {
+ public synchronized void setGuaranteedResourcesUsed(Resource resources) {
maybeInitBuilder();
if (resources == null)
builder.clearUsedResources();
- this.usedResources = resources;
+ this.guaranteedResourceUsed = resources;
+ }
+
+ @Override
+ public synchronized Resource getOpportunisticResourcesUsed() {
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
+ }
+ if (!p.hasUsedOpportunisticResources()) {
+ return null;
+ }
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getUsedOpportunisticResources());
+ return this.opportunisticResourceUsed;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourcesUsed(Resource resources) {
+ maybeInitBuilder();
+ if (resources == null) {
+ builder.clearUsedOpportunisticResources();
+ }
+ this.opportunisticResourceUsed = resources;
}
@Override
@@ -220,14 +266,14 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setMemorySeconds(long memory_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.MEMORY_MB.getName(), memory_seconds);
+ public synchronized void setGuaranteedMemorySeconds(long memorySeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
}
@Override
- public synchronized long getMemorySeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ public synchronized long getGuaranteedMemorySeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
return tmp.get(ResourceInformation.MEMORY_MB.getName());
}
@@ -235,20 +281,50 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setVcoreSeconds(long vcore_seconds) {
- getResourceSecondsMap()
- .put(ResourceInformation.VCORES.getName(), vcore_seconds);
+ @Deprecated
+ public synchronized long getMemorySeconds() {
+ return getGuaranteedMemorySeconds();
+ }
+
+ @Override
+ public synchronized void setGuaranteedVcoreSeconds(long vcoreSeconds) {
+ getGuaranteedResourceSecondsMap()
+ .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
}
@Override
+ public synchronized long getGuaranteedVcoreSeconds() {
+ Map<String, Long> tmp = getGuaranteedResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ @Deprecated
public synchronized long getVcoreSeconds() {
- Map<String, Long> tmp = getResourceSecondsMap();
+ return getGuaranteedVcoreSeconds();
+ }
+
+ @Override
+ public synchronized long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public synchronized long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
return tmp.get(ResourceInformation.VCORES.getName());
}
return 0;
}
-
+
@Override
public synchronized void setPreemptedMemorySeconds(
long preemptedMemorySeconds) {
@@ -314,41 +390,47 @@ extends ApplicationResourceUsageReport {
}
@Override
- public synchronized void setResourceSecondsMap(
+ public synchronized void setGuaranteedResourceSecondsMap(
Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap == null) {
return;
}
if (!resourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(0L);
+ this.setGuaranteedMemorySeconds(0L);
}
if (!resourceSecondsMap.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(0L);
+ this.setGuaranteedVcoreSeconds(0L);
}
}
@Override
+ @Deprecated
public synchronized Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ public synchronized Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils
+ this.guaranteedResourceSecondsMap = ProtoUtils
.convertStringLongMapProtoListToMap(
p.getApplicationResourceUsageMapList());
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.MEMORY_MB.getName())) {
- this.setMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
}
- if (!this.resourceSecondsMap
+ if (!this.guaranteedResourceSecondsMap
.containsKey(ResourceInformation.VCORES.getName())) {
- this.setVcoreSeconds(p.getVcoreSeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
}
- this.setMemorySeconds(p.getMemorySeconds());
- this.setVcoreSeconds(p.getVcoreSeconds());
- return this.resourceSecondsMap;
+ this.setGuaranteedMemorySeconds(p.getMemorySeconds());
+ this.setGuaranteedVcoreSeconds(p.getVcoreSeconds());
+ return this.guaranteedResourceSecondsMap;
}
@Override
@@ -389,4 +471,45 @@ extends ApplicationResourceUsageReport {
this.setPreemptedVcoreSeconds(p.getPreemptedVcoreSeconds());
return this.preemptedResourceSecondsMap;
}
+
+ @Override
+ public synchronized Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationResourceUsageReportProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap = ProtoUtils
+ .convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap.containsKey(
+ ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public synchronized void setOpportunisticResourceSecondsMap(
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ if (opportunisticResourceSecondsMap == null) {
+ return;
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+ if (!opportunisticResourceSecondsMap
+ .containsKey(ResourceInformation.VCORES.getName())) {
+ this.opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 1c08844..010a0de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
+import java.util.Map;
+
/**
* Resources is a computation class which provides a set of apis to do
* mathematical operations on Resource object.
@@ -556,4 +558,22 @@ public class Resources {
Resource resource, Resource factor) {
return calculator.normalizeDown(resource, factor);
}
+
+ /**
+ * Merge resource usage entries from a map to another map.
+ * @param mergeFrom the map to merge from
+ * @param mergeTo the map to merge to
+ */
+ public static void mergeResourceSecondsMap(Map<String, Long> mergeFrom,
+ Map<String, Long> mergeTo) {
+ for (Map.Entry<String, Long> entry : mergeFrom.entrySet()) {
+ Long value = mergeTo.get(entry.getKey());
+ if (value != null) {
+ value += entry.getValue();
+ } else {
+ value = entry.getValue();
+ }
+ mergeTo.put(entry.getKey(), value);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 0b57717..85cc006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.apache.hadoop.yarn.server.timeline.TimelineDataManager;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -330,29 +329,19 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
.toString();
}
- if (entityInfo.containsKey(ApplicationMetricsConstants.APP_CPU_METRICS)) {
- long vcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_METRICS);
- long memorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_METRICS);
- long preemptedMemorySeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
- long preemptedVcoreSeconds = parseLong(entityInfo,
- ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- preemptedMemorySeconds);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
-
+ if (entityInfo.containsKey(
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)) {
+ Map<String, Long> guaranteedResourceSecondsMap =
+ extractGuaranteedResourceSecondsMap(entityInfo);
+ Map<String, Long> preemptedResoureSecondsMap =
+ extractPreemptedResourceSecondsMap(entityInfo);
+ Map<String, Long> opportunisticResourceSecondsMap =
+ extractOpportunisticResourceSecondsMap(entityInfo);
appResources = ApplicationResourceUsageReport
- .newInstance(0, 0, null, null, null, resourceSecondsMap, 0, 0,
- preemptedResoureSecondsMap);
+ .newInstance(0, 0, null, null, null,
+ guaranteedResourceSecondsMap, 0, 0,
+ preemptedResoureSecondsMap, null,
+ opportunisticResourceSecondsMap);
}
if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
@@ -456,6 +445,54 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
amNodeLabelExpression), appViewACLs);
}
+ private static Map<String, Long> extractGuaranteedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+
+ long guaranteedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS);
+ long guaranteedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), guaranteedMemorySeconds);
+ guaranteedResourceSecondsMap.put(ResourceInformation.VCORES.getName(),
+ guaranteedVcoreSeconds);
+
+ return guaranteedResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractOpportunisticResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+
+ long opportunisticVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS);
+ long opportunisticMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(),
+ opportunisticMemorySeconds);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), opportunisticVcoreSeconds);
+
+ return opportunisticResourceSecondsMap;
+ }
+
+ private static Map<String, Long> extractPreemptedResourceSecondsMap(
+ Map<String, Object> entityInfo) {
+ Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
+
+ long preemptedMemorySeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS);
+ long preemptedVcoreSeconds = parseLong(entityInfo,
+ ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS);
+ preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
+ preemptedMemorySeconds);
+ preemptedResoureSecondsMap
+ .put(ResourceInformation.VCORES.getName(), preemptedVcoreSeconds);
+
+ return preemptedResoureSecondsMap;
+ }
private static long parseLong(Map<String, Object> entityInfo,
String infoKey) {
long result = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index 7ef6eca..6cb21ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -152,9 +152,13 @@ public class TestApplicationHistoryClientService {
ApplicationReport appReport = response.getApplicationReport();
Assert.assertNotNull(appReport);
Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
- .getMemorySeconds());
+ .getGuaranteedMemorySeconds());
Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
- .getVcoreSeconds());
+ .getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticMemorySeconds());
+ Assert.assertEquals(345, appReport.getApplicationResourceUsageReport()
+ .getOpportunisticVcoreSeconds());
Assert.assertEquals("application_0_0001", appReport.getApplicationId()
.toString());
Assert.assertEquals("test app type",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
index ecaaf1e..35996aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java
@@ -250,9 +250,13 @@ public class TestApplicationHistoryManagerOnTimelineStore {
ApplicationResourceUsageReport applicationResourceUsageReport =
app.getApplicationResourceUsageReport();
Assert.assertEquals(123,
- applicationResourceUsageReport.getMemorySeconds());
- Assert
- .assertEquals(345, applicationResourceUsageReport.getVcoreSeconds());
+ applicationResourceUsageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123,
+ applicationResourceUsageReport.getOpportunisticMemorySeconds());
+ Assert.assertEquals(345,
+ applicationResourceUsageReport.getOpportunisticVcoreSeconds());
long expectedPreemptMemSecs = 456;
long expectedPreemptVcoreSecs = 789;
if (i == 3) {
@@ -534,8 +538,12 @@ public class TestApplicationHistoryManagerOnTimelineStore {
Priority.newInstance(0));
entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
Integer.MAX_VALUE + 1L);
- entityInfo.put(ApplicationMetricsConstants.APP_MEM_METRICS, 123);
- entityInfo.put(ApplicationMetricsConstants.APP_CPU_METRICS, 345);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, 123);
+ entityInfo.put(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, 345);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS, 123);
+ entityInfo.put(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS, 345);
if (!missingPreemptMetrics) {
entityInfo.put(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, 456);
entityInfo.put(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS, 789);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
index 4cec409..b4123ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java
@@ -70,12 +70,18 @@ public class ApplicationMetricsConstants {
public static final String STATE_EVENT_INFO =
"YARN_APPLICATION_STATE";
- public static final String APP_CPU_METRICS =
+ public static final String APP_GUARANTEED_CPU_METRICS =
"YARN_APPLICATION_CPU";
- public static final String APP_MEM_METRICS =
+ public static final String APP_GUARANTEED_MEM_METRICS =
"YARN_APPLICATION_MEMORY";
+ public static final String APP_OPPORTUNISTIC_CPU_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_CPU";
+
+ public static final String APP_OPPORTUNISTIC_MEM_METRICS =
+ "YARN_APPLICATION_OPPORTUNISTIC_MEMORY";
+
public static final String APP_RESOURCE_PREEMPTED_CPU =
"YARN_APPLICATION_RESOURCE_PREEMPTED_CPU";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 6abc617..860ebae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -466,19 +466,24 @@ public class BuilderUtils {
}
public static ApplicationResourceUsageReport newApplicationResourceUsageReport(
- int numUsedContainers, int numReservedContainers, Resource usedResources,
+ int numUsedContainers, int numReservedContainers,
+ Resource guaranteedResourcesUsed,
Resource reservedResources, Resource neededResources,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Resource opportunisticResourcesUsed,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationResourceUsageReport report =
recordFactory.newRecordInstance(ApplicationResourceUsageReport.class);
report.setNumUsedContainers(numUsedContainers);
report.setNumReservedContainers(numReservedContainers);
- report.setUsedResources(usedResources);
+ report.setGuaranteedResourcesUsed(guaranteedResourcesUsed);
report.setReservedResources(reservedResources);
report.setNeededResources(neededResources);
- report.setResourceSecondsMap(resourceSecondsMap);
+ report.setGuaranteedResourceSecondsMap(guaranteedResourceSecondsMap);
report.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
+ report.setOpportunisticResourcesUsed(opportunisticResourcesUsed);
+ report.setOpportunisticResourceSecondsMap(opportunisticResourcesSecondsMap);
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
index ac2f8da..fb856d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/AppInfo.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.util.Times;
@@ -61,6 +62,8 @@ public class AppInfo {
protected int priority;
private long allocatedCpuVcores;
private long allocatedMemoryMB;
+ private long allocatedOpportunisticCpuVcores;
+ private long allocatedOpportunisticMemoryMB;
private long reservedCpuVcores;
private long reservedMemoryMB;
protected boolean unmanagedApplication;
@@ -98,15 +101,25 @@ public class AppInfo {
if (app.getApplicationResourceUsageReport() != null) {
runningContainers = app.getApplicationResourceUsageReport()
.getNumUsedContainers();
- if (app.getApplicationResourceUsageReport().getUsedResources() != null) {
- allocatedCpuVcores = app.getApplicationResourceUsageReport()
- .getUsedResources().getVirtualCores();
- allocatedMemoryMB = app.getApplicationResourceUsageReport()
- .getUsedResources().getMemorySize();
- reservedCpuVcores = app.getApplicationResourceUsageReport()
- .getReservedResources().getVirtualCores();
- reservedMemoryMB = app.getApplicationResourceUsageReport()
- .getReservedResources().getMemorySize();
+ Resource guaranteedResourceUsed =
+ app.getApplicationResourceUsageReport().getGuaranteedResourcesUsed();
+ if (guaranteedResourceUsed != null) {
+ allocatedCpuVcores = guaranteedResourceUsed.getVirtualCores();
+ allocatedMemoryMB = guaranteedResourceUsed.getMemorySize();
+ }
+ Resource opportunisticResourceUsed = app
+ .getApplicationResourceUsageReport().getOpportunisticResourcesUsed();
+ if (opportunisticResourceUsed != null) {
+ allocatedOpportunisticCpuVcores =
+ opportunisticResourceUsed.getVirtualCores();
+ allocatedOpportunisticMemoryMB =
+ opportunisticResourceUsed.getMemorySize();
+ }
+ Resource resourceReserved =
+ app.getApplicationResourceUsageReport().getReservedResources();
+ if (resourceReserved != null) {
+ reservedCpuVcores = resourceReserved.getVirtualCores();
+ reservedMemoryMB = resourceReserved.getMemorySize();
}
}
progress = app.getProgress() * 100; // in percent
@@ -166,6 +179,14 @@ public class AppInfo {
return allocatedMemoryMB;
}
+ public long getAllocatedOpportunisticCpuVcores() {
+ return allocatedOpportunisticCpuVcores;
+ }
+
+ public long getAllocatedOpportunisticMemoryMB() {
+ return allocatedOpportunisticMemoryMB;
+ }
+
public long getReservedCpuVcores() {
return reservedCpuVcores;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 1e05fec..cb7d426 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -187,19 +187,27 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
.add("startTime", app.getStartTime())
.add("finishTime", app.getFinishTime())
.add("finalStatus", app.getFinalApplicationStatus())
- .add("memorySeconds", metrics.getMemorySeconds())
- .add("vcoreSeconds", metrics.getVcoreSeconds())
+ .add("guaranteedMemorySeconds", metrics.getGuaranteedMemorySeconds())
+ .add("guaranteedVcoreSeconds", metrics.getGuaranteedVcoreSeconds())
+ .add("opportunisticMemorySeconds",
+ metrics.getOpportunisticMemorySeconds())
+ .add("opportunisticVcoreSeconds",
+ metrics.getOpportunisticVcoreSeconds())
.add("preemptedMemorySeconds", metrics.getPreemptedMemorySeconds())
.add("preemptedVcoreSeconds", metrics.getPreemptedVcoreSeconds())
.add("preemptedAMContainers", metrics.getNumAMContainersPreempted())
.add("preemptedNonAMContainers", metrics.getNumNonAMContainersPreempted())
.add("preemptedResources", metrics.getResourcePreempted())
.add("applicationType", app.getApplicationType())
- .add("resourceSeconds", StringHelper
- .getResourceSecondsString(metrics.getResourceSecondsMap()))
+ .add("guaranteedResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getGuaranteedResourceSecondsMap()))
.add("preemptedResourceSeconds", StringHelper
.getResourceSecondsString(
- metrics.getPreemptedResourceSecondsMap()));
+ metrics.getPreemptedResourceSecondsMap()))
+ .add("opportunisticResourceSeconds", StringHelper
+ .getResourceSecondsString(
+ metrics.getOpportunisticResourceSecondsMap()));
return summary;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 4934243..05b37cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -482,7 +482,8 @@ public class RMServerUtils {
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
Resources.createResource(-1, -1), Resources.createResource(-1, -1),
- Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>());
+ Resources.createResource(-1, -1), new HashMap<>(), new HashMap<>(),
+ Resources.createResource(-1, -1), new HashMap<>());
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index 73bb301..efe5239 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -140,10 +140,16 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
latestApplicationAttemptId);
}
RMAppMetrics appMetrics = app.getRMAppMetrics();
- entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
- appMetrics.getVcoreSeconds());
- entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
- appMetrics.getMemorySeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS,
+ appMetrics.getGuaranteedVcoreSeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS,
+ appMetrics.getGuaranteedMemorySeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ appMetrics.getOpportunisticVcoreSeconds());
+ entity.addOtherInfo(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ appMetrics.getOpportunisticMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS,
appMetrics.getPreemptedMemorySeconds());
entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_PREEMPT_METRICS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 7eaa6e7..84f3e9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -190,11 +190,17 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_CPU_METRICS, timestamp,
- appMetrics.getVcoreSeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS, timestamp,
+ appMetrics.getGuaranteedVcoreSeconds()));
entityMetrics.add(getTimelineMetric(
- ApplicationMetricsConstants.APP_MEM_METRICS, timestamp,
- appMetrics.getMemorySeconds()));
+ ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS, timestamp,
+ appMetrics.getGuaranteedMemorySeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS,
+ timestamp, appMetrics.getOpportunisticVcoreSeconds()));
+ entityMetrics.add(getTimelineMetric(
+ ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS,
+ timestamp, appMetrics.getOpportunisticMemorySeconds()));
entityMetrics.add(getTimelineMetric(
ApplicationMetricsConstants.APP_MEM_PREEMPT_METRICS, timestamp,
appMetrics.getPreemptedMemorySeconds()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index f0ab324..de66802 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -853,8 +853,9 @@ public abstract class RMStateStore extends AbstractService {
appAttempt.getAppAttemptId(),
appAttempt.getMasterContainer(),
credentials, appAttempt.getStartTime(),
- resUsage.getResourceUsageSecondsMap(),
- attempMetrics.getPreemptedResourceSecondsMap());
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ attempMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
getRMStateStoreEventHandler().handle(
new RMStateStoreAppAttemptEvent(attemptState));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
index 2de071a..b08d2eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationAttemptStateData.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -45,8 +46,9 @@ public abstract class ApplicationAttemptStateData {
Credentials attemptTokens, long startTime, RMAppAttemptState finalState,
String finalTrackingUrl, String diagnostics,
FinalApplicationStatus amUnregisteredFinalStatus, int exitStatus,
- long finishTime, Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ long finishTime, Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
ApplicationAttemptStateData attemptStateData =
Records.newRecord(ApplicationAttemptStateData.class);
attemptStateData.setAttemptId(attemptId);
@@ -59,19 +61,22 @@ public abstract class ApplicationAttemptStateData {
attemptStateData.setFinalApplicationStatus(amUnregisteredFinalStatus);
attemptStateData.setAMContainerExitStatus(exitStatus);
attemptStateData.setFinishTime(finishTime);
- attemptStateData.setMemorySeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap,
+ attemptStateData.setGuaranteedMemorySeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
- attemptStateData.setVcoreSeconds(RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L));
+ attemptStateData.setGuaranteedVcoreSeconds(RMServerUtils
+ .getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L));
attemptStateData.setPreemptedMemorySeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L));
attemptStateData.setPreemptedVcoreSeconds(RMServerUtils
.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.VCORES.getName(), 0L));
- attemptStateData.setResourceSecondsMap(resourceSecondsMap);
+ attemptStateData.setGuaranteedResourceSecondsMap(
+ guaranteedResourceSecondsMap);
+ attemptStateData.setOpportunisticResourceSecondsMap(
+ opportunisticResourcesSecondsMap);
attemptStateData
.setPreemptedResourceSecondsMap(preemptedResourceSecondsMap);
return attemptStateData;
@@ -80,11 +85,13 @@ public abstract class ApplicationAttemptStateData {
public static ApplicationAttemptStateData newInstance(
ApplicationAttemptId attemptId, Container masterContainer,
Credentials attemptTokens, long startTime,
- Map<String, Long> resourceSeondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourcesSecondsMap) {
return newInstance(attemptId, masterContainer, attemptTokens, startTime,
null, "N/A", "", null, ContainerExitStatus.INVALID, 0,
- resourceSeondsMap, preemptedResourceSecondsMap);
+ guaranteedResourceSecondsMap, preemptedResourceSecondsMap,
+ opportunisticResourcesSecondsMap);
}
@@ -180,30 +187,78 @@ public abstract class ApplicationAttemptStateData {
public abstract void setFinishTime(long finishTime);
/**
- * Get the <em>memory seconds</em> (in MB seconds) of the application.
- * @return <em>memory seconds</em> (in MB seconds) of the application
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getMemorySeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setMemorySeconds(long memorySeconds);
/**
- * Get the <em>vcore seconds</em> of the application.
- * @return <em>vcore seconds</em> of the application
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
*/
@Public
@Unstable
+ @Deprecated
public abstract long getVcoreSeconds();
@Public
@Unstable
+ @Deprecated
public abstract void setVcoreSeconds(long vcoreSeconds);
/**
+ * Get the <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>guaranteed memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedMemorySeconds();
+
+ @Private
+ public abstract void setGuaranteedMemorySeconds(long memorySeconds);
+
+ /**
+ * Get the <em>guaranteed vcore seconds</em> of the application.
+ * @return <em>guaranteed vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getGuaranteedVcoreSeconds();
+
+ @Private
+ public abstract void setGuaranteedVcoreSeconds(long vcoreSeconds);
+
+ /**
+ * Get the <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application.
+ * @return <em>opportunistic memory seconds</em> (in MB seconds) of the
+ * application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticMemorySeconds();
+
+ /**
+ * Get the <em>opportunistic vcore seconds</em> of the application.
+ * @return <em>opportunistic vcore seconds</em> of the application
+ */
+ @Public
+ @Unstable
+ public abstract long getOpportunisticVcoreSeconds();
+
+ /**
* Get the <em>preempted memory seconds</em>
* (in MB seconds) of the application.
* @return <em>preempted memory seconds</em>
@@ -232,28 +287,54 @@ public abstract class ApplicationAttemptStateData {
public abstract void setPreemptedVcoreSeconds(long vcoreSeconds);
/**
- * Get the aggregated number of resources preempted that the application has
+ * Get the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
- * @return map containing the resource name and aggregated preempted
- * resource-seconds
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract Map<String, Long> getResourceSecondsMap();
/**
- * Set the aggregated number of resources that the application has
+ * Set the aggregated number of guaranteed resources that the application has
* allocated times the number of seconds the application has been running.
*
* @param resourceSecondsMap map containing the resource name and aggregated
- * resource-seconds
+ * guaranteed resource-seconds
*/
@Public
@Unstable
+ @Deprecated
public abstract void setResourceSecondsMap(
Map<String, Long> resourceSecondsMap);
+
+ /**
+ * Get the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated guaranteed
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getGuaranteedResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of guaranteed resources that the application has
+ * allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * guaranteed resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
+
/**
* Get the aggregated number of resources preempted that the application has
* allocated times the number of seconds the application has been running.
@@ -276,4 +357,27 @@ public abstract class ApplicationAttemptStateData {
@Unstable
public abstract void setPreemptedResourceSecondsMap(
Map<String, Long> preemptedResourceSecondsMap);
+
+ /**
+ * Get the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @return map containing the resource name and aggregated opportunistic
+ * resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract Map<String, Long> getOpportunisticResourceSecondsMap();
+
+ /**
+ * Set the aggregated number of opportunistic resources that the application
+ * has allocated times the number of seconds the application has been running.
+ *
+ * @param resourceSecondsMap map containing the resource name and aggregated
+ * opportunistic resource-seconds
+ */
+ @Public
+ @Unstable
+ public abstract void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/10] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
index ed71ea2..0243443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationAttemptStateDataPBImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
@@ -56,8 +57,9 @@ public class ApplicationAttemptStateDataPBImpl extends
private Container masterContainer = null;
private ByteBuffer appAttemptTokens = null;
- private Map<String, Long> resourceSecondsMap;
+ private Map<String, Long> guaranteedResourceSecondsMap;
private Map<String, Long> preemptedResourceSecondsMap;
+ private Map<String, Long> opportunisticResourceSecondsMap;
public ApplicationAttemptStateDataPBImpl() {
builder = ApplicationAttemptStateDataProto.newBuilder();
@@ -243,30 +245,72 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public long getMemorySeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getMemorySeconds();
+ return getGuaranteedMemorySeconds();
}
@Override
+ @Deprecated
public long getVcoreSeconds() {
- ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- return p.getVcoreSeconds();
+ return getGuaranteedVcoreSeconds();
}
@Override
+ @Deprecated
public void setMemorySeconds(long memorySeconds) {
- maybeInitBuilder();
- builder.setMemorySeconds(memorySeconds);
+ setGuaranteedMemorySeconds(memorySeconds);
}
@Override
+ @Deprecated
public void setVcoreSeconds(long vcoreSeconds) {
+ setGuaranteedVcoreSeconds(vcoreSeconds);
+ }
+
+ @Override
+ public long getGuaranteedMemorySeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getMemorySeconds();
+ }
+
+ @Override
+ public void setGuaranteedMemorySeconds(long memorySeconds) {
+ maybeInitBuilder();
+ builder.setMemorySeconds(memorySeconds);
+ }
+
+ @Override
+ public long getGuaranteedVcoreSeconds() {
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getVcoreSeconds();
+ }
+
+ @Override
+ public void setGuaranteedVcoreSeconds(long vcoreSeconds) {
maybeInitBuilder();
builder.setVcoreSeconds(vcoreSeconds);
}
@Override
+ public long getOpportunisticMemorySeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.MEMORY_MB.getName())) {
+ return tmp.get(ResourceInformation.MEMORY_MB.getName());
+ }
+ return 0;
+ }
+
+ @Override
+ public long getOpportunisticVcoreSeconds() {
+ Map<String, Long> tmp = getOpportunisticResourceSecondsMap();
+ if (tmp.containsKey(ResourceInformation.VCORES.getName())) {
+ return tmp.get(ResourceInformation.VCORES.getName());
+ }
+ return 0;
+ }
+
+ @Override
public long getPreemptedMemorySeconds() {
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
return p.getPreemptedMemorySeconds();
@@ -410,21 +454,35 @@ public class ApplicationAttemptStateDataPBImpl extends
}
@Override
+ @Deprecated
public Map<String, Long> getResourceSecondsMap() {
- if (this.resourceSecondsMap != null) {
- return this.resourceSecondsMap;
+ return getGuaranteedResourceSecondsMap();
+ }
+
+ @Override
+ @Deprecated
+ public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ setGuaranteedResourceSecondsMap(resourceSecondsMap);
+ }
+
+ @Override
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ if (this.guaranteedResourceSecondsMap != null) {
+ return this.guaranteedResourceSecondsMap;
}
ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
- this.resourceSecondsMap = ProtoUtils.convertStringLongMapProtoListToMap(
- p.getApplicationResourceUsageMapList());
- return this.resourceSecondsMap;
+ this.guaranteedResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationResourceUsageMapList());
+ return this.guaranteedResourceSecondsMap;
}
@Override
- public void setResourceSecondsMap(Map<String, Long> resourceSecondsMap) {
+ public void setGuaranteedResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
maybeInitBuilder();
builder.clearApplicationResourceUsageMap();
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = resourceSecondsMap;
if (resourceSecondsMap != null) {
builder.addAllApplicationResourceUsageMap(
ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
@@ -454,4 +512,28 @@ public class ApplicationAttemptStateDataPBImpl extends
.convertMapToStringLongMapProtoList(preemptedResourceSecondsMap));
}
}
+
+ @Override
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ if (this.opportunisticResourceSecondsMap != null) {
+ return this.opportunisticResourceSecondsMap;
+ }
+ ApplicationAttemptStateDataProtoOrBuilder p = viaProto ? proto : builder;
+ this.opportunisticResourceSecondsMap =
+ ProtoUtils.convertStringLongMapProtoListToMap(
+ p.getApplicationOpportunisticResourceUsageMapList());
+ return this.opportunisticResourceSecondsMap;
+ }
+
+ @Override
+ public void setOpportunisticResourceSecondsMap(
+ Map<String, Long> resourceSecondsMap) {
+ maybeInitBuilder();
+ builder.clearApplicationOpportunisticResourceUsageMap();
+ this.opportunisticResourceSecondsMap = resourceSecondsMap;
+ if (resourceSecondsMap != null) {
+ builder.addAllApplicationOpportunisticResourceUsageMap(
+ ProtoUtils.convertMapToStringLongMapProtoList(resourceSecondsMap));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 61936bb..53728c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -449,6 +449,7 @@ public class RMAppImpl implements RMApp, Recoverable {
} else {
this.startTime = startTime;
}
+ LOG.info(applicationId.toString() + " started at " + this.startTime);
this.applicationType = StringInterner.weakIntern(applicationType);
this.applicationTags = applicationTags;
this.amReqs = amReqs;
@@ -763,10 +764,12 @@ public class RMAppImpl implements RMApp, Recoverable {
}
RMAppMetrics rmAppMetrics = getRMAppMetrics();
- appUsageReport
- .setResourceSecondsMap(rmAppMetrics.getResourceSecondsMap());
+ appUsageReport.setGuaranteedResourceSecondsMap(
+ rmAppMetrics.getGuaranteedResourceSecondsMap());
appUsageReport.setPreemptedResourceSecondsMap(
rmAppMetrics.getPreemptedResourceSecondsMap());
+ appUsageReport.setOpportunisticResourceSecondsMap(
+ rmAppMetrics.getOpportunisticResourceSecondsMap());
}
if (currentApplicationAttemptId == null) {
@@ -1636,8 +1639,9 @@ public class RMAppImpl implements RMApp, Recoverable {
Resource resourcePreempted = Resource.newInstance(0, 0);
int numAMContainerPreempted = 0;
int numNonAMContainerPreempted = 0;
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedSecondsMap = new HashMap<>();
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>(2);
+ Map<String, Long> preemptedSecondsMap = new HashMap<>(2);
+ Map<String, Long> opportunsiticResourceSecondsMap = new HashMap<>(2);
this.readLock.lock();
try {
for (RMAppAttempt attempt : attempts.values()) {
@@ -1653,20 +1657,15 @@ public class RMAppImpl implements RMApp, Recoverable {
// for both running and finished containers.
AggregateAppResourceUsage resUsage =
attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
- for (Map.Entry<String, Long> entry : resUsage
- .getResourceUsageSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- resourceSecondsMap.put(entry.getKey(), value);
- }
- for (Map.Entry<String, Long> entry : attemptMetrics
- .getPreemptedResourceSecondsMap().entrySet()) {
- long value = RMServerUtils
- .getOrDefault(preemptedSecondsMap, entry.getKey(), 0L);
- value += entry.getValue();
- preemptedSecondsMap.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ guaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap(),
+ opportunsiticResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ attemptMetrics.getPreemptedResourceSecondsMap(),
+ preemptedSecondsMap);
}
}
} finally {
@@ -1674,7 +1673,8 @@ public class RMAppImpl implements RMApp, Recoverable {
}
return new RMAppMetrics(resourcePreempted, numNonAMContainerPreempted,
- numAMContainerPreempted, resourceSecondsMap, preemptedSecondsMap);
+ numAMContainerPreempted, guaranteedResourceSecondsMap,
+ preemptedSecondsMap, opportunsiticResourceSecondsMap);
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
index 2bb7fd1..76272e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppMetrics.java
@@ -28,18 +28,21 @@ public class RMAppMetrics {
final Resource resourcePreempted;
final int numNonAMContainersPreempted;
final int numAMContainersPreempted;
- private final Map<String, Long> resourceSecondsMap;
+ private final Map<String, Long> guaranteedResourceSecondsMap;
private final Map<String, Long> preemptedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
public RMAppMetrics(Resource resourcePreempted,
int numNonAMContainersPreempted, int numAMContainersPreempted,
- Map<String, Long> resourceSecondsMap,
- Map<String, Long> preemptedResourceSecondsMap) {
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> preemptedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
this.resourcePreempted = resourcePreempted;
this.numNonAMContainersPreempted = numNonAMContainersPreempted;
this.numAMContainersPreempted = numAMContainersPreempted;
- this.resourceSecondsMap = resourceSecondsMap;
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
this.preemptedResourceSecondsMap = preemptedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
}
public Resource getResourcePreempted() {
@@ -54,17 +57,25 @@ public class RMAppMetrics {
return numAMContainersPreempted;
}
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
}
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
public long getPreemptedMemorySeconds() {
return RMServerUtils.getOrDefault(preemptedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
@@ -75,12 +86,15 @@ public class RMAppMetrics {
ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceSecondsMap() {
- return resourceSecondsMap;
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
+ return guaranteedResourceSecondsMap;
}
public Map<String, Long> getPreemptedResourceSecondsMap() {
return preemptedResourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
index b858712..a75cc57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AggregateAppResourceUsage.java
@@ -27,30 +27,55 @@ import java.util.Map;
@Private
public class AggregateAppResourceUsage {
- private Map<String, Long> resourceSecondsMap = new HashMap<>();
+ private Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
- public AggregateAppResourceUsage(Map<String, Long> resourceSecondsMap) {
- this.resourceSecondsMap.putAll(resourceSecondsMap);
+ public AggregateAppResourceUsage(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap.putAll(
+ guaranteedResourceSecondsMap);
+ this.opportunisticResourceSecondsMap.putAll(
+ opportunisticResourceSecondsMap);
}
/**
- * @return the memorySeconds
+ * @return the guaranteed memory seconds
*/
- public long getMemorySeconds() {
- return RMServerUtils.getOrDefault(resourceSecondsMap,
+ public long getGuaranteedMemorySeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
ResourceInformation.MEMORY_MB.getName(), 0L);
}
/**
- * @return the vcoreSeconds
+ * @return the guaranteed vcore seconds
*/
- public long getVcoreSeconds() {
- return RMServerUtils
- .getOrDefault(resourceSecondsMap, ResourceInformation.VCORES.getName(),
- 0L);
+ public long getGuaranteedVcoreSeconds() {
+ return RMServerUtils.getOrDefault(guaranteedResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
}
- public Map<String, Long> getResourceUsageSecondsMap() {
- return resourceSecondsMap;
+ /**
+ * @return the opportunistic memory seconds
+ */
+ public long getOpportunisticMemorySeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.MEMORY_MB.getName(), 0L);
+ }
+
+ /**
+ * @return the opportunistic vcore seconds
+ */
+ public long getOpportunisticVcoreSeconds() {
+ return RMServerUtils.getOrDefault(opportunisticResourceSecondsMap,
+ ResourceInformation.VCORES.getName(), 0L);
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index cf10be4..6f46b33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -934,15 +934,18 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock.lock();
try {
ApplicationResourceUsageReport report =
- scheduler.getAppResourceUsageReport(this.getAppAttemptId());
+ scheduler.getAppActiveResourceUsageReport(this.getAppAttemptId());
if (report == null) {
report = RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT;
}
AggregateAppResourceUsage resUsage =
this.attemptMetrics.getAggregateAppResourceUsage();
- report.setResourceSecondsMap(resUsage.getResourceUsageSecondsMap());
+ report.setGuaranteedResourceSecondsMap(
+ resUsage.getGuaranteedResourceUsageSecondsMap());
report.setPreemptedResourceSecondsMap(
this.attemptMetrics.getPreemptedResourceSecondsMap());
+ report.setOpportunisticResourceSecondsMap(
+ resUsage.getOpportunisticResourceSecondsMap());
return report;
} finally {
this.readLock.unlock();
@@ -979,10 +982,12 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
this.finishTime = attemptState.getFinishTime();
- this.attemptMetrics
- .updateAggregateAppResourceUsage(attemptState.getResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppGuaranteedResourceUsage(
+ attemptState.getGuaranteedResourceSecondsMap());
this.attemptMetrics.updateAggregatePreemptedAppResourceUsage(
attemptState.getPreemptedResourceSecondsMap());
+ this.attemptMetrics.updateAggregateAppOpportunisticResourceUsage(
+ attemptState.getOpportunisticResourceSecondsMap());
}
public void transferStateFromAttempt(RMAppAttempt attempt) {
@@ -1360,9 +1365,11 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
ApplicationAttemptStateData attemptState = ApplicationAttemptStateData
.newInstance(applicationAttemptId, getMasterContainer(),
rmStore.getCredentialsFromAppAttempt(this), startTime,
- stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus, exitStatus,
- getFinishTime(), resUsage.getResourceUsageSecondsMap(),
- this.attemptMetrics.getPreemptedResourceSecondsMap());
+ stateToBeStored, finalTrackingUrl, diags.toString(), finalStatus,
+ exitStatus, getFinishTime(),
+ resUsage.getGuaranteedResourceUsageSecondsMap(),
+ this.attemptMetrics.getPreemptedResourceSecondsMap(),
+ resUsage.getOpportunisticResourceSecondsMap());
LOG.info("Updating application attempt " + applicationAttemptId
+ " with final state: " + targetedFinalState + ", and exit status: "
+ exitStatus);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 015cff7..192722b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -53,8 +53,11 @@ public class RMAppAttemptMetrics {
private ReadLock readLock;
private WriteLock writeLock;
- private Map<String, AtomicLong> resourceUsageMap = new HashMap<>();
+ private Map<String, AtomicLong> guaranteedResourceUsageMap
+ = new HashMap<>();
private Map<String, AtomicLong> preemptedResourceMap = new HashMap<>();
+ private Map<String, AtomicLong> opportunisticResourceUsageMap
+ = new HashMap<>();
private RMContext rmContext;
private int[][] localityStatistics =
@@ -129,8 +132,10 @@ public class RMAppAttemptMetrics {
}
public AggregateAppResourceUsage getAggregateAppResourceUsage() {
- Map<String, Long> resourcesUsed =
- convertAtomicLongMaptoLongMap(resourceUsageMap);
+ Map<String, Long> guaranteedResourcesUsed =
+ convertAtomicLongMaptoLongMap(guaranteedResourceUsageMap);
+ Map<String, Long> opportunisticResourcesUsed =
+ convertAtomicLongMaptoLongMap(opportunisticResourceUsageMap);
// Only add in the running containers if this is the active attempt.
RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
@@ -139,27 +144,25 @@ public class RMAppAttemptMetrics {
if (currentAttempt != null
&& currentAttempt.getAppAttemptId().equals(attemptId)) {
ApplicationResourceUsageReport appResUsageReport =
- rmContext.getScheduler().getAppResourceUsageReport(attemptId);
+ rmContext.getScheduler().getAppActiveResourceUsageReport(attemptId);
if (appResUsageReport != null) {
- Map<String, Long> tmp = appResUsageReport.getResourceSecondsMap();
- for (Map.Entry<String, Long> entry : tmp.entrySet()) {
- Long value = resourcesUsed.get(entry.getKey());
- if (value != null) {
- value += entry.getValue();
- } else {
- value = entry.getValue();
- }
- resourcesUsed.put(entry.getKey(), value);
- }
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getGuaranteedResourceSecondsMap(),
+ guaranteedResourcesUsed);
+ Resources.mergeResourceSecondsMap(
+ appResUsageReport.getOpportunisticResourceSecondsMap(),
+ opportunisticResourcesUsed);
}
}
}
- return new AggregateAppResourceUsage(resourcesUsed);
+
+ return new AggregateAppResourceUsage(guaranteedResourcesUsed,
+ opportunisticResourcesUsed);
}
public void updateAggregateAppResourceUsage(Resource allocated,
long deltaUsedMillis) {
- updateUsageMap(allocated, deltaUsedMillis, resourceUsageMap);
+ updateUsageMap(allocated, deltaUsedMillis, guaranteedResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(Resource allocated,
@@ -167,9 +170,14 @@ public class RMAppAttemptMetrics {
updateUsageMap(allocated, deltaUsedMillis, preemptedResourceMap);
}
- public void updateAggregateAppResourceUsage(
+ public void updateAggregateAppGuaranteedResourceUsage(
Map<String, Long> resourceSecondsMap) {
- updateUsageMap(resourceSecondsMap, resourceUsageMap);
+ updateUsageMap(resourceSecondsMap, guaranteedResourceUsageMap);
+ }
+
+ public void updateAggregateAppOpportunisticResourceUsage(
+ Map<String, Long> resourceSecondsMap) {
+ updateUsageMap(resourceSecondsMap, opportunisticResourceUsageMap);
}
public void updateAggregatePreemptedAppResourceUsage(
@@ -199,10 +207,9 @@ public class RMAppAttemptMetrics {
if (!targetMap.containsKey(entry.getKey())) {
resourceUsed = new AtomicLong(0);
targetMap.put(entry.getKey(), resourceUsed);
-
}
resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.set(entry.getValue());
+ resourceUsed.addAndGet(entry.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
new file mode 100644
index 0000000..c91dae8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/ContainerResourceUsageReport.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+
+import java.util.Map;
+
+/**
+ * The resource usage information of a <code>Container</code>.
+ */
+@Private
+public class ContainerResourceUsageReport {
+ private final Map<String, Long> guaranteedResourceSecondsMap;
+ private final Map<String, Long> opportunisticResourceSecondsMap;
+
+ public ContainerResourceUsageReport(
+ Map<String, Long> guaranteedResourceSecondsMap,
+ Map<String, Long> opportunisticResourceSecondsMap) {
+ this.guaranteedResourceSecondsMap = guaranteedResourceSecondsMap;
+ this.opportunisticResourceSecondsMap = opportunisticResourceSecondsMap;
+ }
+
+ public Map<String, Long> getGuaranteedResourceUsageSecondsMap() {
+ return guaranteedResourceSecondsMap;
+ }
+
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index f3cbf63..4c43c5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
-import java.util.List;
-
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -29,7 +27,6 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ContainerRequest;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -85,7 +82,9 @@ public interface RMContainer extends EventHandler<RMContainerEvent>,
ContainerState getContainerState();
ContainerReport createContainerReport();
-
+
+ ContainerResourceUsageReport getResourceUsageReport();
+
boolean isAMContainer();
ContainerRequest getContainerRequest();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index e26689e..8f51d1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
import java.util.Collections;
import java.util.EnumSet;
-import java.util.List;
+import java.util.HashMap;
+import java.util.Map;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -39,7 +41,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -176,7 +178,7 @@ public class RMContainerImpl implements RMContainer {
private NodeId reservedNode;
private SchedulerRequestKey reservedSchedulerKey;
private long creationTime;
- private long finishTime;
+ private long finishTime = -1L;
private ContainerStatus finishedStatus;
private boolean isAMContainer;
private ContainerRequest containerRequestForRecovery;
@@ -189,6 +191,10 @@ public class RMContainerImpl implements RMContainer {
private boolean isExternallyAllocated;
private SchedulerRequestKey allocatedSchedulerKey;
+ // This is updated whenever the container instance is updated in
+ // the case of container increase/decrease/promotion/demotion
+ private long lastContainerUpdateTime;
+
public RMContainerImpl(Container container, SchedulerRequestKey schedulerKey,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext) {
@@ -230,6 +236,7 @@ public class RMContainerImpl implements RMContainer {
this.appAttemptId = appAttemptId;
this.user = user;
this.creationTime = creationTime;
+ this.lastContainerUpdateTime = creationTime;
this.rmContext = rmContext;
this.eventHandler = rmContext.getDispatcher().getEventHandler();
this.containerAllocationExpirer = rmContext.getContainerAllocationExpirer();
@@ -277,9 +284,87 @@ public class RMContainerImpl implements RMContainer {
}
public void setContainer(Container container) {
+ // containers are updated by resetting the underlying Container instance,
+ // always update RMContainer resource usage before the update.
+ updateAppAttemptMetrics(true);
this.container = container;
}
+ /**
+ * Collect the resource usage information of the current underlying Container
+ * instance and update its RMAppAttemptMetrics. This is called whenever the
+ * RMContainer is updated (by updating its underlying container) or finishes.
+ * @param containerUpdated if it is called upon container update event
+ */
+ private void updateAppAttemptMetrics(boolean containerUpdated) {
+ RMAppAttempt rmAttempt = rmContext.getRMApps()
+ .get(getApplicationAttemptId().getApplicationId())
+ .getCurrentAppAttempt();
+
+ if (rmAttempt != null) {
+ // collect resource usage information of the current Container instance
+ ContainerResourceUsageReport resourceUsage =
+ getCurrentContainerResourceUsage(containerUpdated);
+
+ // If this is a preempted container, update preemption metrics
+ if (finishedStatus != null &&
+ ContainerExitStatus.PREEMPTED == finishedStatus.getExitStatus()) {
+ rmAttempt.getRMAppAttemptMetrics()
+ .updatePreemptionInfo(container.getResource(), this);
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppOpportunisticResourceUsage(
+ resourceUsage.getOpportunisticResourceSecondsMap());
+ rmAttempt.getRMAppAttemptMetrics()
+ .updateAggregateAppGuaranteedResourceUsage(
+ resourceUsage.getGuaranteedResourceUsageSecondsMap());
+ }
+ }
+
+ /**
+ * Get resource usage of the current underlying Container instance.
+ * @param containerUpdated if this is called upon container update
+ * @return resource usage of the current <code>container</code>
+ */
+ private ContainerResourceUsageReport getCurrentContainerResourceUsage(
+ boolean containerUpdated) {
+ ContainerResourceUsageReport report;
+
+ // A container generates usage until it finishes which is indicated
+ // by a positive finish timestamp that is set once it finishes
+ final long currentTimeMillis =
+ finishTime < 0 ? System.currentTimeMillis() : finishTime;
+ final long usedSeconds = (currentTimeMillis -
+ lastContainerUpdateTime) / DateUtils.MILLIS_PER_SECOND;
+ Resource resource = container.getResource();
+
+ if (container.getExecutionType() == ExecutionType.GUARANTEED) {
+ Map<String, Long> guaranteedResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ guaranteedResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(guaranteedResourceSeconds,
+ Collections.emptyMap());
+ } else {
+ Map<String, Long> opportunisticResourceSeconds = new HashMap<>(2);
+ for (ResourceInformation entry : resource.getResources()) {
+ opportunisticResourceSeconds.put(
+ entry.getName(), entry.getValue() * usedSeconds);
+ }
+ report = new ContainerResourceUsageReport(Collections.emptyMap(),
+ opportunisticResourceSeconds);
+ }
+
+ if (containerUpdated) {
+ lastContainerUpdateTime = currentTimeMillis;
+ }
+ return report;
+ }
+
@Override
public RMContainerState getState() {
this.readLock.lock();
@@ -683,7 +768,7 @@ public class RMContainerImpl implements RMContainer {
// Inform AppAttempt
// container.getContainer() can return null when a RMContainer is a
// reserved container
- updateAttemptMetrics(container);
+ container.updateAppAttemptMetrics(false);
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
container.appAttemptId, finishedEvent.getRemoteContainerStatus(),
@@ -705,27 +790,6 @@ public class RMContainerImpl implements RMContainer {
}
}
-
- private static void updateAttemptMetrics(RMContainerImpl container) {
- Resource resource = container.getContainer().getResource();
- RMAppAttempt rmAttempt = container.rmContext.getRMApps()
- .get(container.getApplicationAttemptId().getApplicationId())
- .getCurrentAppAttempt();
-
- if (rmAttempt != null) {
- long usedMillis = container.finishTime - container.creationTime;
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregateAppResourceUsage(resource, usedMillis);
- // If this is a preempted container, update preemption metrics
- if (ContainerExitStatus.PREEMPTED == container.finishedStatus
- .getExitStatus()) {
- rmAttempt.getRMAppAttemptMetrics()
- .updatePreemptionInfo(resource, container);
- rmAttempt.getRMAppAttemptMetrics()
- .updateAggregatePreemptedAppResourceUsage(resource, usedMillis);
- }
- }
- }
}
private static final class KillTransition extends FinishedTransition {
@@ -764,6 +828,11 @@ public class RMContainerImpl implements RMContainer {
}
@Override
+ public ContainerResourceUsageReport getResourceUsageReport() {
+ return getCurrentContainerResourceUsage(false);
+ }
+
+ @Override
public String getNodeHttpAddress() {
try {
readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index a755a4c..b78e93e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -412,7 +412,7 @@ public abstract class AbstractYarnScheduler
}
@Override
- public ApplicationResourceUsageReport getAppResourceUsageReport(
+ public ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId) {
SchedulerApplicationAttempt attempt = getApplicationAttempt(appAttemptId);
if (attempt == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index f02f113..af4ef54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -60,12 +59,12 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -107,7 +106,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
protected long lastMemoryAggregateAllocationUpdateTime = 0;
- private Map<String, Long> lastResourceSecondsMap = new HashMap<>();
+ private Map<String, Long> lastGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ private Map<String, Long> lastOpportunisticResourceSecondsMap =
+ new HashMap<>();
protected final AppSchedulingInfo appSchedulingInfo;
protected ApplicationAttemptId attemptId;
protected Map<ContainerId, RMContainer> liveContainers =
@@ -1053,34 +1055,47 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// recently.
if ((currentTimeMillis - lastMemoryAggregateAllocationUpdateTime)
> MEM_AGGREGATE_ALLOCATION_CACHE_MSECS) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
+ Map<String, Long> aggregatedGuaranteedResourceSecondsMap =
+ new HashMap<>();
+ Map<String, Long> aggregatedOpportunisticResourceSecondsMap =
+ new HashMap<>();
for (RMContainer rmContainer : this.liveContainers.values()) {
- long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
- Resource resource = rmContainer.getContainer().getResource();
- for (ResourceInformation entry : resource.getResources()) {
- long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
- value += entry.getValue() * usedMillis
- / DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getName(), value);
- }
+ ContainerResourceUsageReport containerResourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ aggregatedGuaranteedResourceSecondsMap);
+ Resources.mergeResourceSecondsMap(
+ containerResourceUsageReport.getOpportunisticResourceSecondsMap(),
+ aggregatedOpportunisticResourceSecondsMap);
}
-
+ lastGuaranteedResourceSecondsMap =
+ aggregatedGuaranteedResourceSecondsMap;
+ lastOpportunisticResourceSecondsMap =
+ aggregatedOpportunisticResourceSecondsMap;
lastMemoryAggregateAllocationUpdateTime = currentTimeMillis;
- lastResourceSecondsMap = resourceSecondsMap;
}
- return new AggregateAppResourceUsage(lastResourceSecondsMap);
+ return new AggregateAppResourceUsage(lastGuaranteedResourceSecondsMap,
+ lastOpportunisticResourceSecondsMap);
}
+ /**
+ * Get the resources that are actively being used by the app attempt.
+ * @return active resource usage
+ */
public ApplicationResourceUsageReport getResourceUsageReport() {
try {
writeLock.lock();
AggregateAppResourceUsage runningResourceUsage =
getRunningAggregateAppResourceUsage();
- Resource usedResourceClone = Resources.clone(
+ Resource guaranteedResourceUsedClone = Resources.clone(
attemptResourceUsage.getAllUsed());
+ Resource opportunisticResourceUsedClone = Resources.clone(
+ attemptOpportunisticResourceUsage.getAllUsed());
Resource reservedResourceClone = Resources.clone(
attemptResourceUsage.getReserved());
+ Resource neededResource = Resources.add(opportunisticResourceUsedClone,
+ Resources.add(guaranteedResourceUsedClone, reservedResourceClone));
Resource cluster = rmContext.getScheduler().getClusterResource();
ResourceCalculator calc =
rmContext.getScheduler().getResourceCalculator();
@@ -1095,18 +1110,20 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
float queueCapacityPerc = queue.getQueueInfo(false, false)
.getCapacity();
if (queueCapacityPerc != 0) {
- queueUsagePerc = calc.divide(cluster, usedResourceClone,
+ queueUsagePerc = calc.divide(cluster, guaranteedResourceUsedClone,
Resources.multiply(cluster, queueCapacityPerc)) * 100;
}
clusterUsagePerc =
- calc.divide(cluster, usedResourceClone, cluster) * 100;
+ calc.divide(cluster, guaranteedResourceUsedClone, cluster) * 100;
}
return ApplicationResourceUsageReport
.newInstance(liveContainers.size(), reservedContainers.size(),
- usedResourceClone, reservedResourceClone,
- Resources.add(usedResourceClone, reservedResourceClone),
- runningResourceUsage.getResourceUsageSecondsMap(), queueUsagePerc,
- clusterUsagePerc, preemptedResourceSecondsMaps);
+ guaranteedResourceUsedClone, reservedResourceClone,
+ neededResource,
+ runningResourceUsage.getGuaranteedResourceUsageSecondsMap(),
+ queueUsagePerc, clusterUsagePerc, preemptedResourceSecondsMaps,
+ opportunisticResourceUsedClone,
+ runningResourceUsage.getOpportunisticResourceSecondsMap());
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 93ca7c2..a8ec8b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -171,7 +171,7 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
*/
@LimitedPrivate("yarn")
@Evolving
- ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationResourceUsageReport getAppActiveResourceUsageReport(
ApplicationAttemptId appAttemptId);
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index cf651bb..75b2f81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -1017,7 +1017,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
.getEffectiveCapacity(getAppAMNodePartitionName());
if (!effCap.equals(Resources.none())) {
queueUsagePerc = calc.divide(totalPartitionRes,
- report.getUsedResources(), effCap) * 100;
+ report.getGuaranteedResourcesUsed(), effCap) * 100;
}
report.setQueueUsagePercentage(queueUsagePerc);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 0719ca6..9ef4d31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -169,8 +169,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
queue.getMetrics().releaseResources(
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
- this.attemptResourceUsage.decUsed(containerResource);
- getQueue().decUsedGuaranteedResource(containerResource);
+ if (rmContainer.getExecutionType().equals(ExecutionType.GUARANTEED)) {
+ this.attemptResourceUsage.decUsed(containerResource);
+ getQueue().decUsedGuaranteedResource(containerResource);
+ } else {
+ this.attemptOpportunisticResourceUsage.decUsed(containerResource);
+ }
+
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
index ac88f86..14dab9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerAppsBlock.java
@@ -95,6 +95,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
th(".runningcontainer", "Running Containers").
th(".allocatedCpu", "Allocated CPU VCores").
th(".allocatedMemory", "Allocated Memory MB").
+ th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU Vcores").
+ th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB").
th(".reservedCpu", "Reserved CPU VCores").
th(".reservedMemory", "Reserved Memory MB").
th(".progress", "Progress").
@@ -144,6 +148,10 @@ public class FairSchedulerAppsBlock extends HtmlBlock {
.valueOf(appInfo.getAllocatedVCores())).append("\",\"")
.append(appInfo.getAllocatedMB() == -1 ? "N/A" : String
.valueOf(appInfo.getAllocatedMB())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticVCores() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticVCores())).append("\",\"")
+ .append(appInfo.getAllocatedOpportunisticMB() == -1 ? "N/A" : String
+ .valueOf(appInfo.getAllocatedOpportunisticMB())).append("\",\"")
.append(appInfo.getReservedVCores() == -1 ? "N/A" : String
.valueOf(appInfo.getReservedVCores())).append("\",\"")
.append(appInfo.getReservedMB() == -1 ? "N/A" : String
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 8553d8c..6cd10f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -107,9 +107,12 @@ public class RMAppBlock extends AppBlock{
attemptResourcePreempted)
.__("Number of Non-AM Containers Preempted from Current Attempt:",
attemptNumNonAMContainerPreempted)
- .__("Aggregate Resource Allocation:", appMetrics == null ? "N/A" :
- StringHelper
- .getResourceSecondsString(appMetrics.getResourceSecondsMap()))
+ .__("Aggregate Guaranteed Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getGuaranteedResourceSecondsMap()))
+ .__("Aggregate Opportunistic Resource Allocation:", appMetrics == null ?
+ "N/A" : StringHelper.getResourceSecondsString(
+ appMetrics.getOpportunisticResourceSecondsMap()))
.__("Aggregate Preempted Resource Allocation:",
appMetrics == null ? "N/A" : StringHelper.getResourceSecondsString(
appMetrics.getPreemptedResourceSecondsMap()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
index d0dccab..0510d62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -69,6 +69,10 @@ public class RMAppsBlock extends AppsBlock {
.th(".allocatedMemory", "Allocated Memory MB")
.th(".reservedCpu", "Reserved CPU VCores")
.th(".reservedMemory", "Reserved Memory MB")
+ .th(".allocatedOpportunisticCpu",
+ "Allocated Opportunistic CPU VCores")
+ .th(".allocatedOpportunisticMemory",
+ "Allocated Opportunistic Memory MB")
.th(".queuePercentage", "% of Queue")
.th(".clusterPercentage", "% of Cluster")
.th(".progress", "Progress")
@@ -155,6 +159,12 @@ public class RMAppsBlock extends AppsBlock {
.append(app.getReservedMemoryMB() == -1 ? "N/A" :
String.valueOf(app.getReservedMemoryMB()))
.append("\",\"")
+ .append(app.getAllocatedOpportunisticCpuVcores() == -1 ? "N/A" : String
+ .valueOf(app.getAllocatedOpportunisticCpuVcores()))
+ .append("\",\"")
+ .append(app.getAllocatedOpportunisticMemoryMB() == -1 ? "N/A" :
+ String.valueOf(app.getAllocatedOpportunisticMemoryMB()))
+ .append("\",\"")
.append(queuePercent)
.append("\",\"")
.append(clusterPercent)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
index 631c908..e7ac27e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppInfo.java
@@ -95,6 +95,8 @@ public class AppInfo {
private String masterNodeId;
private long allocatedMB;
private long allocatedVCores;
+ private long allocatedOpportunisticMB;
+ private long allocatedOpportunisticVCores;
private long reservedMB;
private long reservedVCores;
private int runningContainers;
@@ -103,6 +105,9 @@ public class AppInfo {
protected float queueUsagePercentage;
protected float clusterUsagePercentage;
protected Map<String, Long> resourceSecondsMap;
+ private long opportunisticMemorySeconds;
+ private long opportunisticVcoreSeconds;
+ private Map<String, Long> opportunisticResourceSecondsMap;
// preemption info fields
private long preemptedResourceMB;
@@ -202,10 +207,17 @@ public class AppInfo {
ApplicationResourceUsageReport resourceReport =
attempt.getApplicationResourceUsageReport();
if (resourceReport != null) {
- Resource usedResources = resourceReport.getUsedResources();
+ Resource guaranteedResourcesUsed =
+ resourceReport.getGuaranteedResourcesUsed();
+ Resource opportunisticResourceUsed =
+ resourceReport.getOpportunisticResourcesUsed();
Resource reservedResources = resourceReport.getReservedResources();
- allocatedMB = usedResources.getMemorySize();
- allocatedVCores = usedResources.getVirtualCores();
+ allocatedMB = guaranteedResourcesUsed.getMemorySize();
+ allocatedVCores = guaranteedResourcesUsed.getVirtualCores();
+ allocatedOpportunisticMB =
+ opportunisticResourceUsed.getMemorySize();
+ allocatedOpportunisticVCores =
+ opportunisticResourceUsed.getVirtualCores();
reservedMB = reservedResources.getMemorySize();
reservedVCores = reservedResources.getVirtualCores();
runningContainers = resourceReport.getNumUsedContainers();
@@ -239,9 +251,13 @@ public class AppInfo {
numNonAMContainerPreempted = appMetrics.getNumNonAMContainersPreempted();
preemptedResourceVCores =
appMetrics.getResourcePreempted().getVirtualCores();
- memorySeconds = appMetrics.getMemorySeconds();
- vcoreSeconds = appMetrics.getVcoreSeconds();
- resourceSecondsMap = appMetrics.getResourceSecondsMap();
+ memorySeconds = appMetrics.getGuaranteedMemorySeconds();
+ vcoreSeconds = appMetrics.getGuaranteedVcoreSeconds();
+ resourceSecondsMap = appMetrics.getGuaranteedResourceSecondsMap();
+ opportunisticMemorySeconds = appMetrics.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds = appMetrics.getOpportunisticVcoreSeconds();
+ opportunisticResourceSecondsMap =
+ appMetrics.getOpportunisticResourceSecondsMap();
preemptedMemorySeconds = appMetrics.getPreemptedMemorySeconds();
preemptedVcoreSeconds = appMetrics.getPreemptedVcoreSeconds();
preemptedResourceSecondsMap = appMetrics.getPreemptedResourceSecondsMap();
@@ -454,6 +470,13 @@ public class AppInfo {
return this.allocatedVCores;
}
+ public long getAllocatedOpportunisticMB() {
+ return this.allocatedOpportunisticMB;
+ }
+
+ public long getAllocatedOpportunisticVCores() {
+ return this.allocatedOpportunisticVCores;
+ }
public long getReservedMB() {
return this.reservedMB;
}
@@ -478,18 +501,30 @@ public class AppInfo {
return numAMContainerPreempted;
}
- public long getMemorySeconds() {
+ public long getGuaranteedMemorySeconds() {
return memorySeconds;
}
- public long getVcoreSeconds() {
+ public long getGuaranteedVcoreSeconds() {
return vcoreSeconds;
}
- public Map<String, Long> getResourceSecondsMap() {
+ public long getOpportunisticMemorySeconds() {
+ return opportunisticMemorySeconds;
+ }
+
+ public long getOpportunisticVcoreSeconds() {
+ return opportunisticVcoreSeconds;
+ }
+
+ public Map<String, Long> getGuaranteedResourceSecondsMap() {
return resourceSecondsMap;
}
+ public Map<String, Long> getOpportunisticResourceSecondsMap() {
+ return opportunisticResourceSecondsMap;
+ }
+
public long getPreemptedMemorySeconds() {
return preemptedMemorySeconds;
}
@@ -582,6 +617,15 @@ public class AppInfo {
this.allocatedVCores = allocatedVCores;
}
+ public void setAllocatedOpportunisticMB(long allocatedOpportunisticMB) {
+ this.allocatedOpportunisticMB = allocatedOpportunisticMB;
+ }
+
+ public void setAllocatedOpportunisticVCores(
+ long allocatedOpportunisticVCores) {
+ this.allocatedOpportunisticVCores = allocatedOpportunisticVCores;
+ }
+
public void setReservedMB(long reservedMB) {
this.reservedMB = reservedMB;
}
@@ -594,12 +638,20 @@ public class AppInfo {
this.runningContainers = runningContainers;
}
- public void setMemorySeconds(long memorySeconds) {
- this.memorySeconds = memorySeconds;
+ public void setGuaranteedMemorySeconds(long guaranteedMemorySeconds) {
+ this.memorySeconds = guaranteedMemorySeconds;
+ }
+
+ public void setGuaranteedVcoreSeconds(long guaranteedVcoreSeconds) {
+ this.vcoreSeconds = guaranteedVcoreSeconds;
+ }
+
+ public void setOpportunisticMemorySeconds(long oppMemorySeconds) {
+ this.opportunisticMemorySeconds = oppMemorySeconds;
}
- public void setVcoreSeconds(long vcoreSeconds) {
- this.vcoreSeconds = vcoreSeconds;
+ public void setOpportunisticVcoreSeconds(long oppVcoreSeconds) {
+ this.opportunisticVcoreSeconds = oppVcoreSeconds;
}
public void setAppId(String appId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
index 39a56a8..1721b1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/proto/yarn_server_resourcemanager_recovery.proto
@@ -89,6 +89,7 @@ message ApplicationAttemptStateDataProto {
optional int64 preempted_vcore_seconds = 14;
repeated StringLongMapProto application_resource_usage_map = 15;
repeated StringLongMapProto preempted_resource_usage_map = 16;
+ repeated StringLongMapProto application_opportunistic_resource_usage_map = 17;
}
message EpochProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 140c15f..6ad6006 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -1031,12 +1031,20 @@ public class TestAppManager{
when(app.getState()).thenReturn(RMAppState.RUNNING);
when(app.getApplicationType()).thenReturn("MAPREDUCE");
when(app.getSubmitTime()).thenReturn(1000L);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap.put(ResourceInformation.MEMORY_MB.getName(), 16384L);
- resourceSecondsMap.put(ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ guaranteedResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
+ Map<String, Long> opportunisticResourceSecondsMap = new HashMap<>();
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.MEMORY_MB.getName(), 16384L);
+ opportunisticResourceSecondsMap.put(
+ ResourceInformation.VCORES.getName(), 64L);
RMAppMetrics metrics =
new RMAppMetrics(Resource.newInstance(1234, 56),
- 10, 1, resourceSecondsMap, new HashMap<>());
+ 10, 1, guaranteedResourceSecondsMap,
+ new HashMap<>(), opportunisticResourceSecondsMap);
when(app.getRMAppMetrics()).thenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary =
@@ -1047,16 +1055,19 @@ public class TestAppManager{
Assert.assertFalse(msg.contains("\r"));
String escaped = "\\n\\n\\r\\r";
- assertTrue(msg.contains("Multiline" + escaped +"AppName"));
- assertTrue(msg.contains("Multiline" + escaped +"UserName"));
- assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
- assertTrue(msg.contains("submitTime=1000"));
- assertTrue(msg.contains("memorySeconds=16384"));
- assertTrue(msg.contains("vcoreSeconds=64"));
- assertTrue(msg.contains("preemptedAMContainers=1"));
- assertTrue(msg.contains("preemptedNonAMContainers=10"));
- assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
- assertTrue(msg.contains("applicationType=MAPREDUCE"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
+ Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
+ Assert.assertTrue(msg.contains("submitTime=1000"));
+ Assert.assertTrue(msg.contains("guaranteedMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("guaranteedVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("opportunisticMemorySeconds=16384"));
+ Assert.assertTrue(msg.contains("opportunisticVcoreSeconds=64"));
+ Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
+ Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
+ Assert.assertTrue(
+ msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
+ Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index c7ed02c..698ca90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -426,9 +426,11 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
Assert.assertEquals("Enemy should not see app reserved containers",
-1, usageReport.getNumReservedContainers());
Assert.assertEquals("Enemy should not see app used resources",
- -1, usageReport.getUsedResources().getMemorySize());
+ -1, usageReport.getGuaranteedResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app reserved resources",
-1, usageReport.getReservedResources().getMemorySize());
+ Assert.assertEquals("Enemy should not see app used opportunistic resources",
+ -1, usageReport.getOpportunisticResourcesUsed().getMemorySize());
Assert.assertEquals("Enemy should not see app needed resources",
-1, usageReport.getNeededResources().getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 1c50dd3..a66a91b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -370,8 +370,8 @@ public class TestClientRMService {
ApplicationReport report = response.getApplicationReport();
ApplicationResourceUsageReport usageReport =
report.getApplicationResourceUsageReport();
- Assert.assertEquals(10, usageReport.getMemorySeconds());
- Assert.assertEquals(3, usageReport.getVcoreSeconds());
+ Assert.assertEquals(10, usageReport.getGuaranteedMemorySeconds());
+ Assert.assertEquals(3, usageReport.getGuaranteedVcoreSeconds());
Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
@@ -1402,26 +1402,28 @@ public class TestClientRMService {
when(asContext.getMaxAppAttempts()).thenReturn(1);
when(asContext.getNodeLabelExpression()).thenReturn(appNodeLabelExpression);
when(asContext.getPriority()).thenReturn(Priority.newInstance(0));
- RMAppImpl app =
- spy(new RMAppImpl(applicationId3, rmContext, config, null, null,
+ ResourceRequest resourceRequest = BuilderUtils.newResourceRequest(
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+ Resource.newInstance(1024, 1), 1);
+ RMAppImpl rmApp =
+ new RMAppImpl(applicationId3, rmContext, config, null, null,
queueName, asContext, yarnScheduler, null,
System.currentTimeMillis(), "YARN", null,
- Collections.singletonList(BuilderUtils.newResourceRequest(
- RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- Resource.newInstance(1024, 1), 1))){
- @Override
- public ApplicationReport createAndGetApplicationReport(
- String clientUserName, boolean allowAccess) {
- ApplicationReport report = super.createAndGetApplicationReport(
- clientUserName, allowAccess);
- ApplicationResourceUsageReport usageReport =
- report.getApplicationResourceUsageReport();
- usageReport.setMemorySeconds(memorySeconds);
- usageReport.setVcoreSeconds(vcoreSeconds);
- report.setApplicationResourceUsageReport(usageReport);
- return report;
- }
- });
+ Collections.singletonList(resourceRequest)) {
+ @Override
+ public ApplicationReport createAndGetApplicationReport(
+ String clientUserName, boolean allowAccess) {
+ ApplicationReport report = super.createAndGetApplicationReport(
+ clientUserName, allowAccess);
+ ApplicationResourceUsageReport usageReport =
+ report.getApplicationResourceUsageReport();
+ usageReport.setGuaranteedMemorySeconds(memorySeconds);
+ usageReport.setGuaranteedVcoreSeconds(vcoreSeconds);
+ report.setApplicationResourceUsageReport(usageReport);
+ return report;
+ }
+ };
+ RMAppImpl app = spy(rmApp);
app.getAMResourceRequests().get(0)
.setNodeLabelExpression(amNodeLabelExpression);
ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
@@ -1474,7 +1476,8 @@ public class TestClientRMService {
when(yarnScheduler.getAppsInQueue(QUEUE_2)).thenReturn(
Arrays.asList(getApplicationAttemptId(103)));
ApplicationAttemptId attemptId = getApplicationAttemptId(1);
- when(yarnScheduler.getAppResourceUsageReport(attemptId)).thenReturn(null);
+ when(yarnScheduler.getAppActiveResourceUsageReport(attemptId))
+ .thenReturn(null);
ResourceCalculator rs = mock(ResourceCalculator.class);
when(yarnScheduler.getResourceCalculator()).thenReturn(rs);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/10] hadoop git commit: YARN-7337. Expose per-node over-allocation
info in Node Report Contributed by Haibo Chen.
Posted by sz...@apache.org.
YARN-7337. Expose per-node over-allocation info in Node Report Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a8c2becd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a8c2becd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a8c2becd
Branch: refs/heads/YARN-1011
Commit: a8c2becd1c0a42824845d85351053f115710393c
Parents: 38b6d75
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 08:39:03 2017 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:34:26 2018 -0700
----------------------------------------------------------------------
.../hadoop/mapreduce/v2/TestRMNMInfo.java | 14 ++-
.../hadoop/yarn/api/records/NodeReport.java | 98 +++++++++++++++-----
.../src/main/proto/yarn_protos.proto | 7 +-
.../applications/distributedshell/Client.java | 2 +-
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 49 +++++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 43 ++++++---
.../api/records/impl/pb/NodeReportPBImpl.java | 94 +++++++++++++++----
.../hadoop/yarn/server/utils/BuilderUtils.java | 37 ++++----
.../server/resourcemanager/ClientRMService.java | 21 +++--
.../resourcemanager/DefaultAMSProcessor.java | 22 +++--
.../yarn/server/resourcemanager/RMNMInfo.java | 45 +++++----
.../resourcemanager/rmnode/RMNodeImpl.java | 2 +-
.../scheduler/SchedulerNodeReport.java | 50 +++++++---
.../webapp/dao/FifoSchedulerInfo.java | 9 +-
.../resourcemanager/webapp/dao/NodeInfo.java | 17 ++--
.../server/resourcemanager/NodeManager.java | 4 +-
.../capacity/TestApplicationPriority.java | 66 ++++++++-----
.../capacity/TestCapacityScheduler.java | 32 ++++---
.../TestNodeLabelContainerAllocation.java | 40 ++++----
.../scheduler/fifo/TestFifoScheduler.java | 50 ++++++----
.../webapp/TestRMWebServicesNodes.java | 13 +--
21 files changed, 491 insertions(+), 224 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
index efea709..76b5493 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestRMNMInfo.java
@@ -124,11 +124,17 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNotNull(n.get("NumContainers"));
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected number of used containers",
- 0, n.get("NumContainers").asInt());
+ n.get("NodeId") + ": Unexpected number of guaranteed containers used",
+ 0, n.get("NumContainers").asInt());
+ Assert.assertEquals(n.get("NodeId") +
+ ": Unexpected number of opportunistic containers used",
+ 0, n.get("NumOpportunisticContainers").asInt());
Assert.assertEquals(
- n.get("NodeId") + ": Unexpected amount of used memory",
+ n.get("NodeId") + ": Unexpected amount of guaranteed memory used",
0, n.get("UsedMemoryMB").asInt());
+ Assert.assertEquals(
+ n.get("NodeId") + ": Unexpected amount of used opportunistic memory",
+ 0, n.get("UsedOpportunisticMemoryMB").asInt());
Assert.assertNotNull(n.get("AvailableMemoryMB"));
}
}
@@ -161,6 +167,8 @@ public class TestRMNMInfo {
Assert.assertNotNull(n.get("NodeManagerVersion"));
Assert.assertNull(n.get("NumContainers"));
Assert.assertNull(n.get("UsedMemoryMB"));
+ Assert.assertNull(n.get("NumOpportunisticContainers"));
+ Assert.assertNull(n.get("UsedOpportunisticMemoryMB"));
Assert.assertNull(n.get("AvailableMemoryMB"));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index 3a80641..53e9f92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -36,9 +36,12 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@link NodeId} of the node.</li>
* <li>HTTP Tracking URL of the node.</li>
* <li>Rack name for the node.</li>
- * <li>Used {@link Resource} on the node.</li>
+ * <li>Used guaranteed {@link Resource} on the node.</li>
+ * <li>Used opportunistic {@link Resource} on the node.</li>
* <li>Total available {@link Resource} of the node.</li>
- * <li>Number of running containers on the node.</li>
+ * <li>Number of total running containers on the node.</li>
+ * <li>Number of running guaranteed containers on the node.</li>
+ * <li>Number of running opportunistic containers on the node.</li>
* </ul>
*
* @see ApplicationClientProtocol#getClusterNodes(org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest)
@@ -50,33 +53,37 @@ public abstract class NodeReport {
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newInstance(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime) {
+ return newInstance(nodeId, nodeState, httpAddress, rackName, guaranteedUsed,
+ capability, numGuaranteedContainers, healthReport, lastHealthReportTime,
+ null, null, null, null, 0);
}
@Private
@Unstable
public static NodeReport newInstance(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
+ String httpAddress, String rackName, Resource guaranteedUsed,
+ Resource capability, int numGuaranteedContainers, String healthReport,
+ long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType,
+ Resource opportunisticUsed, int numOpportunisticContainers) {
NodeReport nodeReport = Records.newRecord(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setGuaranteedResourceUsed(guaranteedUsed);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
nodeReport.setDecommissioningTimeout(decommissioningTimeout);
nodeReport.setNodeUpdateType(nodeUpdateType);
+ nodeReport.setOpportunisticResourceUsed(opportunisticUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
return nodeReport;
}
@@ -129,17 +136,43 @@ public abstract class NodeReport {
public abstract void setRackName(String rackName);
/**
- * Get <em>used</em> <code>Resource</code> on the node.
- * @return <em>used</em> <code>Resource</code> on the node
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
*/
@Public
@Stable
+ @Deprecated
public abstract Resource getUsed();
@Private
@Unstable
+ @Deprecated
public abstract void setUsed(Resource used);
-
+
+ /**
+ * Get <em>guaranteed</em> <code>Resource</code> used on the node.
+ * @return <em>guaranteed</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getGuaranteedResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setGuaranteedResourceUsed(Resource guaranteed);
+
+ /**
+ * Get <em>opportunistic</em> <code>Resource</code> used on the node.
+ * @return <em>opportunistic</em> <code>Resource</code> used on the node
+ */
+ @Public
+ @Unstable
+ public abstract Resource getOpportunisticResourceUsed();
+
+ @Private
+ @Unstable
+ public abstract void setOpportunisticResourceUsed(Resource opportunistic);
+
/**
* Get the <em>total</em> <code>Resource</code> on the node.
* @return <em>total</em> <code>Resource</code> on the node
@@ -153,19 +186,40 @@ public abstract class NodeReport {
public abstract void setCapability(Resource capability);
/**
- * Get the <em>number of allocated containers</em> on the node.
- * @return <em>number of allocated containers</em> on the node
+ * Get the <em>number of guaranteed containers</em> allocated on the node.
+ * @return <em>number of guaranteed containers</em> allocated on the node
*/
@Private
@Unstable
- public abstract int getNumContainers();
+ public abstract int getNumGuaranteedContainers();
@Private
@Unstable
- public abstract void setNumContainers(int numContainers);
-
+ public abstract void setNumGuaranteedContainers(int numContainers);
- /**
+ /**
+ * Get the <em>number of opportunistic containers</em> allocated on the node.
+ * @return <em>number of opportunistic containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public abstract int getNumOpportunisticContainers();
+
+ @Private
+ @Unstable
+ public abstract void setNumOpportunisticContainers(int numContainers);
+
+ /**
+ * Get the <em>number of containers</em> allocated on the node.
+ * @return <em>number of containers</em> allocated on the node
+ */
+ @Private
+ @Unstable
+ public int getNumTotalContainers() {
+ return getNumGuaranteedContainers() + getNumOpportunisticContainers();
+ }
+
+ /**
* Get the <em>diagnostic health report</em> of the node.
* @return <em>diagnostic health report</em> of the node
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index b6ea5f9..44e1b88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -342,9 +342,9 @@ message NodeReportProto {
optional NodeIdProto nodeId = 1;
optional string httpAddress = 2;
optional string rackName = 3;
- optional ResourceProto used = 4;
+ optional ResourceProto guaranteedResourceUsed = 4;
optional ResourceProto capability = 5;
- optional int32 numContainers = 6;
+ optional int32 numGuaranteedContainers = 6;
optional NodeStateProto node_state = 7;
optional string health_report = 8;
optional int64 last_health_report_time = 9;
@@ -353,6 +353,9 @@ message NodeReportProto {
optional ResourceUtilizationProto node_utilization = 12;
optional uint32 decommissioning_timeout = 13;
optional NodeUpdateTypeProto node_update_type = 14;
+ optional int32 numOpportunisticContainers = 15;
+ optional ResourceProto opportunisticResourceUsed = 16;
+ optional int32 numTotalContainers = 17;
}
message NodeIdToLabelsProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index ef635d3..7520d30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -600,7 +600,7 @@ public class Client {
+ ", nodeId=" + node.getNodeId()
+ ", nodeAddress=" + node.getHttpAddress()
+ ", nodeRackName=" + node.getRackName()
- + ", nodeNumContainers=" + node.getNumContainers());
+ + ", nodeNumContainers=" + node.getNumGuaranteedContainers());
}
QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index 288a5d2..be466fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.ConverterUtils;
@Private
@Unstable
@@ -185,7 +185,7 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(), nodeReport
.getNodeState(), nodeReport.getHttpAddress(), nodeReport
- .getNumContainers());
+ .getNumGuaranteedContainers());
}
writer.flush();
}
@@ -210,13 +210,18 @@ public class NodeCLI extends YarnCLI {
for (NodeReport nodeReport : nodesReport) {
writer.printf(NODES_PATTERN, nodeReport.getNodeId(),
nodeReport.getNodeState(), nodeReport.getHttpAddress(),
- nodeReport.getNumContainers());
+ nodeReport.getNumGuaranteedContainers());
writer.println("Detailed Node Information :");
writer.print("\tConfigured Resources : ");
writer.println(nodeReport.getCapability());
- writer.print("\tAllocated Resources : ");
- if (nodeReport.getUsed() != null) {
- writer.print(nodeReport.getUsed());
+ writer.print("\tAllocated Guaranteed Resources : ");
+ if (nodeReport.getGuaranteedResourceUsed() != null) {
+ writer.print(nodeReport.getGuaranteedResourceUsed());
+ }
+ writer.println();
+ writer.print("\tAllocated Opportunistic Resources : ");
+ if (nodeReport.getOpportunisticResourceUsed() != null) {
+ writer.print(nodeReport.getOpportunisticResourceUsed());
}
writer.println();
@@ -287,16 +292,32 @@ public class NodeCLI extends YarnCLI {
nodeReportStr.print("\tHealth-Report : ");
nodeReportStr
.println(nodeReport.getHealthReport());
- nodeReportStr.print("\tContainers : ");
- nodeReportStr.println(nodeReport.getNumContainers());
- nodeReportStr.print("\tMemory-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0MB"
- : (nodeReport.getUsed().getMemorySize() + "MB"));
+ nodeReportStr.print("\tGuaranteed Containers : ");
+ nodeReportStr.println(nodeReport.getNumGuaranteedContainers());
+ nodeReportStr.print("\tOpportunistic Containers : ");
+ nodeReportStr.println(nodeReport.getNumOpportunisticContainers());
+ Resource guaranteedResourceUsed =
+ nodeReport.getGuaranteedResourceUsed();
+ Resource opportunisticResourceUsed =
+ nodeReport.getOpportunisticResourceUsed();
+ nodeReportStr.print("\tGuaranteed-Memory-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0MB" :
+ (guaranteedResourceUsed.getMemorySize() + "MB"));
+ nodeReportStr.print("\tOpportunistic-Memory-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0MB" :
+ (opportunisticResourceUsed.getMemorySize() + "MB"));
nodeReportStr.print("\tMemory-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getMemorySize() + "MB");
- nodeReportStr.print("\tCPU-Used : ");
- nodeReportStr.println((nodeReport.getUsed() == null) ? "0 vcores"
- : (nodeReport.getUsed().getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tGuaranteed-CPU-Used : ");
+ nodeReportStr.println(
+ (guaranteedResourceUsed == null) ? "0 vcores" :
+ (guaranteedResourceUsed.getVirtualCores() + " vcores"));
+ nodeReportStr.print("\tOpportunistic-CPU-Used : ");
+ nodeReportStr.println(
+ (opportunisticResourceUsed == null) ? "0 vcores" :
+ (opportunisticResourceUsed.getVirtualCores() + " vcores"));
nodeReportStr.print("\tCPU-Capacity : ");
nodeReportStr.println(nodeReport.getCapability().getVirtualCores() + " vcores");
nodeReportStr.print("\tNode-Labels : ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index fdd3fc8..e172acd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -1352,7 +1352,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1360,7 +1361,8 @@ public class TestYarnCLI {
pw.println(" 0");
pw.println("Detailed Node Information :");
pw.println("\tConfigured Resources : <memory:0, vCores:0>");
- pw.println("\tAllocated Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Guaranteed Resources : <memory:0, vCores:0>");
+ pw.println("\tAllocated Opportunistic Resources : <memory:0, vCores:0>");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.println("\tNode-Labels : ");
@@ -1561,10 +1563,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
@@ -1597,10 +1602,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : ");
pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
@@ -1633,10 +1641,13 @@ public class TestYarnCLI {
pw.println("\tLast-Health-Update : "
+ DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
pw.println("\tHealth-Report : ");
- pw.println("\tContainers : 0");
- pw.println("\tMemory-Used : 0MB");
+ pw.println("\tGuaranteed Containers : 0");
+ pw.println("\tOpportunistic Containers : 0");
+ pw.println("\tGuaranteed-Memory-Used : 0MB");
+ pw.println("\tOpportunistic-Memory-Used : 0MB");
pw.println("\tMemory-Capacity : 0MB");
- pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tGuaranteed-CPU-Used : 0 vcores");
+ pw.println("\tOpportunistic-CPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
pw.println("\tResource Utilization by Node : ");
@@ -1989,10 +2000,12 @@ public class TestYarnCLI {
// ordered
nodeLabels = ImmutableSet.of("c", "b", "a", "x", "z", "y");
}
- NodeReport nodeReport = NodeReport.newInstance(NodeId
- .newInstance("host" + i, 0), state, "host" + 1 + ":8888",
- "rack1", Records.newRecord(Resource.class), Records
- .newRecord(Resource.class), 0, "", 0, nodeLabels, null, null);
+ NodeReport nodeReport = NodeReport.newInstance(
+ NodeId.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
+ "rack1", Records.newRecord(Resource.class),
+ Records.newRecord(Resource.class), 0, "",
+ 0, nodeLabels, null, null,
+ Records.newRecord(Resource.class), 0);
if (!emptyResourceUtilization) {
ResourceUtilization containersUtilization = ResourceUtilization
.newInstance(1024, 2048, 4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index ced588d..0960eb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -45,7 +45,8 @@ public class NodeReportPBImpl extends NodeReport {
private NodeReportProto.Builder builder = null;
private boolean viaProto = false;
private NodeId nodeId;
- private Resource used;
+ private Resource guaranteedResourceUsed;
+ private Resource opportunisticResourceUsed;
private Resource capability;
private ResourceUtilization containersUtilization = null;
private ResourceUtilization nodeUtilization = null;
@@ -109,9 +110,17 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public int getNumContainers() {
+ public int getNumGuaranteedContainers() {
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- return (p.hasNumContainers()) ? p.getNumContainers() : 0;
+ return (p.hasNumGuaranteedContainers()) ?
+ p.getNumGuaranteedContainers() : 0;
+ }
+
+ @Override
+ public int getNumOpportunisticContainers() {
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasNumOpportunisticContainers()) ?
+ p.getNumOpportunisticContainers() : 0;
}
@Override
@@ -120,18 +129,25 @@ public class NodeReportPBImpl extends NodeReport {
return (p.hasRackName()) ? p.getRackName() : null;
}
+ @Deprecated
@Override
public Resource getUsed() {
- if (this.used != null) {
- return this.used;
+ return getGuaranteedResourceUsed();
+ }
+
+ @Override
+ public Resource getOpportunisticResourceUsed() {
+ if (this.opportunisticResourceUsed != null) {
+ return this.opportunisticResourceUsed;
}
NodeReportProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasUsed()) {
+ if (!p.hasOpportunisticResourceUsed()) {
return null;
}
- this.used = convertFromProtoFormat(p.getUsed());
- return this.used;
+ this.opportunisticResourceUsed =
+ convertFromProtoFormat(p.getOpportunisticResourceUsed());
+ return this.opportunisticResourceUsed;
}
@Override
@@ -196,13 +212,24 @@ public class NodeReportPBImpl extends NodeReport {
}
@Override
- public void setNumContainers(int numContainers) {
+ public void setNumGuaranteedContainers(int numContainers) {
+ maybeInitBuilder();
+ if (numContainers == 0) {
+ builder.clearNumGuaranteedContainers();
+ return;
+ }
+ builder.setNumGuaranteedContainers(numContainers);
+ }
+
+ @Override
+ public void setNumOpportunisticContainers(int numContainers) {
maybeInitBuilder();
if (numContainers == 0) {
- builder.clearNumContainers();
+ builder.clearNumOpportunisticContainers();
return;
}
- builder.setNumContainers(numContainers);
+ builder.setNumOpportunisticContainers(numContainers);
+
}
@Override
@@ -215,13 +242,43 @@ public class NodeReportPBImpl extends NodeReport {
builder.setRackName(rackName);
}
+ @Deprecated
@Override
public void setUsed(Resource used) {
+ setGuaranteedResourceUsed(used);
+ }
+
+ @Override
+ public Resource getGuaranteedResourceUsed() {
+ if (this.guaranteedResourceUsed != null) {
+ return this.guaranteedResourceUsed;
+ }
+
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasGuaranteedResourceUsed()) {
+ return null;
+ }
+ this.guaranteedResourceUsed =
+ convertFromProtoFormat(p.getGuaranteedResourceUsed());
+ return this.guaranteedResourceUsed;
+ }
+
+ @Override
+ public void setGuaranteedResourceUsed(Resource guaranteed) {
+ maybeInitBuilder();
+ if (guaranteedResourceUsed == null) {
+ builder.clearGuaranteedResourceUsed();
+ }
+ this.guaranteedResourceUsed = guaranteed;
+ }
+
+ @Override
+ public void setOpportunisticResourceUsed(Resource opportunisticUsed) {
maybeInitBuilder();
- if (used == null) {
- builder.clearUsed();
+ if (opportunisticUsed == null) {
+ builder.clearOpportunisticResourceUsed();
}
- this.used = used;
+ this.opportunisticResourceUsed = opportunisticUsed;
}
public NodeReportProto getProto() {
@@ -258,8 +315,13 @@ public class NodeReportPBImpl extends NodeReport {
builder.getNodeId())) {
builder.setNodeId(convertToProtoFormat(this.nodeId));
}
- if (this.used != null) {
- builder.setUsed(convertToProtoFormat(this.used));
+ if (this.guaranteedResourceUsed != null) {
+ builder.setGuaranteedResourceUsed(
+ convertToProtoFormat(this.guaranteedResourceUsed));
+ }
+ if (this.opportunisticResourceUsed != null) {
+ builder.setOpportunisticResourceUsed(
+ convertToProtoFormat(this.opportunisticResourceUsed));
}
if (this.capability != null) {
builder.setCapability(convertToProtoFormat(this.capability));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 83f912f..63333ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -185,26 +185,23 @@ public class BuilderUtils {
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- null, null, null);
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainer,
+ String healthReport, long lastHealthReportTime, Set<String> nodeLabels,
+ Integer decommissioningTimeout, NodeUpdateType nodeUpdateType) {
+ return newNodeReport(nodeId, nodeState, httpAddress, rackName,
+ guaranteedResourceUsed, capability, numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainer, healthReport,
+ lastHealthReportTime, nodeLabels, null, null,
+ decommissioningTimeout, nodeUpdateType);
}
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
- Set<String> nodeLabels, Integer decommissioningTimeout,
- NodeUpdateType nodeUpdateType) {
- return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
- capability, numContainers, healthReport, lastHealthReportTime,
- nodeLabels, null, null, decommissioningTimeout, nodeUpdateType);
- }
-
- public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
- String httpAddress, String rackName, Resource used, Resource capability,
- int numContainers, String healthReport, long lastHealthReportTime,
+ String httpAddress, String rackName, Resource guaranteedResourceUsed,
+ Resource capability, int numGuaranteedContainers,
+ Resource opportunisticResourceUsed, int numOpportunisticContainers,
+ String healthReport, long lastHealthReportTime,
Set<String> nodeLabels, ResourceUtilization containersUtilization,
ResourceUtilization nodeUtilization, Integer decommissioningTimeout,
NodeUpdateType nodeUpdateType) {
@@ -213,9 +210,11 @@ public class BuilderUtils {
nodeReport.setNodeState(nodeState);
nodeReport.setHttpAddress(httpAddress);
nodeReport.setRackName(rackName);
- nodeReport.setUsed(used);
+ nodeReport.setGuaranteedResourceUsed(guaranteedResourceUsed);
nodeReport.setCapability(capability);
- nodeReport.setNumContainers(numContainers);
+ nodeReport.setNumGuaranteedContainers(numGuaranteedContainers);
+ nodeReport.setOpportunisticResourceUsed(opportunisticResourceUsed);
+ nodeReport.setNumOpportunisticContainers(numOpportunisticContainers);
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 55a3f0b..7b560fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -1027,17 +1027,26 @@ public class ClientRMService extends AbstractService implements
private NodeReport createNodeReports(RMNode rmNode) {
SchedulerNodeReport schedulerNodeReport =
scheduler.getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ guaranteedResourceUsed = schedulerNodeReport.getGuaranteedResourceUsed();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeReport report =
BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed,
+ rmNode.getTotalCapability(), numGuaranteedContainers,
+ opportunisticResourceUsed, numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
rmNode.getNodeUtilization(), rmNode.getDecommissioningTimeout(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 713947f..cea00c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -340,17 +340,27 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
RMNode rmNode = rmNodeEntry.getKey();
SchedulerNodeReport schedulerNodeReport =
getScheduler().getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0, 0);
- int numContainers = 0;
+ Resource guaranteedResourceUsed = BuilderUtils.newResource(0, 0);
+ int numGuaranteedContainers = 0;
+ Resource opportunisticResourceUsed = BuilderUtils.newResource(0, 0);
+ int numOpportunisticContainers = 0;
if (schedulerNodeReport != null) {
- used = schedulerNodeReport.getUsedResource();
- numContainers = schedulerNodeReport.getNumContainers();
+ opportunisticResourceUsed =
+ schedulerNodeReport.getOpportunisticResourceUsed();
+ guaranteedResourceUsed =
+ schedulerNodeReport.getGuaranteedResourceUsed();
+ numGuaranteedContainers =
+ schedulerNodeReport.getNumGuaranteedContainers();
+ numOpportunisticContainers =
+ schedulerNodeReport.getNumOpportunisticContainers();
}
NodeId nodeId = rmNode.getNodeID();
NodeReport report =
BuilderUtils.newNodeReport(nodeId, rmNode.getState(),
- rmNode.getHttpAddress(), rmNode.getRackName(), used,
- rmNode.getTotalCapability(), numContainers,
+ rmNode.getHttpAddress(), rmNode.getRackName(),
+ guaranteedResourceUsed, rmNode.getTotalCapability(),
+ numGuaranteedContainers, opportunisticResourceUsed,
+ numOpportunisticContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
rmNode.getNodeLabels(), rmNode.getDecommissioningTimeout(),
rmNodeEntry.getValue());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
index 1b7ddd3..849db5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMNMInfo.java
@@ -79,27 +79,32 @@ public class RMNMInfo implements RMNMInfoBeans {
List<InfoMap> nodesInfo = new ArrayList<InfoMap>();
for (final RMNode ni : nodes) {
- SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
- InfoMap info = new InfoMap();
- info.put("HostName", ni.getHostName());
- info.put("Rack", ni.getRackName());
- info.put("State", ni.getState().toString());
- info.put("NodeId", ni.getNodeID());
- info.put("NodeHTTPAddress", ni.getHttpAddress());
- info.put("LastHealthUpdate",
- ni.getLastHealthReportTime());
- info.put("HealthReport",
- ni.getHealthReport());
- info.put("NodeManagerVersion",
- ni.getNodeManagerVersion());
- if(report != null) {
- info.put("NumContainers", report.getNumContainers());
- info.put("UsedMemoryMB", report.getUsedResource().getMemorySize());
- info.put("AvailableMemoryMB",
- report.getAvailableResource().getMemorySize());
- }
+ SchedulerNodeReport report = scheduler.getNodeReport(ni.getNodeID());
+ InfoMap info = new InfoMap();
+ info.put("HostName", ni.getHostName());
+ info.put("Rack", ni.getRackName());
+ info.put("State", ni.getState().toString());
+ info.put("NodeId", ni.getNodeID());
+ info.put("NodeHTTPAddress", ni.getHttpAddress());
+ info.put("LastHealthUpdate",
+ ni.getLastHealthReportTime());
+ info.put("HealthReport",
+ ni.getHealthReport());
+ info.put("NodeManagerVersion",
+ ni.getNodeManagerVersion());
+ if(report != null) {
+ info.put("NumContainers", report.getNumGuaranteedContainers());
+ info.put("NumOpportunisticContainers",
+ report.getNumOpportunisticContainers());
+ info.put("UsedMemoryMB",
+ report.getGuaranteedResourceUsed().getMemorySize());
+ info.put("UsedOpportunisticMemoryMB",
+ report.getOpportunisticResourceUsed().getMemorySize());
+ info.put("AvailableMemoryMB",
+ report.getAvailableGuaranteedResource().getMemorySize());
+ }
- nodesInfo.add(info);
+ nodesInfo.add(info);
}
return JSON.toString(nodesInfo);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 3cac38f..9e2781f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -139,7 +139,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
/** Physical resources in the node. */
private volatile Resource physicalResource;
- /* Container Queue Information for the node.. Used by Distributed Scheduler */
+ /* Container Queue Information for the node..*/
private OpportunisticContainersStatus opportunisticContainersStatus;
private final ContainerAllocationExpirer containerAllocationExpirer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
index ea30d78..25621a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNodeReport.java
@@ -28,34 +28,54 @@ import org.apache.hadoop.yarn.api.records.Resource;
@Private
@Stable
public class SchedulerNodeReport {
- private final Resource used;
- private final Resource avail;
- private final int num;
+ private final Resource guaranteedResourceUsage;
+ private final Resource opportunisticResourceUsage;
+ private final Resource guaranteedResourceAvail;
+ private final int numOfGuaranteedContainers;
+ private final int numOfOpportunisticContainers;
public SchedulerNodeReport(SchedulerNode node) {
- this.used = node.getAllocatedResource();
- this.avail = node.getUnallocatedResource();
- this.num = node.getNumGuaranteedContainers();
+ this.guaranteedResourceUsage = node.getAllocatedResource();
+ this.opportunisticResourceUsage = node.getOpportunisticResourceAllocated();
+ this.guaranteedResourceAvail = node.getUnallocatedResource();
+ this.numOfGuaranteedContainers = node.getNumGuaranteedContainers();
+ this.numOfOpportunisticContainers = node.getNumOpportunisticContainers();
}
/**
- * @return the amount of resources currently used by the node.
+ * @return the amount of guaranteed resources currently used by the node.
*/
- public Resource getUsedResource() {
- return used;
+ public Resource getGuaranteedResourceUsed() {
+ return guaranteedResourceUsage;
}
/**
- * @return the amount of resources currently available on the node
+ * @return the amount of opportunistic resources currently used by the node.
*/
- public Resource getAvailableResource() {
- return avail;
+ public Resource getOpportunisticResourceUsed() {
+ return opportunisticResourceUsage;
}
/**
- * @return the number of containers currently running on this node.
+ * @return the amount of guaranteed resources currently available on the node
*/
- public int getNumContainers() {
- return num;
+ public Resource getAvailableGuaranteedResource() {
+ return guaranteedResourceAvail;
+ }
+
+ /**
+ * @return the number of guaranteed containers currently running on
+ * this node.
+ */
+ public int getNumGuaranteedContainers() {
+ return numOfGuaranteedContainers;
+ }
+
+ /**
+ * @return the number of opportunistic containers currently running on
+ * this node.
+ */
+ public int getNumOpportunisticContainers() {
+ return numOfOpportunisticContainers;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
index 1752546..f5d8d07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FifoSchedulerInfo.java
@@ -79,10 +79,13 @@ public class FifoSchedulerInfo extends SchedulerInfo {
for (RMNode ni : rmContext.getRMNodes().values()) {
SchedulerNodeReport report = fs.getNodeReport(ni.getNodeID());
- this.usedNodeCapacity += report.getUsedResource().getMemorySize();
- this.availNodeCapacity += report.getAvailableResource().getMemorySize();
+ this.usedNodeCapacity +=
+ report.getGuaranteedResourceUsed().getMemorySize();
+ this.availNodeCapacity +=
+ report.getAvailableGuaranteedResource().getMemorySize();
this.totalNodeCapacity += ni.getTotalCapability().getMemorySize();
- this.numContainers += fs.getNodeReport(ni.getNodeID()).getNumContainers();
+ this.numContainers +=
+ fs.getNodeReport(ni.getNodeID()).getNumGuaranteedContainers();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 3cec215..2c67dfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -71,14 +71,17 @@ public class NodeInfo {
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
if (report != null) {
- this.numContainers = report.getNumContainers();
- this.usedMemoryMB = report.getUsedResource().getMemorySize();
- this.availMemoryMB = report.getAvailableResource().getMemorySize();
- this.usedVirtualCores = report.getUsedResource().getVirtualCores();
+ this.numContainers = report.getNumGuaranteedContainers();
+ this.usedMemoryMB = report.getGuaranteedResourceUsed().getMemorySize();
+ this.availMemoryMB =
+ report.getAvailableGuaranteedResource().getMemorySize();
+ this.usedVirtualCores =
+ report.getGuaranteedResourceUsed().getVirtualCores();
this.availableVirtualCores =
- report.getAvailableResource().getVirtualCores();
- this.usedResource = new ResourceInfo(report.getUsedResource());
- this.availableResource = new ResourceInfo(report.getAvailableResource());
+ report.getAvailableGuaranteedResource().getVirtualCores();
+ this.usedResource = new ResourceInfo(report.getGuaranteedResourceUsed());
+ this.availableResource =
+ new ResourceInfo(report.getAvailableGuaranteedResource());
}
this.id = id.toString();
this.rack = ni.getRackName();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
index ee974e3..96613fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
@@ -225,10 +225,10 @@ public class NodeManager implements ContainerManagementProtocol {
LOG.info("Checking resource usage for " + containerManagerAddress);
Assert.assertEquals(available.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getAvailableResource().getMemorySize());
+ this.nodeId).getAvailableGuaranteedResource().getMemorySize());
Assert.assertEquals(used.getMemorySize(),
resourceManager.getResourceScheduler().getNodeReport(
- this.nodeId).getUsedResource().getMemorySize());
+ this.nodeId).getGuaranteedResourceUsed().getMemorySize());
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
index cad0151..9c317e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationPriority.java
@@ -176,8 +176,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -189,8 +191,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -210,8 +214,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// send updated request for App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -226,8 +232,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -261,8 +269,10 @@ public class TestApplicationPriority {
// check node report, 8 GB used (1 AM and 7 containers) and 0 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(8 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(8 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 7
Priority appPriority2 = Priority.newInstance(7);
@@ -288,8 +298,10 @@ public class TestApplicationPriority {
// check node report, 1 GB used and 7 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(7 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(7 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
@@ -486,8 +498,10 @@ public class TestApplicationPriority {
// check node report, 15 GB used (1 AM and 7 containers) and 1 GB available
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
- Assert.assertEquals(15 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(1 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(15 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Submit the second app App2 with priority 8 (Higher than App1)
Priority appPriority2 = Priority.newInstance(8);
@@ -499,8 +513,10 @@ public class TestApplicationPriority {
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// get scheduler
CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
@@ -522,8 +538,10 @@ public class TestApplicationPriority {
// check node report, 12 GB used and 4 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(12 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(12 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers App1
am1.allocate("127.0.0.1", 2 * GB, 10, new ArrayList<ContainerId>());
@@ -535,8 +553,10 @@ public class TestApplicationPriority {
Assert.assertEquals(2, allocated2.size());
// check node report, 16 GB used and 0 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(16 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(16 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// kill 1 more
counter = 0;
@@ -552,8 +572,10 @@ public class TestApplicationPriority {
// check node report, 14 GB used and 2 GB available
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(14 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(14 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Change the priority of App1 to 3 (lowest)
Priority appPriority3 = Priority.newInstance(3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 8ab63c1..849a2c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -1267,8 +1267,10 @@ public class TestCapacityScheduler {
SchedulerNodeReport report_nm1 = rm.getResourceScheduler().getNodeReport(
nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1289,8 +1291,10 @@ public class TestCapacityScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1309,7 +1313,7 @@ public class TestCapacityScheduler {
waitCount = 0;
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- if (report_nm1.getAvailableResource().getMemorySize() != 0) {
+ if (report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1318,8 +1322,10 @@ public class TestCapacityScheduler {
}
// Now, the used resource is still 4 GB, and available resource is minus value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource over-commitment.
ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
@@ -1335,9 +1341,11 @@ public class TestCapacityScheduler {
Assert.assertEquals(1, attempt1.getJustFinishedContainers().size());
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses().size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Verify no NPE is trigger in schedule after resource is updated.
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 3 * GB, 1, 1);
@@ -2837,8 +2845,10 @@ public class TestCapacityScheduler {
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report
- Assert.assertEquals(1 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(9 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(9 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 1 * GB, 1, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 02c0cc5..8023457 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -1947,15 +1947,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(5 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(5 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
assertEquals(5 * GB, leafQueue.getMetrics().getAvailableMB());
@@ -2047,15 +2049,17 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(3 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(3 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(7 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
SchedulerNodeReport reportNm2 = rm1.getResourceScheduler()
.getNodeReport(nm2.getNodeId());
- Assert.assertEquals(1 * GB, reportNm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(1 * GB,
+ reportNm2.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(9 * GB,
- reportNm2.getAvailableResource().getMemorySize());
+ reportNm2.getAvailableGuaranteedResource().getMemorySize());
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
double delta = 0.0001;
@@ -2177,9 +2181,10 @@ public class TestNodeLabelContainerAllocation {
SchedulerNodeReport reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(6 * GB, reportNm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
Assert.assertEquals(14 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Try to launch app2 in a2, asked 2GB, should success
// app2 -> a2
@@ -2196,9 +2201,10 @@ public class TestNodeLabelContainerAllocation {
reportNm1 = rm1.getResourceScheduler()
.getNodeReport(nm1.getNodeId());
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a2
cs.killAllAppsInQueue("a2");
@@ -2217,9 +2223,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(4, nm1.getNodeId(),
cs.getApplicationAttempt(am3.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
// Kill all apps in queue a1
cs.killAllAppsInQueue("a1");
@@ -2240,9 +2247,10 @@ public class TestNodeLabelContainerAllocation {
checkNumOfContainersInAnAppOnGivenNode(6, nm1.getNodeId(),
cs.getApplicationAttempt(am4.getApplicationAttemptId()));
- Assert.assertEquals(10 * GB, reportNm1.getUsedResource().getMemorySize());
Assert.assertEquals(10 * GB,
- reportNm1.getAvailableResource().getMemorySize());
+ reportNm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(10 * GB,
+ reportNm1.getAvailableGuaranteedResource().getMemorySize());
rm1.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 63da0c3..0640004 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -697,7 +697,8 @@ public class TestFifoScheduler {
am1.registerAppAttempt();
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
RMApp app2 = rm.submitApp(2048);
// kick the scheduling, 2GB given to AM, remaining 2 GB on nm2
@@ -707,7 +708,8 @@ public class TestFifoScheduler {
am2.registerAppAttempt();
SchedulerNodeReport report_nm2 =
rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, GB, 1, 1);
@@ -743,11 +745,15 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
report_nm2 = rm.getResourceScheduler().getNodeReport(nm2.getNodeId());
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getAvailableGuaranteedResource().getMemorySize());
- Assert.assertEquals(6 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm2.getUsedResource().getMemorySize());
+ Assert.assertEquals(6 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm2.getGuaranteedResourceUsed().getMemorySize());
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemorySize());
@@ -765,7 +771,8 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(5 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ 5 * GB, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -822,7 +829,8 @@ public class TestFifoScheduler {
int checkAlloc =
conf.getInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- Assert.assertEquals(checkAlloc, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(
+ checkAlloc, report_nm1.getGuaranteedResourceUsed().getMemorySize());
rm.stop();
}
@@ -1113,8 +1121,10 @@ public class TestFifoScheduler {
SchedulerNodeReport report_nm1 =
rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 2 GB used and 2 GB available
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// add request for containers
am1.addRequests(new String[] { "127.0.0.1", "127.0.0.2" }, 2 * GB, 1, 1);
@@ -1135,8 +1145,10 @@ public class TestFifoScheduler {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
// check node report, 4 GB used and 0 GB available
- Assert.assertEquals(0, report_nm1.getAvailableResource().getMemorySize());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(0,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// check container is assigned with 2 GB.
Container c1 = allocated1.get(0);
@@ -1155,7 +1167,7 @@ public class TestFifoScheduler {
while (waitCount++ != 20) {
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
if (null != report_nm1 &&
- report_nm1.getAvailableResource().getMemorySize() != 0) {
+ report_nm1.getAvailableGuaranteedResource().getMemorySize() != 0) {
break;
}
LOG.info("Waiting for RMNodeResourceUpdateEvent to be handled... Tried "
@@ -1165,8 +1177,10 @@ public class TestFifoScheduler {
// Now, the used resource is still 4 GB, and available resource is minus
// value.
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(4 * GB, report_nm1.getUsedResource().getMemorySize());
- Assert.assertEquals(-2 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(4 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
+ Assert.assertEquals(-2 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
// Check container can complete successfully in case of resource
// over-commitment.
@@ -1184,9 +1198,11 @@ public class TestFifoScheduler {
Assert.assertEquals(1, am1.schedule().getCompletedContainersStatuses()
.size());
report_nm1 = rm.getResourceScheduler().getNodeReport(nm1.getNodeId());
- Assert.assertEquals(2 * GB, report_nm1.getUsedResource().getMemorySize());
+ Assert.assertEquals(2 * GB,
+ report_nm1.getGuaranteedResourceUsed().getMemorySize());
// As container return 2 GB back, the available resource becomes 0 again.
- Assert.assertEquals(0 * GB, report_nm1.getAvailableResource().getMemorySize());
+ Assert.assertEquals(0 * GB,
+ report_nm1.getAvailableGuaranteedResource().getMemorySize());
rm.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a8c2becd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index fb597fc..f972681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -811,15 +811,16 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
if (report != null) {
assertEquals("numContainers doesn't match: " + numContainers,
- report.getNumContainers(), numContainers);
+ report.getNumGuaranteedContainers(), numContainers);
assertEquals("usedMemoryMB doesn't match: " + usedMemoryMB, report
- .getUsedResource().getMemorySize(), usedMemoryMB);
+ .getGuaranteedResourceUsed().getMemorySize(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
- .getAvailableResource().getMemorySize(), availMemoryMB);
+ .getAvailableGuaranteedResource().getMemorySize(), availMemoryMB);
assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
- .getUsedResource().getVirtualCores(), usedVirtualCores);
- assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
- .getAvailableResource().getVirtualCores(), availVirtualCores);
+ .getGuaranteedResourceUsed().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores,
+ report.getAvailableGuaranteedResource().getVirtualCores(),
+ availVirtualCores);
}
if (opportunisticStatus != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/10] hadoop git commit: YARN-5473. Expose per-application
over-allocation info in the Resource Manager. Contributed by Haibo Chen.
Posted by sz...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
index 3508ab4..ad3413f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestContainerResourceUsage.java
@@ -23,17 +23,12 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
-import java.util.Map;
-import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
@@ -43,6 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerResourceUsageReport;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.log4j.Level;
@@ -84,13 +80,13 @@ public class TestContainerResourceUsage {
RMAppMetrics rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
- "Before app submittion, memory seconds should have been 0 but was "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() == 0);
+ "Before app submission, memory seconds should have been 0 but was "
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() == 0);
Assert.assertTrue(
"Before app submission, vcore seconds should have been 0 but was "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() == 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.getCurrentAppAttempt();
@@ -105,7 +101,8 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (rmAppMetrics.getMemorySeconds() <= 0 && cumulativeSleepTime < 5000) {
+ while (rmAppMetrics.getGuaranteedMemorySeconds() <= 0
+ && cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
@@ -113,27 +110,35 @@ public class TestContainerResourceUsage {
rmAppMetrics = app0.getRMAppMetrics();
Assert.assertTrue(
"While app is running, memory seconds should be >0 but is "
- + rmAppMetrics.getMemorySeconds(),
- rmAppMetrics.getMemorySeconds() > 0);
+ + rmAppMetrics.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds() > 0);
Assert.assertTrue(
"While app is running, vcore seconds should be >0 but is "
- + rmAppMetrics.getVcoreSeconds(),
- rmAppMetrics.getVcoreSeconds() > 0);
+ + rmAppMetrics.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds() > 0);
MockRM.finishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- ru.getMemorySeconds(), rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- ru.getVcoreSeconds(), rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ ru.getGuaranteedMemorySeconds(),
+ rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ ru.getGuaranteedVcoreSeconds(),
+ rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ ru.getOpportunisticMemorySeconds(),
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ ru.getOpportunisticVcoreSeconds(),
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
}
- @Test (timeout = 120000)
+ @Test ()
public void testUsageWithMultipleContainersAndRMRestart() throws Exception {
// Set max attempts to 1 so that when the first attempt fails, the app
// won't try to start a new one.
@@ -191,7 +196,7 @@ public class TestContainerResourceUsage {
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
- while (app0.getRMAppMetrics().getMemorySeconds() <= 0
+ while (app0.getRMAppMetrics().getGuaranteedMemorySeconds() <= 0
&& cumulativeSleepTime < 5000) {
Thread.sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
@@ -216,19 +221,29 @@ public class TestContainerResourceUsage {
rm0.waitForState(nm, cId, RMContainerState.COMPLETED);
// Check that the container metrics match those from the app usage report.
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
for (RMContainer c : rmContainers) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics metricsBefore = app0.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, metricsBefore.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, metricsBefore.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, metricsBefore.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, metricsBefore.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ metricsBefore.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ metricsBefore.getOpportunisticVcoreSeconds());
// create new RM to represent RM restart. Load up the state store.
MockRM rm1 = new MockRM(conf, memStore);
@@ -238,10 +253,22 @@ public class TestContainerResourceUsage {
// Compare container resource usage metrics from before and after restart.
RMAppMetrics metricsAfter = app0After.getRMAppMetrics();
- Assert.assertEquals("Vcore seconds were not the same after RM Restart",
- metricsBefore.getVcoreSeconds(), metricsAfter.getVcoreSeconds());
- Assert.assertEquals("Memory seconds were not the same after RM Restart",
- metricsBefore.getMemorySeconds(), metricsAfter.getMemorySeconds());
+ Assert.assertEquals(
+ "Guaranteed vcore seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedVcoreSeconds(),
+ metricsAfter.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(
+ "Guaranteed memory seconds were not the same after RM Restart",
+ metricsBefore.getGuaranteedMemorySeconds(),
+ metricsAfter.getGuaranteedMemorySeconds());
+ Assert.assertEquals(
+ "Opportunistic vcore seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticVcoreSeconds(),
+ metricsAfter.getOpportunisticVcoreSeconds());
+ Assert.assertEquals(
+ "Opportunistic memory seconds were not the same after RM Restart",
+ metricsBefore.getOpportunisticMemorySeconds(),
+ metricsAfter.getOpportunisticMemorySeconds());
rm0.stop();
rm0.close();
@@ -264,10 +291,8 @@ public class TestContainerResourceUsage {
MockRM rm = new MockRM(conf);
rm.start();
- RMApp app =
- rm.submitApp(200, "name", "user",
- new HashMap<ApplicationAccessType, String>(), false, "default", -1,
- null, "MAPREDUCE", false, keepRunningContainers);
+ RMApp app = rm.submitApp(200, "name", "user", new HashMap<>(), false,
+ "default", -1, null, "MAPREDUCE", false, keepRunningContainers);
MockNM nm =
new MockNM("127.0.0.1:1234", 10240, rm.getResourceTrackerService());
nm.registerNode();
@@ -275,18 +300,17 @@ public class TestContainerResourceUsage {
MockAM am0 = MockRM.launchAndRegisterAM(app, rm, nm);
int NUM_CONTAINERS = 1;
// allocate NUM_CONTAINERS containers
- am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am0.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
List<Container> containers =
- am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ am0.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
- containers.addAll(am0.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers());
+ containers.addAll(am0.allocate(new ArrayList<>(),
+ new ArrayList<>()).getAllocatedContainers());
Thread.sleep(200);
}
@@ -294,26 +318,27 @@ public class TestContainerResourceUsage {
ContainerId containerId2 =
ContainerId.newContainerId(am0.getApplicationAttemptId(), 2);
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- containerId2.getContainerId(), ContainerState.RUNNING);
+ containerId2.getContainerId(), ContainerState.RUNNING);
rm.waitForState(nm, containerId2, RMContainerState.RUNNING);
// Capture the containers here so the metrics can be calculated after the
// app has completed.
- Collection<RMContainer> rmContainers =
- rm.scheduler
- .getSchedulerAppInfo(am0.getApplicationAttemptId())
- .getLiveContainers();
+ Collection<RMContainer> rmContainers = rm.scheduler
+ .getSchedulerAppInfo(am0.getApplicationAttemptId())
+ .getLiveContainers();
// fail the first app attempt by sending CONTAINER_FINISHED event without
// registering.
ContainerId amContainerId =
app.getCurrentAppAttempt().getMasterContainer().getId();
nm.nodeHeartbeat(am0.getApplicationAttemptId(),
- amContainerId.getContainerId(), ContainerState.COMPLETE);
+ amContainerId.getContainerId(), ContainerState.COMPLETE);
rm.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.FAILED);
rm.drainEvents();
- long memorySeconds = 0;
- long vcoreSeconds = 0;
+ long guaranteedMemorySeconds = 0;
+ long guaranteedVcoreSeconds = 0;
+ long opportunisticMemorySeconds = 0;
+ long opportunisticVcoreSeconds = 0;
// Calculate container usage metrics for first attempt.
if (keepRunningContainers) {
@@ -321,8 +346,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
if (c.getContainerId().equals(amContainerId)) {
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
} else {
// The remaining container should be RUNNING.
Assert.assertTrue("After first attempt failed, remaining container "
@@ -336,8 +363,10 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
}
@@ -355,14 +384,12 @@ public class TestContainerResourceUsage {
am1.registerAppAttempt();
rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
// allocate NUM_CONTAINERS containers
- am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS,
- new ArrayList<ContainerId>());
+ am1.allocate("127.0.0.1", 1024, NUM_CONTAINERS, new ArrayList<>());
nm.nodeHeartbeat(true);
// wait for containers to be allocated.
- containers =
- am1.allocate(new ArrayList<ResourceRequest>(),
- new ArrayList<ContainerId>()).getAllocatedContainers();
+ containers = am1.allocate(new ArrayList<>(), new ArrayList<>())
+ .getAllocatedContainers();
while (containers.size() != NUM_CONTAINERS) {
nm.nodeHeartbeat(true);
containers.addAll(am1.allocate(new ArrayList<ResourceRequest>(),
@@ -388,19 +415,26 @@ public class TestContainerResourceUsage {
for (RMContainer c : rmContainers) {
waitforContainerCompletion(rm, nm, amContainerId, c);
AggregateAppResourceUsage ru = calculateContainerResourceMetrics(c);
- memorySeconds += ru.getMemorySeconds();
- vcoreSeconds += ru.getVcoreSeconds();
+ guaranteedMemorySeconds += ru.getGuaranteedMemorySeconds();
+ guaranteedVcoreSeconds += ru.getGuaranteedVcoreSeconds();
+ opportunisticMemorySeconds += ru.getOpportunisticMemorySeconds();
+ opportunisticVcoreSeconds += ru.getOpportunisticVcoreSeconds();
}
RMAppMetrics rmAppMetrics = app.getRMAppMetrics();
- Assert.assertEquals("Unexpected MemorySeconds value",
- memorySeconds, rmAppMetrics.getMemorySeconds());
- Assert.assertEquals("Unexpected VcoreSeconds value",
- vcoreSeconds, rmAppMetrics.getVcoreSeconds());
+ Assert.assertEquals("Unexpected GuaranteedMemorySeconds value",
+ guaranteedMemorySeconds, rmAppMetrics.getGuaranteedMemorySeconds());
+ Assert.assertEquals("Unexpected GuaranteedVcoreSeconds value",
+ guaranteedVcoreSeconds, rmAppMetrics.getGuaranteedVcoreSeconds());
+ Assert.assertEquals("Unexpected OpportunisticMemorySeconds value",
+ opportunisticMemorySeconds,
+ rmAppMetrics.getOpportunisticMemorySeconds());
+ Assert.assertEquals("Unexpected OpportunisticVcoreSeconds value",
+ opportunisticVcoreSeconds,
+ rmAppMetrics.getOpportunisticVcoreSeconds());
rm.stop();
- return;
}
private void waitforContainerCompletion(MockRM rm, MockNM nm,
@@ -419,16 +453,10 @@ public class TestContainerResourceUsage {
private AggregateAppResourceUsage calculateContainerResourceMetrics(
RMContainer rmContainer) {
- Resource resource = rmContainer.getContainer().getResource();
- long usedMillis =
- rmContainer.getFinishTime() - rmContainer.getCreationTime();
- long memorySeconds = resource.getMemorySize()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- long vcoreSeconds = resource.getVirtualCores()
- * usedMillis / DateUtils.MILLIS_PER_SECOND;
- Map<String, Long> map = new HashMap<>();
- map.put(ResourceInformation.MEMORY_MB.getName(), memorySeconds);
- map.put(ResourceInformation.VCORES.getName(), vcoreSeconds);
- return new AggregateAppResourceUsage(map);
+ ContainerResourceUsageReport resourceUsageReport =
+ rmContainer.getResourceUsageReport();
+ return new AggregateAppResourceUsage(
+ resourceUsageReport.getGuaranteedResourceUsageSecondsMap(),
+ resourceUsageReport.getOpportunisticResourceSecondsMap());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 72de27c..3c2db3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -193,7 +193,7 @@ public abstract class MockAsm extends MockApps {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
}
@Override
@@ -353,7 +353,7 @@ public abstract class MockAsm extends MockApps {
ApplicationResourceUsageReport usageReport =
ApplicationResourceUsageReport
.newInstance(0, 0, null, null, null, new HashMap<>(), 0, 0,
- new HashMap<>());
+ new HashMap<>(), null, new HashMap<>());
ApplicationReport report = ApplicationReport.newInstance(
getApplicationId(), appAttemptId, getUser(), getQueue(),
getName(), null, 0, null, null, getDiagnostics().toString(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
index 830d01c..d561cab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestCombinedSystemMetricsPublisher.java
@@ -453,7 +453,7 @@ public class TestCombinedSystemMetricsPublisher {
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(rmApp.getRMAppMetrics()).thenReturn(
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceMap,
- preemptedMap));
+ preemptedMap, new HashMap<>()));
when(rmApp.getApplicationTags()).thenReturn(
Collections.<String> emptySet());
ApplicationSubmissionContext appSubmissionContext =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 2287617..c9e2d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -219,13 +219,25 @@ public class TestSystemMetricsPublisher {
entity.getOtherInfo().get(
ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO));
Assert.assertEquals(
- app.getRMAppMetrics().getMemorySeconds(),
+ app.getRMAppMetrics().getGuaranteedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_MEM_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getGuaranteedVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_GUARANTEED_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
- app.getRMAppMetrics().getVcoreSeconds(),
+ app.getRMAppMetrics().getOpportunisticMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
- .get(ApplicationMetricsConstants.APP_CPU_METRICS).toString()));
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_MEM_METRICS)
+ .toString()));
+ Assert.assertEquals(
+ app.getRMAppMetrics().getOpportunisticVcoreSeconds(),
+ Long.parseLong(entity.getOtherInfo()
+ .get(ApplicationMetricsConstants.APP_OPPORTUNISTIC_CPU_METRICS)
+ .toString()));
Assert.assertEquals(
app.getRMAppMetrics().getPreemptedMemorySeconds(),
Long.parseLong(entity.getOtherInfo()
@@ -517,7 +529,8 @@ public class TestSystemMetricsPublisher {
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
preemptedMap.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics())
- .thenReturn(new RMAppMetrics(null, 0, 0, resourceMap, preemptedMap));
+ .thenReturn(new RMAppMetrics(null, 0, 0,
+ resourceMap, preemptedMap, new HashMap<>()));
Set<String> appTags = new HashSet<String>();
appTags.add("test");
appTags.add("tags");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 68bb325..7aa1a60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -221,7 +221,7 @@ public class TestSystemMetricsPublisherForV2 {
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
verifyEntity(
- appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 8, 0);
+ appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE, 10, 0);
}
@Test(timeout = 10000)
@@ -365,14 +365,14 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
when(app.getFinalApplicationStatus())
.thenReturn(FinalApplicationStatus.UNDEFINED);
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- resourceSecondsMap
+ Map<String, Long> guaranteedResourceSecondsMap = new HashMap<>();
+ guaranteedResourceSecondsMap
.put(ResourceInformation.MEMORY_MB.getName(), (long) Integer.MAX_VALUE);
- resourceSecondsMap
+ guaranteedResourceSecondsMap
.put(ResourceInformation.VCORES.getName(), Long.MAX_VALUE);
when(app.getRMAppMetrics()).thenReturn(
- new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, resourceSecondsMap,
- new HashMap<>()));
+ new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
+ guaranteedResourceSecondsMap, new HashMap<>(), new HashMap<>()));
when(app.getApplicationTags()).thenReturn(Collections.<String>emptySet());
ApplicationSubmissionContext appSubmissionContext =
mock(ApplicationSubmissionContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
index 453d805..98199a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
@@ -194,7 +194,8 @@ public class RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
dispatcher.attemptId = attemptId;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
@@ -292,7 +293,8 @@ public class RMStateStoreTestBase {
when(mockRemovedAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(
+ new HashMap<>(), new HashMap<>()));
attempts.put(attemptIdRemoved, mockRemovedAttempt);
store.removeApplication(mockRemovedApp);
@@ -369,7 +371,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 100,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
// test updating the state of an app/attempt whose initial state was not
@@ -393,7 +396,8 @@ public class RMStateStoreTestBase {
oldAttemptState.getStartTime(), RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics",
FinalApplicationStatus.SUCCEEDED, 111,
- oldAttemptState.getFinishTime(), new HashMap<>(), new HashMap<>());
+ oldAttemptState.getFinishTime(), new HashMap<>(),
+ new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(dummyAttempt);
// let things settle down
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
index 6a8f47d..4d2c2da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
@@ -543,7 +543,8 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
when(mockAttempt.getRMAppAttemptMetrics())
.thenReturn(mockRmAppAttemptMetrics);
when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
- .thenReturn(new AggregateAppResourceUsage(new HashMap<>()));
+ .thenReturn(new AggregateAppResourceUsage(new HashMap<>(),
+ new HashMap<>(0)));
store.storeNewApplicationAttempt(mockAttempt);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -555,7 +556,7 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
store.getCredentialsFromAppAttempt(mockAttempt),
startTime, RMAppAttemptState.FINISHED, "testUrl",
"test", FinalApplicationStatus.SUCCEEDED, 100,
- finishTime, new HashMap<>(), new HashMap<>());
+ finishTime, new HashMap<>(), new HashMap<>(), new HashMap<>());
store.updateApplicationAttemptState(newAttemptState);
assertEquals("RMStateStore should have been in fenced state",
true, store.isFencedState());
@@ -792,20 +793,33 @@ public class TestZKRMStateStore extends RMStateStoreTestBase {
private static ApplicationAttemptStateData createFinishedAttempt(
ApplicationAttemptId attemptId, Container container, long startTime,
int amExitStatus) {
- Map<String, Long> resourceSecondsMap = new HashMap<>();
- Map<String, Long> preemptedResoureSecondsMap = new HashMap<>();
- resourceSecondsMap
- .put(ResourceInformation.MEMORY_MB.getName(), 0L);
- resourceSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
- preemptedResoureSecondsMap.put(ResourceInformation.MEMORY_MB.getName(),
- 0L);
- preemptedResoureSecondsMap
- .put(ResourceInformation.VCORES.getName(), 0L);
+ Map<String, Long> guaranteedResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> opportunisticResourceSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+ Map<String, Long> preemptedResoureSecondsMap =
+ new HashMap<String, Long>() {
+ {
+ put(ResourceInformation.MEMORY_MB.getName(), 0L);
+ put(ResourceInformation.VCORES.getName(), 0L);
+ }
+ };
+
return ApplicationAttemptStateData.newInstance(attemptId,
container, null, startTime, RMAppAttemptState.FINISHED,
"myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.SUCCEEDED,
- amExitStatus, 0, resourceSecondsMap, preemptedResoureSecondsMap);
+ amExitStatus, 0, guaranteedResourceSecondsMap,
+ preemptedResoureSecondsMap, opportunisticResourceSecondsMap);
}
private ApplicationAttemptId storeAttempt(RMStateStore store,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 920bd4f..dde74a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -243,7 +242,7 @@ public class TestRMAppTransitions {
ResourceScheduler resourceScheduler = mock(ResourceScheduler.class);
doReturn(null).when(resourceScheduler)
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
+ .getAppActiveResourceUsageReport(Matchers.any());
doReturn(resourceScheduler).when(rmContext).getScheduler();
doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
@@ -1249,7 +1248,8 @@ public class TestRMAppTransitions {
assertAppState(RMAppState.NEW, app);
ApplicationReport report = app.createAndGetApplicationReport(null, true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
- Assert.assertEquals(report.getApplicationResourceUsageReport(),RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT);
+ Assert.assertEquals(RMServerUtils.DUMMY_APPLICATION_RESOURCE_USAGE_REPORT,
+ report.getApplicationResourceUsageReport());
report = app.createAndGetApplicationReport("clientuser", true);
Assert.assertNotNull(report.getApplicationResourceUsageReport());
Assert.assertTrue("bad proxy url for app",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 0e4f308..d889158 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -299,10 +299,12 @@ public class TestRMAppAttemptTransitions {
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(0L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(0L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(0L);
when(resourceScheduler
- .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any()))
+ .getAppActiveResourceUsageReport((ApplicationAttemptId)Matchers.any()))
.thenReturn(appResUsgRpt);
spyRMContext = spy(rmContext);
Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler();
@@ -762,10 +764,12 @@ public class TestRMAppAttemptTransitions {
ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
ApplicationResourceUsageReport appResUsgRpt =
mock(ApplicationResourceUsageReport.class);
- when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
- when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
- .thenReturn(appResUsgRpt);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(55544L);
+ when(appResUsgRpt.getOpportunisticMemorySeconds()).thenReturn(123456L);
+ when(appResUsgRpt.getOpportunisticVcoreSeconds()).thenReturn(55544L);
+ when(scheduler.getAppActiveResourceUsageReport(
+ any(ApplicationAttemptId.class))).thenReturn(appResUsgRpt);
// start and finish the attempt
Container amContainer = allocateApplicationAttempt();
@@ -777,12 +781,14 @@ public class TestRMAppAttemptTransitions {
// expect usage stats to come from the scheduler report
ApplicationResourceUsageReport report =
applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(123456L, report.getMemorySeconds());
- Assert.assertEquals(55544L, report.getVcoreSeconds());
+ Assert.assertEquals(123456L, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(55544L, report.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(123456L, report.getOpportunisticMemorySeconds());
+ Assert.assertEquals(55544L, report.getOpportunisticVcoreSeconds());
// finish app attempt and remove it from scheduler
- when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
- when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
+ when(appResUsgRpt.getGuaranteedMemorySeconds()).thenReturn(223456L);
+ when(appResUsgRpt.getGuaranteedVcoreSeconds()).thenReturn(75544L);
sendAttemptUpdateSavedEvent(applicationAttempt);
NodeId anyNodeId = NodeId.newInstance("host", 1234);
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
@@ -793,8 +799,8 @@ public class TestRMAppAttemptTransitions {
when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);
report = applicationAttempt.getApplicationResourceUsageReport();
- Assert.assertEquals(223456, report.getMemorySeconds());
- Assert.assertEquals(75544, report.getVcoreSeconds());
+ Assert.assertEquals(223456, report.getGuaranteedMemorySeconds());
+ Assert.assertEquals(75544, report.getGuaranteedVcoreSeconds());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 849a2c3..7748b1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -103,7 +103,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMW
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.placement.UserGroupMappingPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
@@ -2859,7 +2858,7 @@ public class TestCapacityScheduler {
Thread.sleep(1000);
AllocateResponse allocResponse = am1.schedule();
ApplicationResourceUsageReport report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(0, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
@@ -2872,7 +2871,7 @@ public class TestCapacityScheduler {
allocResponse = am1.schedule();
}
report =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
attempt1.getAppAttemptId());
Assert.assertEquals(1, allocResponse.getAllocatedContainers().size());
Assert.assertEquals(0, report.getNumReservedContainers());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index b4ebd15..a3ae7bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -223,23 +223,23 @@ public class TestCapacitySchedulerNodeLabelUpdate {
RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
ApplicationResourceUsageReport appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(1024, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(1, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(1024, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(1, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
// request a container.
am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED);
appResourceUsageReport =
- rm.getResourceScheduler().getAppResourceUsageReport(
+ rm.getResourceScheduler().getAppActiveResourceUsageReport(
am1.getApplicationAttemptId());
- Assert.assertEquals(2048, appResourceUsageReport.getUsedResources()
- .getMemorySize());
- Assert.assertEquals(2, appResourceUsageReport.getUsedResources()
- .getVirtualCores());
+ Assert.assertEquals(2048, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getMemorySize());
+ Assert.assertEquals(2, appResourceUsageReport
+ .getGuaranteedResourcesUsed().getVirtualCores());
LeafQueue queue =
(LeafQueue) ((CapacityScheduler) rm.getResourceScheduler())
.getQueue("a");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
index f88ac8b..de97ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestAppPage.java
@@ -66,7 +66,7 @@ public class TestAppPage {
RMAppMetrics appMetrics =
new RMAppMetrics(Resource.newInstance(0, 0), 0, 0, new HashMap<>(),
- new HashMap<>());
+ new HashMap<>(), new HashMap<>());
when(app.getRMAppMetrics()).thenReturn(appMetrics);
// initialize RM Context, and create RMApp, without creating RMAppAttempt
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index b125608..93efe23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Token;
@@ -259,7 +258,7 @@ public class TestRMWebApp {
app.getDiagnostics().toString(), (String) null,
app.getStartTime(), app.getFinishTime(),
app.getFinalApplicationStatus(),
- (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
+ null, app.getTrackingUrl(),
app.getProgress(), app.getApplicationType(), (Token) null);
appReports.add(appReport);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
index cea0088..87846c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebAppFairScheduler.java
@@ -138,7 +138,7 @@ public class TestRMWebAppFairScheduler {
@Override
public RMAppMetrics getRMAppMetrics() {
return new RMAppMetrics(Resource.newInstance(0, 0), 0, 0,
- new HashMap<>(), new HashMap<>());
+ new HashMap<>(), new HashMap<>(), new HashMap<>());
}
@Override
public YarnApplicationState createApplicationState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index a381ed4..754d8b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1603,7 +1603,7 @@ public class TestRMWebServicesApps extends JerseyTestBase {
public void verifyAppInfo(JSONObject info, RMApp app, boolean hasResourceReqs)
throws JSONException, Exception {
- int expectedNumberOfElements = 39 + (hasResourceReqs ? 2 : 0);
+ int expectedNumberOfElements = 44 + (hasResourceReqs ? 2 : 0);
String appNodeLabelExpression = null;
String amNodeLabelExpression = null;
if (app.getApplicationSubmissionContext()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
index 40bdbd8..6f1ee3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/RouterWebServiceUtil.java
@@ -339,12 +339,22 @@ public final class RouterWebServiceUtil {
am.setAllocatedMB(am.getAllocatedMB() + uam.getAllocatedMB());
am.setAllocatedVCores(am.getAllocatedVCores() + uam.getAllocatedVCores());
+ am.setAllocatedOpportunisticMB(am.getAllocatedOpportunisticMB() +
+ uam.getAllocatedOpportunisticMB());
+ am.setAllocatedOpportunisticVCores(am.getAllocatedOpportunisticVCores() +
+ uam.getAllocatedOpportunisticVCores());
am.setReservedMB(am.getReservedMB() + uam.getReservedMB());
am.setReservedVCores(am.getReservedVCores() + uam.getReservedMB());
am.setRunningContainers(
am.getRunningContainers() + uam.getRunningContainers());
- am.setMemorySeconds(am.getMemorySeconds() + uam.getMemorySeconds());
- am.setVcoreSeconds(am.getVcoreSeconds() + uam.getVcoreSeconds());
+ am.setGuaranteedMemorySeconds(am.getGuaranteedMemorySeconds() +
+ uam.getGuaranteedMemorySeconds());
+ am.setGuaranteedVcoreSeconds(am.getGuaranteedVcoreSeconds() +
+ uam.getGuaranteedVcoreSeconds());
+ am.setOpportunisticVcoreSeconds(am.getOpportunisticVcoreSeconds() +
+ uam.getOpportunisticVcoreSeconds());
+ am.setOpportunisticMemorySeconds(am.getOpportunisticMemorySeconds() +
+ uam.getOpportunisticMemorySeconds());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29de8de8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
index edf3804..1a5c42e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestRouterWebServiceUtil.java
@@ -249,8 +249,10 @@ public class TestRouterWebServiceUtil {
Assert.assertEquals(value * 3, app.getReservedMB());
Assert.assertEquals(value * 3, app.getReservedVCores());
Assert.assertEquals(value * 3, app.getRunningContainers());
- Assert.assertEquals(value * 3, app.getMemorySeconds());
- Assert.assertEquals(value * 3, app.getVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedMemorySeconds());
+ Assert.assertEquals(value * 3, app.getGuaranteedVcoreSeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticMemorySeconds());
+ Assert.assertEquals(value * 3, app.getOpportunisticVcoreSeconds());
Assert.assertEquals(3, app.getResourceRequests().size());
}
@@ -258,11 +260,15 @@ public class TestRouterWebServiceUtil {
am.getResourceRequests().add(new ResourceRequestInfo());
am.setAllocatedMB(value);
am.setAllocatedVCores(value);
+ am.setAllocatedOpportunisticVCores(value);
+ am.setAllocatedOpportunisticMB(value);
am.setReservedMB(value);
am.setReservedVCores(value);
am.setRunningContainers(value);
- am.setMemorySeconds(value);
- am.setVcoreSeconds(value);
+ am.setGuaranteedMemorySeconds(value);
+ am.setGuaranteedVcoreSeconds(value);
+ am.setOpportunisticMemorySeconds(value);
+ am.setOpportunisticVcoreSeconds(value);
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/10] hadoop git commit: MAPREDUCE-6926. Allow MR jobs to opt out
of oversubscription. Contributed by Haibo Chen.
Posted by sz...@apache.org.
MAPREDUCE-6926. Allow MR jobs to opt out of oversubscription. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83f03dc5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83f03dc5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83f03dc5
Branch: refs/heads/YARN-1011
Commit: 83f03dc50852f69fe479b77846b6899eaa92d9b5
Parents: c483d49
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Jan 10 13:21:11 2018 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:35:01 2018 -0700
----------------------------------------------------------------------
.../v2/app/rm/RMContainerRequestor.java | 48 ++---
.../v2/app/rm/TestRMContainerAllocator.java | 192 +++++++++++++++++++
.../apache/hadoop/mapreduce/MRJobConfig.java | 6 +
.../src/main/resources/mapred-default.xml | 8 +
4 files changed, 231 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f03dc5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
index bb3e1fa..d996690 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerRequestor.java
@@ -111,6 +111,7 @@ public abstract class RMContainerRequestor extends RMCommunicator {
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
private final Set<String> blacklistRemovals = Collections
.newSetFromMap(new ConcurrentHashMap<String, Boolean>());
+ private boolean optOutOfOversubscription;
public RMContainerRequestor(ClientService clientService, AppContext context) {
super(clientService, context);
@@ -136,9 +137,11 @@ public abstract class RMContainerRequestor extends RMCommunicator {
public ContainerRequest(ContainerRequestEvent event, Priority priority,
String nodeLabelExpression) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
- event.getRacks(), priority, nodeLabelExpression);
+ event.getRacks(), priority, System.currentTimeMillis(),
+ nodeLabelExpression);
}
+ @VisibleForTesting
public ContainerRequest(ContainerRequestEvent event, Priority priority,
long requestTimeMs) {
this(event.getAttemptID(), event.getCapability(), event.getHosts(),
@@ -146,13 +149,6 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
public ContainerRequest(TaskAttemptId attemptID,
- Resource capability, String[] hosts, String[] racks,
- Priority priority, String nodeLabelExpression) {
- this(attemptID, capability, hosts, racks, priority,
- System.currentTimeMillis(), nodeLabelExpression);
- }
-
- public ContainerRequest(TaskAttemptId attemptID,
Resource capability, String[] hosts, String[] racks,
Priority priority, long requestTimeMs,String nodeLabelExpression) {
this.attemptID = attemptID;
@@ -186,6 +182,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
MRJobConfig.MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERECENT,
MRJobConfig.DEFAULT_MR_AM_IGNORE_BLACKLISTING_BLACKLISTED_NODE_PERCENT);
LOG.info("maxTaskFailuresPerNode is " + maxTaskFailuresPerNode);
+ optOutOfOversubscription = conf.getBoolean(
+ MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ MRJobConfig.DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT);
+ LOG.info("optOutOfOversubscription is " + optOutOfOversubscription);
if (blacklistDisablePercent < -1 || blacklistDisablePercent > 100) {
throw new YarnRuntimeException("Invalid blacklistDisablePercent: "
+ blacklistDisablePercent
@@ -398,20 +398,20 @@ public abstract class RMContainerRequestor extends RMCommunicator {
for (String host : req.hosts) {
// Data-local
if (!isNodeBlacklisted(host)) {
- addResourceRequest(req.priority, host, req.capability,
+ addGuaranteedResourceRequest(req.priority, host, req.capability,
null);
}
}
// Nothing Rack-local for now
for (String rack : req.racks) {
- addResourceRequest(req.priority, rack, req.capability,
+ addGuaranteedResourceRequest(req.priority, rack, req.capability,
null);
}
// Off-switch
- addResourceRequest(req.priority, ResourceRequest.ANY, req.capability,
- req.nodeLabelExpression);
+ addGuaranteedResourceRequest(req.priority, ResourceRequest.ANY,
+ req.capability, req.nodeLabelExpression);
}
protected void decContainerReq(ContainerRequest req) {
@@ -430,18 +430,18 @@ public abstract class RMContainerRequestor extends RMCommunicator {
protected void addOpportunisticResourceRequest(Priority priority,
Resource capability) {
addResourceRequest(priority, ResourceRequest.ANY, capability, null,
- ExecutionType.OPPORTUNISTIC);
+ ExecutionType.OPPORTUNISTIC, true);
}
- private void addResourceRequest(Priority priority, String resourceName,
- Resource capability, String nodeLabelExpression) {
+ private void addGuaranteedResourceRequest(Priority priority,
+ String resourceName, Resource capability, String nodeLabelExpression) {
addResourceRequest(priority, resourceName, capability, nodeLabelExpression,
- ExecutionType.GUARANTEED);
+ ExecutionType.GUARANTEED, optOutOfOversubscription);
}
private void addResourceRequest(Priority priority, String resourceName,
Resource capability, String nodeLabelExpression,
- ExecutionType executionType) {
+ ExecutionType executionType, boolean enforceExecutionType) {
Map<String, Map<Resource, ResourceRequest>> remoteRequests =
this.remoteRequestsTable.get(priority);
if (remoteRequests == null) {
@@ -464,8 +464,8 @@ public abstract class RMContainerRequestor extends RMCommunicator {
remoteRequest.setCapability(capability);
remoteRequest.setNumContainers(0);
remoteRequest.setNodeLabelExpression(nodeLabelExpression);
- remoteRequest.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(executionType, true));
+ remoteRequest.setExecutionTypeRequest(ExecutionTypeRequest.
+ newInstance(executionType, enforceExecutionType));
reqMap.put(capability, remoteRequest);
}
remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);
@@ -473,9 +473,10 @@ public abstract class RMContainerRequestor extends RMCommunicator {
// Note this down for next interaction with ResourceManager
addResourceRequestToAsk(remoteRequest);
if (LOG.isDebugEnabled()) {
- LOG.debug("addResourceRequest:" + " applicationId="
+ LOG.debug("addGuaranteedResourceRequest:" + " applicationId="
+ applicationId.getId() + " priority=" + priority.getPriority()
- + " resourceName=" + resourceName + " numContainers="
+ + " resourceName=" + resourceName + " ExecutionType=" + executionType
+ + " enforceExecutionType=" + enforceExecutionType + " numContainers="
+ remoteRequest.getNumContainers() + " #asks=" + ask.size());
}
}
@@ -559,8 +560,9 @@ public abstract class RMContainerRequestor extends RMCommunicator {
}
}
String[] hosts = newHosts.toArray(new String[newHosts.size()]);
- ContainerRequest newReq = new ContainerRequest(orig.attemptID, orig.capability,
- hosts, orig.racks, orig.priority, orig.nodeLabelExpression);
+ ContainerRequest newReq = new ContainerRequest(orig.attemptID,
+ orig.capability, hosts, orig.racks, orig.priority,
+ System.currentTimeMillis(), orig.nodeLabelExpression);
return newReq;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f03dc5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 85e4181..d95793d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -105,6 +105,8 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -844,6 +846,196 @@ public class TestRMContainerAllocator {
allocator.close();
}
+ /**
+ * Test A MapReduce job can be configured to opt out of oversubscription,
+ * that is, it always wait for guaranteed resources to execute its tasks.
+ * This is done by setting up a MapReduce job with 2 mappers and 1 reducers
+ * and capturing all ResourceRequests sent from the AM to RM, then checking
+ * if all ResourceRequests are guaranteed and their enforceExecutionType is
+ * true.
+ */
+ @Test
+ public void testMapReduceOptingOutOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(true);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or not enforced.");
+ }
+ }
+ }
+
+ /**
+ * Test a MapReduce job can be configured to opt in oversubscription (
+ * true by default). This is done by setting up a MapReduce job with 2
+ * mappers and 1 reducers and capturing all ResourceRequests sent from
+ * the AM to RM, then checking if all ResourceRequests are guaranteed
+ * but their enforceExecutionType is always set to false.
+ */
+ @Test
+ public void testMapReduceOptingInOversubscription() throws Exception {
+ List<ResourceRequest> resourceRequests = captureResourceRequests(false);
+
+ for(ResourceRequest resourceRequest : resourceRequests) {
+ ExecutionTypeRequest executionTypeRequest =
+ resourceRequest.getExecutionTypeRequest();
+ if (!executionTypeRequest.equals(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, false))) {
+ Assert.fail("The execution type of ResourceRequest " + resourceRequest +
+ " is not guaranteed or it is enforced.");
+ }
+ }
+ }
+
+ /**
+ * Set up a mapreduce job with 2 mappers and 1 reducer and return
+ * all ResourceRequests sent from the AM to RM.
+ */
+ private List<ResourceRequest> captureResourceRequests(
+ boolean optOutOfOversubscription) throws Exception {
+ List<ResourceRequest> resourceRequests = new ArrayList<>();
+
+ final Configuration conf = new Configuration();
+ conf.setBoolean(MRJobConfig.MR_OVERSUBSCRIPTION_OPT_OUT,
+ optOutOfOversubscription);
+
+ // start the resource manager
+ final MyResourceManager rm = new MyResourceManager(conf);
+ rm.start();
+ DrainDispatcher rmDispatcher = (DrainDispatcher) rm.getRMContext()
+ .getDispatcher();
+
+ // submit an application
+ RMApp rmApp = rm.submitApp(1024);
+ rm.drainEvents();
+
+ MockNM amNodeManager = rm.registerNode("amNM:1234", 11264);
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ final ApplicationAttemptId appAttemptId = rmApp.getCurrentAppAttempt()
+ .getAppAttemptId();
+ rm.sendAMLaunched(appAttemptId);
+ rm.drainEvents();
+
+ // start the MR AM and wait until it is in running state
+ MRApp mrApp = new MRApp(appAttemptId, ContainerId.newContainerId(
+ appAttemptId, 0), 2, 1, false,
+ this.getClass().getName(), true, 1) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return new DrainDispatcher();
+ }
+ protected ContainerAllocator createContainerAllocator(
+ ClientService clientService, AppContext context) {
+ return new MyContainerAllocator(rm, appAttemptId, context);
+ };
+ };
+ mrApp.submit(conf);
+
+ Job job = mrApp.getContext().getAllJobs().entrySet().iterator().next()
+ .getValue();
+ DrainDispatcher amDispatcher = (DrainDispatcher) mrApp.getDispatcher();
+ MyContainerAllocator allocator = (MyContainerAllocator) mrApp
+ .getContainerAllocator();
+ mrApp.waitForInternalState((JobImpl)job, JobStateInternal.RUNNING);
+ amDispatcher.await();
+
+ // wait until all attempts request for containers
+ for (Task t : job.getTasks().values()) {
+ mrApp.waitForInternalState((TaskAttemptImpl) t.getAttempts().values()
+ .iterator().next(), TaskAttemptStateInternal.UNASSIGNED);
+ }
+ amDispatcher.await();
+
+ // send map resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for both map tasks to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.MAP) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish both map tasks so that the reduce task can be scheduled
+ Iterator<Task> it = job.getTasks().values().iterator();
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 2);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // send the reduce resource requests to RM
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ // wait for the reduce task to be running
+ amNodeManager.nodeHeartbeat(true);
+ rm.drainEvents();
+
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ for (Task t : job.getTasks().values()) {
+ if (t.getType() == TaskType.REDUCE) {
+ mrApp.waitForState(t, TaskState.RUNNING);
+ }
+ }
+
+ // finish the reduce task
+ finishNextNTasks(rmDispatcher, amNodeManager, mrApp, it, 1);
+ allocator.schedule();
+ rm.drainEvents();
+ for (ResourceRequest rr : rm.getMyFifoScheduler().lastAsk) {
+ resourceRequests.add(ResourceRequest.newInstance(
+ rr.getPriority(), rr.getResourceName(), rr.getCapability(),
+ rr.getNumContainers(), rr.getRelaxLocality(),
+ rr.getNodeLabelExpression(), rr.getExecutionTypeRequest()));
+ }
+
+ return resourceRequests;
+ }
+
@Test
public void testMapReduceScheduling() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f03dc5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index ca18bfe..48887c5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1179,6 +1179,12 @@ public interface MRJobConfig {
public static final int DEFAULT_MR_NUM_OPPORTUNISTIC_MAPS_PERCENT = 0;
/**
+ * Opt out of YARN oversubscription so that the job always waits for
+ * GUARANTEED resources available in the cluster.
+ */
+ String MR_OVERSUBSCRIPTION_OPT_OUT = "mapreduce.job.oversubscription-opt-out";
+ boolean DEFAULT_MR_OVERSUBSCRIPTION_OPT_OUT = false;
+ /**
* A comma-separated list of properties whose value will be redacted.
*/
String MR_JOB_REDACTED_PROPERTIES = "mapreduce.job.redacted-properties";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83f03dc5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 72f509c..6d63a15 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -2098,4 +2098,12 @@
</description>
</property>
+<property>
+ <description>
+ Opts out of YARN oversubscription so that the job always waits for GUARANTEED
+ resources available.
+ </description>
+ <name>mapreduce.job.oversubscription-opt-out</name>
+ <value>false</value>
+</property>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/10] hadoop git commit: YARN-6921. Allow resource request to opt
out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Posted by sz...@apache.org.
YARN-6921. Allow resource request to opt out of oversubscription in Fair Scheduler. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e48809ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e48809ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e48809ca
Branch: refs/heads/YARN-1011
Commit: e48809ca7a657d2e0eafa9a231b08c67c75bfc18
Parents: a8c2bec
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Nov 22 09:03:05 2017 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:34:35 2018 -0700
----------------------------------------------------------------------
.../scheduler/common/PendingAsk.java | 14 +++-
.../scheduler/fair/FSAppAttempt.java | 5 ++
.../LocalityAppPlacementAllocator.java | 27 ++++++-
.../scheduler/fair/FairSchedulerTestBase.java | 32 +++++++-
.../scheduler/fair/TestFairScheduler.java | 77 ++++++++++++++++++++
5 files changed, 148 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48809ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
index 85d8715..e9931a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/PendingAsk.java
@@ -29,11 +29,15 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class PendingAsk {
private final Resource perAllocationResource;
private final int count;
- public final static PendingAsk ZERO = new PendingAsk(Resources.none(), 0);
+ private final boolean isGuaranteedTypeEnforced;
- public PendingAsk(Resource res, int num) {
+ public final static PendingAsk ZERO =
+ new PendingAsk(Resources.none(), 0, false);
+
+ public PendingAsk(Resource res, int num, boolean guaranteedTypeEnforced) {
this.perAllocationResource = res;
this.count = num;
+ this.isGuaranteedTypeEnforced = guaranteedTypeEnforced;
}
public Resource getPerAllocationResource() {
@@ -44,11 +48,17 @@ public class PendingAsk {
return count;
}
+ public boolean isGuaranteedTypeEnforced() {
+ return isGuaranteedTypeEnforced;
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("<per-allocation-resource=");
sb.append(getPerAllocationResource());
+ sb.append(", isGuaranteedEnforced=");
+ sb.append(isGuaranteedTypeEnforced());
sb.append(",repeat=");
sb.append(getCount());
sb.append(">");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48809ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 970ffcb..0719ca6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -860,6 +860,11 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
FSSchedulerNode node, PendingAsk pendingAsk, NodeType type,
boolean reserved, boolean opportunistic,
SchedulerRequestKey schedulerKey) {
+ if (pendingAsk.isGuaranteedTypeEnforced() && opportunistic) {
+ // do not attempt to assign an OPPORTUNISTIC container to a resource
+ // request that has explicitly opted out of oversubscription
+ return Resources.none();
+ }
// How much does this request need?
Resource capability = pendingAsk.getPerAllocationResource();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48809ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
index be1c1cc..dcc0f8b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/placement/LocalityAppPlacementAllocator.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement;
import org.apache.commons.collections.IteratorUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AppSchedulingInfo;
@@ -165,13 +167,16 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
PendingAsk lastPendingAsk =
lastRequest == null ? null : new PendingAsk(
- lastRequest.getCapability(), lastRequest.getNumContainers());
+ lastRequest.getCapability(), lastRequest.getNumContainers(),
+ enforceGuaranteedExecutionType(lastRequest));
String lastRequestedNodePartition =
lastRequest == null ? null : lastRequest.getNodeLabelExpression();
updateResult = new PendingAskUpdateResult(lastPendingAsk,
new PendingAsk(request.getCapability(),
- request.getNumContainers()), lastRequestedNodePartition,
+ request.getNumContainers(),
+ enforceGuaranteedExecutionType(request)),
+ lastRequestedNodePartition,
request.getNodeLabelExpression());
}
}
@@ -198,8 +203,9 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
if (null == request) {
return PendingAsk.ZERO;
} else{
+ boolean guaranteedEnforced = enforceGuaranteedExecutionType(request);
return new PendingAsk(request.getCapability(),
- request.getNumContainers());
+ request.getNumContainers(), guaranteedEnforced);
}
} finally {
readLock.unlock();
@@ -207,6 +213,21 @@ public class LocalityAppPlacementAllocator <N extends SchedulerNode>
}
+ /**
+ * Check for a given ResourceRequest, if its guaranteed execution type
+ * needs to be enforced.
+ * @param request resource request
+ * @return true if its guaranteed execution type is to be enforced.
+ * false otherwise
+ */
+ private static boolean enforceGuaranteedExecutionType(
+ ResourceRequest request) {
+ ExecutionTypeRequest executionType = request.getExecutionTypeRequest();
+ return executionType != null &&
+ executionType.getExecutionType() == ExecutionType.GUARANTEED &&
+ executionType.getEnforceExecutionType();
+ }
+
@Override
public int getOutstandingAsksCount(String resourceName) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48809ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
index 5f29186..459b5b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -115,9 +117,15 @@ public class FairSchedulerTestBase {
relaxLocality);
}
+ protected ResourceRequest createResourceRequest(int memory, int vcores,
+ String host, int priority, int numContainers, boolean relaxLocality) {
+ return createResourceRequest(memory, vcores, host, priority,
+ numContainers, relaxLocality, false);
+ }
+
protected ResourceRequest createResourceRequest(
int memory, int vcores, String host, int priority, int numContainers,
- boolean relaxLocality) {
+ boolean relaxLocality, boolean guaranteedExecutionEnforced) {
ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
request.setCapability(BuilderUtils.newResource(memory, vcores));
request.setResourceName(host);
@@ -127,6 +135,11 @@ public class FairSchedulerTestBase {
request.setPriority(prio);
request.setRelaxLocality(relaxLocality);
request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
+ if (guaranteedExecutionEnforced) {
+ ExecutionTypeRequest executionType = ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true);
+ request.setExecutionTypeRequest(executionType);
+ }
return request;
}
@@ -150,6 +163,13 @@ public class FairSchedulerTestBase {
}
protected ApplicationAttemptId createSchedulingRequest(
+ int memory, String queueId, String userId,
+ int numContainers, boolean guaranteedExecutionEnforced) {
+ return createSchedulingRequest(memory, 1, queueId,
+ userId, numContainers, 1, guaranteedExecutionEnforced);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers) {
return createSchedulingRequest(memory, vcores, queueId, userId, numContainers, 1);
}
@@ -163,6 +183,13 @@ public class FairSchedulerTestBase {
protected ApplicationAttemptId createSchedulingRequest(
int memory, int vcores, String queueId, String userId, int numContainers,
int priority) {
+ return createSchedulingRequest(memory, vcores, queueId, userId,
+ numContainers, priority, false);
+ }
+
+ protected ApplicationAttemptId createSchedulingRequest(
+ int memory, int vcores, String queueId, String userId, int numContainers,
+ int priority, boolean guaranteedExecutionEnforced) {
ApplicationAttemptId id = createAppAttemptId(this.APP_ID++,
this.ATTEMPT_ID++);
scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
@@ -174,7 +201,8 @@ public class FairSchedulerTestBase {
}
List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
ResourceRequest request = createResourceRequest(memory, vcores,
- ResourceRequest.ANY, priority, numContainers, true);
+ ResourceRequest.ANY, priority, numContainers, true,
+ guaranteedExecutionEnforced);
ask.add(request);
RMApp rmApp = mock(RMApp.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e48809ca/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 8756c7d..1f89ace 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2706,6 +2706,83 @@ public class TestFairScheduler extends FairSchedulerTestBase {
getPriority().getPriority());
}
+ @Test
+ public void testResourceRequestOptOutOfOversubscription() throws Exception {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ FairSchedulerConfiguration.
+ DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB, 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request that leaves some unallocated resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request that opts out of oversubscription and
+ // asks for more than what's left unallocated on the node.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1536, "queue2", "user1", 1, true);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 0);
+
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(1536, 1)));
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt(
+ FairSchedulerConfiguration.RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
+ memoryAllocationIncrement);
+ }
+ }
+
/**
* Test that NO OPPORTUNISTIC containers can be allocated on a node that
* is fully allocated and with a very high utilization.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/10] hadoop git commit: YARN-7491. Make sure AM is not scheduled
on an opportunistic container. Contributed by Haibo Chen.
Posted by sz...@apache.org.
YARN-7491. Make sure AM is not scheduled on an opportunistic container. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c483d491
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c483d491
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c483d491
Branch: refs/heads/YARN-1011
Commit: c483d491367ddb3d24bab97223a367989103e168
Parents: 2109c36
Author: Miklos Szegedi <sz...@apache.org>
Authored: Tue Nov 28 14:01:30 2017 -0800
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:34:53 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +++-
.../yarn/server/utils/TestBuilderUtils.java | 54 +++++++++++++++
.../server/resourcemanager/RMAppManager.java | 6 +-
.../server/resourcemanager/TestAppManager.java | 73 ++++++++++++++++----
4 files changed, 130 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c483d491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 63333ac..6abc617 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -349,15 +349,28 @@ public class BuilderUtils {
return priority;
}
+ public static ResourceRequest newAMResourceRequest(Priority priority,
+ String hostName, Resource capability) {
+ return newResourceRequest(priority, hostName, capability, 1,
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
+ }
+
public static ResourceRequest newResourceRequest(Priority priority,
String hostName, Resource capability, int numContainers) {
+ return newResourceRequest(priority, hostName, capability,
+ numContainers, ExecutionTypeRequest.newInstance());
+ }
+
+ public static ResourceRequest newResourceRequest(Priority priority,
+ String hostName, Resource capability, int numContainers,
+ ExecutionTypeRequest executionTypeRequest) {
ResourceRequest request = recordFactory
.newRecordInstance(ResourceRequest.class);
request.setPriority(priority);
request.setResourceName(hostName);
request.setCapability(capability);
request.setNumContainers(numContainers);
- request.setExecutionTypeRequest(ExecutionTypeRequest.newInstance());
+ request.setExecutionTypeRequest(executionTypeRequest);
return request;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c483d491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
new file mode 100644
index 0000000..4d5ab79
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/utils/TestBuilderUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.utils;
+
+import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Unit tests for BuilderUtils.
+ */
+public class TestBuilderUtils {
+ @Test
+ public void testNewAMResourceRequest() {
+ final Priority priority = Priority.newInstance(0);
+ final String hostName = "ResourceA";
+ final Resource capacity = Resource.newInstance(1024, 1);
+
+ ResourceRequest resourceRequest =
+ BuilderUtils.newAMResourceRequest(priority, hostName, capacity);
+
+ Assert.assertEquals("unexpected priority", priority,
+ resourceRequest.getPriority());
+ Assert.assertEquals("unexpected host name", hostName,
+ resourceRequest.getResourceName());
+ Assert.assertEquals("unexpected capacity", capacity,
+ resourceRequest.getCapability());
+ Assert.assertEquals("unexpected number of containers", 1,
+ resourceRequest.getNumContainers());
+ Assert.assertEquals("unexpected execution type",
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true),
+ resourceRequest.getExecutionTypeRequest());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c483d491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 2983077..1e05fec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -488,8 +488,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
if (amReqs == null || amReqs.isEmpty()) {
if (submissionContext.getResource() != null) {
amReqs = Collections.singletonList(BuilderUtils
- .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, submissionContext.getResource(), 1));
+ .newAMResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
+ ResourceRequest.ANY, submissionContext.getResource()));
} else {
throw new InvalidResourceRequestException("Invalid resource request, "
+ "no resources requested");
@@ -520,7 +520,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
for (ResourceRequest amReq : amReqs) {
amReq.setCapability(anyReq.getCapability());
amReq.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
amReq.setNumContainers(1);
amReq.setPriority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c483d491/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 7c2ce64..140c15f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -691,15 +691,45 @@ public class TestAppManager{
public void testRMAppSubmitAMContainerResourceRequests() throws Exception {
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequest(
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1024), 1, true));
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build());
List<ResourceRequest> reqs = new ArrayList<>();
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack", Resources.createResource(1025), 1, false));
- reqs.add(ResourceRequest.newInstance(Priority.newInstance(0),
- "/rack/node", Resources.createResource(1025), 1, true));
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(false)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
+ reqs.add(
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName("/rack/node")
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()
+ );
asContext.setAMContainerResourceRequests(cloneResourceRequests(reqs));
// getAMContainerResourceRequest uses the first entry of
// getAMContainerResourceRequests
@@ -720,8 +750,14 @@ public class TestAppManager{
asContext.setResource(Resources.createResource(1024));
asContext.setAMContainerResourceRequests(null);
ResourceRequest req =
- ResourceRequest.newInstance(Priority.newInstance(0),
- ResourceRequest.ANY, Resources.createResource(1025), 1, true);
+ ResourceRequest.newBuilder().priority(Priority.newInstance(0))
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1025))
+ .numContainers(1)
+ .relaxLocality(true)
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build();
asContext.setAMContainerResourceRequest(cloneResourceRequest(req));
// getAMContainerResourceRequests uses a singleton list of
// getAMContainerResourceRequest
@@ -742,8 +778,16 @@ public class TestAppManager{
RMApp app = testRMAppSubmit();
// setResource
Assert.assertEquals(Collections.singletonList(
- ResourceRequest.newInstance(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
- ResourceRequest.ANY, Resources.createResource(1024), 1, true, "")),
+ ResourceRequest.newBuilder()
+ .priority(RMAppAttemptImpl.AM_CONTAINER_PRIORITY)
+ .resourceName(ResourceRequest.ANY)
+ .capability(Resources.createResource(1024))
+ .numContainers(1)
+ .relaxLocality(true)
+ .nodeLabelExpression("")
+ .executionTypeRequest(ExecutionTypeRequest.newInstance(
+ ExecutionType.GUARANTEED, true))
+ .build()),
app.getAMResourceRequests());
}
@@ -783,7 +827,7 @@ public class TestAppManager{
for (ResourceRequest req : reqs) {
req.setCapability(anyReq.getCapability());
req.setExecutionTypeRequest(
- ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED));
+ ExecutionTypeRequest.newInstance(ExecutionType.GUARANTEED, true));
req.setNumContainers(1);
req.setPriority(Priority.newInstance(0));
}
@@ -1100,7 +1144,8 @@ public class TestAppManager{
req.getNodeLabelExpression() != null
? new String(req.getNodeLabelExpression()) : null,
ExecutionTypeRequest.newInstance(
- req.getExecutionTypeRequest().getExecutionType()));
+ req.getExecutionTypeRequest().getExecutionType(),
+ req.getExecutionTypeRequest().getEnforceExecutionType()));
}
private static List<ResourceRequest> cloneResourceRequests(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/10] hadoop git commit: YARN-6739. Crash NM at start time if
oversubscription is on but LinuxContainerExcutor or cgroup is off.
Contributed by Haibo Chen.
Posted by sz...@apache.org.
YARN-6739. Crash NM at start time if oversubscription is on but LinuxContainerExcutor or cgroup is off. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dece162b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dece162b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dece162b
Branch: refs/heads/YARN-1011
Commit: dece162b0872c7356f3dc4293300ae2698584493
Parents: 29de8de
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Apr 2 15:09:52 2018 -0700
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Mon Apr 2 15:35:06 2018 -0700
----------------------------------------------------------------------
.../monitor/ContainersMonitorImpl.java | 27 ++++++++++++++++++++
1 file changed, 27 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dece162b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 4863ee2..540e8eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerModule;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -184,6 +187,7 @@ public class ContainersMonitorImpl extends AbstractService implements
initializeOverAllocation(conf);
if (context.isOverAllocationEnabled()) {
+ checkOverAllocationPrerequisites();
pmemCheckEnabled = true;
LOG.info("Force enabling physical memory checks because " +
"overallocation is enabled");
@@ -223,6 +227,29 @@ public class ContainersMonitorImpl extends AbstractService implements
super.serviceInit(this.conf);
}
+ /**
+ * Check all prerequisites for NM over-allocation.
+ */
+ private void checkOverAllocationPrerequisites() throws YarnException {
+ // LinuxContainerExecutor is required to enable overallocation
+ if (!(containerExecutor instanceof LinuxContainerExecutor)) {
+ throw new YarnException(LinuxContainerExecutor.class.getName() +
+ " is required for overallocation");
+ }
+ if (ResourceHandlerModule.getCGroupsHandler() == null) {
+ throw new YarnException("CGroups must be enabled to support" +
+ " overallocation");
+ }
+ if (ResourceHandlerModule.getCpuResourceHandler() == null) {
+ throw new YarnException(
+ "CGroups cpu isolation must be enabled to support overallocation");
+ }
+ if (ResourceHandlerModule.getMemoryResourceHandler() == null) {
+ throw new YarnException(
+ "CGroups memory isolation must be enabled for overallocation");
+ }
+ }
+
private boolean isContainerMonitorEnabled() {
return conf.getBoolean(YarnConfiguration.NM_CONTAINER_MONITOR_ENABLED,
YarnConfiguration.DEFAULT_NM_CONTAINER_MONITOR_ENABLED);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org