You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2015/07/23 17:20:21 UTC
[09/14] hadoop git commit: YARN-3932.
SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based
on total-used-resources. (Bibin A Chundatt via wangda)
YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on total-used-resources. (Bibin A Chundatt via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ec26de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ec26de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ec26de
Branch: refs/heads/HADOOP-12111
Commit: 76ec26de8099dc48ce3812c595b7ab857a600442
Parents: 1b3bceb
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Jul 22 11:54:02 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Jul 22 11:54:02 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/SchedulerApplicationAttempt.java | 2 +-
.../scheduler/capacity/LeafQueue.java | 8 ++-
.../TestCapacitySchedulerNodeLabelUpdate.java | 64 ++++++++++++++++++++
4 files changed, 74 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5100cdf..f751862 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.8.0 - UNRELEASED
YARN-3885. ProportionalCapacityPreemptionPolicy doesn't preempt if queue is
more than 2 level. (Ajith S via wangda)
+ YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo
+ should based on total-used-resources. (Bibin A Chundatt via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index cf543bd..317e61c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -598,7 +598,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
AggregateAppResourceUsage runningResourceUsage =
getRunningAggregateAppResourceUsage();
Resource usedResourceClone =
- Resources.clone(attemptResourceUsage.getUsed());
+ Resources.clone(attemptResourceUsage.getAllUsed());
Resource reservedResourceClone =
Resources.clone(attemptResourceUsage.getReserved());
return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 0ce4d68..5c283f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -439,7 +439,7 @@ public class LeafQueue extends AbstractCSQueue {
for (Map.Entry<String, User> entry : users.entrySet()) {
User user = entry.getValue();
usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user
- .getUsed()), user.getActiveApplications(), user
+ .getAllUsed()), user.getActiveApplications(), user
.getPendingApplications(), Resources.clone(user
.getConsumedAMResources()), Resources.clone(user
.getUserResourceLimit())));
@@ -1894,7 +1894,11 @@ public class LeafQueue extends AbstractCSQueue {
public Resource getUsed() {
return userResourceUsage.getUsed();
}
-
+
+ public Resource getAllUsed() {
+ return userResourceUsage.getAllUsed();
+ }
+
public Resource getUsed(String label) {
return userResourceUsage.getUsed(label);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index e60e496..0a701d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -95,6 +97,68 @@ public class TestCapacitySchedulerNodeLabelUpdate {
.getMemory());
}
+ @Test(timeout = 60000)
+ public void testResourceUsage() throws Exception {
+ // set node -> label
+ mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y",
+ "z"));
+
+ // set mapping:
+ // h1 -> x
+ // h2 -> y
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
+
+ // inject node label manager
+ MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+ rm.getRMContext().setNodeLabelManager(mgr);
+ rm.start();
+ MockNM nm1 = rm.registerNode("h1:1234", 2048);
+ MockNM nm2 = rm.registerNode("h2:1234", 2048);
+ MockNM nm3 = rm.registerNode("h3:1234", 2048);
+
+ ContainerId containerId;
+ // launch an app to queue a1 (label = x), and check all container will
+ // be allocated in h1
+ RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
+ ApplicationResourceUsageReport appResourceUsageReport =
+ rm.getResourceScheduler().getAppResourceUsageReport(
+ am1.getApplicationAttemptId());
+ Assert.assertEquals(1024, appResourceUsageReport.getUsedResources()
+ .getMemory());
+ Assert.assertEquals(1, appResourceUsageReport.getUsedResources()
+ .getVirtualCores());
+ // request a container.
+ am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
+ containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED, 10 * 1000);
+ appResourceUsageReport =
+ rm.getResourceScheduler().getAppResourceUsageReport(
+ am1.getApplicationAttemptId());
+ Assert.assertEquals(2048, appResourceUsageReport.getUsedResources()
+ .getMemory());
+ Assert.assertEquals(2, appResourceUsageReport.getUsedResources()
+ .getVirtualCores());
+ LeafQueue queue =
+ (LeafQueue) ((CapacityScheduler) rm.getResourceScheduler())
+ .getQueue("a");
+ ArrayList<UserInfo> users = queue.getUsers();
+ for (UserInfo userInfo : users) {
+ if (userInfo.getUsername().equals("user")) {
+ ResourceInfo resourcesUsed = userInfo.getResourcesUsed();
+ Assert.assertEquals(2048, resourcesUsed.getMemory());
+ Assert.assertEquals(2, resourcesUsed.getvCores());
+ }
+ }
+ rm.stop();
+ }
+
@Test (timeout = 60000)
public void testNodeUpdate() throws Exception {
// set node -> label