You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2017/10/24 18:31:47 UTC
[13/50] [abbrv] hadoop git commit: YARN-4090. Make Collections.sort()
more efficient by caching resource usage. (Contributed by Yufei Gu,
Shilong Zhang and Xianyin Xin)
YARN-4090. Make Collections.sort() more efficient by caching resource usage. (Contributed by Yufei Gu, Shilong Zhang and Xianyin Xin)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f4cdf10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f4cdf10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f4cdf10
Branch: refs/heads/HDFS-9806
Commit: 1f4cdf10681b6903207a63fb5c306c9665ed9464
Parents: 4afd308
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Oct 20 01:32:20 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Oct 20 01:32:20 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/FSAppAttempt.java | 19 ++++++++++
.../scheduler/fair/FSLeafQueue.java | 19 +---------
.../scheduler/fair/FSParentQueue.java | 14 -------
.../resourcemanager/scheduler/fair/FSQueue.java | 34 +++++++++++++++++
.../scheduler/fair/TestFairScheduler.java | 39 +++++++++++++++++++-
5 files changed, 93 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 3024558..8ab6e13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
@@ -167,6 +168,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
rmContainer.getNodeLabelExpression(),
getUser(), 1, containerResource);
this.attemptResourceUsage.decUsed(containerResource);
+ getQueue().decUsedResource(containerResource);
// Clear resource utilization metrics cache.
lastMemoryAggregateAllocationUpdateTime = -1;
@@ -461,6 +463,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
type, node, schedulerKey, container);
this.attemptResourceUsage.incUsed(container.getResource());
+ getQueue().incUsedResource(container.getResource());
// Update resource requests related to "request" and store in RMContainer
((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
@@ -644,6 +647,22 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
schedulerKey.getAllocationRequestId());
}
+ @Override
+ public synchronized void recoverContainer(SchedulerNode node,
+ RMContainer rmContainer) {
+ try {
+ writeLock.lock();
+
+ super.recoverContainer(node, rmContainer);
+
+ if (!rmContainer.getState().equals(RMContainerState.COMPLETED)) {
+ getQueue().incUsedResource(rmContainer.getContainer().getResource());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
/**
* Reserve a spot for {@code container} on this {@code node}. If
* the container is {@code alreadyReserved} on the node, simply
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 60477d4..49d2166 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -89,6 +89,7 @@ public class FSLeafQueue extends FSQueue {
} else {
nonRunnableApps.add(app);
}
+ incUsedResource(app.getResourceUsage());
} finally {
writeLock.unlock();
}
@@ -123,6 +124,7 @@ public class FSLeafQueue extends FSQueue {
getMetrics().setAMResourceUsage(amResourceUsage);
}
+ decUsedResource(app.getResourceUsage());
return runnable;
}
@@ -292,23 +294,6 @@ public class FSLeafQueue extends FSQueue {
return demand;
}
- @Override
- public Resource getResourceUsage() {
- Resource usage = Resources.createResource(0);
- readLock.lock();
- try {
- for (FSAppAttempt app : runnableApps) {
- Resources.addTo(usage, app.getResourceUsage());
- }
- for (FSAppAttempt app : nonRunnableApps) {
- Resources.addTo(usage, app.getResourceUsage());
- }
- } finally {
- readLock.unlock();
- }
- return usage;
- }
-
Resource getAmResourceUsage() {
return amResourceUsage;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index e42c9f5..a8e53fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -119,20 +119,6 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public Resource getResourceUsage() {
- Resource usage = Resources.createResource(0);
- readLock.lock();
- try {
- for (FSQueue child : childQueues) {
- Resources.addTo(usage, child.getResourceUsage());
- }
- } finally {
- readLock.unlock();
- }
- return usage;
- }
-
- @Override
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index e0df480..4babfd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -57,6 +57,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private Resource fairShare = Resources.createResource(0, 0);
private Resource steadyFairShare = Resources.createResource(0, 0);
private Resource reservedResource = Resources.createResource(0, 0);
+ private final Resource resourceUsage = Resource.newInstance(0, 0);
private final String name;
protected final FairScheduler scheduler;
private final YarnAuthorizationProvider authorizer;
@@ -479,6 +480,39 @@ public abstract class FSQueue implements Queue, Schedulable {
}
@Override
+ public Resource getResourceUsage() {
+ return resourceUsage;
+ }
+
+ /**
+ * Increase resource usage for this queue and all parent queues.
+ *
+ * @param res the resource to increase
+ */
+ protected void incUsedResource(Resource res) {
+ synchronized (resourceUsage) {
+ Resources.addTo(resourceUsage, res);
+ if (parent != null) {
+ parent.incUsedResource(res);
+ }
+ }
+ }
+
+ /**
+ * Decrease resource usage for this queue and all parent queues.
+ *
+ * @param res the resource to decrease
+ */
+ protected void decUsedResource(Resource res) {
+ synchronized (resourceUsage) {
+ Resources.subtractFrom(resourceUsage, res);
+ if (parent != null) {
+ parent.decUsedResource(res);
+ }
+ }
+ }
+
+ @Override
public Priority getDefaultApplicationPriority() {
// TODO add implementation for FSParentQueue
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f4cdf10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 6b2109d..944db1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4585,6 +4585,43 @@ public class TestFairScheduler extends FairSchedulerTestBase {
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
scheduler.getSchedulerApplications(), scheduler, "default");
}
+
+ @Test
+ public void testResourceUsageByMoveApp() throws Exception {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMNode node1 = MockNodes.newNodeInfo(
+ 1, Resources.createResource(1 * GB, 4), 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ ApplicationAttemptId appAttId =
+ createSchedulingRequest(1 * GB, 2, "parent1.queue1", "user1", 2);
+ scheduler.update();
+
+ NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
+ scheduler.handle(updateEvent);
+
+ QueueManager queueMgr = scheduler.getQueueManager();
+ FSQueue parent1 = queueMgr.getParentQueue("parent1", true);
+ FSQueue parent2 = queueMgr.getParentQueue("parent2", true);
+ FSQueue queue2 = queueMgr.getLeafQueue("parent2.queue2", true);
+ FSQueue queue1 = queueMgr.getLeafQueue("parent1.queue1", true);
+
+ Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 1 * GB);
+
+ scheduler.moveApplication(appAttId.getApplicationId(), "parent2.queue2");
+
+ Assert.assertEquals(parent2.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(queue2.getResourceUsage().getMemorySize(), 1 * GB);
+ Assert.assertEquals(parent1.getResourceUsage().getMemorySize(), 0);
+ Assert.assertEquals(queue1.getResourceUsage().getMemorySize(), 0);
+ }
@Test (expected = YarnException.class)
public void testMoveWouldViolateMaxAppsConstraints() throws Exception {
@@ -4598,7 +4635,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
-
+
scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org