You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/09/18 04:19:29 UTC
[1/2] hadoop git commit: YARN-7149. Cross-queue preemption sometimes
starves an underserved queue. (Eric Payne via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 31cdd260f -> e3a010498
YARN-7149. Cross-queue preemption sometimes starves an underserved queue. (Eric Payne via wangda)
Change-Id: Ib269991dbebce160378e8372ee6d24849c4a5ed6
(cherry picked from commit 3dfa937a1fadfc62947755872515f549b3b15e6a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3a01049
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3a01049
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3a01049
Branch: refs/heads/branch-3.0
Commit: e3a01049807fa7694a3ee3b086ff70653c1a611f
Parents: 03b6e95
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Sep 15 21:25:21 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Sun Sep 17 21:19:20 2017 -0700
----------------------------------------------------------------------
.../scheduler/capacity/UsersManager.java | 4 +-
.../capacity/TestContainerAllocation.java | 50 ++++++++++++++++++++
.../scheduler/capacity/TestLeafQueue.java | 8 ++--
3 files changed, 57 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3a01049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
index 5f7d185..33f30b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java
@@ -731,7 +731,9 @@ public class UsersManager implements AbstractUsersManager {
* should be higher than queue-hard-limit * ulMin
*/
float usersSummedByWeight = activeUsersTimesWeights;
- Resource resourceUsed = totalResUsageForActiveUsers.getUsed(nodePartition);
+ Resource resourceUsed = Resources.add(
+ totalResUsageForActiveUsers.getUsed(nodePartition),
+ required);
// For non-activeUser calculation, consider all users count.
if (!activeUser) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3a01049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index dd6b25b..906febf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.SecurityUtilTestHelper;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.Container;
@@ -887,4 +888,53 @@ public class TestContainerAllocation {
rm1.close();
}
+
+
+
+ @Test(timeout = 60000)
+ public void testUserLimitAllocationMultipleContainers() throws Exception {
+ CapacitySchedulerConfiguration newConf =
+ (CapacitySchedulerConfiguration) TestUtils
+ .getConfigurationWithMultipleQueues(conf);
+ newConf.setUserLimit("root.c", 50);
+ MockRM rm1 = new MockRM(newConf);
+
+ rm1.getRMContext().setNodeLabelManager(mgr);
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("h1:1234", 1000 * GB);
+
+ // launch app from 1st user to queue C, AM container should be launched in nm1
+ RMApp app1 = rm1.submitApp(2 * GB, "app", "user1", null, "c");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // launch app from 2nd user to queue C, AM container should be launched in nm1
+ RMApp app2 = rm1.submitApp(2 * GB, "app", "user2", null, "c");
+ MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
+
+ // Each application asks 1000 * 5GB containers
+ am1.allocate("*", 5 * GB, 1000, null);
+ am1.allocate("h1", 5 * GB, 1000, null);
+ am1.allocate(NetworkTopology.DEFAULT_RACK, 5 * GB, 1000, null);
+
+ // Each application asks 1000 * 5GB containers
+ am2.allocate("*", 5 * GB, 1000, null);
+ am2.allocate("h1", 5 * GB, 1000, null);
+ am2.allocate(NetworkTopology.DEFAULT_RACK, 5 * GB, 1000, null);
+
+ CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
+ RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ FiCaSchedulerApp schedulerApp1 =
+ cs.getApplicationAttempt(am1.getApplicationAttemptId());
+ FiCaSchedulerApp schedulerApp2 =
+ cs.getApplicationAttempt(am2.getApplicationAttemptId());
+
+ // container will be allocated to am1
+ // App1 will get 2 container allocated (plus AM container)
+ cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
+ Assert.assertEquals(101, schedulerApp1.getLiveContainers().size());
+ Assert.assertEquals(100, schedulerApp2.getLiveContainers().size());
+
+ rm1.close();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3a01049/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index d45f756..a32352b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -1252,7 +1252,7 @@ public class TestLeafQueue {
//app4 is user 0
//maxqueue 16G, userlimit 7G, used 8G, headroom 5G
//(8G used is 6G from this test case - app4, 2 from last test case, app_1)
- assertEquals(0*GB, app_4.getHeadroom().getMemorySize());
+ assertEquals(1*GB, app_4.getHeadroom().getMemorySize());
}
@Test
@@ -1436,7 +1436,7 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize());
// TODO, fix headroom in the future patch
- assertEquals(0*GB, app_0.getHeadroom().getMemorySize());
+ assertEquals(1*GB, app_0.getHeadroom().getMemorySize());
// User limit = 2G, 2 in use
assertEquals(0*GB, app_1.getHeadroom().getMemorySize());
// the application is not yet active
@@ -1449,8 +1449,8 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getUsedResources().getMemorySize());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize());
- assertEquals(0*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G
- assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
+ assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G
+ assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-6977. Node information is not provided
for non am containers in RM logs. (Suma Shivaprasad via wangda)
Posted by wa...@apache.org.
YARN-6977. Node information is not provided for non am containers in RM logs. (Suma Shivaprasad via wangda)
Change-Id: I0c44d09a560446dee2ba68c2b9ae69fce0ec1d3e
(cherry picked from commit 8a42e922fad613f3cf1cc6cb0f3fa72546a9cc56)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03b6e95f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03b6e95f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03b6e95f
Branch: refs/heads/branch-3.0
Commit: 03b6e95fa5aedc875b43e2b8b8d001d1bb5614c5
Parents: 31cdd26
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Sep 15 21:24:11 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Sun Sep 17 21:19:20 2017 -0700
----------------------------------------------------------------------
.../resourcemanager/scheduler/SchedulerNode.java | 8 --------
.../scheduler/common/fica/FiCaSchedulerNode.java | 13 +++++++++++++
.../scheduler/fair/FSSchedulerNode.java | 10 ++++++++++
3 files changed, 23 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03b6e95f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index 272537c..90fa3e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -170,14 +170,6 @@ public abstract class SchedulerNode {
launchedContainers.put(container.getId(),
new ContainerInfo(rmContainer, launchedOnNode));
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Assigned container " + container.getId() + " of capacity "
- + container.getResource() + " on host " + rmNode.getNodeAddress()
- + ", which has " + numContainers + " containers, "
- + getAllocatedResource() + " used and " + getUnallocatedResource()
- + " available after allocation");
- }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03b6e95f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index c26a11b..7277779 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -160,4 +160,17 @@ public class FiCaSchedulerNode extends SchedulerNode {
public synchronized Map<ContainerId, RMContainer> getKillableContainers() {
return Collections.unmodifiableMap(killableContainers);
}
+
+ protected synchronized void allocateContainer(RMContainer rmContainer,
+ boolean launchedOnNode) {
+ super.allocateContainer(rmContainer, launchedOnNode);
+
+ final Container container = rmContainer.getContainer();
+ LOG.info("Assigned container " + container.getId() + " of capacity "
+ + container.getResource() + " on host " + getRMNode().getNodeAddress()
+ + ", which has " + getNumContainers() + " containers, "
+ + getAllocatedResource() + " used and " + getUnallocatedResource()
+ + " available after allocation");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03b6e95f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index 93646f4..44ec9c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -239,6 +240,15 @@ public class FSSchedulerNode extends SchedulerNode {
protected synchronized void allocateContainer(RMContainer rmContainer,
boolean launchedOnNode) {
super.allocateContainer(rmContainer, launchedOnNode);
+ if (LOG.isDebugEnabled()) {
+ final Container container = rmContainer.getContainer();
+ LOG.debug("Assigned container " + container.getId() + " of capacity "
+ + container.getResource() + " on host " + getRMNode().getNodeAddress()
+ + ", which has " + getNumContainers() + " containers, "
+ + getAllocatedResource() + " used and " + getUnallocatedResource()
+ + " available after allocation");
+ }
+
Resource allocated = rmContainer.getAllocatedResource();
if (!Resources.isNone(allocated)) {
// check for satisfied preemption request and update bookkeeping
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org