You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by jl...@apache.org on 2013/04/05 17:30:24 UTC
svn commit: r1465009 - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/
hadoop-yarn/hadoop-yarn-server/hado...
Author: jlowe
Date: Fri Apr 5 15:30:24 2013
New Revision: 1465009
URL: http://svn.apache.org/r1465009
Log:
YARN-525. make CS node-locality-delay refreshable. Contributed by Thomas Graves
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1465009&r1=1465008&r2=1465009&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Fri Apr 5 15:30:24 2013
@@ -509,6 +509,8 @@ Release 0.23.7 - UNRELEASED
YARN-200. yarn log does not output all needed information, and is in a
binary format (Ravi Prakash via jlowe)
+ YARN-525. make CS node-locality-delay refreshable (Thomas Graves via jlowe)
+
OPTIMIZATIONS
YARN-357. App submission should not be synchronized (daryn)
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1465009&r1=1465008&r2=1465009&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Fri Apr 5 15:30:24 2013
@@ -89,6 +89,8 @@ public class LeafQueue implements CSQueu
private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
private int maxActiveApplicationsPerUser;
+ private int nodeLocalityDelay;
+
private Resource usedResources = Resources.createResource(0, 0);
private float usedCapacity = 0.0f;
private volatile int numContainers;
@@ -123,8 +125,6 @@ public class LeafQueue implements CSQueu
private final ActiveUsersManager activeUsersManager;
- private final int nodeLocalityDelay;
-
private final ResourceCalculator resourceCalculator;
public LeafQueue(CapacitySchedulerContext cs,
@@ -196,9 +196,6 @@ public class LeafQueue implements CSQueu
Map<QueueACL, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
- this.nodeLocalityDelay =
- cs.getConfiguration().getNodeLocalityDelay();
-
setupQueueConfigs(
cs.getClusterResources(),
capacity, absoluteCapacity,
@@ -206,7 +203,7 @@ public class LeafQueue implements CSQueu
userLimit, userLimitFactor,
maxApplications, maxApplicationsPerUser,
maxActiveApplications, maxActiveApplicationsPerUser,
- state, acls);
+ state, acls, cs.getConfiguration().getNodeLocalityDelay());
if(LOG.isDebugEnabled()) {
LOG.debug("LeafQueue:" + " name=" + queueName
@@ -227,7 +224,8 @@ public class LeafQueue implements CSQueu
int userLimit, float userLimitFactor,
int maxApplications, int maxApplicationsPerUser,
int maxActiveApplications, int maxActiveApplicationsPerUser,
- QueueState state, Map<QueueACL, AccessControlList> acls)
+ QueueState state, Map<QueueACL, AccessControlList> acls,
+ int nodeLocalityDelay)
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
@@ -256,6 +254,8 @@ public class LeafQueue implements CSQueu
this.queueInfo.setCapacity(this.capacity);
this.queueInfo.setMaximumCapacity(this.maximumCapacity);
this.queueInfo.setQueueState(this.state);
+
+ this.nodeLocalityDelay = nodeLocalityDelay;
StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
@@ -319,7 +319,8 @@ public class LeafQueue implements CSQueu
"state = " + state +
" [= configuredState ]" + "\n" +
"acls = " + aclsString +
- " [= configuredAcls ]" + "\n");
+ " [= configuredAcls ]" + "\n" +
+ "nodeLocalityDelay = " + nodeLocalityDelay + "\n");
}
@Override
@@ -605,7 +606,8 @@ public class LeafQueue implements CSQueu
newlyParsedLeafQueue.getMaxApplicationsPerUser(),
newlyParsedLeafQueue.getMaximumActiveApplications(),
newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
- newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls);
+ newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls,
+ newlyParsedLeafQueue.getNodeLocalityDelay());
// queue metrics are updated, more resource may be available
// activate the pending applications if possible
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1465009&r1=1465008&r2=1465009&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Fri Apr 5 15:30:24 2013
@@ -1623,6 +1623,30 @@ public class TestLeafQueue {
assertEquals(3, e.activeApplications.size());
assertEquals(0, e.pendingApplications.size());
}
+
+ @Test (timeout = 30000)
+ public void testNodeLocalityAfterQueueRefresh() throws Exception {
+
+ // Manipulate queue 'e'
+ LeafQueue e = stubLeafQueue((LeafQueue)queues.get(E));
+
+ // before reinitialization
+ assertEquals(0, e.getNodeLocalityDelay());
+
+ csConf.setInt(CapacitySchedulerConfiguration
+ .NODE_LOCALITY_DELAY, 60);
+ Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
+ CSQueue newRoot =
+ CapacityScheduler.parseQueue(csContext, csConf, null,
+ CapacitySchedulerConfiguration.ROOT,
+ newQueues, queues,
+ TestUtils.spyHook);
+ queues = newQueues;
+ root.reinitialize(newRoot, cs.getClusterResources());
+
+ // after reinitialization
+ assertEquals(60, e.getNodeLocalityDelay());
+ }
@Test (timeout = 30000)
public void testActivateApplicationByUpdatingClusterResource()