You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2015/09/29 17:13:13 UTC
hadoop git commit: YARN-4066. Large number of queues choke fair
scheduler. (Johan Gustavsson via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 715dbddf7 -> a0b5a0a41
YARN-4066. Large number of queues choke fair scheduler. (Johan Gustavsson via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a0b5a0a4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a0b5a0a4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a0b5a0a4
Branch: refs/heads/trunk
Commit: a0b5a0a419dfc07b7ac45c06b11b4c8dc7e79958
Parents: 715dbdd
Author: Karthik Kambatla <ka...@apache.org>
Authored: Tue Sep 29 07:55:34 2015 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Tue Sep 29 07:55:34 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/fair/QueueManager.java | 80 ++++++++++++++------
2 files changed, 60 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0b5a0a4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e9d04d3..43c501f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -509,6 +509,9 @@ Release 2.8.0 - UNRELEASED
YARN-3635. Refactored current queue mapping implementation in CapacityScheduler
to use a generic PlacementManager framework. (Wangda Tan via jianhe)
+ YARN-4066. Large number of queues choke fair scheduler.
+ (Johan Gustavsson via kasha)
+
BUG FIXES
YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a0b5a0a4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 0092845..51a298b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -87,7 +87,19 @@ public class QueueManager {
* could be referred to as just "parent1.queue2".
*/
public FSLeafQueue getLeafQueue(String name, boolean create) {
- FSQueue queue = getQueue(name, create, FSQueueType.LEAF);
+ return getLeafQueue(name, create, true);
+ }
+
+ public FSLeafQueue getLeafQueue(
+ String name,
+ boolean create,
+ boolean recomputeSteadyShares) {
+ FSQueue queue = getQueue(
+ name,
+ create,
+ FSQueueType.LEAF,
+ recomputeSteadyShares
+ );
if (queue instanceof FSParentQueue) {
return null;
}
@@ -117,28 +129,46 @@ public class QueueManager {
* could be referred to as just "parent1.queue2".
*/
public FSParentQueue getParentQueue(String name, boolean create) {
- FSQueue queue = getQueue(name, create, FSQueueType.PARENT);
+ return getParentQueue(name, create, true);
+ }
+
+ public FSParentQueue getParentQueue(
+ String name,
+ boolean create,
+ boolean recomputeSteadyShares) {
+ FSQueue queue = getQueue(
+ name,
+ create,
+ FSQueueType.PARENT,
+ recomputeSteadyShares
+ );
if (queue instanceof FSLeafQueue) {
return null;
}
return (FSParentQueue) queue;
}
-
- private FSQueue getQueue(String name, boolean create, FSQueueType queueType) {
+
+ private FSQueue getQueue(
+ String name,
+ boolean create,
+ FSQueueType queueType,
+ boolean recomputeSteadyShares) {
+ boolean recompute = recomputeSteadyShares;
name = ensureRootPrefix(name);
+ FSQueue queue;
synchronized (queues) {
- FSQueue queue = queues.get(name);
+ queue = queues.get(name);
if (queue == null && create) {
// if the queue doesn't exist,create it and return
queue = createQueue(name, queueType);
-
- // Update steady fair share for all queues
- if (queue != null) {
- rootQueue.recomputeSteadyShares();
- }
+ } else {
+ recompute = false;
}
- return queue;
}
+ if (recompute) {
+ rootQueue.recomputeSteadyShares();
+ }
+ return queue;
}
/**
@@ -376,21 +406,25 @@ public class QueueManager {
public void updateAllocationConfiguration(AllocationConfiguration queueConf) {
// Create leaf queues and the parent queues in a leaf's ancestry if they do not exist
- for (String name : queueConf.getConfiguredQueues().get(FSQueueType.LEAF)) {
- if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) {
- getLeafQueue(name, true);
+ synchronized (queues) {
+ for (String name : queueConf.getConfiguredQueues().get(
+ FSQueueType.LEAF)) {
+ if (removeEmptyIncompatibleQueues(name, FSQueueType.LEAF)) {
+ getLeafQueue(name, true, false);
+ }
}
- }
-
- // At this point all leaves and 'parents with at least one child' would have been created.
- // Now create parents with no configured leaf.
- for (String name : queueConf.getConfiguredQueues().get(
- FSQueueType.PARENT)) {
- if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) {
- getParentQueue(name, true);
+ // At this point all leaves and 'parents with
+ // at least one child' would have been created.
+ // Now create parents with no configured leaf.
+ for (String name : queueConf.getConfiguredQueues().get(
+ FSQueueType.PARENT)) {
+ if (removeEmptyIncompatibleQueues(name, FSQueueType.PARENT)) {
+ getParentQueue(name, true, false);
+ }
}
}
-
+ rootQueue.recomputeSteadyShares();
+
for (FSQueue queue : queues.values()) {
// Update queue metrics
FSQueueMetrics queueMetrics = queue.getMetrics();