You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xg...@apache.org on 2017/06/05 20:31:18 UTC
[29/50] [abbrv] hadoop git commit: YARN-6246. Identifying starved
apps does not need the scheduler writelock (Contributed by Karthik Kambatla
via Daniel Templeton)
YARN-6246. Identifying starved apps does not need the scheduler writelock
(Contributed by Karthik Kambatla via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e648f18a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e648f18a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e648f18a
Branch: refs/heads/YARN-5734
Commit: e648f18a082aaec9c2a132bca0ea6a5741f01c5b
Parents: 9560fd5
Author: Daniel Templeton <te...@apache.org>
Authored: Wed May 31 15:48:04 2017 -0700
Committer: Xuan <xg...@apache.org>
Committed: Mon Jun 5 13:29:41 2017 -0700
----------------------------------------------------------------------
.../scheduler/fair/FSLeafQueue.java | 9 +++----
.../scheduler/fair/FSParentQueue.java | 4 +--
.../resourcemanager/scheduler/fair/FSQueue.java | 19 +++++++++-----
.../scheduler/fair/FairScheduler.java | 27 ++++++++++++++------
4 files changed, 38 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648f18a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 10f1e28..1de0e30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -198,13 +198,10 @@ public class FSLeafQueue extends FSQueue {
}
@Override
- public void updateInternal(boolean checkStarvation) {
+ void updateInternal() {
readLock.lock();
try {
policy.computeShares(runnableApps, getFairShare());
- if (checkStarvation) {
- updateStarvedApps();
- }
} finally {
readLock.unlock();
}
@@ -283,8 +280,10 @@ public class FSLeafQueue extends FSQueue {
* If this queue is starving due to fairshare, there must be at least
* one application that is starved. And, even if the queue is not
* starved due to fairshare, there might still be starved applications.
+ *
+ * Caller does not need read/write lock on the leaf queue.
*/
- private void updateStarvedApps() {
+ void updateStarvedApps() {
// Fetch apps with pending demand
TreeSet<FSAppAttempt> appsWithDemand = fetchAppsWithDemand(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648f18a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index b062c58..5b4e4dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -79,13 +79,13 @@ public class FSParentQueue extends FSQueue {
}
@Override
- public void updateInternal(boolean checkStarvation) {
+ void updateInternal() {
readLock.lock();
try {
policy.computeShares(childQueues, getFairShare());
for (FSQueue childQueue : childQueues) {
childQueue.getMetrics().setFairShare(childQueue.getFairShare());
- childQueue.updateInternal(checkStarvation);
+ childQueue.updateInternal();
}
} finally {
readLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648f18a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index e131140..12b1b83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -326,16 +326,23 @@ public abstract class FSQueue implements Queue, Schedulable {
/**
* Recomputes the shares for all child queues and applications based on this
- * queue's current share, and checks for starvation.
+ * queue's current share.
*
- * @param checkStarvation whether to check for fairshare or minshare
- * starvation on update
+ * To be called holding the scheduler writelock.
*/
- abstract void updateInternal(boolean checkStarvation);
+ abstract void updateInternal();
- public void update(Resource fairShare, boolean checkStarvation) {
+ /**
+ * Set the queue's fairshare and update the demand/fairshare of child
+ * queues/applications.
+ *
+ * To be called holding the scheduler writelock.
+ *
+ * @param fairShare
+ */
+ public void update(Resource fairShare) {
setFairShare(fairShare);
- updateInternal(checkStarvation);
+ updateInternal();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e648f18a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index d1a237a..d779159 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -366,20 +366,31 @@ public class FairScheduler extends
*/
@VisibleForTesting
public void update() {
- try {
- writeLock.lock();
-
- FSQueue rootQueue = queueMgr.getRootQueue();
+ FSQueue rootQueue = queueMgr.getRootQueue();
+ // Update demands and fairshares
+ writeLock.lock();
+ try {
// Recursively update demands for all queues
rootQueue.updateDemand();
-
- Resource clusterResource = getClusterResource();
- rootQueue.update(clusterResource, shouldAttemptPreemption());
+ rootQueue.update(getClusterResource());
// Update metrics
updateRootQueueMetrics();
+ } finally {
+ writeLock.unlock();
+ }
+
+ readLock.lock();
+ try {
+ // Update starvation stats and identify starved applications
+ if (shouldAttemptPreemption()) {
+ for (FSLeafQueue queue : queueMgr.getLeafQueues()) {
+ queue.updateStarvedApps();
+ }
+ }
+ // Log debug information
if (LOG.isDebugEnabled()) {
if (--updatesToSkipForDebug < 0) {
updatesToSkipForDebug = UPDATE_DEBUG_FREQUENCY;
@@ -387,7 +398,7 @@ public class FairScheduler extends
}
}
} finally {
- writeLock.unlock();
+ readLock.unlock();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org