You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/12/09 05:50:25 UTC
hadoop git commit: YARN-7591. NPE in async-scheduling mode of
CapacityScheduler. (Tao Yang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/branch-3.0 ba874d678 -> 41796ea60
YARN-7591. NPE in async-scheduling mode of CapacityScheduler. (Tao Yang via wangda)
Change-Id: I67e086d6d35c3166ec0f6305490ea6527bd6d799
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41796ea6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41796ea6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41796ea6
Branch: refs/heads/branch-3.0
Commit: 41796ea60ab9dbd6a41dd8fd3c6ff50990bba2f5
Parents: ba874d6
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Dec 8 21:43:28 2017 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 8 21:43:28 2017 -0800
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 4 ++++
.../scheduler/capacity/LeafQueue.java | 21 +++++++++++++++++++-
.../scheduler/common/fica/FiCaSchedulerApp.java | 5 +++++
3 files changed, 29 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41796ea6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index f0855fc..ac9f374 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1260,6 +1260,10 @@ public class CapacityScheduler extends
if (reservedContainer != null) {
FiCaSchedulerApp reservedApplication = getCurrentAttemptForContainer(
reservedContainer.getContainerId());
+ if (reservedApplication == null) {
+ LOG.error("Trying to schedule for a finished app, please double check.");
+ return null;
+ }
// Try to fulfill the reservation
LOG.info(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41796ea6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 096d087..2deb5b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1176,7 +1176,14 @@ public class LeafQueue extends AbstractCSQueue {
allocation.getSchedulingMode(), null);
// Deduct resources that we can release
- Resource usedResource = Resources.clone(getUser(username).getUsed(p));
+ User user = getUser(username);
+ if (user == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User " + username + " has been removed!");
+ }
+ return false;
+ }
+ Resource usedResource = Resources.clone(user.getUsed(p));
Resources.subtractFrom(usedResource,
request.getTotalReleasedResource());
@@ -1381,6 +1388,12 @@ public class LeafQueue extends AbstractCSQueue {
SchedulingMode schedulingMode, Resource userLimit) {
String user = application.getUser();
User queueUser = getUser(user);
+ if (queueUser == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User " + user + " has been removed!");
+ }
+ return Resources.none();
+ }
// Compute user limit respect requested labels,
// TODO, need consider headroom respect labels also
@@ -1475,6 +1488,12 @@ public class LeafQueue extends AbstractCSQueue {
try {
readLock.lock();
User user = getUser(userName);
+ if (user == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User " + userName + " has been removed!");
+ }
+ return false;
+ }
currentResourceLimits.setAmountNeededUnreserve(Resources.none());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41796ea6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index e9bee14..f78da4f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -320,6 +320,11 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
RMContainer reservedContainerOnNode =
schedulerContainer.getSchedulerNode().getReservedContainer();
if (reservedContainerOnNode != null) {
+ // adding NP check as this proposal could not be allocated from reserved
+ // container in async-scheduling mode
+ if (allocation.getAllocateFromReservedContainer() == null) {
+ return false;
+ }
RMContainer fromReservedContainer =
allocation.getAllocateFromReservedContainer().getRmContainer();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org