You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ka...@apache.org on 2017/02/23 22:29:38 UTC
hadoop git commit: YARN-6175. FairScheduler: Negative vcore for
resource needed to preempt. (Yufei Gu via kasha)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 b1ec79066 -> 1d156c875
YARN-6175. FairScheduler: Negative vcore for resource needed to preempt. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d156c87
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d156c87
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d156c87
Branch: refs/heads/branch-2.8
Commit: 1d156c8758ec3dc45da810d9abaecdb4e811891f
Parents: b1ec790
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Thu Feb 23 14:29:30 2017 -0800
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Thu Feb 23 14:29:30 2017 -0800
----------------------------------------------------------------------
.../hadoop/yarn/util/resource/Resources.java | 18 ++++++++++++++++++
.../scheduler/fair/FSAppAttempt.java | 8 +-------
.../scheduler/fair/FairScheduler.java | 8 ++++----
3 files changed, 23 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d156c87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index fdd9ec7..d62e558 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -172,6 +172,24 @@ public class Resources {
return subtractFrom(clone(lhs), rhs);
}
+ /**
+ * Subtract <code>rhs</code> from <code>lhs</code> and reset any negative
+ * values to zero.
+ * @param lhs {@link Resource} to subtract from
+ * @param rhs {@link Resource} to subtract
+ * @return the value of lhs after subtraction
+ */
+ public static Resource subtractFromNonNegative(Resource lhs, Resource rhs) {
+ subtractFrom(lhs, rhs);
+ if (lhs.getMemorySize() < 0) {
+ lhs.setMemorySize(0);
+ }
+ if (lhs.getVirtualCores() < 0) {
+ lhs.setVirtualCores(0);
+ }
+ return lhs;
+ }
+
public static Resource negate(Resource resource) {
return subtract(NONE, resource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d156c87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 44d2b42..802479d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -194,16 +194,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
for (NodeId nodeId: blacklistNodeIds) {
SchedulerNode node = scheduler.getSchedulerNode(nodeId);
if (node != null) {
- Resources.subtractFrom(availableResources,
+ Resources.subtractFromNonNegative(availableResources,
node.getAvailableResource());
}
}
- if (availableResources.getMemory() < 0) {
- availableResources.setMemory(0);
- }
- if (availableResources.getVirtualCores() < 0) {
- availableResources.setVirtualCores(0);
- }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d156c87/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 1586a49..8e7d299 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -531,14 +531,14 @@ public class FairScheduler extends
if (curTime - sched.getLastTimeAtMinShare() > minShareTimeout) {
Resource target = Resources.componentwiseMin(
sched.getMinShare(), sched.getDemand());
- resDueToMinShare = Resources.max(calc, clusterResource,
- Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
+ resDueToMinShare =
+ Resources.subtractFromNonNegative(target, sched.getResourceUsage());
}
if (curTime - sched.getLastTimeAtFairShareThreshold() > fairShareTimeout) {
Resource target = Resources.componentwiseMin(
sched.getFairShare(), sched.getDemand());
- resDueToFairShare = Resources.max(calc, clusterResource,
- Resources.none(), Resources.subtract(target, sched.getResourceUsage()));
+ resDueToFairShare =
+ Resources.subtractFromNonNegative(target, sched.getResourceUsage());
}
Resource deficit = Resources.max(calc, clusterResource,
resDueToMinShare, resDueToFairShare);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org