You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ww...@apache.org on 2018/05/01 01:56:24 UTC
hadoop git commit: YARN-8212. Pending backlog for async allocation
threads should be configurable. Contributed by Tao Yang.
Repository: hadoop
Updated Branches:
refs/heads/trunk a966ec6e2 -> 2d319e379
YARN-8212. Pending backlog for async allocation threads should be configurable. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d319e37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d319e37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d319e37
Branch: refs/heads/trunk
Commit: 2d319e37937c1e20c6a7dc4477ef88defd1f8464
Parents: a966ec6
Author: Weiwei Yang <ww...@apache.org>
Authored: Tue May 1 09:47:10 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Tue May 1 09:47:10 2018 +0800
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 9 ++++++++-
.../scheduler/capacity/CapacitySchedulerConfiguration.java | 8 ++++++++
2 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d319e37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 776e512..1d6c104 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -261,6 +261,7 @@ public class CapacityScheduler extends
CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX
+ ".scheduling-interval-ms";
private static final long DEFAULT_ASYNC_SCHEDULER_INTERVAL = 5;
+ private long asyncMaxPendingBacklogs;
public CapacityScheduler() {
super(CapacityScheduler.class.getName());
@@ -379,6 +380,11 @@ public class CapacityScheduler extends
asyncSchedulerThreads.add(new AsyncScheduleThread(this));
}
resourceCommitterService = new ResourceCommitterService(this);
+ asyncMaxPendingBacklogs = this.conf.getInt(
+ CapacitySchedulerConfiguration.
+ SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS,
+ CapacitySchedulerConfiguration.
+ DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS);
}
// Setup how many containers we can allocate for each round
@@ -573,7 +579,8 @@ public class CapacityScheduler extends
Thread.sleep(100);
} else {
// Don't run schedule if we have some pending backlogs already
- if (cs.getAsyncSchedulingPendingBacklogs() > 100) {
+ if (cs.getAsyncSchedulingPendingBacklogs()
+ > cs.asyncMaxPendingBacklogs) {
Thread.sleep(1);
} else{
schedule(cs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d319e37/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index c41bd96..76eaac0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -246,6 +246,14 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-threads";
@Private
+ public static final String SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS =
+ SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".maximum-pending-backlogs";
+
+ @Private
+ public static final Integer
+ DEFAULT_SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_PENDING_BACKLOGS = 100;
+
+ @Private
public static final boolean DEFAULT_SCHEDULE_ASYNCHRONOUSLY_ENABLE = false;
@Private
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org