You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2017/07/18 08:13:37 UTC
hadoop git commit: Addendum patch for YARN-5731
Repository: hadoop
Updated Branches:
refs/heads/trunk ccaf03666 -> 0b7afc060
Addendum patch for YARN-5731
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b7afc06
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b7afc06
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b7afc06
Branch: refs/heads/trunk
Commit: 0b7afc060c2024a882bd1934d0f722bfca731742
Parents: ccaf036
Author: Sunil G <su...@apache.org>
Authored: Tue Jul 18 11:49:09 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Jul 18 11:49:09 2017 +0530
----------------------------------------------------------------------
.../ProportionalCapacityPreemptionPolicy.java | 25 ++++--------------
.../CapacitySchedulerConfiguration.java | 27 ++++++++++++++++++++
...TestCapacitySchedulerSurgicalPreemption.java | 6 +++--
3 files changed, 36 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b7afc06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 719d2eb..fc8ad2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -231,28 +231,13 @@ public class ProportionalCapacityPreemptionPolicy
.add(new ReservedContainerCandidatesSelector(this));
}
+ boolean additionalPreemptionBasedOnReservedResource = csConfig.getBoolean(
+ CapacitySchedulerConfiguration.ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS,
+ CapacitySchedulerConfiguration.DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS);
+
// initialize candidates preemption selection policies
- // When select candidates for reserved containers is enabled, exclude reserved
- // resource in fifo policy (less aggressive). Otherwise include reserved
- // resource.
- //
- // Why doing this? In YARN-4390, we added preemption-based-on-reserved-container
- // Support. To reduce unnecessary preemption for large containers. We will
- // not include reserved resources while calculating ideal-allocation in
- // FifoCandidatesSelector.
- //
- // Changes in YARN-4390 will significantly reduce number of containers preempted
- // When cluster has heterogeneous container requests. (Please check test
- // report: https://issues.apache.org/jira/secure/attachment/12796197/YARN-4390-test-results.pdf
- //
- // However, on the other hand, in some corner cases, especially for
- // fragmented cluster. It could lead to preemption cannot kick in in some
- // cases. Please see YARN-5731.
- //
- // So to solve the problem, we will include reserved when surgical preemption
- // for reserved container, which reverts behavior when YARN-4390 is disabled.
candidatesSelectionPolicies.add(new FifoCandidatesSelector(this,
- !selectCandidatesForResevedContainers));
+ additionalPreemptionBasedOnReservedResource));
// Do we need to specially consider intra queue
boolean isIntraQueuePreemptionEnabled = csConfig.getBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b7afc06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 90a7e65..1e29d50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -1206,6 +1206,33 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
0.2f;
/**
+ * By default, reserved resource will be excluded while balancing capacities
+ * of queues.
+ *
+ * Why doing this? In YARN-4390, we added preemption-based-on-reserved-container
+ * Support. To reduce unnecessary preemption for large containers. We will
+ * not include reserved resources while calculating ideal-allocation in
+ * FifoCandidatesSelector.
+ *
+ * Changes in YARN-4390 will significantly reduce number of containers preempted
+ * When cluster has heterogeneous container requests. (Please check test
+ * report: https://issues.apache.org/jira/secure/attachment/12796197/YARN-4390-test-results.pdf
+ *
+ * However, on the other hand, in some corner cases, especially for
+ * fragmented cluster. It could lead to preemption cannot kick in in some
+ * cases. Please see YARN-5731.
+ *
+ * So to solve the problem, make this change to be configurable, and please
+ * note that it is an experimental option.
+ */
+ public static final String
+ ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS =
+ PREEMPTION_CONFIG_PREFIX
+ + "additional_res_balance_based_on_reserved_containers";
+ public static final boolean
+ DEFAULT_ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS = false;
+
+ /**
* When calculating which containers to be preempted, we will try to preempt
* containers for reserved containers first. By default is false.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b7afc06/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
index afd2f82..9146373 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerSurgicalPreemption.java
@@ -813,9 +813,11 @@ public class TestCapacitySchedulerSurgicalPreemption
@Test(timeout = 60000)
public void testPreemptionForFragmentatedCluster() throws Exception {
+ // Set additional_balance_queue_based_on_reserved_res to true to get
+ // additional preemptions.
conf.setBoolean(
- CapacitySchedulerConfiguration.PREEMPTION_SELECT_CANDIDATES_FOR_RESERVED_CONTAINERS,
- false);
+ CapacitySchedulerConfiguration.ADDITIONAL_RESOURCE_BALANCE_BASED_ON_RESERVED_CONTAINERS,
+ true);
/**
* Two queues, a/b, each of them are 50/50
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org