You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/02/16 17:07:23 UTC
[hadoop] branch trunk updated: YARN-10625. FairScheduler: add
global flag to disable AM-preemption. Contributed by Peter Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9882a22 YARN-10625. FairScheduler: add global flag to disable AM-preemption. Contributed by Peter Bacsko
9882a22 is described below
commit 9882a223652bf5d959e749b5d19d2ac3b53b00f1
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Tue Feb 16 18:06:57 2021 +0100
YARN-10625. FairScheduler: add global flag to disable AM-preemption. Contributed by Peter Bacsko
---
.../scheduler/fair/FairScheduler.java | 7 +++++++
.../scheduler/fair/FairSchedulerConfiguration.java | 14 ++++++++++++-
.../fair/TestFairSchedulerPreemption.java | 24 +++++++++++++++++++++-
3 files changed, 43 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 813f87b..bc21a17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1442,6 +1442,13 @@ public class FairScheduler extends
+ " ms instead");
}
+ boolean globalAmPreemption = conf.getBoolean(
+ FairSchedulerConfiguration.AM_PREEMPTION,
+ FairSchedulerConfiguration.DEFAULT_AM_PREEMPTION);
+ if (!globalAmPreemption) {
+ LOG.info("AM preemption is DISABLED globally");
+ }
+
rootMetrics = FSQueueMetrics.forQueue("root", null, true, conf);
fsOpDurations = FSOpDurations.getInstance(true);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 4de6a0b..80fb144 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -176,6 +176,8 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String PREEMPTION = CONF_PREFIX + "preemption";
public static final boolean DEFAULT_PREEMPTION = false;
+ protected static final String AM_PREEMPTION =
+ CONF_PREFIX + "am.preemption";
protected static final String AM_PREEMPTION_PREFIX =
CONF_PREFIX + "am.preemption.";
protected static final boolean DEFAULT_AM_PREEMPTION = true;
@@ -407,7 +409,17 @@ public class FairSchedulerConfiguration extends Configuration {
}
public boolean getAMPreemptionEnabled(String queueName) {
- return getBoolean(AM_PREEMPTION_PREFIX + queueName, DEFAULT_AM_PREEMPTION);
+ String propertyName = AM_PREEMPTION_PREFIX + queueName;
+
+ if (get(propertyName) != null) {
+ boolean amPreemptionEnabled =
+ getBoolean(propertyName, DEFAULT_AM_PREEMPTION);
+ LOG.debug("AM preemption enabled for queue {}: {}",
+ queueName, amPreemptionEnabled);
+ return amPreemptionEnabled;
+ }
+
+ return getBoolean(AM_PREEMPTION, DEFAULT_AM_PREEMPTION);
}
public float getPreemptionUtilizationThreshold() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index 0240132..8d7665a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -35,7 +35,9 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -53,6 +55,8 @@ import java.util.List;
public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
private static final File ALLOC_FILE = new File(TEST_DIR, "test-queues");
private static final int GB = 1024;
+ private static final String TC_DISABLE_AM_PREEMPTION_GLOBALLY =
+ "testDisableAMPreemptionGlobally";
// Scheduler clock
private final ControlledClock clock = new ControlledClock();
@@ -69,6 +73,9 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
// Starving app that is expected to instigate preemption
private FSAppAttempt starvingApp;
+ @Rule
+ public TestName testName = new TestName();
+
@Parameterized.Parameters(name = "{0}")
public static Collection<Object[]> getParameters() {
return Arrays.asList(new Object[][] {
@@ -95,6 +102,10 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
conf.setFloat(FairSchedulerConfiguration.PREEMPTION_THRESHOLD, 0f);
conf.setInt(FairSchedulerConfiguration.WAIT_TIME_BEFORE_KILL, 0);
conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 60_000L);
+ String testMethod = testName.getMethodName();
+ if (testMethod.startsWith(TC_DISABLE_AM_PREEMPTION_GLOBALLY)) {
+ conf.setBoolean(FairSchedulerConfiguration.AM_PREEMPTION, false);
+ }
setupCluster();
}
@@ -417,13 +428,24 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
@Test
public void testDisableAMPreemption() {
+ testDisableAMPreemption(false);
+ }
+
+ @Test
+ public void testDisableAMPreemptionGlobally() {
+ testDisableAMPreemption(true);
+ }
+
+ private void testDisableAMPreemption(boolean global) {
takeAllResources("root.preemptable.child-1");
setNumAMContainersPerNode(2);
RMContainer container = greedyApp.getLiveContainers().stream()
.filter(rmContainer -> rmContainer.isAMContainer())
.findFirst()
.get();
- greedyApp.setEnableAMPreemption(false);
+ if (!global) {
+ greedyApp.setEnableAMPreemption(false);
+ }
assertFalse(greedyApp.canContainerBePreempted(container, null));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org