You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by yu...@apache.org on 2019/11/12 17:40:58 UTC
[hadoop] branch trunk updated: YARN-9537. Add configuration to
disable AM preemption. Contributed by ZhouKang
This is an automated email from the ASF dual-hosted git repository.
yufei pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new b83b9ab YARN-9537. Add configuration to disable AM preemption. Contributed by ZhouKang
b83b9ab is described below
commit b83b9ab41874646e92eb28b7f9153eaba858f4d0
Author: Yufei Gu <yu...@apache.org>
AuthorDate: Tue Nov 12 09:40:16 2019 -0800
YARN-9537. Add configuration to disable AM preemption. Contributed by ZhouKang
---
.../resourcemanager/scheduler/fair/FSAppAttempt.java | 15 +++++++++++++++
.../scheduler/fair/FairSchedulerConfiguration.java | 8 ++++++++
.../resourcemanager/scheduler/fair/TestFSAppAttempt.java | 2 ++
.../scheduler/fair/TestFairSchedulerPreemption.java | 12 ++++++++++++
4 files changed, 37 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index d6fb544..62c1142 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -99,6 +100,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
private final Map<String, Set<String>> reservations = new HashMap<>();
private final List<FSSchedulerNode> blacklistNodeIds = new ArrayList<>();
+
+ private boolean enableAMPreemption;
+
/**
* Delay scheduling: We often want to prioritize scheduling of node-local
* containers over rack-local or off-switch containers. To achieve this
@@ -121,6 +125,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
this.startTime = scheduler.getClock().getTime();
this.lastTimeAtFairShare = this.startTime;
this.appPriority = Priority.newInstance(1);
+ this.enableAMPreemption = scheduler.getConf()
+ .getAMPreemptionEnabled(getQueue().getQueueName());
}
/**
@@ -589,6 +595,10 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return false;
}
+ if (container.isAMContainer() && !enableAMPreemption) {
+ return false;
+ }
+
// Sanity check that the app owns this container
if (!getLiveContainersMap().containsKey(container.getContainerId()) &&
!newlyAllocatedContainers.contains(container)) {
@@ -1416,4 +1426,9 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
public boolean isPreemptable() {
return getQueue().isPreemptable();
}
+
+ @VisibleForTesting
+ public void setEnableAMPreemption(boolean enableAMPreemption) {
+ this.enableAMPreemption = enableAMPreemption;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
index 45e1395..23f4482 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
@@ -169,6 +169,10 @@ public class FairSchedulerConfiguration extends Configuration {
public static final String PREEMPTION = CONF_PREFIX + "preemption";
public static final boolean DEFAULT_PREEMPTION = false;
+ protected static final String AM_PREEMPTION_PREFIX =
+ CONF_PREFIX + "am.preemption.";
+ protected static final boolean DEFAULT_AM_PREEMPTION = true;
+
protected static final String PREEMPTION_THRESHOLD =
CONF_PREFIX + "preemption.cluster-utilization-threshold";
protected static final float DEFAULT_PREEMPTION_THRESHOLD = 0.8f;
@@ -395,6 +399,10 @@ public class FairSchedulerConfiguration extends Configuration {
return getBoolean(PREEMPTION, DEFAULT_PREEMPTION);
}
+ public boolean getAMPreemptionEnabled(String queueName) {
+ return getBoolean(AM_PREEMPTION_PREFIX + queueName, DEFAULT_AM_PREEMPTION);
+ }
+
public float getPreemptionUtilizationThreshold() {
return getFloat(PREEMPTION_THRESHOLD, DEFAULT_PREEMPTION_THRESHOLD);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
index 6d11898..28c98fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppAttempt.java
@@ -238,6 +238,8 @@ public class TestFSAppAttempt extends FairSchedulerTestBase {
(clusterUsage);
Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn
(fakeRootQueueMetrics);
+ Mockito.when(mockScheduler.getConf()).thenReturn
+ (Mockito.mock(FairSchedulerConfiguration.class));
ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
RMContext rmContext = resourceManager.getRMContext();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index 67cb600..f86987f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -422,6 +422,18 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
}
@Test
+ public void testDisableAMPreemption() {
+ takeAllResources("root.preemptable.child-1");
+ setNumAMContainersPerNode(2);
+ RMContainer container = greedyApp.getLiveContainers().stream()
+ .filter(rmContainer -> rmContainer.isAMContainer())
+ .findFirst()
+ .get();
+ greedyApp.setEnableAMPreemption(false);
+ assertFalse(greedyApp.canContainerBePreempted(container, null));
+ }
+
+ @Test
public void testPreemptionBetweenSiblingQueuesWithParentAtFairShare()
throws InterruptedException {
// Run this test only for fairshare preemption
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org