You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ha...@apache.org on 2018/09/24 19:08:05 UTC
[43/50] [abbrv] hadoop git commit: YARN-8240. Add queue-level control
to allow all applications in a queue to opt-out. Contributed by Haibo Chen.
YARN-8240. Add queue-level control to allow all applications in a queue to opt-out. Contributed by Haibo Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23ae827c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23ae827c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23ae827c
Branch: refs/heads/YARN-1011
Commit: 23ae827cff9521025dc267ed7e0c3ccb3d090ed7
Parents: 4cf8e47
Author: Miklos Szegedi <sz...@apache.org>
Authored: Mon Jun 4 11:20:32 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Fri Sep 21 17:28:58 2018 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationConfiguration.java | 9 ++
.../scheduler/fair/FSLeafQueue.java | 5 +
.../resourcemanager/scheduler/fair/FSQueue.java | 21 +++-
.../allocation/AllocationFileQueueParser.java | 4 +
.../fair/allocation/QueueProperties.java | 13 +++
.../webapp/dao/FairSchedulerQueueInfo.java | 6 ++
.../fair/TestAllocationFileLoaderService.java | 61 +++++++++++
.../scheduler/fair/TestFairScheduler.java | 103 +++++++++++++++++++
.../scheduler/fair/TestQueueManager.java | 70 +++++++++++++
.../src/site/markdown/FairScheduler.md | 1 +
10 files changed, 292 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index e48e04b..14e2aeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -103,6 +104,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
private ReservationQueueConfiguration globalReservationQueueConfig;
private final Set<String> nonPreemptableQueues;
+ private final Map<String, Boolean> queueOversubscriptionSetting;
public AllocationConfiguration(QueueProperties queueProperties,
AllocationFileParser allocationFileParser,
@@ -138,6 +140,8 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
this.placementPolicy = newPlacementPolicy;
this.configuredQueues = queueProperties.getConfiguredQueues();
this.nonPreemptableQueues = queueProperties.getNonPreemptableQueues();
+ this.queueOversubscriptionSetting =
+ queueProperties.getQueueOversubscriptionSetting();
}
public AllocationConfiguration(Configuration conf) {
@@ -167,6 +171,7 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
placementPolicy =
QueuePlacementPolicy.fromConfiguration(conf, configuredQueues);
nonPreemptableQueues = new HashSet<>();
+ queueOversubscriptionSetting = new HashMap<>(0);
}
/**
@@ -221,6 +226,10 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return !nonPreemptableQueues.contains(queueName);
}
+ public Optional<Boolean> isOversubscriptionAllowed(String queueName) {
+ return Optional.ofNullable(queueOversubscriptionSetting.get(queueName));
+ }
+
private float getQueueWeight(String queue) {
Float weight = queueWeights.get(queue);
return (weight == null) ? 1.0f : weight;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 3834130..48836b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -378,6 +378,11 @@ public class FSLeafQueue extends FSQueue {
getName() + " fairShare: " + getFairShare());
}
+ if (opportunistic && !isOversubscriptionAllowed()) {
+ // do not proceed if the queue is opted out of over-subscription
+ return assigned;
+ }
+
if (!assignContainerPreCheck(node, opportunistic)) {
return assigned;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 52c76f4..3a9ffae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import java.util.Optional;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -75,6 +76,7 @@ public abstract class FSQueue implements Queue, Schedulable {
private ConfigurableResource maxShare;
protected int maxRunningApps;
private ConfigurableResource maxChildQueueResource;
+ private boolean isOversubscriptionAllowed;
// maxAMShare is a value between 0 and 1.
protected float maxAMShare;
@@ -93,6 +95,7 @@ public abstract class FSQueue implements Queue, Schedulable {
this.queueEntity = new PrivilegedEntity(EntityType.QUEUE, name);
this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
this.parent = parent;
+ this.isOversubscriptionAllowed = true;
setPolicy(scheduler.getAllocationConfiguration().getSchedulingPolicy(name));
reinit(false);
}
@@ -112,6 +115,14 @@ public abstract class FSQueue implements Queue, Schedulable {
allocConf.initFSQueue(this);
updatePreemptionVariables();
+ Optional<Boolean> allowed = allocConf.isOversubscriptionAllowed(name);
+ if (allowed.isPresent()) {
+ isOversubscriptionAllowed = allowed.get();
+ } else if (getParent() != null){
+ // inherit from the parent queue if it is not explicitly set
+ isOversubscriptionAllowed = getParent().isOversubscriptionAllowed();
+ }
+
if (recursive) {
for (FSQueue child : getChildQueues()) {
child.reinit(recursive);
@@ -213,6 +224,15 @@ public abstract class FSQueue implements Queue, Schedulable {
return 0;
}
+ public boolean isOversubscriptionAllowed() {
+ return isOversubscriptionAllowed;
+ }
+
+ @VisibleForTesting
+ public void allowOversubscription(boolean allowed) {
+ this.isOversubscriptionAllowed = allowed;
+ }
+
@Override
public Priority getPriority() {
Priority p = recordFactory.newRecordInstance(Priority.class);
@@ -584,7 +604,6 @@ public abstract class FSQueue implements Queue, Schedulable {
return sb.toString();
}
-
/**
* Recursively dump states of all queues.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
index 441c34a..62a6060 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/AllocationFileQueueParser.java
@@ -68,6 +68,7 @@ public class AllocationFileQueueParser {
private static final String ACL_SUBMIT_RESERVATIONS = "aclSubmitReservations";
private static final String RESERVATION = "reservation";
private static final String ALLOW_PREEMPTION_FROM = "allowPreemptionFrom";
+ private static final String ALLOW_OVERSUBSCRIPTION ="allowOversubscription";
private static final String QUEUE = "queue";
private static final String POOL = "pool";
@@ -207,6 +208,9 @@ public class AllocationFileQueueParser {
if (!Boolean.parseBoolean(text)) {
builder.nonPreemptableQueues(queueName);
}
+ } else if (ALLOW_OVERSUBSCRIPTION.equals(field.getTagName())) {
+ Boolean allow = Boolean.parseBoolean(getTrimmedTextData(field));
+ builder.addQueueOversubscriptionSetting(queueName, allow);
} else if (QUEUE.endsWith(field.getTagName())
|| POOL.equals(field.getTagName())) {
loadQueue(queueName, field, builder);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
index ee5f179..f15f912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocation/QueueProperties.java
@@ -52,6 +52,7 @@ public class QueueProperties {
reservationAcls;
private final Set<String> reservableQueues;
private final Set<String> nonPreemptableQueues;
+ private final Map<String, Boolean> queueOversubscriptionSetting;
private final Map<FSQueueType, Set<String>> configuredQueues;
QueueProperties(Builder builder) {
@@ -70,6 +71,7 @@ public class QueueProperties {
this.maxChildQueueResources = builder.maxChildQueueResources;
this.reservationAcls = builder.reservationAcls;
this.queueAcls = builder.queueAcls;
+ this.queueOversubscriptionSetting = builder.queueOversubscriptionSetting;
}
public Map<FSQueueType, Set<String>> getConfiguredQueues() {
@@ -133,6 +135,9 @@ public class QueueProperties {
return nonPreemptableQueues;
}
+ public Map<String, Boolean> getQueueOversubscriptionSetting() {
+ return queueOversubscriptionSetting;
+ }
/**
* Builder class for {@link QueueProperties}.
* All methods are adding queue properties to the maps of this builder
@@ -160,6 +165,8 @@ public class QueueProperties {
reservationAcls = new HashMap<>();
private Set<String> reservableQueues = new HashSet<>();
private Set<String> nonPreemptableQueues = new HashSet<>();
+ private Map<String, Boolean> queueOversubscriptionSetting =
+ new HashMap<>(0);
// Remember all queue names so we can display them on web UI, etc.
// configuredQueues is segregated based on whether it is a leaf queue
// or a parent queue. This information is used for creating queues
@@ -253,6 +260,12 @@ public class QueueProperties {
return this;
}
+ public Builder addQueueOversubscriptionSetting(
+ String queue, Boolean allow) {
+ this.queueOversubscriptionSetting.put(queue, allow);
+ return this;
+ }
+
public void configuredQueues(FSQueueType queueType, String queueName) {
this.configuredQueues.get(queueType).add(queueName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index fe313a6..45b9ea9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -68,6 +68,7 @@ public class FairSchedulerQueueInfo {
private String schedulingPolicy;
private boolean preemptable;
+ private boolean allowOversubscription;
private FairSchedulerQueueInfoList childQueues;
@@ -118,6 +119,7 @@ public class FairSchedulerQueueInfo {
}
preemptable = queue.isPreemptable();
+ allowOversubscription = queue.isOversubscriptionAllowed();
childQueues = getChildQueues(queue, scheduler);
}
@@ -255,4 +257,8 @@ public class FairSchedulerQueueInfo {
public boolean isPreemptable() {
return preemptable;
}
+
+ public boolean isOversubscriptionAllowed() {
+ return allowOversubscription;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index 50a003e..35144cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -538,6 +538,67 @@ public class TestAllocationFileLoaderService {
}
@Test
+ public void testQueueOptOutOfOversubscription() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ AllocationFileLoaderService allocLoader = new AllocationFileLoaderService();
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ // Queue A has the default value of allowOversubscription
+ out.println("<queue name=\"queueA\">");
+ // The child queue of A, queueA.child1 has overridden the parent's setting
+ out.println("<queue name=\"child1\">");
+ out.println("<allowOversubscription>false</allowOversubscription>");
+ out.println("</queue>");
+ // The child queue of A, queueA.child2 inherits the setting from its parent
+ out.println("<queue name=\"child2\" />");
+ out.println("</queue>");
+ // Queue A has opted out of oversubscription
+ out.println("<queue name=\"queueB\">");
+ // The child queue of B, queueB.child has overridden the parent's setting
+ out.println("<allowOversubscription>true</allowOversubscription>");
+ // The child queue of A, queueA.child1 has overridden the parent's setting
+ out.println("<queue name=\"child1\" />");
+ // The child queue of B, queueB.child2 inherits the setting from its parent
+ out.println("<queue name=\"child2\">");
+ out.println("<allowOversubscription>false</allowOversubscription>");
+ out.println("</queue>");
+ out.println("</queue>");
+ out.println("</allocations>");
+ out.close();
+
+ allocLoader.init(conf);
+ ReloadListener confHolder = new ReloadListener();
+ allocLoader.setReloadListener(confHolder);
+ allocLoader.reloadAllocations();
+ AllocationConfiguration queueConf = confHolder.allocConf;
+
+ assertTrue("allowOversubscription is not set for queueA",
+ !queueConf.isOversubscriptionAllowed("root.queueA").isPresent());
+ assertTrue("allowOversubscription is not set for queueA.child2",
+ !queueConf.isOversubscriptionAllowed("root.queueA.child2").isPresent());
+ assertTrue("allowOversubscription is not set for queueB.child1",
+ !queueConf.isOversubscriptionAllowed("root.queueB.child1").isPresent());
+
+ assertTrue("allowOversubscription is set for queueB",
+ queueConf.isOversubscriptionAllowed("root.queueB").isPresent());
+ assertTrue("queueB does allow oversubscription",
+ queueConf.isOversubscriptionAllowed("root.queueB").get());
+ assertTrue("allowOversubscription is set for queueB.child2",
+ queueConf.isOversubscriptionAllowed("root.queueB.child2").isPresent());
+ assertTrue("queueB.child2 does not allow oversubscription",
+ !queueConf.isOversubscriptionAllowed("root.queueB.child2").get());
+
+ assertTrue("allowOversubscription is set for queueA.child1",
+ queueConf.isOversubscriptionAllowed("root.queueA.child1").isPresent());
+ assertTrue("queueA.child1 does not allow oversubscription",
+ !queueConf.isOversubscriptionAllowed("root.queueA.child1").get());
+
+ }
+
+ @Test
public void testSimplePlacementPolicyFromConf() throws Exception {
Configuration conf = new Configuration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 9302f88..5878ccd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -2789,6 +2789,109 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
}
+ @Test
+ public void testQueueOptOutOfOversubscription() throws IOException {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ true);
+ // disable resource request normalization in fair scheduler
+ int memoryAllocationIncrement = conf.getInt(
+ "yarn.resource-types.memory-mb.increment-allocation", 1024);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation", 1);
+ int memoryAllocationMinimum = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1);
+
+ try {
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ FSLeafQueue queue1 =
+ scheduler.getQueueManager().getLeafQueue("queue1", true);
+ // queue 2 has opted out of oversubscription
+ FSLeafQueue queue2 =
+ scheduler.getQueueManager().getLeafQueue("queue2", true);
+ queue2.allowOversubscription(false);
+
+ FSParentQueue queue3
+ = scheduler.getQueueManager().getParentQueue("queue3", true);
+ // queue 3 has opted out of oversubscription
+ queue3.allowOversubscription(false);
+ FSLeafQueue queue4 =
+ scheduler.getQueueManager().getLeafQueue("queue3.queue4", true);
+ assertTrue("queue 4 should inherit the setting from queue 3",
+ !queue4.isOversubscriptionAllowed());
+
+
+ // Add a node with 4G of memory and 4 vcores and an overallocation
+ // threshold of 0.75f and 0.75f for memory and cpu respectively
+ OverAllocationInfo overAllocationInfo = OverAllocationInfo.newInstance(
+ ResourceThresholds.newInstance(0.75f, 0.75f));
+ MockNodes.MockRMNodeImpl node = MockNodes.newNodeInfo(1,
+ Resources.createResource(4096, 4), overAllocationInfo);
+ scheduler.handle(new NodeAddedSchedulerEvent(node));
+
+ // create a scheduling request in queue1 that leaves some unallocated
+ // resources
+ ApplicationAttemptId appAttempt1 =
+ createSchedulingRequest(3600, "queue1", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(3600, scheduler.getQueueManager().getQueue("queue1").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers1 =
+ scheduler.getSchedulerApp(appAttempt1).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers1.size() == 1);
+ assertEquals("unexpected container execution type",
+ ExecutionType.GUARANTEED,
+ allocatedContainers1.get(0).getExecutionType());
+
+ // node utilization is low after the container runs on the node
+ ContainerStatus containerStatus = ContainerStatus.newInstance(
+ allocatedContainers1.get(0).getId(), ContainerState.RUNNING, "",
+ ContainerExitStatus.SUCCESS);
+ node.updateContainersAndNodeUtilization(
+ new UpdatedContainerInfo(Collections.singletonList(containerStatus),
+ Collections.emptyList()),
+ ResourceUtilization.newInstance(1024, 0, 0.1f));
+
+ // create another scheduling request in queue 2 that has opted out of
+ // oversubscription.
+ ApplicationAttemptId appAttempt2 =
+ createSchedulingRequest(1536, "queue2", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue2").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers2 =
+ scheduler.getSchedulerApp(appAttempt2).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers2.size() == 0);
+
+ // verify that a reservation is made for the second request
+ assertTrue("A reservation should be made for the second request",
+ scheduler.getNode(node.getNodeID()).getReservedContainer().
+ getReservedResource().equals(Resource.newInstance(1536, 1)));
+
+ // create another scheduling request in queue 4 that has opted out of
+ // oversubscription.
+ ApplicationAttemptId appAttempt3 =
+ createSchedulingRequest(1536, "queue3.queue4", "user1", 1, false);
+ scheduler.handle(new NodeUpdateSchedulerEvent(node));
+ assertEquals(0, scheduler.getQueueManager().getQueue("queue3.queue4").
+ getGuaranteedResourceUsage().getMemorySize());
+ List<Container> allocatedContainers3 =
+ scheduler.getSchedulerApp(appAttempt3).pullNewlyAllocatedContainers();
+ assertTrue(allocatedContainers3.size() == 0);
+ } finally {
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_OVERSUBSCRIPTION_ENABLED,
+ false);
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ memoryAllocationMinimum);
+ conf.setInt("yarn.resource-types.memory-mb.increment-allocation",
+ memoryAllocationIncrement);
+ }
+
+ }
+
/**
* Test that NO OPPORTUNISTIC containers can be allocated on a node that
* is fully allocated and with a very high utilization.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index 3674ffb..10111f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -642,4 +642,74 @@ public class TestQueueManager {
assertNotNull("root.a does not exist", p);
}
+ /**
+ * Test queues created dynamically inherit the oversubscription setting
+ * from its parent queue, in which oversubscription is enabled.
+ */
+ @Test
+ public void testCreateQueueWithParentOversubscriptionSettingsEnabled() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+ queueManager.updateAllocationConfiguration(allocConf);
+
+ FSQueue q1 = queueManager.createQueue("root.queue1",
+ FSQueueType.PARENT);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+ assertTrue("root queue should always allow oversubscription",
+ queueManager.getParentQueue("root", false).isOversubscriptionAllowed());
+ assertTrue("root.queue1 should allow oversubscription as its parent",
+ queueManager.getParentQueue("root.queue1", false)
+ .isOversubscriptionAllowed());
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.LEAF);
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getLeafQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ assertTrue("root.queue1.queue2 should allow oversubscription as its parent",
+ queueManager.getLeafQueue("root.queue1.queue2", false)
+ .isOversubscriptionAllowed());
+
+ }
+
+ /**
+ * Test queues created dynamically inherit the oversubscription setting
+ * from its parent queue, in which oversubscription is disabled.
+ */
+ @Test
+ public void testCreateQueueWithParentOversubscriptionSettingsDisabled() {
+ AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
+
+ queueManager.updateAllocationConfiguration(allocConf);
+ FSQueue q1 = queueManager.createQueue("root.queue1",
+ FSQueueType.PARENT);
+ q1.allowOversubscription(false);
+
+ assertNotNull("Parent queue root.queue1 was not created",
+ queueManager.getParentQueue("root.queue1", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1", q1.getName());
+
+ assertTrue("root queue should always allow oversubscription",
+ queueManager.getParentQueue("root", false).isOversubscriptionAllowed());
+ assertTrue("root.queue1 should not allow oversubscription",
+ !queueManager.getParentQueue("root.queue1", false)
+ .isOversubscriptionAllowed());
+
+ FSQueue q2 = queueManager.createQueue("root.queue1.queue2",
+ FSQueueType.LEAF);
+ assertNotNull("Leaf queue root.queue1.queue2 was not created",
+ queueManager.getLeafQueue("root.queue1.queue2", false));
+ assertEquals("createQueue() returned wrong queue",
+ "root.queue1.queue2", q2.getName());
+ assertTrue(
+ "root.queue1.queue2 should not allow oversubscription as its parent",
+ !queueManager.getLeafQueue("root.queue1.queue2", false)
+ .isOversubscriptionAllowed());
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23ae827c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
index b5bcbf5..920b2ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/FairScheduler.md
@@ -111,6 +111,7 @@ The allocation file must be in XML format. The format contains five types of ele
* **fairSharePreemptionThreshold**: the fair share preemption threshold for the queue. If the queue waits fairSharePreemptionTimeout without receiving fairSharePreemptionThreshold\*fairShare resources, it is allowed to preempt containers to take resources from other queues. If not set, the queue will inherit the value from its parent queue. Default value is 0.5f.
* **allowPreemptionFrom**: determines whether the scheduler is allowed to preempt resources from the queue. The default is true. If a queue has this property set to false, this property will apply recursively to all child queues.
+ * **allowOversubscription**: determines whether the scheduler is allowed to assign OPPORTUNISTIC resources to applications running in the queue when over-subscription is turned on. The default is true.
* **reservation**: indicates to the `ReservationSystem` that the queue's resources is available for users to reserve. This only applies for leaf queues. A leaf queue is not reservable if this property isn't configured.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org