You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/02/15 13:50:31 UTC
[hadoop] branch trunk updated: YARN-10622. Fix preemption policy to
exclude childless ParentQueues. Contributed by Andras Gyori
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bad6038 YARN-10622. Fix preemption policy to exclude childless ParentQueues. Contributed by Andras Gyori
bad6038 is described below
commit bad6038a4879be7b93eb52cfb54ddfd4ce7111cd
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Mon Feb 15 14:48:42 2021 +0100
YARN-10622. Fix preemption policy to exclude childless ParentQueues. Contributed by Andras Gyori
---
.../ProportionalCapacityPreemptionPolicy.java | 9 +++--
.../TestProportionalCapacityPreemptionPolicy.java | 42 ++++++++++++++++++++++
2 files changed, 48 insertions(+), 3 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index ccbaf69..b343678 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.thirdparty.com.google.common.collect.ImmutableSet;
import org.apache.commons.lang3.StringUtils;
@@ -409,9 +410,11 @@ public class ProportionalCapacityPreemptionPolicy
}
private Set<String> getLeafQueueNames(TempQueuePerPartition q) {
- // If its a ManagedParentQueue, it might not have any children
- if ((q.children == null || q.children.isEmpty())
- && !(q.parentQueue instanceof ManagedParentQueue)) {
+ // Also exclude ParentQueues, which might be without children
+ if (CollectionUtils.isEmpty(q.children)
+ && !(q.parentQueue instanceof ManagedParentQueue)
+ && (q.parentQueue == null
+ || !q.parentQueue.isEligibleForAutoQueueCreation())) {
return ImmutableSet.of(q.queueName);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index 34b86bb..fe89a69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
@@ -81,6 +81,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_KILLABLE;
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType.MARK_CONTAINER_FOR_PREEMPTION;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -1071,6 +1072,47 @@ public class TestProportionalCapacityPreemptionPolicy {
assertEquals(newObserveOnly, policy.isObserveOnly());
}
+ @Test
+ public void testLeafQueueNameExtraction() throws Exception {
+ ProportionalCapacityPreemptionPolicy policy =
+ buildPolicy(Q_DATA_FOR_IGNORE);
+ ParentQueue root = (ParentQueue) mCS.getRootQueue();
+ root.addDynamicParentQueue("childlessFlexible");
+ List<CSQueue> queues = root.getChildQueues();
+ ArrayList<CSQueue> extendedQueues = new ArrayList<>();
+ LinkedList<ParentQueue> pqs = new LinkedList<>();
+ ParentQueue dynamicParent = mockParentQueue(
+ null, 0, pqs);
+ when(dynamicParent.getQueuePath()).thenReturn("root.dynamicParent");
+ when(dynamicParent.getQueueCapacities()).thenReturn(
+ new QueueCapacities(false));
+ QueueResourceQuotas dynamicParentQr = new QueueResourceQuotas();
+ dynamicParentQr.setEffectiveMaxResource(Resource.newInstance(1, 1));
+ dynamicParentQr.setEffectiveMinResource(Resources.createResource(1));
+ dynamicParentQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL,
+ Resource.newInstance(1, 1));
+ dynamicParentQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL,
+ Resources.createResource(1));
+ when(dynamicParent.getQueueResourceQuotas()).thenReturn(dynamicParentQr);
+ when(dynamicParent.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resources.createResource(1));
+ when(dynamicParent.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL))
+ .thenReturn(Resource.newInstance(1, 1));
+ ResourceUsage resUsage = new ResourceUsage();
+ resUsage.setUsed(Resources.createResource(1024));
+ resUsage.setReserved(Resources.createResource(1024));
+ when(dynamicParent.getQueueResourceUsage()).thenReturn(resUsage);
+ when(dynamicParent.isEligibleForAutoQueueCreation()).thenReturn(true);
+ extendedQueues.add(dynamicParent);
+ extendedQueues.addAll(queues);
+ when(root.getChildQueues()).thenReturn(extendedQueues);
+
+ policy.editSchedule();
+
+ assertFalse("dynamicParent should not be a LeafQueue " +
+ "candidate", policy.getLeafQueueNames().contains("root.dynamicParent"));
+ }
+
static class IsPreemptionRequestFor
implements ArgumentMatcher<ContainerPreemptEvent> {
private final ApplicationAttemptId appAttId;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org