You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/02/10 10:16:09 UTC
[hadoop] branch trunk updated: YARN-10620. fs2cs: parentQueue for
certain placement rules are not set during conversion. Contributed by Peter
Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a8bd516 YARN-10620. fs2cs: parentQueue for certain placement rules are not set during conversion. Contributed by Peter Bacsko
a8bd516 is described below
commit a8bd516e39fa82eea2a42b8085f3171fb0e1a883
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Feb 10 11:15:53 2021 +0100
YARN-10620. fs2cs: parentQueue for certain placement rules are not set during conversion. Contributed by Peter Bacsko
---
.../fair/converter/QueuePlacementConverter.java | 20 ++++++++++
.../converter/TestQueuePlacementConverter.java | 45 ++++++++++++++++++++++
2 files changed, 65 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
index 0dd6ba4..65d9505 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/QueuePlacementConverter.java
@@ -17,7 +17,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import java.util.ArrayList;
import java.util.List;
+import java.util.Set;
+import org.apache.hadoop.thirdparty.com.google.common.collect.Sets;
import org.apache.hadoop.yarn.server.resourcemanager.placement.DefaultPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.FSPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
@@ -38,6 +40,12 @@ class QueuePlacementConverter {
private static final FallbackResult SKIP_RESULT = FallbackResult.SKIP;
private static final String DEFAULT_QUEUE = "root.default";
private static final String MATCH_ALL_USER = "*";
+ private static final Set<Policy> NEED_ROOT_PARENT = Sets.newHashSet(
+ Policy.USER,
+ Policy.PRIMARY_GROUP,
+ Policy.PRIMARY_GROUP_USER,
+ Policy.SECONDARY_GROUP,
+ Policy.SECONDARY_GROUP_USER);
MappingRulesDescription convertPlacementPolicy(
PlacementManager placementManager,
@@ -162,6 +170,16 @@ class QueuePlacementConverter {
}
}
+ // Need to set the parent queue in weight mode.
+ //
+ // We *don't* set in pct mode, because auto-creation under "root"
+ // is not possible and probably it can cause the validation step to fail
+ // if create=true.
+ if (!usePercentages &&
+ NEED_ROOT_PARENT.contains(policy)) {
+ rule.setParentQueue("root");
+ }
+
return rule;
}
@@ -175,6 +193,8 @@ class QueuePlacementConverter {
Rule rule = createRule(policy, create, ruleHandler, usePercentages);
+ // "parent" is already set to "root" at this point,
+ // so we override it if necessary
if (parentQueue != null) {
rule.setParentQueue(parentQueue);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
index 2f041c2..a1a19f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestQueuePlacementConverter.java
@@ -369,6 +369,51 @@ public class TestQueuePlacementConverter {
}
@Test
+ public void testParentSetToRootInWeightModeUserPolicy() {
+ UserPlacementRule fsRule = mock(UserPlacementRule.class);
+ testParentSetToRootInWeightMode(fsRule);
+ }
+
+ @Test
+ public void testParentSetToRootInWeightModePrimaryGroupPolicy() {
+ PrimaryGroupPlacementRule fsRule = mock(PrimaryGroupPlacementRule.class);
+ testParentSetToRootInWeightMode(fsRule);
+ }
+
+ @Test
+ public void testParentSetToRootInWeightModePrimaryGroupUserPolicy() {
+ UserPlacementRule fsRule = mock(UserPlacementRule.class);
+ PrimaryGroupPlacementRule parent = mock(PrimaryGroupPlacementRule.class);
+ when(fsRule.getParentRule()).thenReturn(parent);
+ testParentSetToRootInWeightMode(fsRule);
+ }
+
+ @Test
+ public void testParentSetToRootInWeightModeSecondaryGroupPolicy() {
+ SecondaryGroupExistingPlacementRule fsRule =
+ mock(SecondaryGroupExistingPlacementRule.class);
+ testParentSetToRootInWeightMode(fsRule);
+ }
+
+ @Test
+ public void testParentSetToRootInWeightModeSecondaryGroupUserPolicy() {
+ UserPlacementRule fsRule = mock(UserPlacementRule.class);
+ SecondaryGroupExistingPlacementRule parent =
+ mock(SecondaryGroupExistingPlacementRule.class);
+ when(fsRule.getParentRule()).thenReturn(parent);
+ testParentSetToRootInWeightMode(fsRule);
+ }
+
+ private void testParentSetToRootInWeightMode(FSPlacementRule fsRule) {
+ initPlacementManagerMock(fsRule);
+
+ MappingRulesDescription desc = convertInWeightMode();
+ Rule rule = desc.getRules().get(0);
+
+ assertEquals("Parent queue", "root", rule.getParentQueue());
+ }
+
+ @Test
public void testConvertNestedPrimaryGroupRuleWithParentCreate() {
UserPlacementRule fsRule = mock(UserPlacementRule.class);
PrimaryGroupPlacementRule parent = mock(PrimaryGroupPlacementRule.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org