You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2021/01/27 14:49:46 UTC
[hadoop] branch trunk updated: YARN-10599. fs2cs should generate
new 'auto-queue-creation-v2.enabled' properties for all parents.
Contributed by Peter Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7c4ef42 YARN-10599. fs2cs should generate new 'auto-queue-creation-v2.enabled' properties for all parents. Contributed by Peter Bacsko
7c4ef42 is described below
commit 7c4ef42837955a531400005f824b27ca64c11cbd
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed Jan 27 15:49:26 2021 +0100
YARN-10599. fs2cs should generate new 'auto-queue-creation-v2.enabled' properties for all parents. Contributed by Peter Bacsko
---
.../weightconversion/WeightToWeightConverter.java | 12 +++++++--
.../fair/converter/TestFSQueueConverter.java | 29 ++++++++++++++++++++
.../TestWeightToWeightConverter.java | 31 ++++++++++++++++++++--
.../weightconversion/WeightConverterTestBase.java | 10 ++++---
4 files changed, 75 insertions(+), 7 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
index e5ce0b7..6dcd05c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightToWeightConverter.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.C
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
public class WeightToWeightConverter
@@ -33,14 +34,21 @@ public class WeightToWeightConverter
Configuration csConfig) {
List<FSQueue> children = queue.getChildQueues();
- children.forEach(fsQueue -> csConfig.set(
- getProperty(fsQueue), getWeightString(fsQueue)));
+ if (queue instanceof FSParentQueue || !children.isEmpty()) {
+ children.forEach(fsQueue -> csConfig.set(
+ getProperty(fsQueue), getWeightString(fsQueue)));
+ csConfig.setBoolean(getAutoCreateV2EnabledProperty(queue), true);
+ }
}
private String getProperty(FSQueue queue) {
return PREFIX + queue.getName() + ".capacity";
}
+ private String getAutoCreateV2EnabledProperty(FSQueue queue) {
+ return PREFIX + queue.getName() + ".auto-queue-creation-v2.enabled";
+ }
+
private String getWeightString(FSQueue queue) {
return Float.toString(queue.getWeight()) + "w";
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
index 7807092..e648c1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/TestFSQueueConverter.java
@@ -335,6 +335,35 @@ public class TestFSQueueConverter {
}
@Test
+ public void testAutoCreateV2FlagsInWeightMode() {
+ converter = builder.withPercentages(false).build();
+
+ converter.convertQueueHierarchy(rootQueue);
+
+ assertTrue("root autocreate v2 flag",
+ csConfig.getBoolean(
+ PREFIX + "root.auto-queue-creation-v2.enabled", false));
+ assertTrue("root.admins autocreate v2 flag",
+ csConfig.getBoolean(
+ PREFIX + "root.admins.auto-queue-creation-v2.enabled", false));
+ assertTrue("root.users autocreate v2 flag",
+ csConfig.getBoolean(
+ PREFIX + "root.users.auto-queue-creation-v2.enabled", false));
+ assertTrue("root.misc autocreate v2 flag",
+ csConfig.getBoolean(
+ PREFIX + "root.misc.auto-queue-creation-v2.enabled", false));
+
+ Set<String> leafs = Sets.difference(ALL_QUEUES,
+ Sets.newHashSet("root",
+ "root.default",
+ "root.admins",
+ "root.users",
+ "root.misc"));
+ assertNoValueForQueues(leafs, "auto-queue-creation-v2.enabled",
+ csConfig);
+ }
+
+ @Test
public void testZeroSumCapacityValidation() {
converter = builder.withPercentages(true).build();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
index c605a84..5f9b2d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/TestWeightToWeightConverter.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter.w
import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.PREFIX;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
@@ -41,7 +44,7 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
FSQueue root = createFSQueues();
converter.convertWeightsForChildQueues(root, config);
- assertEquals("Converted items", 0,
+ assertEquals("Converted items", 1,
config.getPropsWithPrefix(PREFIX).size());
}
@@ -52,6 +55,8 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
assertEquals("root.a weight", "1.0w",
config.get(PREFIX + "root.a.capacity"));
+ assertEquals("Number of properties", 2,
+ config.getPropsWithPrefix(PREFIX).size());
}
@Test
@@ -60,7 +65,7 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
converter.convertWeightsForChildQueues(root, config);
- assertEquals("Number of properties", 3,
+ assertEquals("Number of properties", 4,
config.getPropsWithPrefix(PREFIX).size());
assertEquals("root.a weight", "1.0w",
config.get(PREFIX + "root.a.capacity"));
@@ -69,4 +74,26 @@ public class TestWeightToWeightConverter extends WeightConverterTestBase {
assertEquals("root.c weight", "3.0w",
config.get(PREFIX + "root.c.capacity"));
}
+
+ @Test
+ public void testAutoCreateV2FlagOnParent() {
+ FSQueue root = createFSQueues(1);
+ converter.convertWeightsForChildQueues(root, config);
+
+ assertTrue("root autocreate v2 enabled",
+ config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled",
+ false));
+ }
+
+ @Test
+ public void testAutoCreateV2FlagOnParentWithoutChildren() {
+ FSQueue root = createParent(new ArrayList<>());
+ converter.convertWeightsForChildQueues(root, config);
+
+ assertEquals("Number of properties", 1,
+ config.getPropsWithPrefix(PREFIX).size());
+ assertTrue("root autocreate v2 enabled",
+ config.getBoolean(PREFIX + "root.auto-queue-creation-v2.enabled",
+ false));
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java
index 7ea6104..6dbdc12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/weightconversion/WeightConverterTestBase.java
@@ -24,6 +24,7 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSQueue;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -44,12 +45,15 @@ public abstract class WeightConverterTestBase {
queues.add(queue);
}
- FSQueue root = mock(FSQueue.class);
+ return createParent(queues);
+ }
+
+ protected FSParentQueue createParent(List<FSQueue> children) {
+ FSParentQueue root = mock(FSParentQueue.class);
when(root.getWeight()).thenReturn(1.0f);
when(root.getName()).thenReturn("root");
when(root.getMinShare()).thenReturn(Resources.none());
- when(root.getChildQueues()).thenReturn(queues);
-
+ when(root.getChildQueues()).thenReturn(children);
return root;
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org