You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/05/27 11:06:43 UTC
[hadoop] branch trunk updated (c30c23c -> 593af87)
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git.
from c30c23c YARN-6492. Generate queue metrics for each partition. Contributed by Manikandan R
new fb030f3 Revert "FS-CS converter: nestedUserQueue with default rule results in invalid queue mapping. Contributed by Gergely Pollak"
new 593af87 YARN-10108. FS-CS converter: nestedUserQueue with default rule results in invalid queue mapping. Contributed by Gergely Pollak
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 02/02: YARN-10108. FS-CS converter: nestedUserQueue with
default rule results in invalid queue mapping. Contributed by Gergely
Pollak
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 593af878c006953a2df5da504e59aa65505db438
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Thu May 21 13:11:44 2020 +0200
YARN-10108. FS-CS converter: nestedUserQueue with default rule results in invalid queue mapping. Contributed by Gergely Pollak
---
.../scheduler/capacity/CapacityScheduler.java | 6 +++--
.../TestCapacitySchedulerAutoQueueCreation.java | 29 ++++++++++++++++++++++
2 files changed, 33 insertions(+), 2 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 309fb60..5cef57a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -992,12 +992,14 @@ public class CapacityScheduler extends
// not auto-created above, then its parent queue should match
// the parent queue specified in queue mapping
} else if (!queue.getParent().getQueueShortName().equals(
- placementContext.getParentQueue())) {
+ placementContext.getParentQueue())
+ && !queue.getParent().getQueuePath().equals(
+ placementContext.getParentQueue())) {
String message =
"Auto created Leaf queue " + placementContext.getQueue() + " "
+ "already exists under queue : " + queue
.getParent().getQueueShortName()
- + ".But Queue mapping configuration " +
+ + ". But Queue mapping configuration " +
CapacitySchedulerConfiguration.QUEUE_MAPPING + " has been "
+ "updated to a different parent queue : "
+ placementContext.getParentQueue()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index efe5c02..caf1df4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -162,6 +162,35 @@ public class TestCapacitySchedulerAutoQueueCreation
}
}
+ @Test(timeout = 20000)
+ public void testAutoCreateLeafQueueCreationUsingFullParentPath()
+ throws Exception {
+
+ try {
+ setupGroupQueueMappings("root.d", cs.getConfiguration(), "%user");
+ cs.reinitialize(cs.getConfiguration(), mockRM.getRMContext());
+
+ submitApp(mockRM, cs.getQueue("d"), TEST_GROUPUSER, TEST_GROUPUSER, 1, 1);
+ AutoCreatedLeafQueue autoCreatedLeafQueue =
+ (AutoCreatedLeafQueue) cs.getQueue(TEST_GROUPUSER);
+ ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue("d");
+ assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
+
+ Map<String, Float> expectedChildQueueAbsCapacity =
+ new HashMap<String, Float>() {{
+ put(NO_LABEL, 0.02f);
+ }};
+
+ validateInitialQueueEntitlement(parentQueue, TEST_GROUPUSER,
+ expectedChildQueueAbsCapacity,
+ new HashSet<String>() {{ add(NO_LABEL); }});
+
+ } finally {
+ cleanupQueue(USER0);
+ cleanupQueue(TEST_GROUPUSER);
+ }
+ }
+
@Test
public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[hadoop] 01/02: Revert "FS-CS converter: nestedUserQueue with
default rule results in invalid queue mapping. Contributed by Gergely
Pollak"
Posted by sn...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit fb030f3239ec0cdc5d2de9e0a53fb31b1d89aeec
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Wed May 27 13:04:48 2020 +0200
Revert "FS-CS converter: nestedUserQueue with default rule results in invalid queue mapping. Contributed by Gergely Pollak"
This reverts commit ac4540dd8e2872b113a61107b0a4a66beeec8649.
Reverting because of missing Jira-ID in the commit message.
---
.../scheduler/capacity/CapacityScheduler.java | 6 ++---
.../TestCapacitySchedulerAutoQueueCreation.java | 29 ----------------------
2 files changed, 2 insertions(+), 33 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 5cef57a..309fb60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -992,14 +992,12 @@ public class CapacityScheduler extends
// not auto-created above, then its parent queue should match
// the parent queue specified in queue mapping
} else if (!queue.getParent().getQueueShortName().equals(
- placementContext.getParentQueue())
- && !queue.getParent().getQueuePath().equals(
- placementContext.getParentQueue())) {
+ placementContext.getParentQueue())) {
String message =
"Auto created Leaf queue " + placementContext.getQueue() + " "
+ "already exists under queue : " + queue
.getParent().getQueueShortName()
- + ". But Queue mapping configuration " +
+ + ".But Queue mapping configuration " +
CapacitySchedulerConfiguration.QUEUE_MAPPING + " has been "
+ "updated to a different parent queue : "
+ placementContext.getParentQueue()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
index caf1df4..efe5c02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerAutoQueueCreation.java
@@ -162,35 +162,6 @@ public class TestCapacitySchedulerAutoQueueCreation
}
}
- @Test(timeout = 20000)
- public void testAutoCreateLeafQueueCreationUsingFullParentPath()
- throws Exception {
-
- try {
- setupGroupQueueMappings("root.d", cs.getConfiguration(), "%user");
- cs.reinitialize(cs.getConfiguration(), mockRM.getRMContext());
-
- submitApp(mockRM, cs.getQueue("d"), TEST_GROUPUSER, TEST_GROUPUSER, 1, 1);
- AutoCreatedLeafQueue autoCreatedLeafQueue =
- (AutoCreatedLeafQueue) cs.getQueue(TEST_GROUPUSER);
- ManagedParentQueue parentQueue = (ManagedParentQueue) cs.getQueue("d");
- assertEquals(parentQueue, autoCreatedLeafQueue.getParent());
-
- Map<String, Float> expectedChildQueueAbsCapacity =
- new HashMap<String, Float>() {{
- put(NO_LABEL, 0.02f);
- }};
-
- validateInitialQueueEntitlement(parentQueue, TEST_GROUPUSER,
- expectedChildQueueAbsCapacity,
- new HashSet<String>() {{ add(NO_LABEL); }});
-
- } finally {
- cleanupQueue(USER0);
- cleanupQueue(TEST_GROUPUSER);
- }
- }
-
@Test
public void testReinitializeStoppedAutoCreatedLeafQueue() throws Exception {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org