You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2016/10/21 20:57:12 UTC
[06/50] [abbrv] hadoop git commit: YARN-4911. Bad placement policy in
FairScheduler causes the RM to crash
YARN-4911. Bad placement policy in FairScheduler causes the RM to crash
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a064865a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a064865a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a064865a
Branch: refs/heads/YARN-3368
Commit: a064865abf7dceee46d3c42eca67a04a25af9d4e
Parents: d7d87de
Author: Karthik Kambatla <ka...@apache.org>
Authored: Thu Oct 20 20:57:04 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Thu Oct 20 20:57:04 2016 -0700
----------------------------------------------------------------------
.../scheduler/fair/FairScheduler.java | 6 +++++
.../scheduler/fair/TestFairScheduler.java | 28 ++++++++++++++++++++
.../fair/TestQueuePlacementPolicy.java | 9 +++++--
3 files changed, 41 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a064865a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 8daf0f3..d33c214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -774,6 +774,12 @@ public class FairScheduler extends
appRejectMsg = queueName + " is not a leaf queue";
}
}
+ } catch (IllegalStateException se) {
+ appRejectMsg = "Unable to match app " + rmApp.getApplicationId() +
+ " to a queue placement policy, and no valid terminal queue " +
+ " placement rule is configured. Please contact an administrator " +
+ " to confirm that the fair scheduler configuration contains a " +
+ " valid terminal queue placement rule.";
} catch (InvalidQueueNameException qne) {
appRejectMsg = qne.getMessage();
} catch (IOException ioe) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a064865a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 98af8b9..7535f69 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -1605,6 +1605,34 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
@Test
+ public void testAssignToBadDefaultQueue() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queuePlacementPolicy>");
+ out.println("<rule name=\"specified\" create=\"false\" />");
+ out.println("<rule name=\"default\" create=\"false\" />");
+ out.println("</queuePlacementPolicy>");
+ out.println("</allocations>");
+ out.close();
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
+
+ try {
+ FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default",
+ "asterix");
+ } catch (IllegalStateException ise) {
+ fail("Bad queue placement policy terminal rule should not throw " +
+ "exception ");
+ }
+ }
+
+ @Test
public void testAssignToNonLeafQueueReturnsNull() throws Exception {
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
scheduler.init(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a064865a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
index 0dbc99b..3fe9ce3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
@@ -131,9 +131,14 @@ public class TestQueuePlacementPolicy {
StringBuffer sb = new StringBuffer();
sb.append("<queuePlacementPolicy>");
sb.append(" <rule name='secondaryGroupExistingQueue' create='true'/>");
- sb.append(" <rule name='default' create='false'/>");
+ sb.append(" <rule name='default' queue='otherdefault' create='false'/>");
sb.append("</queuePlacementPolicy>");
- parse(sb.toString());
+ QueuePlacementPolicy policy = parse(sb.toString());
+ try {
+ policy.assignAppToQueue("root.otherdefault", "user1");
+ fail("Expect exception from having default rule with create=\'false\'");
+ } catch (IllegalStateException se) {
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org