You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by um...@apache.org on 2014/05/12 14:44:10 UTC
svn commit: r1593948 [2/2] - in
/hadoop/common/branches/HDFS-2006/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-app...
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Mon May 12 12:43:59 2014
@@ -350,8 +350,7 @@ public class TestLeafQueue {
// Submit applications
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 1);
- FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, null,
- rmContext);
+
AppAddedSchedulerEvent addAppEvent =
new AppAddedSchedulerEvent(appAttemptId_0.getApplicationId(),
a.getQueueName(), user_0);
@@ -687,7 +686,7 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G
- assertEquals(0*GB, app_0.getHeadroom().getMemory()); // User limit = 2G
+ assertEquals(0*GB, app_1.getHeadroom().getMemory()); // User limit = 2G
// Again one to user_0 since he hasn't exceeded user limit yet
a.assignContainers(clusterResource, node_0);
@@ -695,7 +694,7 @@ public class TestLeafQueue {
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G
- assertEquals(0*GB, app_0.getHeadroom().getMemory()); // 3G - 2G
+ assertEquals(0*GB, app_1.getHeadroom().getMemory()); // 3G - 2G
// Submit requests for app_1 and set max-cap
a.setMaxCapacity(.1f);
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java Mon May 12 12:43:59 2014
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.*;
-import static org.junit.Assert.assertEquals;
import java.io.File;
import java.io.FileWriter;
@@ -28,6 +27,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.QueuePlacementRule.NestedUserQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.util.Clock;
@@ -99,9 +99,12 @@ public class TestAllocationFileLoaderSer
assertEquals(1, rules.size());
assertEquals(QueuePlacementRule.Default.class, rules.get(0).getClass());
assertEquals(1, allocConf.getQueueMaxApps("root.queueA"));
- assertEquals(2, allocConf.getQueueNames().size());
- assertTrue(allocConf.getQueueNames().contains("root.queueA"));
- assertTrue(allocConf.getQueueNames().contains("root.queueB"));
+ assertEquals(2, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .size());
+ assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueA"));
+ assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueB"));
confHolder.allocConf = null;
@@ -114,6 +117,9 @@ public class TestAllocationFileLoaderSer
out.println(" </queue>");
out.println(" <queuePlacementPolicy>");
out.println(" <rule name='specified' />");
+ out.println(" <rule name='nestedUserQueue' >");
+ out.println(" <rule name='primaryGroup' />");
+ out.println(" </rule>");
out.println(" <rule name='default' />");
out.println(" </queuePlacementPolicy>");
out.println("</allocations>");
@@ -131,12 +137,18 @@ public class TestAllocationFileLoaderSer
allocConf = confHolder.allocConf;
policy = allocConf.getPlacementPolicy();
rules = policy.getRules();
- assertEquals(2, rules.size());
+ assertEquals(3, rules.size());
assertEquals(QueuePlacementRule.Specified.class, rules.get(0).getClass());
- assertEquals(QueuePlacementRule.Default.class, rules.get(1).getClass());
+ assertEquals(QueuePlacementRule.NestedUserQueue.class, rules.get(1)
+ .getClass());
+ assertEquals(QueuePlacementRule.PrimaryGroup.class,
+ ((NestedUserQueue) (rules.get(1))).nestedRule.getClass());
+ assertEquals(QueuePlacementRule.Default.class, rules.get(2).getClass());
assertEquals(3, allocConf.getQueueMaxApps("root.queueB"));
- assertEquals(1, allocConf.getQueueNames().size());
- assertTrue(allocConf.getQueueNames().contains("root.queueB"));
+ assertEquals(1, allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .size());
+ assertTrue(allocConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueB"));
}
@Test
@@ -170,6 +182,14 @@ public class TestAllocationFileLoaderSer
out.println("<queue name=\"queueE\">");
out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
out.println("</queue>");
+ //Make queue F a parent queue without configured leaf queues using the 'type' attribute
+ out.println("<queue name=\"queueF\" type=\"parent\" >");
+ out.println("</queue>");
+ //Create hierarchical queues G,H
+ out.println("<queue name=\"queueG\">");
+ out.println(" <queue name=\"queueH\">");
+ out.println(" </queue>");
+ out.println("</queue>");
// Set default limit of apps per queue to 15
out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>");
// Set default limit of apps per user to 5
@@ -194,7 +214,7 @@ public class TestAllocationFileLoaderSer
allocLoader.reloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
- assertEquals(5, queueConf.getQueueNames().size());
+ assertEquals(6, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
@@ -250,6 +270,14 @@ public class TestAllocationFileLoaderSer
assertEquals(60000, queueConf.getMinSharePreemptionTimeout("root.queueE"));
assertEquals(300000, queueConf.getFairSharePreemptionTimeout());
+ assertTrue(queueConf.getConfiguredQueues()
+ .get(FSQueueType.PARENT)
+ .contains("root.queueF"));
+ assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.PARENT)
+ .contains("root.queueG"));
+ assertTrue(queueConf.getConfiguredQueues().get(FSQueueType.LEAF)
+ .contains("root.queueG.queueH"));
+
// Verify existing queues have default scheduling policy
assertEquals(DominantResourceFairnessPolicy.NAME,
queueConf.getSchedulingPolicy("root").getName());
@@ -315,7 +343,7 @@ public class TestAllocationFileLoaderSer
allocLoader.reloadAllocations();
AllocationConfiguration queueConf = confHolder.allocConf;
- assertEquals(5, queueConf.getQueueNames().size());
+ assertEquals(5, queueConf.getConfiguredQueues().get(FSQueueType.LEAF).size());
assertEquals(Resources.createResource(0),
queueConf.getMinResources("root." + YarnConfiguration.DEFAULT_QUEUE_NAME));
assertEquals(Resources.createResource(0),
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Mon May 12 12:43:59 2014
@@ -34,6 +34,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -43,7 +44,6 @@ import java.util.Set;
import javax.xml.parsers.ParserConfigurationException;
import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
@@ -737,8 +738,11 @@ public class TestFairScheduler {
rules.add(new QueuePlacementRule.Default().initialize(true, null));
Set<String> queues = Sets.newHashSet("root.user1", "root.user3group",
"root.user4subgroup1", "root.user4subgroup2" , "root.user5subgroup2");
+ Map<FSQueueType, Set<String>> configuredQueues = new HashMap<FSQueueType, Set<String>>();
+ configuredQueues.put(FSQueueType.LEAF, queues);
+ configuredQueues.put(FSQueueType.PARENT, new HashSet<String>());
scheduler.getAllocationConfiguration().placementPolicy =
- new QueuePlacementPolicy(rules, queues, conf);
+ new QueuePlacementPolicy(rules, configuredQueues, conf);
appId = createSchedulingRequest(1024, "somequeue", "user1");
assertEquals("root.somequeue", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "default", "user1");
@@ -758,7 +762,7 @@ public class TestFairScheduler {
rules.add(new QueuePlacementRule.Specified().initialize(true, null));
rules.add(new QueuePlacementRule.Default().initialize(true, null));
scheduler.getAllocationConfiguration().placementPolicy =
- new QueuePlacementPolicy(rules, queues, conf);
+ new QueuePlacementPolicy(rules, configuredQueues, conf);
appId = createSchedulingRequest(1024, "somequeue", "user1");
assertEquals("root.user1", scheduler.getSchedulerApp(appId).getQueueName());
appId = createSchedulingRequest(1024, "somequeue", "otheruser");
@@ -809,7 +813,89 @@ public class TestFairScheduler {
}
}
}
+
+ @Test
+ public void testNestedUserQueue() throws IOException {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+ conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
+ SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"user1group\" type=\"parent\">");
+ out.println("<minResources>1024mb,0vcores</minResources>");
+ out.println("</queue>");
+ out.println("<queuePlacementPolicy>");
+ out.println("<rule name=\"specified\" create=\"false\" />");
+ out.println("<rule name=\"nestedUserQueue\">");
+ out.println(" <rule name=\"primaryGroup\" create=\"false\" />");
+ out.println("</rule>");
+ out.println("<rule name=\"default\" />");
+ out.println("</queuePlacementPolicy>");
+ out.println("</allocations>");
+ out.close();
+
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+ RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
+
+ FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default",
+ "user1");
+
+ assertEquals("root.user1group.user1", user1Leaf.getName());
+ }
+
+ @Test
+ public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception {
+ conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
+ out.println("<?xml version=\"1.0\"?>");
+ out.println("<allocations>");
+ out.println("<queue name=\"parentq\" type=\"parent\">");
+ out.println("<minResources>1024mb,0vcores</minResources>");
+ out.println("</queue>");
+ out.println("<queuePlacementPolicy>");
+ out.println("<rule name=\"nestedUserQueue\">");
+ out.println(" <rule name=\"specified\" create=\"false\" />");
+ out.println("</rule>");
+ out.println("<rule name=\"default\" />");
+ out.println("</queuePlacementPolicy>");
+ out.println("</allocations>");
+ out.close();
+
+ RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
+ RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
+
+ scheduler.reinitialize(conf, resourceManager.getRMContext());
+
+ int capacity = 16 * 1024;
+ // create node with 16 G
+ RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity),
+ 1, "127.0.0.1");
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+
+ // user1,user2 submit their apps to parentq and create user queues
+ scheduler.assignToQueue(rmApp1, "root.parentq", "user1");
+ scheduler.assignToQueue(rmApp2, "root.parentq", "user2");
+
+ scheduler.update();
+
+ Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
+ .getLeafQueues();
+
+ for (FSLeafQueue leaf : leafQueues) {
+ if (leaf.getName().equals("root.parentq.user1")
+ || leaf.getName().equals("root.parentq.user2")) {
+ // assert that the fair share is 1/4th node1's capacity
+ assertEquals(capacity / 4, leaf.getFairShare().getMemory());
+ // assert weights are equal for both the user queues
+ assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0);
+ }
+ }
+ }
+
/**
* Make allocation requests and ensure they are reflected in queue demand.
*/
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java Mon May 12 12:43:59 2014
@@ -17,8 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
import java.util.HashSet;
@@ -57,45 +56,77 @@ public class TestQueueManager {
@Test
public void testReloadTurnsLeafQueueIntoParent() throws Exception {
- updateConfiguredQueues(queueManager, "queue1");
+ updateConfiguredLeafQueues(queueManager, "queue1");
// When no apps are running in the leaf queue, should be fine turning it
// into a parent.
- updateConfiguredQueues(queueManager, "queue1.queue2");
+ updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1", false));
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
// When leaf queues are empty, should be ok deleting them and
// turning parent into a leaf.
- updateConfiguredQueues(queueManager, "queue1");
+ updateConfiguredLeafQueues(queueManager, "queue1");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queue, we shouldn't be able to create
// children under it, but things should work otherwise.
notEmptyQueues.add(queueManager.getLeafQueue("queue1", false));
- updateConfiguredQueues(queueManager, "queue1.queue2");
+ updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queues under a parent queue, shouldn't be
// able to turn it into a leaf queue, but things should work otherwise.
notEmptyQueues.clear();
- updateConfiguredQueues(queueManager, "queue1.queue2");
+ updateConfiguredLeafQueues(queueManager, "queue1.queue2");
notEmptyQueues.add(queueManager.getQueue("root.queue1"));
- updateConfiguredQueues(queueManager, "queue1");
+ updateConfiguredLeafQueues(queueManager, "queue1");
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNull(queueManager.getLeafQueue("queue1", false));
// Should never to be able to create a queue under the default queue
- updateConfiguredQueues(queueManager, "default.queue3");
+ updateConfiguredLeafQueues(queueManager, "default.queue3");
assertNull(queueManager.getLeafQueue("default.queue3", false));
assertNotNull(queueManager.getLeafQueue("default", false));
}
- private void updateConfiguredQueues(QueueManager queueMgr, String... confQueues) {
+ @Test
+ public void testReloadTurnsLeafToParentWithNoLeaf() {
+ AllocationConfiguration allocConf = new AllocationConfiguration(conf);
+ // Create a leaf queue1
+ allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1");
+ queueManager.updateAllocationConfiguration(allocConf);
+ assertNotNull(queueManager.getLeafQueue("root.queue1", false));
+
+ // Lets say later on admin makes queue1 a parent queue by
+ // specifying "type=parent" in the alloc xml and lets say apps running in
+ // queue1
+ notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false));
+ allocConf = new AllocationConfiguration(conf);
+ allocConf.configuredQueues.get(FSQueueType.PARENT)
+ .add("root.queue1");
+
+ // When allocs are reloaded queue1 shouldn't be converter to parent
+ queueManager.updateAllocationConfiguration(allocConf);
+ assertNotNull(queueManager.getLeafQueue("root.queue1", false));
+ assertNull(queueManager.getParentQueue("root.queue1", false));
+
+ // Now lets assume apps completed and there are no apps in queue1
+ notEmptyQueues.clear();
+ // We should see queue1 transform from leaf queue to parent queue.
+ queueManager.updateAllocationConfiguration(allocConf);
+ assertNull(queueManager.getLeafQueue("root.queue1", false));
+ assertNotNull(queueManager.getParentQueue("root.queue1", false));
+ // this parent should not have any children
+ assertTrue(queueManager.getParentQueue("root.queue1", false)
+ .getChildQueues().isEmpty());
+ }
+
+ private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
- allocConf.queueNames = Sets.newHashSet(confQueues);
+ allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
queueMgr.updateAllocationConfiguration(allocConf);
}
}
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java Mon May 12 12:43:59 2014
@@ -17,8 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
@@ -28,16 +31,15 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-import com.google.common.collect.Sets;
-
public class TestQueuePlacementPolicy {
private final static Configuration conf = new Configuration();
- private final static Set<String> configuredQueues = Sets.newHashSet("root.someuser");
+ private Map<FSQueueType, Set<String>> configuredQueues;
@BeforeClass
public static void setup() {
@@ -45,6 +47,14 @@ public class TestQueuePlacementPolicy {
SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
}
+ @Before
+ public void initTest() {
+ configuredQueues = new HashMap<FSQueueType, Set<String>>();
+ for (FSQueueType type : FSQueueType.values()) {
+ configuredQueues.put(type, new HashSet<String>());
+ }
+ }
+
@Test
public void testSpecifiedUserPolicy() throws Exception {
StringBuffer sb = new StringBuffer();
@@ -53,9 +63,12 @@ public class TestQueuePlacementPolicy {
sb.append(" <rule name='user' />");
sb.append("</queuePlacementPolicy>");
QueuePlacementPolicy policy = parse(sb.toString());
- assertEquals("root.specifiedq",policy.assignAppToQueue("specifiedq", "someuser"));
- assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser"));
- assertEquals("root.otheruser", policy.assignAppToQueue("default", "otheruser"));
+ assertEquals("root.specifiedq",
+ policy.assignAppToQueue("specifiedq", "someuser"));
+ assertEquals("root.someuser",
+ policy.assignAppToQueue("default", "someuser"));
+ assertEquals("root.otheruser",
+ policy.assignAppToQueue("default", "otheruser"));
}
@Test
@@ -66,6 +79,8 @@ public class TestQueuePlacementPolicy {
sb.append(" <rule name='user' create=\"false\" />");
sb.append(" <rule name='default' />");
sb.append("</queuePlacementPolicy>");
+
+ configuredQueues.get(FSQueueType.LEAF).add("root.someuser");
QueuePlacementPolicy policy = parse(sb.toString());
assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser"));
assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser"));
@@ -81,7 +96,8 @@ public class TestQueuePlacementPolicy {
sb.append(" <rule name='reject' />");
sb.append("</queuePlacementPolicy>");
QueuePlacementPolicy policy = parse(sb.toString());
- assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser"));
+ assertEquals("root.specifiedq",
+ policy.assignAppToQueue("specifiedq", "someuser"));
assertEquals(null, policy.assignAppToQueue("default", "someuser"));
}
@@ -117,10 +133,188 @@ public class TestQueuePlacementPolicy {
parse(sb.toString());
}
+ @Test
+ public void testNestedUserQueueParsingErrors() {
+ // No nested rule specified in hierarchical user queue
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' />");
+ sb.append(" <rule name='nestedUserQueue'/>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ assertIfExceptionThrown(sb);
+
+ // Specified nested rule is not a QueuePlacementRule
+ sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='unknownRule'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ assertIfExceptionThrown(sb);
+ }
+
+ private void assertIfExceptionThrown(StringBuffer sb) {
+ Throwable th = null;
+ try {
+ parse(sb.toString());
+ } catch (Exception e) {
+ th = e;
+ }
+
+ assertTrue(th instanceof AllocationConfigurationException);
+ }
+
+ @Test
+ public void testNestedUserQueueParsing() throws Exception {
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='primaryGroup'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ Throwable th = null;
+ try {
+ parse(sb.toString());
+ } catch (Exception e) {
+ th = e;
+ }
+
+ assertNull(th);
+ }
+
+ @Test
+ public void testNestedUserQueuePrimaryGroup() throws Exception {
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='primaryGroup'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ // User queue would be created under primary group queue
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.user1group.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+ // Other rules above and below hierarchical user queue rule should work as
+ // usual
+ configuredQueues.get(FSQueueType.LEAF).add("root.specifiedq");
+ // test if specified rule(above nestedUserQueue rule) works ok
+ assertEquals("root.specifiedq",
+ policy.assignAppToQueue("root.specifiedq", "user2"));
+
+ // test if default rule(below nestedUserQueue rule) works
+ configuredQueues.get(FSQueueType.LEAF).add("root.user3group");
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user3"));
+ }
+
+ @Test
+ public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception {
+ // Primary group rule has create='false'
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='primaryGroup' create='false'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+
+ // Should return root.default since primary group 'root.user1group' is not
+ // configured
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user1"));
+
+ // Let's configure primary group and check if user queue is created
+ configuredQueues.get(FSQueueType.PARENT).add("root.user1group");
+ policy = parse(sb.toString());
+ assertEquals("root.user1group.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+
+ // Both Primary group and nestedUserQueue rule has create='false'
+ sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue' create='false'>");
+ sb.append(" <rule name='primaryGroup' create='false'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ // Should return root.default since primary group and user queue for user 2
+ // are not configured.
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user2"));
+
+ // Now configure both primary group and the user queue for user2
+ configuredQueues.get(FSQueueType.PARENT).add("root.user2group");
+ configuredQueues.get(FSQueueType.LEAF).add("root.user2group.user2");
+ policy = parse(sb.toString());
+
+ assertEquals("root.user2group.user2",
+ policy.assignAppToQueue("root.default", "user2"));
+ }
+
+ @Test
+ public void testNestedUserQueueSecondaryGroup() throws Exception {
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='secondaryGroupExistingQueue'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ // Should return root.default since secondary groups are not configured
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user1"));
+
+ // configure secondary group for user1
+ configuredQueues.get(FSQueueType.PARENT).add("root.user1subgroup1");
+ policy = parse(sb.toString());
+ // user queue created should be created under secondary group
+ assertEquals("root.user1subgroup1.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
+ @Test
+ public void testNestedUserQueueSpecificRule() throws Exception {
+ // This test covers the use case where users can specify different parent
+ // queues and want user queues under those.
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='specified' create='false'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ // Let's create couple of parent queues
+ configuredQueues.get(FSQueueType.PARENT).add("root.parent1");
+ configuredQueues.get(FSQueueType.PARENT).add("root.parent2");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.parent1.user1",
+ policy.assignAppToQueue("root.parent1", "user1"));
+ assertEquals("root.parent2.user2",
+ policy.assignAppToQueue("root.parent2", "user2"));
+ }
+
private QueuePlacementPolicy parse(String str) throws Exception {
// Read and parse the allocations file.
- DocumentBuilderFactory docBuilderFactory =
- DocumentBuilderFactory.newInstance();
+ DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
+ .newInstance();
docBuilderFactory.setIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
Document doc = builder.parse(IOUtils.toInputStream(str));
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Mon May 12 12:43:59 2014
@@ -37,24 +37,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Mon May 12 12:43:59 2014
@@ -43,24 +43,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
Modified: hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1593948&r1=1593947&r2=1593948&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-2006/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Mon May 12 12:43:59 2014
@@ -206,8 +206,10 @@ Allocation file format
The allocation file must be in XML format. The format contains five types of
elements:
- * <<Queue elements>>, which represent queues. Each may contain the following
- properties:
+ * <<Queue elements>>, which represent queues. Queue elements can take an optional
+ attribute âtypeâ,which when set to âparentâ makes it a parent queue. This is useful
+ when we want to create a parent queue without configuring any leaf queues.
+ Each queue element may contain the following properties:
* minResources: minimum resources the queue is entitled to, in the form
"X mb, Y vcores". For the single-resource fairness policy, the vcores
@@ -299,6 +301,15 @@ Allocation file format
that matches a secondary group of the user who submitted it. The first
secondary group that matches a configured queue will be selected.
+ * nestedUserQueue : the app is placed into a queue with the name of the user
+ under the queue suggested by the nested rule. This is similar to âuserâ
+ rule,the difference being in ânestedUserQueueâ rule,user queues can be created
+ under any parent queue, while âuserâ rule creates user queues only under root queue.
+ Note that nestedUserQueue rule would be applied only if the nested rule returns a
+ parent queue.One can configure a parent queue either by setting âtypeâ attribute of queue
+ to âparentâ or by configuring at least one leaf under that queue which makes it a parent.
+ See example allocation for a sample use case.
+
* default: the app is placed into the queue named "default".
* reject: the app is rejected.
@@ -319,6 +330,12 @@ Allocation file format
<minResources>5000 mb,0vcores</minResources>
</queue>
</queue>
+
+ <!â- Queue âsecondary_group_queueâ is a parent queue and may have
+ user queues under it -â>
+ <queue name=âsecondary_group_queueâ type=âparentâ>
+ <weight>3.0</weight>
+ </queue>
<user name="sample_user">
<maxRunningApps>30</maxRunningApps>
@@ -328,6 +345,9 @@ Allocation file format
<queuePlacementPolicy>
<rule name="specified" />
<rule name="primaryGroup" create="false" />
+ <rule name=ânestedUserQueueâ>
+ <rule name=âsecondaryGroupExistingQueueâ create=âfalseâ />
+ </rule>
<rule name="default" />
</queuePlacementPolicy>
</allocations>