You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:51:01 UTC
svn commit: r1619012 [24/26] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerEventLog.java Tue Aug 19 23:49:39 2014
@@ -51,6 +51,8 @@ public class TestFairSchedulerEventLog {
resourceManager = new ResourceManager();
resourceManager.init(conf);
((AsyncDispatcher)resourceManager.getRMContext().getDispatcher()).start();
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
}
@@ -69,7 +71,13 @@ public class TestFairSchedulerEventLog {
public void tearDown() {
logFile.delete();
logFile.getParentFile().delete(); // fairscheduler/
- scheduler = null;
- resourceManager = null;
+ if (scheduler != null) {
+ scheduler.stop();
+ scheduler = null;
+ }
+ if (resourceManager != null) {
+ resourceManager.stop();
+ resourceManager = null;
+ }
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java Tue Aug 19 23:49:39 2014
@@ -28,6 +28,7 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.junit.Before;
@@ -40,12 +41,14 @@ public class TestMaxRunningAppsEnforcer
private MaxRunningAppsEnforcer maxAppsEnforcer;
private int appNum;
private TestFairScheduler.MockClock clock;
+ private RMContext rmContext;
+ private FairScheduler scheduler;
@Before
public void setup() throws Exception {
Configuration conf = new Configuration();
clock = new TestFairScheduler.MockClock();
- FairScheduler scheduler = mock(FairScheduler.class);
+ scheduler = mock(FairScheduler.class);
when(scheduler.getConf()).thenReturn(
new FairSchedulerConfiguration(conf));
when(scheduler.getClock()).thenReturn(clock);
@@ -59,13 +62,16 @@ public class TestMaxRunningAppsEnforcer
userMaxApps = allocConf.userMaxApps;
maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
appNum = 0;
+ rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(0);
}
- private FSSchedulerApp addApp(FSLeafQueue queue, String user) {
+ private FSAppAttempt addApp(FSLeafQueue queue, String user) {
ApplicationId appId = ApplicationId.newInstance(0l, appNum++);
ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 0);
boolean runnable = maxAppsEnforcer.canAppBeRunnable(queue, user);
- FSSchedulerApp app = new FSSchedulerApp(attId, user, queue, null, null);
+ FSAppAttempt app = new FSAppAttempt(scheduler, attId, user, queue, null,
+ rmContext);
queue.addApp(app, runnable);
if (runnable) {
maxAppsEnforcer.trackRunnableApp(app);
@@ -75,7 +81,7 @@ public class TestMaxRunningAppsEnforcer
return app;
}
- private void removeApp(FSSchedulerApp app) {
+ private void removeApp(FSAppAttempt app) {
app.getQueue().removeApp(app);
maxAppsEnforcer.untrackRunnableApp(app);
maxAppsEnforcer.updateRunnabilityOnAppRemoval(app, app.getQueue());
@@ -88,7 +94,7 @@ public class TestMaxRunningAppsEnforcer
queueMaxApps.put("root", 2);
queueMaxApps.put("root.queue1", 1);
queueMaxApps.put("root.queue2", 1);
- FSSchedulerApp app1 = addApp(leaf1, "user");
+ FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
assertEquals(1, leaf1.getRunnableAppSchedulables().size());
@@ -105,7 +111,7 @@ public class TestMaxRunningAppsEnforcer
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
queueMaxApps.put("root.queue1", 2);
- FSSchedulerApp app1 = addApp(leaf1, "user");
+ FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
assertEquals(1, leaf1.getRunnableAppSchedulables().size());
@@ -123,7 +129,7 @@ public class TestMaxRunningAppsEnforcer
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true);
queueMaxApps.put("root.queue1.leaf1", 2);
userMaxApps.put("user1", 1);
- FSSchedulerApp app1 = addApp(leaf1, "user1");
+ FSAppAttempt app1 = addApp(leaf1, "user1");
addApp(leaf1, "user2");
addApp(leaf1, "user3");
addApp(leaf2, "user1");
@@ -142,7 +148,7 @@ public class TestMaxRunningAppsEnforcer
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
queueMaxApps.put("root.queue1", 2);
- FSSchedulerApp app1 = addApp(leaf1, "user");
+ FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
clock.tick(20);
@@ -162,7 +168,7 @@ public class TestMaxRunningAppsEnforcer
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
queueMaxApps.put("root.queue1", 2);
- FSSchedulerApp app1 = addApp(leaf1, "user");
+ FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
@@ -177,21 +183,18 @@ public class TestMaxRunningAppsEnforcer
@Test
public void testMultiListStartTimeIteratorEmptyAppLists() {
- List<List<AppSchedulable>> lists = new ArrayList<List<AppSchedulable>>();
- lists.add(Arrays.asList(mockAppSched(1)));
- lists.add(Arrays.asList(mockAppSched(2)));
- Iterator<FSSchedulerApp> iter =
+ List<List<FSAppAttempt>> lists = new ArrayList<List<FSAppAttempt>>();
+ lists.add(Arrays.asList(mockAppAttempt(1)));
+ lists.add(Arrays.asList(mockAppAttempt(2)));
+ Iterator<FSAppAttempt> iter =
new MaxRunningAppsEnforcer.MultiListStartTimeIterator(lists);
- assertEquals(1, iter.next().getAppSchedulable().getStartTime());
- assertEquals(2, iter.next().getAppSchedulable().getStartTime());
+ assertEquals(1, iter.next().getStartTime());
+ assertEquals(2, iter.next().getStartTime());
}
- private AppSchedulable mockAppSched(long startTime) {
- AppSchedulable appSched = mock(AppSchedulable.class);
- when(appSched.getStartTime()).thenReturn(startTime);
- FSSchedulerApp schedApp = mock(FSSchedulerApp.class);
- when(schedApp.getAppSchedulable()).thenReturn(appSched);
- when(appSched.getApp()).thenReturn(schedApp);
- return appSched;
+ private FSAppAttempt mockAppAttempt(long startTime) {
+ FSAppAttempt schedApp = mock(FSAppAttempt.class);
+ when(schedApp.getStartTime()).thenReturn(startTime);
+ return schedApp;
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java Tue Aug 19 23:49:39 2014
@@ -17,8 +17,7 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
+import static org.junit.Assert.*;
import static org.mockito.Mockito.*;
import java.util.HashSet;
@@ -57,45 +56,77 @@ public class TestQueueManager {
@Test
public void testReloadTurnsLeafQueueIntoParent() throws Exception {
- updateConfiguredQueues(queueManager, "queue1");
+ updateConfiguredLeafQueues(queueManager, "queue1");
// When no apps are running in the leaf queue, should be fine turning it
// into a parent.
- updateConfiguredQueues(queueManager, "queue1.queue2");
+ updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1", false));
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
// When leaf queues are empty, should be ok deleting them and
// turning parent into a leaf.
- updateConfiguredQueues(queueManager, "queue1");
+ updateConfiguredLeafQueues(queueManager, "queue1");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queue, we shouldn't be able to create
// children under it, but things should work otherwise.
notEmptyQueues.add(queueManager.getLeafQueue("queue1", false));
- updateConfiguredQueues(queueManager, "queue1.queue2");
+ updateConfiguredLeafQueues(queueManager, "queue1.queue2");
assertNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNotNull(queueManager.getLeafQueue("queue1", false));
// When apps exist in leaf queues under a parent queue, shouldn't be
// able to turn it into a leaf queue, but things should work otherwise.
notEmptyQueues.clear();
- updateConfiguredQueues(queueManager, "queue1.queue2");
+ updateConfiguredLeafQueues(queueManager, "queue1.queue2");
notEmptyQueues.add(queueManager.getQueue("root.queue1"));
- updateConfiguredQueues(queueManager, "queue1");
+ updateConfiguredLeafQueues(queueManager, "queue1");
assertNotNull(queueManager.getLeafQueue("queue1.queue2", false));
assertNull(queueManager.getLeafQueue("queue1", false));
// Should never to be able to create a queue under the default queue
- updateConfiguredQueues(queueManager, "default.queue3");
+ updateConfiguredLeafQueues(queueManager, "default.queue3");
assertNull(queueManager.getLeafQueue("default.queue3", false));
assertNotNull(queueManager.getLeafQueue("default", false));
}
- private void updateConfiguredQueues(QueueManager queueMgr, String... confQueues) {
+ @Test
+ public void testReloadTurnsLeafToParentWithNoLeaf() {
+ AllocationConfiguration allocConf = new AllocationConfiguration(conf);
+ // Create a leaf queue1
+ allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.queue1");
+ queueManager.updateAllocationConfiguration(allocConf);
+ assertNotNull(queueManager.getLeafQueue("root.queue1", false));
+
+ // Lets say later on admin makes queue1 a parent queue by
+ // specifying "type=parent" in the alloc xml and lets say apps running in
+ // queue1
+ notEmptyQueues.add(queueManager.getLeafQueue("root.queue1", false));
+ allocConf = new AllocationConfiguration(conf);
+ allocConf.configuredQueues.get(FSQueueType.PARENT)
+ .add("root.queue1");
+
+ // When allocs are reloaded queue1 shouldn't be converter to parent
+ queueManager.updateAllocationConfiguration(allocConf);
+ assertNotNull(queueManager.getLeafQueue("root.queue1", false));
+ assertNull(queueManager.getParentQueue("root.queue1", false));
+
+ // Now lets assume apps completed and there are no apps in queue1
+ notEmptyQueues.clear();
+ // We should see queue1 transform from leaf queue to parent queue.
+ queueManager.updateAllocationConfiguration(allocConf);
+ assertNull(queueManager.getLeafQueue("root.queue1", false));
+ assertNotNull(queueManager.getParentQueue("root.queue1", false));
+ // this parent should not have any children
+ assertTrue(queueManager.getParentQueue("root.queue1", false)
+ .getChildQueues().isEmpty());
+ }
+
+ private void updateConfiguredLeafQueues(QueueManager queueMgr, String... confLeafQueues) {
AllocationConfiguration allocConf = new AllocationConfiguration(conf);
- allocConf.queueNames = Sets.newHashSet(confQueues);
+ allocConf.configuredQueues.get(FSQueueType.LEAF).addAll(Sets.newHashSet(confLeafQueues));
queueMgr.updateAllocationConfiguration(allocConf);
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueuePlacementPolicy.java Tue Aug 19 23:49:39 2014
@@ -17,8 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
@@ -28,16 +31,15 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.security.GroupMappingServiceProvider;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
-import com.google.common.collect.Sets;
-
public class TestQueuePlacementPolicy {
private final static Configuration conf = new Configuration();
- private final static Set<String> configuredQueues = Sets.newHashSet("root.someuser");
+ private Map<FSQueueType, Set<String>> configuredQueues;
@BeforeClass
public static void setup() {
@@ -45,6 +47,14 @@ public class TestQueuePlacementPolicy {
SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
}
+ @Before
+ public void initTest() {
+ configuredQueues = new HashMap<FSQueueType, Set<String>>();
+ for (FSQueueType type : FSQueueType.values()) {
+ configuredQueues.put(type, new HashSet<String>());
+ }
+ }
+
@Test
public void testSpecifiedUserPolicy() throws Exception {
StringBuffer sb = new StringBuffer();
@@ -53,9 +63,12 @@ public class TestQueuePlacementPolicy {
sb.append(" <rule name='user' />");
sb.append("</queuePlacementPolicy>");
QueuePlacementPolicy policy = parse(sb.toString());
- assertEquals("root.specifiedq",policy.assignAppToQueue("specifiedq", "someuser"));
- assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser"));
- assertEquals("root.otheruser", policy.assignAppToQueue("default", "otheruser"));
+ assertEquals("root.specifiedq",
+ policy.assignAppToQueue("specifiedq", "someuser"));
+ assertEquals("root.someuser",
+ policy.assignAppToQueue("default", "someuser"));
+ assertEquals("root.otheruser",
+ policy.assignAppToQueue("default", "otheruser"));
}
@Test
@@ -66,6 +79,8 @@ public class TestQueuePlacementPolicy {
sb.append(" <rule name='user' create=\"false\" />");
sb.append(" <rule name='default' />");
sb.append("</queuePlacementPolicy>");
+
+ configuredQueues.get(FSQueueType.LEAF).add("root.someuser");
QueuePlacementPolicy policy = parse(sb.toString());
assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser"));
assertEquals("root.someuser", policy.assignAppToQueue("default", "someuser"));
@@ -81,7 +96,8 @@ public class TestQueuePlacementPolicy {
sb.append(" <rule name='reject' />");
sb.append("</queuePlacementPolicy>");
QueuePlacementPolicy policy = parse(sb.toString());
- assertEquals("root.specifiedq", policy.assignAppToQueue("specifiedq", "someuser"));
+ assertEquals("root.specifiedq",
+ policy.assignAppToQueue("specifiedq", "someuser"));
assertEquals(null, policy.assignAppToQueue("default", "someuser"));
}
@@ -117,10 +133,223 @@ public class TestQueuePlacementPolicy {
parse(sb.toString());
}
+ @Test
+ public void testDefaultRuleWithQueueAttribute() throws Exception {
+ // This test covers the use case where we would like default rule
+ // to point to a different queue by default rather than root.default
+ configuredQueues.get(FSQueueType.LEAF).add("root.someDefaultQueue");
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='default' queue='root.someDefaultQueue'/>");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.someDefaultQueue",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
+ @Test
+ public void testNestedUserQueueParsingErrors() {
+ // No nested rule specified in hierarchical user queue
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' />");
+ sb.append(" <rule name='nestedUserQueue'/>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ assertIfExceptionThrown(sb);
+
+ // Specified nested rule is not a QueuePlacementRule
+ sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='unknownRule'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ assertIfExceptionThrown(sb);
+ }
+
+ private void assertIfExceptionThrown(StringBuffer sb) {
+ Throwable th = null;
+ try {
+ parse(sb.toString());
+ } catch (Exception e) {
+ th = e;
+ }
+
+ assertTrue(th instanceof AllocationConfigurationException);
+ }
+
+ @Test
+ public void testNestedUserQueueParsing() throws Exception {
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='primaryGroup'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ Throwable th = null;
+ try {
+ parse(sb.toString());
+ } catch (Exception e) {
+ th = e;
+ }
+
+ assertNull(th);
+ }
+
+ @Test
+ public void testNestedUserQueuePrimaryGroup() throws Exception {
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='primaryGroup'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ // User queue would be created under primary group queue
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.user1group.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+ // Other rules above and below hierarchical user queue rule should work as
+ // usual
+ configuredQueues.get(FSQueueType.LEAF).add("root.specifiedq");
+ // test if specified rule(above nestedUserQueue rule) works ok
+ assertEquals("root.specifiedq",
+ policy.assignAppToQueue("root.specifiedq", "user2"));
+
+ // test if default rule(below nestedUserQueue rule) works
+ configuredQueues.get(FSQueueType.LEAF).add("root.user3group");
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user3"));
+ }
+
+ @Test
+ public void testNestedUserQueuePrimaryGroupNoCreate() throws Exception {
+ // Primary group rule has create='false'
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='primaryGroup' create='false'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+
+ // Should return root.default since primary group 'root.user1group' is not
+ // configured
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user1"));
+
+ // Let's configure primary group and check if user queue is created
+ configuredQueues.get(FSQueueType.PARENT).add("root.user1group");
+ policy = parse(sb.toString());
+ assertEquals("root.user1group.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+
+ // Both Primary group and nestedUserQueue rule has create='false'
+ sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue' create='false'>");
+ sb.append(" <rule name='primaryGroup' create='false'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ // Should return root.default since primary group and user queue for user 2
+ // are not configured.
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user2"));
+
+ // Now configure both primary group and the user queue for user2
+ configuredQueues.get(FSQueueType.PARENT).add("root.user2group");
+ configuredQueues.get(FSQueueType.LEAF).add("root.user2group.user2");
+ policy = parse(sb.toString());
+
+ assertEquals("root.user2group.user2",
+ policy.assignAppToQueue("root.default", "user2"));
+ }
+
+ @Test
+ public void testNestedUserQueueSecondaryGroup() throws Exception {
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='secondaryGroupExistingQueue'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ // Should return root.default since secondary groups are not configured
+ assertEquals("root.default",
+ policy.assignAppToQueue("root.default", "user1"));
+
+ // configure secondary group for user1
+ configuredQueues.get(FSQueueType.PARENT).add("root.user1subgroup1");
+ policy = parse(sb.toString());
+ // user queue created should be created under secondary group
+ assertEquals("root.user1subgroup1.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
+ @Test
+ public void testNestedUserQueueSpecificRule() throws Exception {
+ // This test covers the use case where users can specify different parent
+ // queues and want user queues under those.
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='specified' create='false'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ // Let's create couple of parent queues
+ configuredQueues.get(FSQueueType.PARENT).add("root.parent1");
+ configuredQueues.get(FSQueueType.PARENT).add("root.parent2");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.parent1.user1",
+ policy.assignAppToQueue("root.parent1", "user1"));
+ assertEquals("root.parent2.user2",
+ policy.assignAppToQueue("root.parent2", "user2"));
+ }
+
+ @Test
+ public void testNestedUserQueueDefaultRule() throws Exception {
+ // This test covers the use case where we would like user queues to be
+ // created under a default parent queue
+ configuredQueues.get(FSQueueType.PARENT).add("root.parentq");
+ StringBuffer sb = new StringBuffer();
+ sb.append("<queuePlacementPolicy>");
+ sb.append(" <rule name='specified' create='false' />");
+ sb.append(" <rule name='nestedUserQueue'>");
+ sb.append(" <rule name='default' queue='root.parentq'/>");
+ sb.append(" </rule>");
+ sb.append(" <rule name='default' />");
+ sb.append("</queuePlacementPolicy>");
+
+ QueuePlacementPolicy policy = parse(sb.toString());
+ assertEquals("root.parentq.user1",
+ policy.assignAppToQueue("root.default", "user1"));
+ }
+
private QueuePlacementPolicy parse(String str) throws Exception {
// Read and parse the allocations file.
- DocumentBuilderFactory docBuilderFactory =
- DocumentBuilderFactory.newInstance();
+ DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory
+ .newInstance();
docBuilderFactory.setIgnoringComments(true);
DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();
Document doc = builder.parse(IOUtils.toInputStream(str));
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Tue Aug 19 23:49:39 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.re
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-
import static org.mockito.Mockito.mock;
import java.io.IOException;
@@ -30,8 +29,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -60,13 +57,13 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.Task;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestCapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -78,6 +75,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -147,9 +145,13 @@ public class TestFifoScheduler {
RMContext rmContext = new RMContextImpl(dispatcher, null,
null, null, null, null, null, null, null, writer);
- FifoScheduler schedular = new FifoScheduler();
- schedular.reinitialize(new Configuration(), rmContext);
- QueueMetrics metrics = schedular.getRootQueueMetrics();
+ FifoScheduler scheduler = new FifoScheduler();
+ Configuration conf = new Configuration();
+ scheduler.setRMContext(rmContext);
+ scheduler.init(conf);
+ scheduler.start();
+ scheduler.reinitialize(conf, rmContext);
+ QueueMetrics metrics = scheduler.getRootQueueMetrics();
int beforeAppsSubmitted = metrics.getAppsSubmitted();
ApplicationId appId = BuilderUtils.newApplicationId(200, 1);
@@ -157,18 +159,19 @@ public class TestFifoScheduler {
appId, 1);
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
- schedular.handle(appEvent);
+ scheduler.handle(appEvent);
SchedulerEvent attemptEvent =
new AppAttemptAddedSchedulerEvent(appAttemptId, false);
- schedular.handle(attemptEvent);
+ scheduler.handle(attemptEvent);
appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 2);
SchedulerEvent attemptEvent2 =
new AppAttemptAddedSchedulerEvent(appAttemptId, false);
- schedular.handle(attemptEvent2);
+ scheduler.handle(attemptEvent2);
int afterAppsSubmitted = metrics.getAppsSubmitted();
Assert.assertEquals(1, afterAppsSubmitted - beforeAppsSubmitted);
+ scheduler.stop();
}
@Test(timeout=2000)
@@ -186,6 +189,9 @@ public class TestFifoScheduler {
null, containerTokenSecretManager, nmTokenSecretManager, null, writer);
FifoScheduler scheduler = new FifoScheduler();
+ scheduler.setRMContext(rmContext);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.newNodeInfo(1,
@@ -234,6 +240,7 @@ public class TestFifoScheduler {
//Also check that the containers were scheduled
SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
Assert.assertEquals(3, info.getLiveContainers().size());
+ scheduler.stop();
}
@Test(timeout=2000)
@@ -256,6 +263,9 @@ public class TestFifoScheduler {
return nodes;
}
};
+ scheduler.setRMContext(rmContext);
+ scheduler.init(conf);
+ scheduler.start();
scheduler.reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.newNodeInfo(1,
Resources.createResource(2048, 4), 1, "127.0.0.1");
@@ -594,9 +604,12 @@ public class TestFifoScheduler {
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
Configuration conf = new Configuration();
conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
- ResourceScheduler.class);
+ ResourceScheduler.class);
MockRM rm = new MockRM(conf);
- FifoScheduler fs = (FifoScheduler)rm.getResourceScheduler();
+ @SuppressWarnings("unchecked")
+ AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
+ (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
+ .getResourceScheduler();
TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
fs.getSchedulerApplications(), fs, "queue");
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java Tue Aug 19 23:49:39 2014
@@ -18,30 +18,34 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
import java.util.Arrays;
import java.util.Collection;
-
-import javax.crypto.SecretKey;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
@@ -50,7 +54,9 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
+import org.apache.hadoop.yarn.server.security.MasterKeyData;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -65,6 +71,8 @@ public class TestAMRMTokens {
private final Configuration conf;
private static final int maxWaitAttempts = 50;
+ private static final int rolling_interval_sec = 13;
+ private static final long am_expire_ms = 4000;
@Parameters
public static Collection<Object[]> configs() {
@@ -180,8 +188,8 @@ public class TestAMRMTokens {
// The exception will still have the earlier appAttemptId as it picks it
// up from the token.
Assert.assertTrue(t.getCause().getMessage().contains(
- "Password not found for ApplicationAttempt " +
- applicationAttemptId.toString()));
+ applicationAttemptId.toString()
+ + " not found in AMRMTokenSecretManager."));
}
} finally {
@@ -201,15 +209,22 @@ public class TestAMRMTokens {
@Test
public void testMasterKeyRollOver() throws Exception {
+ conf.setLong(
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+ rolling_interval_sec);
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
MyContainerManager containerManager = new MyContainerManager();
final MockRMWithAMS rm =
new MockRMWithAMS(conf, containerManager);
rm.start();
-
+ Long startTime = System.currentTimeMillis();
final Configuration conf = rm.getConfig();
final YarnRPC rpc = YarnRPC.create(conf);
ApplicationMasterProtocol rmClient = null;
-
+ AMRMTokenSecretManager appTokenSecretManager =
+ rm.getRMContext().getAMRMTokenSecretManager();
+ MasterKeyData oldKey = appTokenSecretManager.getMasterKey();
+ Assert.assertNotNull(oldKey);
try {
MockNM nm1 = rm.registerNode("localhost:1234", 5120);
@@ -218,7 +233,7 @@ public class TestAMRMTokens {
nm1.nodeHeartbeat(true);
int waitCount = 0;
- while (containerManager.containerTokens == null && waitCount++ < 20) {
+ while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) {
LOG.info("Waiting for AM Launch to happen..");
Thread.sleep(1000);
}
@@ -250,21 +265,65 @@ public class TestAMRMTokens {
Assert.assertTrue(
rmClient.allocate(allocateRequest).getAMCommand() == null);
- // Simulate a master-key-roll-over
- AMRMTokenSecretManager appTokenSecretManager =
- rm.getRMContext().getAMRMTokenSecretManager();
- SecretKey oldKey = appTokenSecretManager.getMasterKey();
- appTokenSecretManager.rollMasterKey();
- SecretKey newKey = appTokenSecretManager.getMasterKey();
+ // Wait for enough time and make sure the roll_over happens
+ // At mean time, the old AMRMToken should continue to work
+ while(System.currentTimeMillis() - startTime < rolling_interval_sec*1000) {
+ rmClient.allocate(allocateRequest);
+ Thread.sleep(500);
+ }
+
+ MasterKeyData newKey = appTokenSecretManager.getMasterKey();
+ Assert.assertNotNull(newKey);
Assert.assertFalse("Master key should have changed!",
oldKey.equals(newKey));
+ // Another allocate call with old AMRMToken. Should continue to work.
+ rpc.stopProxy(rmClient, conf); // To avoid using cached client
+ rmClient = createRMClient(rm, conf, rpc, currentUser);
+ Assert
+ .assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
+
+ waitCount = 0;
+ while(waitCount++ <= maxWaitAttempts) {
+ if (appTokenSecretManager.getCurrnetMasterKeyData() != oldKey) {
+ break;
+ }
+ try {
+ rmClient.allocate(allocateRequest);
+ } catch (Exception ex) {
+ break;
+ }
+ Thread.sleep(200);
+ }
+ // active the nextMasterKey, and replace the currentMasterKey
+ Assert.assertTrue(appTokenSecretManager.getCurrnetMasterKeyData().equals(newKey));
+ Assert.assertTrue(appTokenSecretManager.getMasterKey().equals(newKey));
+ Assert.assertTrue(appTokenSecretManager.getNextMasterKeyData() == null);
+
+ // Create a new Token
+ Token<AMRMTokenIdentifier> newToken =
+ appTokenSecretManager.createAndGetAMRMToken(applicationAttemptId);
+ SecurityUtil.setTokenService(newToken, rmBindAddress);
+ currentUser.addToken(newToken);
// Another allocate call. Should continue to work.
rpc.stopProxy(rmClient, conf); // To avoid using cached client
rmClient = createRMClient(rm, conf, rpc, currentUser);
allocateRequest = Records.newRecord(AllocateRequest.class);
- Assert.assertTrue(
- rmClient.allocate(allocateRequest).getAMCommand() == null);
+ Assert
+ .assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
+
+ // Should not work by using the old AMRMToken.
+ rpc.stopProxy(rmClient, conf); // To avoid using cached client
+ try {
+ currentUser.addToken(amRMToken);
+ rmClient = createRMClient(rm, conf, rpc, currentUser);
+ allocateRequest = Records.newRecord(AllocateRequest.class);
+ Assert
+ .assertTrue(rmClient.allocate(allocateRequest).getAMCommand() == null);
+ Assert.fail("The old Token should not work");
+ } catch (Exception ex) {
+ // expect exception
+ }
} finally {
rm.stop();
if (rmClient != null) {
@@ -273,6 +332,51 @@ public class TestAMRMTokens {
}
}
+ @Test (timeout = 20000)
+ public void testAMRMMasterKeysUpdate() throws Exception {
+ MockRM rm = new MockRM(conf) {
+ @Override
+ protected void doSecureLogin() throws IOException {
+ // Skip the login.
+ }
+ };
+ rm.start();
+ MockNM nm = rm.registerNode("127.0.0.1:1234", 8000);
+ RMApp app = rm.submitApp(200);
+ MockAM am = MockRM.launchAndRegisterAM(app, rm, nm);
+
+ // Do allocate. Should not update AMRMToken
+ AllocateResponse response =
+ am.allocate(Records.newRecord(AllocateRequest.class));
+ Assert.assertNull(response.getAMRMToken());
+
+ // roll over the master key
+ // Do allocate again. the AM should get the latest AMRMToken
+ rm.getRMContext().getAMRMTokenSecretManager().rollMasterKey();
+ response = am.allocate(Records.newRecord(AllocateRequest.class));
+ Assert.assertNotNull(response.getAMRMToken());
+
+ Token<AMRMTokenIdentifier> amrmToken =
+ ConverterUtils.convertFromYarn(response.getAMRMToken(), new Text(
+ response.getAMRMToken().getService()));
+
+ Assert.assertEquals(amrmToken.decodeIdentifier().getKeyId(), rm
+ .getRMContext().getAMRMTokenSecretManager().getMasterKey().getMasterKey()
+ .getKeyId());
+
+ // Do allocate again. The master key does not update.
+ // AM should not update its AMRMToken either
+ response = am.allocate(Records.newRecord(AllocateRequest.class));
+ Assert.assertNull(response.getAMRMToken());
+
+ // Activate the next master key. Since there is new master key generated
+ // in AMRMTokenSecretManager. The AMRMToken will not get updated for AM
+ rm.getRMContext().getAMRMTokenSecretManager().activateNextMasterKey();
+ response = am.allocate(Records.newRecord(AllocateRequest.class));
+ Assert.assertNull(response.getAMRMToken());
+ rm.stop();
+ }
+
private ApplicationMasterProtocol createRMClient(final MockRM rm,
final Configuration conf, final YarnRPC rpc,
UserGroupInformation currentUser) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java Tue Aug 19 23:49:39 2014
@@ -24,6 +24,7 @@ import static org.junit.Assert.fail;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -673,7 +674,40 @@ public class TestDelegationTokenRenewer
Thread.sleep(200);
}
}
-
+
+ @Test(timeout=20000)
+ public void testDTRonAppSubmission()
+ throws IOException, InterruptedException, BrokenBarrierException {
+ final Credentials credsx = new Credentials();
+ final Token<?> tokenx = mock(Token.class);
+ credsx.addToken(new Text("token"), tokenx);
+ doReturn(true).when(tokenx).isManaged();
+ doThrow(new IOException("boom"))
+ .when(tokenx).renew(any(Configuration.class));
+ // fire up the renewer
+ final DelegationTokenRenewer dtr =
+ createNewDelegationTokenRenewer(conf, counter);
+ RMContext mockContext = mock(RMContext.class);
+ ClientRMService mockClientRMService = mock(ClientRMService.class);
+ when(mockContext.getClientRMService()).thenReturn(mockClientRMService);
+ InetSocketAddress sockAddr =
+ InetSocketAddress.createUnresolved("localhost", 1234);
+ when(mockClientRMService.getBindAddress()).thenReturn(sockAddr);
+ dtr.setRMContext(mockContext);
+ when(mockContext.getDelegationTokenRenewer()).thenReturn(dtr);
+ dtr.init(conf);
+ dtr.start();
+
+ try {
+ dtr.addApplicationSync(mock(ApplicationId.class), credsx, false);
+ fail("Catch IOException on app submission");
+ } catch (IOException e){
+ Assert.assertTrue(e.getMessage().contains(tokenx.toString()));
+ Assert.assertTrue(e.getCause().toString().contains("boom"));
+ }
+
+ }
+
@Test(timeout=20000)
public void testConcurrentAddApplication()
throws IOException, InterruptedException, BrokenBarrierException {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java Tue Aug 19 23:49:39 2014
@@ -48,8 +48,8 @@ public class TestNodesPage {
// Number of Actual Table Headers for NodesPage.NodesBlock might change in
// future. In that case this value should be adjusted to the new value.
- final int numberOfThInMetricsTable = 13;
- final int numberOfActualTableHeaders = 10;
+ final int numberOfThInMetricsTable = 16;
+ final int numberOfActualTableHeaders = 12;
private Injector injector;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java Tue Aug 19 23:49:39 2014
@@ -203,10 +203,11 @@ public class TestRMWebApp {
CapacityScheduler cs = new CapacityScheduler();
cs.setConf(new YarnConfiguration());
- cs.reinitialize(conf, new RMContextImpl(null, null, null, null, null,
+ cs.setRMContext(new RMContextImpl(null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), null));
+ cs.init(conf);
return cs;
}
@@ -269,19 +270,21 @@ public class TestRMWebApp {
ResourceManager rm = mock(ResourceManager.class);
RMContext rmContext = mockRMContext(apps, racks, nodes,
mbsPerNode);
- ResourceScheduler rs = mockFifoScheduler();
+ ResourceScheduler rs = mockFifoScheduler(rmContext);
when(rm.getResourceScheduler()).thenReturn(rs);
when(rm.getRMContext()).thenReturn(rmContext);
return rm;
}
- public static FifoScheduler mockFifoScheduler() throws Exception {
+ public static FifoScheduler mockFifoScheduler(RMContext rmContext)
+ throws Exception {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
setupFifoQueueConfiguration(conf);
FifoScheduler fs = new FifoScheduler();
fs.setConf(new YarnConfiguration());
- fs.reinitialize(conf, null);
+ fs.setRMContext(rmContext);
+ fs.init(conf);
return fs;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Tue Aug 19 23:49:39 2014
@@ -389,6 +389,10 @@ public class TestRMWebServices extends J
WebServicesTestUtils.getXmlInt(element, "reservedMB"),
WebServicesTestUtils.getXmlInt(element, "availableMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
+ WebServicesTestUtils.getXmlInt(element, "reservedVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "availableVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "allocatedVirtualCores"),
+ WebServicesTestUtils.getXmlInt(element, "totalVirtualCores"),
WebServicesTestUtils.getXmlInt(element, "containersAllocated"),
WebServicesTestUtils.getXmlInt(element, "totalMB"),
WebServicesTestUtils.getXmlInt(element, "totalNodes"),
@@ -404,11 +408,13 @@ public class TestRMWebServices extends J
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject clusterinfo = json.getJSONObject("clusterMetrics");
- assertEquals("incorrect number of elements", 19, clusterinfo.length());
+ assertEquals("incorrect number of elements", 23, clusterinfo.length());
verifyClusterMetrics(
clusterinfo.getInt("appsSubmitted"), clusterinfo.getInt("appsCompleted"),
clusterinfo.getInt("reservedMB"), clusterinfo.getInt("availableMB"),
clusterinfo.getInt("allocatedMB"),
+ clusterinfo.getInt("reservedVirtualCores"), clusterinfo.getInt("availableVirtualCores"),
+ clusterinfo.getInt("allocatedVirtualCores"), clusterinfo.getInt("totalVirtualCores"),
clusterinfo.getInt("containersAllocated"),
clusterinfo.getInt("totalMB"), clusterinfo.getInt("totalNodes"),
clusterinfo.getInt("lostNodes"), clusterinfo.getInt("unhealthyNodes"),
@@ -418,7 +424,9 @@ public class TestRMWebServices extends J
public void verifyClusterMetrics(int submittedApps, int completedApps,
int reservedMB, int availableMB,
- int allocMB, int containersAlloc, int totalMB, int totalNodes,
+ int allocMB, int reservedVirtualCores, int availableVirtualCores,
+ int allocVirtualCores, int totalVirtualCores,
+ int containersAlloc, int totalMB, int totalNodes,
int lostNodes, int unhealthyNodes, int decommissionedNodes,
int rebootedNodes, int activeNodes) throws JSONException, Exception {
@@ -428,7 +436,8 @@ public class TestRMWebServices extends J
long totalMBExpect =
metrics.getAvailableMB() + metrics.getAllocatedMB();
-
+ long totalVirtualCoresExpect =
+ metrics.getAvailableVirtualCores() + metrics.getAllocatedVirtualCores();
assertEquals("appsSubmitted doesn't match",
metrics.getAppsSubmitted(), submittedApps);
assertEquals("appsCompleted doesn't match",
@@ -439,6 +448,12 @@ public class TestRMWebServices extends J
metrics.getAvailableMB(), availableMB);
assertEquals("allocatedMB doesn't match",
metrics.getAllocatedMB(), allocMB);
+ assertEquals("reservedVirtualCores doesn't match",
+ metrics.getReservedVirtualCores(), reservedVirtualCores);
+ assertEquals("availableVirtualCores doesn't match",
+ metrics.getAvailableVirtualCores(), availableVirtualCores);
+ assertEquals("allocatedVirtualCores doesn't match",
+ totalVirtualCoresExpect, allocVirtualCores);
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
assertEquals(
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Tue Aug 19 23:49:39 2014
@@ -1310,33 +1310,44 @@ public class TestRMWebServicesApps exten
WebServicesTestUtils.getXmlString(element, "amContainerLogs"),
WebServicesTestUtils.getXmlInt(element, "allocatedMB"),
WebServicesTestUtils.getXmlInt(element, "allocatedVCores"),
- WebServicesTestUtils.getXmlInt(element, "runningContainers"));
+ WebServicesTestUtils.getXmlInt(element, "runningContainers"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedResourceMB"),
+ WebServicesTestUtils.getXmlInt(element, "preemptedResourceVCores"),
+ WebServicesTestUtils.getXmlInt(element, "numNonAMContainerPreempted"),
+ WebServicesTestUtils.getXmlInt(element, "numAMContainerPreempted"));
}
}
public void verifyAppInfo(JSONObject info, RMApp app) throws JSONException,
Exception {
- // 20 because trackingUrl not assigned yet
- assertEquals("incorrect number of elements", 20, info.length());
+ // 28 because trackingUrl not assigned yet
+ assertEquals("incorrect number of elements", 24, info.length());
verifyAppInfoGeneric(app, info.getString("id"), info.getString("user"),
- info.getString("name"), info.getString("applicationType"), info.getString("queue"),
- info.getString("state"), info.getString("finalStatus"),
- (float) info.getDouble("progress"), info.getString("trackingUI"),
- info.getString("diagnostics"), info.getLong("clusterId"),
- info.getLong("startedTime"), info.getLong("finishedTime"),
- info.getLong("elapsedTime"), info.getString("amHostHttpAddress"),
- info.getString("amContainerLogs"), info.getInt("allocatedMB"),
- info.getInt("allocatedVCores"), info.getInt("runningContainers"));
+ info.getString("name"), info.getString("applicationType"),
+ info.getString("queue"), info.getString("state"),
+ info.getString("finalStatus"), (float) info.getDouble("progress"),
+ info.getString("trackingUI"), info.getString("diagnostics"),
+ info.getLong("clusterId"), info.getLong("startedTime"),
+ info.getLong("finishedTime"), info.getLong("elapsedTime"),
+ info.getString("amHostHttpAddress"), info.getString("amContainerLogs"),
+ info.getInt("allocatedMB"), info.getInt("allocatedVCores"),
+ info.getInt("runningContainers"),
+ info.getInt("preemptedResourceMB"),
+ info.getInt("preemptedResourceVCores"),
+ info.getInt("numNonAMContainerPreempted"),
+ info.getInt("numAMContainerPreempted"));
}
public void verifyAppInfoGeneric(RMApp app, String id, String user,
- String name, String applicationType, String queue, String state, String finalStatus,
- float progress, String trackingUI, String diagnostics, long clusterId,
- long startedTime, long finishedTime, long elapsedTime,
- String amHostHttpAddress, String amContainerLogs, int allocatedMB,
- int allocatedVCores, int numContainers) throws JSONException,
+ String name, String applicationType, String queue, String state,
+ String finalStatus, float progress, String trackingUI,
+ String diagnostics, long clusterId, long startedTime, long finishedTime,
+ long elapsedTime, String amHostHttpAddress, String amContainerLogs,
+ int allocatedMB, int allocatedVCores, int numContainers,
+ int preemptedResourceMB, int preemptedResourceVCores,
+ int numNonAMContainerPreempted, int numAMContainerPreempted) throws JSONException,
Exception {
WebServicesTestUtils.checkStringMatch("id", app.getApplicationId()
@@ -1371,6 +1382,18 @@ public class TestRMWebServicesApps exten
assertEquals("allocatedMB doesn't match", 1024, allocatedMB);
assertEquals("allocatedVCores doesn't match", 1, allocatedVCores);
assertEquals("numContainers doesn't match", 1, numContainers);
+ assertEquals("preemptedResourceMB doesn't match", app
+ .getRMAppMetrics().getResourcePreempted().getMemory(),
+ preemptedResourceMB);
+ assertEquals("preemptedResourceVCores doesn't match", app
+ .getRMAppMetrics().getResourcePreempted().getVirtualCores(),
+ preemptedResourceVCores);
+ assertEquals("numNonAMContainerPreempted doesn't match", app
+ .getRMAppMetrics().getNumNonAMContainersPreempted(),
+ numNonAMContainerPreempted);
+ assertEquals("numAMContainerPreempted doesn't match", app
+ .getRMAppMetrics().getNumAMContainersPreempted(),
+ numAMContainerPreempted);
}
@Test
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Tue Aug 19 23:49:39 2014
@@ -656,13 +656,15 @@ public class TestRMWebServicesNodes exte
WebServicesTestUtils.getXmlInt(element, "numContainers"),
WebServicesTestUtils.getXmlLong(element, "usedMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
+ WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
+ WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
WebServicesTestUtils.getXmlString(element, "version"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 11, nodeInfo.length());
+ assertEquals("incorrect number of elements", 13, nodeInfo.length());
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
@@ -671,6 +673,7 @@ public class TestRMWebServicesNodes exte
nodeInfo.getLong("lastHealthUpdate"),
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
+ nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
nodeInfo.getString("version"));
}
@@ -678,7 +681,8 @@ public class TestRMWebServicesNodes exte
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
- int numContainers, long usedMemoryMB, long availMemoryMB, String version)
+ int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
+ long availVirtualCores, String version)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@@ -712,6 +716,10 @@ public class TestRMWebServicesNodes exte
.getUsedResource().getMemory(), usedMemoryMB);
assertEquals("availMemoryMB doesn't match: " + availMemoryMB, report
.getAvailableResource().getMemory(), availMemoryMB);
+ assertEquals("usedVirtualCores doesn't match: " + usedVirtualCores, report
+ .getUsedResource().getVirtualCores(), usedVirtualCores);
+ assertEquals("availVirtualCores doesn't match: " + availVirtualCores, report
+ .getAvailableResource().getVirtualCores(), availVirtualCores);
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml Tue Aug 19 23:49:39 2014
@@ -37,24 +37,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Tue Aug 19 23:49:39 2014
@@ -25,6 +25,7 @@ import java.net.UnknownHostException;
import java.util.Collection;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -56,8 +57,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.MemoryTimelineStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -69,6 +69,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
+import org.apache.hadoop.yarn.server.timeline.MemoryTimelineStore;
+import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -504,12 +506,6 @@ public class MiniYARNCluster extends Com
String logDirsString = prepareDirs("log", numLogDirs);
config.set(YarnConfiguration.NM_LOG_DIRS, logDirsString);
- File remoteLogDir =
- new File(testWorkDir, MiniYARNCluster.this.getName()
- + "-remoteLogDir-nm-" + index);
- remoteLogDir.mkdir();
- config.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
- remoteLogDir.getAbsolutePath());
// By default AM + 2 containers
config.setInt(YarnConfiguration.NM_PMEM_MB, 4*1024);
config.set(YarnConfiguration.NM_ADDRESS,
@@ -658,12 +654,14 @@ public class MiniYARNCluster extends Com
*/
public boolean waitForNodeManagersToConnect(long timeout)
throws YarnException, InterruptedException {
- ResourceManager rm = getResourceManager();
GetClusterMetricsRequest req = GetClusterMetricsRequest.newInstance();
-
for (int i = 0; i < timeout / 100; i++) {
- if (nodeManagers.length == rm.getClientRMService().getClusterMetrics(req)
- .getClusterMetrics().getNumNodeManagers()) {
+ ResourceManager rm = getResourceManager();
+ if (rm == null) {
+ throw new YarnException("Can not find the active RM.");
+ }
+ else if (nodeManagers.length == rm.getClientRMService()
+ .getClusterMetrics(req).getClusterMetrics().getNumNodeManagers()) {
return true;
}
Thread.sleep(100);
@@ -723,6 +721,7 @@ public class MiniYARNCluster extends Com
if (appHistoryServer != null) {
appHistoryServer.stop();
}
+ AHSWebApp.resetInstance();
super.serviceStop();
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Tue Aug 19 23:49:39 2014
@@ -29,8 +29,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.List;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -52,6 +50,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.Token;
@@ -60,15 +59,18 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -200,8 +202,6 @@ public class TestContainerManagerSecurit
ApplicationId appId = ApplicationId.newInstance(1, 1);
ApplicationAttemptId validAppAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
- ApplicationAttemptId invalidAppAttemptId =
- ApplicationAttemptId.newInstance(appId, 2);
ContainerId validContainerId =
ContainerId.newInstance(validAppAttemptId, 0);
@@ -215,7 +215,11 @@ public class TestContainerManagerSecurit
org.apache.hadoop.yarn.api.records.Token validContainerToken =
containerTokenSecretManager.createContainerToken(validContainerId,
- validNode, user, r);
+ validNode, user, r, Priority.newInstance(10), 1234);
+ ContainerTokenIdentifier identifier =
+ BuilderUtils.newContainerTokenIdentifier(validContainerToken);
+ Assert.assertEquals(Priority.newInstance(10), identifier.getPriority());
+ Assert.assertEquals(1234, identifier.getCreationTime());
StringBuilder sb;
// testInvalidNMToken ... creating NMToken using different secret manager.
@@ -263,27 +267,14 @@ public class TestContainerManagerSecurit
testStartContainer(rpc, validAppAttemptId, validNode,
validContainerToken, invalidNMToken, true)));
- // using appAttempt-2 token for launching container for appAttempt-1.
- invalidNMToken =
- nmTokenSecretManagerRM.createNMToken(invalidAppAttemptId, validNode,
- user);
- sb = new StringBuilder("\nNMToken for application attempt : ");
- sb.append(invalidAppAttemptId.toString())
- .append(" was used for starting container with container token")
- .append(" issued for application attempt : ")
- .append(validAppAttemptId.toString());
- Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
- validContainerToken, invalidNMToken, true).contains(sb.toString()));
-
// using correct tokens. nmtoken for app attempt should get saved.
conf.setInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
4 * 60 * 1000);
validContainerToken =
containerTokenSecretManager.createContainerToken(validContainerId,
- validNode, user, r);
-
- testStartContainer(rpc, validAppAttemptId, validNode, validContainerToken,
- validNMToken, false);
+ validNode, user, r, Priority.newInstance(0), 0);
+ Assert.assertTrue(testStartContainer(rpc, validAppAttemptId, validNode,
+ validContainerToken, validNMToken, false).isEmpty());
Assert.assertTrue(nmTokenSecretManagerNM
.isAppAttemptNMTokenKeyPresent(validAppAttemptId));
@@ -325,6 +316,18 @@ public class TestContainerManagerSecurit
Assert.assertTrue(testGetContainer(rpc, validAppAttemptId, validNode,
validContainerId, validNMToken, false).contains(sb.toString()));
+ // using appAttempt-1 NMtoken for launching container for appAttempt-2 should
+ // succeed.
+ ApplicationAttemptId attempt2 = ApplicationAttemptId.newInstance(appId, 2);
+ Token attempt1NMToken =
+ nmTokenSecretManagerRM
+ .createNMToken(validAppAttemptId, validNode, user);
+ org.apache.hadoop.yarn.api.records.Token newContainerToken =
+ containerTokenSecretManager.createContainerToken(
+ ContainerId.newInstance(attempt2, 1), validNode, user, r,
+ Priority.newInstance(0), 0);
+ Assert.assertTrue(testStartContainer(rpc, attempt2, validNode,
+ newContainerToken, attempt1NMToken, false).isEmpty());
}
private void waitForContainerToFinishOnNM(ContainerId containerId) {
@@ -590,7 +593,7 @@ public class TestContainerManagerSecurit
// Creating modified containerToken
Token containerToken =
tamperedContainerTokenSecretManager.createContainerToken(cId, nodeId,
- user, r);
+ user, r, Priority.newInstance(0), 0);
Token nmToken =
nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
YarnRPC rpc = YarnRPC.create(conf);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml Tue Aug 19 23:49:39 2014
@@ -43,24 +43,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServer.java Tue Aug 19 23:49:39 2014
@@ -25,9 +25,12 @@ import org.apache.hadoop.service.Service
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.net.InetSocketAddress;
+
public class TestWebAppProxyServer {
private WebAppProxyServer webAppProxy = null;
private final String proxyAddress = "0.0.0.0:8888";
@@ -56,4 +59,14 @@ public class TestWebAppProxyServer {
}
assertEquals(STATE.STARTED, webAppProxy.getServiceState());
}
+
+ @Test
+ public void testBindAddress() {
+ YarnConfiguration conf = new YarnConfiguration();
+
+ InetSocketAddress defaultBindAddress = WebAppProxyServer.getBindAddress(conf);
+ Assert.assertEquals("Web Proxy default bind address port is incorrect",
+ YarnConfiguration.DEFAULT_PROXY_PORT,
+ defaultBindAddress.getPort());
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm Tue Aug 19 23:49:39 2014
@@ -18,8 +18,6 @@
Hadoop MapReduce Next Generation - Capacity Scheduler
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* {Purpose}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Tue Aug 19 23:49:39 2014
@@ -18,8 +18,6 @@
Hadoop MapReduce Next Generation - Fair Scheduler
- \[ {{{./index.html}Go Back}} \]
-
%{toc|section=1|fromDepth=0}
* {Purpose}
@@ -156,6 +154,12 @@ Properties that can be placed in yarn-si
* Whether to use preemption. Note that preemption is experimental in the current
version. Defaults to false.
+ * <<<yarn.scheduler.fair.preemption.cluster-utilization-threshold>>>
+
+ * The utilization threshold after which preemption kicks in. The
+ utilization is computed as the maximum ratio of usage to capacity among
+ all resources. Defaults to 0.8f.
+
* <<<yarn.scheduler.fair.sizebasedweight>>>
* Whether to assign shares to individual apps based on their size, rather than
@@ -201,13 +205,21 @@ Properties that can be placed in yarn-si
instead. Defaults to true. If a queue placement policy is given in the
allocations file, this property is ignored.
+ * <<<yarn.scheduler.fair.update-interval-ms>>>
+
+ * The interval at which to lock the scheduler and recalculate fair shares,
+ recalculate demand, and check whether anything is due for preemption.
+ Defaults to 500 ms.
+
Allocation file format
The allocation file must be in XML format. The format contains five types of
elements:
- * <<Queue elements>>, which represent queues. Each may contain the following
- properties:
+ * <<Queue elements>>, which represent queues. Queue elements can take an optional
+ attribute âtypeâ,which when set to âparentâ makes it a parent queue. This is useful
+ when we want to create a parent queue without configuring any leaf queues.
+ Each queue element may contain the following properties:
* minResources: minimum resources the queue is entitled to, in the form
"X mb, Y vcores". For the single-resource fairness policy, the vcores
@@ -231,6 +243,12 @@ Allocation file format
* maxRunningApps: limit the number of apps from the queue to run at once
+ * maxAMShare: limit the fraction of the queue's fair share that can be used
+ to run application masters. This property can only be used for leaf queues.
+ For example, if set to 1.0f, then AMs in the leaf queue can take up to 100%
+ of both the memory and CPU fair share. The default value is -1.0f, which
+ means that this check is disabled.
+
* weight: to share the cluster non-proportionally with other queues. Weights
default to 1, and a queue with weight 2 should receive approximately twice
as many resources as a queue with the default weight.
@@ -273,6 +291,9 @@ Allocation file format
* <<A queueMaxAppsDefault element>>, which sets the default running app limit
for queues; overriden by maxRunningApps element in each queue.
+ * <<A queueMaxAMShareDefault element>>, which sets the default AM resource
+ limit for queue; overriden by maxAMShare element in each queue.
+
* <<A defaultQueueSchedulingPolicy element>>, which sets the default scheduling
policy for queues; overriden by the schedulingPolicy element in each queue
if specified. Defaults to "fair".
@@ -299,7 +320,17 @@ Allocation file format
that matches a secondary group of the user who submitted it. The first
secondary group that matches a configured queue will be selected.
- * default: the app is placed into the queue named "default".
+ * nestedUserQueue : the app is placed into a queue with the name of the user
+ under the queue suggested by the nested rule. This is similar to âuserâ
+ rule,the difference being in ânestedUserQueueâ rule,user queues can be created
+ under any parent queue, while âuserâ rule creates user queues only under root queue.
+ Note that nestedUserQueue rule would be applied only if the nested rule returns a
+ parent queue.One can configure a parent queue either by setting âtypeâ attribute of queue
+ to âparentâ or by configuring at least one leaf under that queue which makes it a parent.
+ See example allocation for a sample use case.
+
+ * default: the app is placed into the queue specified in the âqueueâ attribute of the
+ default rule. If âqueueâ attribute is not specified, the app is placed into âroot.defaultâ queue.
* reject: the app is rejected.
@@ -312,6 +343,7 @@ Allocation file format
<minResources>10000 mb,0vcores</minResources>
<maxResources>90000 mb,0vcores</maxResources>
<maxRunningApps>50</maxRunningApps>
+ <maxAMShare>0.1</maxAMShare>
<weight>2.0</weight>
<schedulingPolicy>fair</schedulingPolicy>
<queue name="sample_sub_queue">
@@ -319,6 +351,14 @@ Allocation file format
<minResources>5000 mb,0vcores</minResources>
</queue>
</queue>
+
+ <queueMaxAMShareDefault>0.5</queueMaxAMShareDefault>
+
+ <!â- Queue âsecondary_group_queueâ is a parent queue and may have
+ user queues under it -â>
+ <queue name=âsecondary_group_queueâ type=âparentâ>
+ <weight>3.0</weight>
+ </queue>
<user name="sample_user">
<maxRunningApps>30</maxRunningApps>
@@ -328,7 +368,10 @@ Allocation file format
<queuePlacementPolicy>
<rule name="specified" />
<rule name="primaryGroup" create="false" />
- <rule name="default" />
+ <rule name=ânestedUserQueueâ>
+ <rule name=âsecondaryGroupExistingQueueâ create=âfalseâ />
+ </rule>
+ <rule name="default" queue=âsample_queueâ />
</queuePlacementPolicy>
</allocations>
---