You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2012/10/11 08:14:43 UTC
svn commit: r1396918 [3/3] - in
/hadoop/common/branches/HDFS-3077/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/...
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java Thu Oct 11 06:14:26 2012
@@ -112,7 +112,7 @@ public class MockAM {
ResourceRequest hostReq = createResourceReq(host, memory, priority,
containers);
reqs.add(hostReq);
- ResourceRequest rackReq = createResourceReq("default-rack", memory,
+ ResourceRequest rackReq = createResourceReq("/default-rack", memory,
priority, containers);
reqs.add(rackReq);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Thu Oct 11 06:14:26 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
@@ -58,12 +59,12 @@ import org.apache.log4j.Logger;
public class MockRM extends ResourceManager {
public MockRM() {
- this(new Configuration());
+ this(new YarnConfiguration());
}
public MockRM(Configuration conf) {
- super(StoreFactory.getStore(conf));
- init(conf);
+ super(StoreFactory.getStore(conf));
+ init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java Thu Oct 11 06:14:26 2012
@@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.resourcemanager.TestApplicationMasterLauncher.MockRMWithCustomAMLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java Thu Oct 11 06:14:26 2012
@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -49,6 +48,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.Service;
import org.junit.Test;
@@ -140,7 +140,7 @@ public class TestAppManager{
}
public TestRMAppManager(RMContext context,
- ClientToAMSecretManager clientToAMSecretManager,
+ ClientToAMTokenSecretManagerInRM clientToAMSecretManager,
YarnScheduler scheduler, ApplicationMasterService masterService,
ApplicationACLsManager applicationACLsManager, Configuration conf) {
super(context, clientToAMSecretManager, scheduler, masterService,
@@ -342,7 +342,7 @@ public class TestAppManager{
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
- new ClientToAMSecretManager(), scheduler, masterService,
+ new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
new ApplicationACLsManager(conf), conf);
ApplicationId appID = MockApps.newAppID(1);
@@ -390,7 +390,7 @@ public class TestAppManager{
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
- new ClientToAMSecretManager(), scheduler, masterService,
+ new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
new ApplicationACLsManager(conf), conf);
ApplicationId appID = MockApps.newAppID(10);
@@ -438,7 +438,7 @@ public class TestAppManager{
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
- new ClientToAMSecretManager(), scheduler, masterService,
+ new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
new ApplicationACLsManager(conf), conf);
ApplicationId appID = MockApps.newAppID(0);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java Thu Oct 11 06:14:26 2012
@@ -22,7 +22,6 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
@@ -35,9 +34,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -106,40 +102,6 @@ public class TestApplicationMasterLaunch
}
- static class MockRMWithCustomAMLauncher extends MockRM {
-
- private final ContainerManager containerManager;
-
- public MockRMWithCustomAMLauncher(ContainerManager containerManager) {
- this(new Configuration(), containerManager);
- }
-
- public MockRMWithCustomAMLauncher(Configuration conf,
- ContainerManager containerManager) {
- super(conf);
- this.containerManager = containerManager;
- }
-
- @Override
- protected ApplicationMasterLauncher createAMLauncher() {
- return new ApplicationMasterLauncher(super.clientToAMSecretManager,
- getRMContext()) {
- @Override
- protected Runnable createRunnableLauncher(RMAppAttempt application,
- AMLauncherEventType event) {
- return new AMLauncher(context, application, event,
- clientToAMSecretManager, getConfig()) {
- @Override
- protected ContainerManager getContainerMgrProxy(
- ContainerId containerId) {
- return containerManager;
- }
- };
- }
- };
- }
- }
-
@Test
public void testAMLaunchAndCleanup() throws Exception {
Logger rootLogger = LogManager.getRootLogger();
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Thu Oct 11 06:14:26 2012
@@ -52,18 +52,27 @@ import org.apache.hadoop.yarn.util.Build
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestFifoScheduler {
private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class);
private final int GB = 1024;
+ private static YarnConfiguration conf;
+
+ @BeforeClass
+ public static void setup() {
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ FifoScheduler.class, ResourceScheduler.class);
+ }
@Test
public void test() throws Exception {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
- MockRM rm = new MockRM();
+ MockRM rm = new MockRM(conf);
rm.start();
MockNM nm1 = rm.registerNode("h1:1234", 6 * GB);
MockNM nm2 = rm.registerNode("h2:5678", 4 * GB);
@@ -178,15 +187,15 @@ public class TestFifoScheduler {
public void testDefaultMinimumAllocation() throws Exception {
// Test with something lesser than default
testMinimumAllocation(
- new YarnConfiguration(),
+ new YarnConfiguration(TestFifoScheduler.conf),
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB / 2);
}
@Test
public void testNonDefaultMinimumAllocation() throws Exception {
// Set custom min-alloc to test tweaking it
- int allocMB = 512;
- YarnConfiguration conf = new YarnConfiguration();
+ int allocMB = 1536;
+ YarnConfiguration conf = new YarnConfiguration(TestFifoScheduler.conf);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, allocMB);
// Test for something lesser than this.
testMinimumAllocation(conf, allocMB / 2);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java Thu Oct 11 06:14:26 2012
@@ -30,6 +30,7 @@ import org.apache.hadoop.net.NetworkTopo
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
@@ -45,7 +46,7 @@ public class TestResourceManager {
@Before
public void setUp() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = new YarnConfiguration();
Store store = StoreFactory.getStore(conf);
resourceManager = new ResourceManager(store);
resourceManager.init(conf);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Thu Oct 11 06:14:26 2012
@@ -54,7 +54,7 @@ public class TestResourceTrackerService
private MockRM rm;
/**
- * decommissioning using a include hosts file
+ * Decommissioning using a pre-configured include hosts file
*/
@Test
public void testDecommissionWithIncludeHosts() throws Exception {
@@ -86,7 +86,7 @@ public class TestResourceTrackerService
String ip = NetUtils.normalizeHostName("localhost");
writeToHostsFile("host1", ip);
- rm.getNodesListManager().refreshNodes();
+ rm.getNodesListManager().refreshNodes(conf);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
@@ -106,7 +106,7 @@ public class TestResourceTrackerService
}
/**
- * decommissioning using a exclude hosts file
+ * Decommissioning using a pre-configured exclude hosts file
*/
@Test
public void testDecommissionWithExcludeHosts() throws Exception {
@@ -133,7 +133,7 @@ public class TestResourceTrackerService
String ip = NetUtils.normalizeHostName("localhost");
writeToHostsFile("host2", ip);
- rm.getNodesListManager().refreshNodes();
+ rm.getNodesListManager().refreshNodes(conf);
nodeHeartbeat = nm1.nodeHeartbeat(true);
Assert.assertTrue(NodeAction.NORMAL.equals(nodeHeartbeat.getNodeAction()));
@@ -147,7 +147,81 @@ public class TestResourceTrackerService
NodeAction.SHUTDOWN.equals(nodeHeartbeat.getNodeAction()));
checkDecommissionedNMCount(rm, ++metricCount);
}
+
+ /**
+ * Decommissioning using a post-configured include hosts file
+ */
+ @Test
+ public void testAddNewIncludePathToConfiguration() throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ assert(metrics != null);
+ int initialMetricCount = metrics.getNumDecommisionedNMs();
+ HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ writeToHostsFile("host1");
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ rm.getNodesListManager().refreshNodes(conf);
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ "Node should not have been decomissioned.",
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals("Node should have been decomissioned but is in state" +
+ nodeHeartbeat.getNodeAction(),
+ NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
+ checkDecommissionedNMCount(rm, ++initialMetricCount);
+ }
+ /**
+ * Decommissioning using a post-configured exclude hosts file
+ */
+ @Test
+ public void testAddNewExcludePathToConfiguration() throws Exception {
+ Configuration conf = new Configuration();
+ rm = new MockRM(conf);
+ rm.start();
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+ ClusterMetrics metrics = ClusterMetrics.getMetrics();
+ assert(metrics != null);
+ int initialMetricCount = metrics.getNumDecommisionedNMs();
+ HeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ writeToHostsFile("host2");
+ conf.set(YarnConfiguration.RM_NODES_EXCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ rm.getNodesListManager().refreshNodes(conf);
+ nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(
+ "Node should not have been decomissioned.",
+ NodeAction.NORMAL,
+ nodeHeartbeat.getNodeAction());
+ nodeHeartbeat = nm2.nodeHeartbeat(true);
+ Assert.assertEquals("Node should have been decomissioned but is in state" +
+ nodeHeartbeat.getNodeAction(),
+ NodeAction.SHUTDOWN, nodeHeartbeat.getNodeAction());
+ checkDecommissionedNMCount(rm, ++initialMetricCount);
+ }
+
@Test
public void testNodeRegistrationFailure() throws Exception {
writeToHostsFile("host1");
@@ -268,10 +342,12 @@ public class TestResourceTrackerService
MockNM nm2 = rm.registerNode("host2:5678", 5120);
nm1.nodeHeartbeat(true);
nm2.nodeHeartbeat(false);
+ dispatcher.await();
checkUnealthyNMCount(rm, nm2, true, 1);
final int expectedNMs = ClusterMetrics.getMetrics().getNumActiveNMs();
QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
- Assert.assertEquals(5120 + 5120, metrics.getAvailableMB());
+ // TODO Metrics incorrect in case of the FifoScheduler
+ Assert.assertEquals(5120, metrics.getAvailableMB());
// reconnect of healthy node
nm1 = rm.registerNode("host1:1234", 5120);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java Thu Oct 11 06:14:26 2012
@@ -90,10 +90,10 @@ public class TestAMRMRPCNodeUpdates {
@Test
public void testAMRMUnusableNodes() throws Exception {
- MockNM nm1 = rm.registerNode("h1:1234", 5000);
- MockNM nm2 = rm.registerNode("h2:1234", 5000);
- MockNM nm3 = rm.registerNode("h3:1234", 5000);
- MockNM nm4 = rm.registerNode("h4:1234", 5000);
+ MockNM nm1 = rm.registerNode("h1:1234", 10000);
+ MockNM nm2 = rm.registerNode("h2:1234", 10000);
+ MockNM nm3 = rm.registerNode("h3:1234", 10000);
+ MockNM nm4 = rm.registerNode("h4:1234", 10000);
RMApp app1 = rm.submitApp(2000);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Thu Oct 11 06:14:26 2012
@@ -262,6 +262,10 @@ public class TestRMAppAttemptTransitions
assertEquals(0, applicationAttempt.getRanNodes().size());
assertNull(applicationAttempt.getFinalApplicationStatus());
+ // Check events
+ verify(masterService).
+ unregisterAttempt(applicationAttempt.getAppAttemptId());
+
// this works for unmanaged and managed AM's because this is actually doing
// verify(application).handle(anyObject());
verify(application).handle(any(RMAppRejectedEvent.class));
@@ -527,7 +531,8 @@ public class TestRMAppAttemptTransitions
// launch AM and verify attempt failed
applicationAttempt.handle(new RMAppAttemptRegistrationEvent(
applicationAttempt.getAppAttemptId(), "host", 8042, "oldtrackingurl"));
- testAppAttemptSubmittedToFailedState("Unmanaged AM must register after AM attempt reaches LAUNCHED state.");
+ testAppAttemptSubmittedToFailedState(
+ "Unmanaged AM must register after AM attempt reaches LAUNCHED state.");
}
@Test
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java Thu Oct 11 06:14:26 2012
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
@@ -93,7 +94,7 @@ public class TestFairScheduler {
@Before
public void setUp() throws IOException {
scheduler = new FairScheduler();
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
// All tests assume only one assignment per node update
conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
Store store = StoreFactory.getStore(conf);
@@ -109,6 +110,13 @@ public class TestFairScheduler {
resourceManager = null;
}
+ private Configuration createConfiguration() {
+ Configuration conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
+ ResourceScheduler.class);
+ return conf;
+ }
+
private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
ApplicationAttemptId attId = recordFactory.newRecordInstance(ApplicationAttemptId.class);
ApplicationId appIdImpl = recordFactory.newRecordInstance(ApplicationId.class);
@@ -217,7 +225,9 @@ public class TestFairScheduler {
new ArrayList<ContainerStatus>(), new ArrayList<ContainerStatus>());
scheduler.handle(updateEvent);
- assertEquals(512, scheduler.getQueueManager().getQueue("queue1").
+ // Asked for less than min_allocation.
+ assertEquals(YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ scheduler.getQueueManager().getQueue("queue1").
getQueueSchedulable().getResourceUsage().getMemory());
NodeUpdateSchedulerEvent updateEvent2 = new NodeUpdateSchedulerEvent(node2,
@@ -278,7 +288,7 @@ public class TestFairScheduler {
@Test
public void testUserAsDefaultQueue() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
scheduler.reinitialize(conf, resourceManager.getRMContext());
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(
@@ -299,7 +309,7 @@ public class TestFairScheduler {
@Test
public void testFairShareWithMinAlloc() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -354,31 +364,35 @@ public class TestFairScheduler {
ApplicationAttemptId id22 = createAppAttemptId(2, 2);
scheduler.addApplication(id22, "queue2", "user1");
- // First ask, queue1 requests 1024
+ int minReqSize = YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB;
+
+ // First ask, queue1 requests 1 large (minReqSize * 2).
List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
- ResourceRequest request1 = createResourceRequest(1024, "*", 1, 1);
+ ResourceRequest request1 = createResourceRequest(minReqSize * 2, "*", 1, 1);
ask1.add(request1);
scheduler.allocate(id11, ask1, new ArrayList<ContainerId>());
- // Second ask, queue2 requests 1024 + (2 * 512)
+ // Second ask, queue2 requests 1 large + (2 * minReqSize)
List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
- ResourceRequest request2 = createResourceRequest(1024, "foo", 1, 1);
- ResourceRequest request3 = createResourceRequest(512, "bar", 1, 2);
+ ResourceRequest request2 = createResourceRequest(2 * minReqSize, "foo", 1, 1);
+ ResourceRequest request3 = createResourceRequest(minReqSize, "bar", 1, 2);
ask2.add(request2);
ask2.add(request3);
scheduler.allocate(id21, ask2, new ArrayList<ContainerId>());
- // Third ask, queue2 requests 1024
+ // Third ask, queue2 requests 1 large
List<ResourceRequest> ask3 = new ArrayList<ResourceRequest>();
- ResourceRequest request4 = createResourceRequest(1024, "*", 1, 1);
+ ResourceRequest request4 = createResourceRequest(2 * minReqSize, "*", 1, 1);
ask3.add(request4);
scheduler.allocate(id22, ask3, new ArrayList<ContainerId>());
scheduler.update();
- assertEquals(1024, scheduler.getQueueManager().getQueue("queue1").getQueueSchedulable().getDemand().getMemory());
- assertEquals(1024 + 1024 + (2 * 512), scheduler.getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand().getMemory());
-
+ assertEquals(2 * minReqSize, scheduler.getQueueManager().getQueue("queue1")
+ .getQueueSchedulable().getDemand().getMemory());
+ assertEquals(2 * minReqSize + 2 * minReqSize + (2 * minReqSize), scheduler
+ .getQueueManager().getQueue("queue2").getQueueSchedulable().getDemand()
+ .getMemory());
}
@Test
@@ -405,7 +419,7 @@ public class TestFairScheduler {
@Test
public void testAllocationFileParsing() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -508,7 +522,7 @@ public class TestFairScheduler {
@Test
public void testBackwardsCompatibleAllocationFileParsing() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -611,7 +625,7 @@ public class TestFairScheduler {
@Test
public void testIsStarvedForMinShare() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -670,7 +684,7 @@ public class TestFairScheduler {
@Test
public void testIsStarvedForFairShare() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -734,7 +748,7 @@ public class TestFairScheduler {
* now this means decreasing order of priority.
*/
public void testChoiceOfPreemptedContainers() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE + ".allocation.file", ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -867,7 +881,7 @@ public class TestFairScheduler {
* Tests the timing of decision to preempt tasks.
*/
public void testPreemptionDecision() throws Exception {
- Configuration conf = new Configuration();
+ Configuration conf = createConfiguration();
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
MockClock clock = new MockClock();
scheduler.setClock(clock);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Thu Oct 11 06:14:26 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.InlineDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -59,7 +61,10 @@ public class TestFifoScheduler {
public void setUp() throws Exception {
Store store = StoreFactory.getStore(new Configuration());
resourceManager = new ResourceManager(store);
- resourceManager.init(new Configuration());
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ FifoScheduler.class, ResourceScheduler.class);
+ resourceManager.init(conf);
}
@After
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Thu Oct 11 06:14:26 2012
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -74,7 +75,10 @@ public class TestRMWebServices extends J
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
- rm = new MockRM(new Configuration());
+ Configuration conf = new Configuration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+ ResourceScheduler.class);
+ rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
bind(RMContext.class).toInstance(rm.getRMContext());
bind(ApplicationACLsManager.class).toInstance(
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java Thu Oct 11 06:14:26 2012
@@ -45,6 +45,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppFailedAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
@@ -82,6 +84,8 @@ public class TestRMWebServicesApps exten
bind(GenericExceptionHandler.class);
Configuration conf = new Configuration();
conf.setInt(YarnConfiguration.RM_AM_MAX_RETRIES, 2);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+ ResourceScheduler.class);
rm = new MockRM(conf);
bind(ResourceManager.class).toInstance(rm);
bind(RMContext.class).toInstance(rm.getRMContext());
@@ -276,9 +280,9 @@ public class TestRMWebServicesApps exten
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
- .checkStringMatch(
+ .checkStringContains(
"exception message",
- "No enum const class org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test",
+ "org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState.INVALID_test",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
@@ -355,9 +359,9 @@ public class TestRMWebServicesApps exten
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
- .checkStringMatch(
+ .checkStringContains(
"exception message",
- "No enum const class org.apache.hadoop.yarn.api.records.FinalApplicationStatus.INVALID_test",
+ "org.apache.hadoop.yarn.api.records.FinalApplicationStatus.INVALID_test",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java Thu Oct 11 06:14:26 2012
@@ -229,9 +229,9 @@ public class TestRMWebServicesNodes exte
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
WebServicesTestUtils
- .checkStringMatch(
+ .checkStringContains(
"exception message",
- "No enum const class org.apache.hadoop.yarn.api.records.NodeState.BOGUSSTATE",
+ "org.apache.hadoop.yarn.api.records.NodeState.BOGUSSTATE",
message);
WebServicesTestUtils.checkStringMatch("exception type",
"IllegalArgumentException", type);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Thu Oct 11 06:14:26 2012
@@ -102,6 +102,12 @@ public class MiniYARNCluster extends Com
nodeManagers[index] = new CustomNodeManager();
}
}
+
+ @Override
+ public void init(Configuration conf) {
+ super.init(conf instanceof YarnConfiguration ? conf
+ : new YarnConfiguration(conf));
+ }
public File getTestWorkDir() {
return testWorkDir;
@@ -201,7 +207,7 @@ public class MiniYARNCluster extends Com
}
public synchronized void init(Configuration conf) {
- Configuration config = new Configuration(conf);
+ Configuration config = new YarnConfiguration(conf);
super.init(config);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Thu Oct 11 06:14:26 2012
@@ -219,9 +219,10 @@ public class TestContainerManagerSecurit
// Malice user modifies the resource amount
Resource modifiedResource = BuilderUtils.newResource(2048);
- ContainerTokenIdentifier modifiedIdentifier = new ContainerTokenIdentifier(
- dummyIdentifier.getContainerID(), dummyIdentifier.getNmHostAddress(),
- modifiedResource, Long.MAX_VALUE, dummyIdentifier.getMasterKeyId());
+ ContainerTokenIdentifier modifiedIdentifier =
+ new ContainerTokenIdentifier(dummyIdentifier.getContainerID(),
+ dummyIdentifier.getNmHostAddress(), "testUser", modifiedResource,
+ Long.MAX_VALUE, dummyIdentifier.getMasterKeyId());
Token<ContainerTokenIdentifier> modifiedToken = new Token<ContainerTokenIdentifier>(
modifiedIdentifier.getBytes(), containerToken.getPassword().array(),
new Text(containerToken.getKind()), new Text(containerToken
@@ -320,12 +321,14 @@ public class TestContainerManagerSecurit
callWithIllegalContainerID(client, tokenId);
callWithIllegalResource(client, tokenId);
+ callWithIllegalUserName(client, tokenId);
return client;
}
});
- /////////// End of testing for illegal containerIDs and illegal Resources
+ // ///////// End of testing for illegal containerIDs, illegal Resources and
+ // illegal users
/////////// Test calls with expired tokens
RPC.stopProxy(client);
@@ -336,7 +339,7 @@ public class TestContainerManagerSecurit
resourceManager.getRMContainerTokenSecretManager();
final ContainerTokenIdentifier newTokenId =
new ContainerTokenIdentifier(tokenId.getContainerID(),
- tokenId.getNmHostAddress(), tokenId.getResource(),
+ tokenId.getNmHostAddress(), "testUser", tokenId.getResource(),
System.currentTimeMillis() - 1,
containerTokenSecreteManager.getCurrentKey().getKeyId());
byte[] passowrd =
@@ -346,9 +349,7 @@ public class TestContainerManagerSecurit
token = new Token<ContainerTokenIdentifier>(
newTokenId.getBytes(), passowrd, new Text(
containerToken.getKind()), new Text(containerToken.getService()));
-
-
-
+
unauthorizedUser.addToken(token);
unauthorizedUser.doAs(new PrivilegedAction<Void>() {
@Override
@@ -567,6 +568,29 @@ public class TestContainerManagerSecurit
}
}
+ void callWithIllegalUserName(ContainerManager client,
+ ContainerTokenIdentifier tokenId) {
+ StartContainerRequest request = recordFactory
+ .newRecordInstance(StartContainerRequest.class);
+ // Authenticated but unauthorized, due to wrong resource
+ ContainerLaunchContext context =
+ createContainerLaunchContextForTest(tokenId);
+ context.setUser("Saruman"); // Set a different user-name.
+ request.setContainerLaunchContext(context);
+ try {
+ client.startContainer(request);
+ fail("Connection initiation with unauthorized "
+ + "access is expected to fail.");
+ } catch (YarnRemoteException e) {
+ LOG.info("Got exception : ", e);
+ Assert.assertTrue(e.getMessage().contains(
+ "Unauthorized request to start container. "));
+ Assert.assertTrue(e.getMessage().contains(
+ "Expected user-name " + tokenId.getApplicationSubmitter()
+ + " but found " + context.getUser()));
+ }
+ }
+
private ContainerLaunchContext createContainerLaunchContextForTest(
ContainerTokenIdentifier tokenId) {
ContainerLaunchContext context =
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/CapacityScheduler.apt.vm Thu Oct 11 06:14:26 2012
@@ -278,17 +278,17 @@ Hadoop MapReduce Next Generation - Capac
| | Existing applications continue to completion, thus the queue can be
| | <drained> gracefully. |
*--------------------------------------+--------------------------------------+
-| <<<yarn.scheduler.capacity.root.<queue-path>.acl_submit_jobs>>> | |
-| | The <ACL> which controls who can <submit> jobs to the given queue. |
+| <<<yarn.scheduler.capacity.root.<queue-path>.acl_submit_applications>>> | |
+| | The <ACL> which controls who can <submit> applications to the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
-| | <one of the parent queues in the hierarchy> they can submit jobs. |
+| | <one of the parent queues in the hierarchy> they can submit applications. |
| | <ACLs> for this property <are> inherited from the parent queue |
| | if not specified. |
*--------------------------------------+--------------------------------------+
-| <<<yarn.scheduler.capacity.root.<queue-path>.acl_administer_jobs>>> | |
-| | The <ACL> which controls who can <administer> jobs on the given queue. |
+| <<<yarn.scheduler.capacity.root.<queue-path>.acl_administer_queue>>> | |
+| | The <ACL> which controls who can <administer> applications on the given queue. |
| | If the given user/group has necessary ACLs on the given queue or |
-| | <one of the parent queues in the hierarchy> they can administer jobs. |
+| | <one of the parent queues in the hierarchy> they can administer applications. |
| | <ACLs> for this property <are> inherited from the parent queue |
| | if not specified. |
*--------------------------------------+--------------------------------------+
@@ -318,7 +318,7 @@ Hadoop MapReduce Next Generation - Capac
----
$ vi $HADOOP_CONF_DIR/capacity-scheduler.xml
-$ $YARN_HOME/bin/yarn rmadmin -refreshQueues
+$ $HADOOP_YARN_HOME/bin/yarn rmadmin -refreshQueues
----
<Note:> Queues cannot be <deleted>, only addition of new queues is supported -
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ClusterSetup.apt.vm Thu Oct 11 06:14:26 2012
@@ -497,20 +497,20 @@ Hadoop MapReduce Next Generation - Clust
ResourceManager:
----
- $ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
+ $ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
----
Run a script to start NodeManagers on all slaves:
----
- $ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
+ $ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
----
Start a standalone WebAppProxy server. If multiple servers
are used with load balancing it should be run on each of them:
----
- $ $YARN_HOME/bin/yarn start proxyserver --config $HADOOP_CONF_DIR
+ $ $HADOOP_YARN_HOME/bin/yarn start proxyserver --config $HADOOP_CONF_DIR
----
Start the MapReduce JobHistory Server with the following command, run on the
@@ -539,20 +539,20 @@ Hadoop MapReduce Next Generation - Clust
ResourceManager:
----
- $ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
+ $ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
----
Run a script to stop NodeManagers on all slaves:
----
- $ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
+ $ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
----
Stop the WebAppProxy server. If multiple servers are used with load
balancing it should be run on each of them:
----
- $ $YARN_HOME/bin/yarn stop proxyserver --config $HADOOP_CONF_DIR
+ $ $HADOOP_YARN_HOME/bin/yarn stop proxyserver --config $HADOOP_CONF_DIR
----
@@ -883,7 +883,7 @@ KVNO Timestamp Principal
The path passed in <<<-Dcontainer-executor.conf.dir>>> should be the
path on the cluster nodes where a configuration file for the setuid
executable should be located. The executable should be installed in
- $YARN_HOME/bin.
+ $HADOOP_YARN_HOME/bin.
The executable must have specific permissions: 6050 or --Sr-s---
permissions user-owned by <root> (super-user) and group-owned by a
@@ -1040,13 +1040,13 @@ KVNO Timestamp Principal
ResourceManager as <yarn>:
----
-[yarn]$ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
+[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start resourcemanager
----
Run a script to start NodeManagers on all slaves as <yarn>:
----
-[yarn]$ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
+[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR start nodemanager
----
Start a standalone WebAppProxy server. Run on the WebAppProxy
@@ -1054,7 +1054,7 @@ KVNO Timestamp Principal
it should be run on each of them:
----
-[yarn]$ $YARN_HOME/bin/yarn start proxyserver --config $HADOOP_CONF_DIR
+[yarn]$ $HADOOP_YARN_HOME/bin/yarn start proxyserver --config $HADOOP_CONF_DIR
----
Start the MapReduce JobHistory Server with the following command, run on the
@@ -1083,13 +1083,13 @@ KVNO Timestamp Principal
ResourceManager as <yarn>:
----
-[yarn]$ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
+[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop resourcemanager
----
Run a script to stop NodeManagers on all slaves as <yarn>:
----
-[yarn]$ $YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
+[yarn]$ $HADOOP_YARN_HOME/sbin/yarn-daemon.sh --config $HADOOP_CONF_DIR stop nodemanager
----
Stop the WebAppProxy server. Run on the WebAppProxy server as
@@ -1097,7 +1097,7 @@ KVNO Timestamp Principal
should be run on each of them:
----
-[yarn]$ $YARN_HOME/bin/yarn stop proxyserver --config $HADOOP_CONF_DIR
+[yarn]$ $HADOOP_YARN_HOME/bin/yarn stop proxyserver --config $HADOOP_CONF_DIR
----
Stop the MapReduce JobHistory Server with the following command, run on the
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SingleCluster.apt.vm Thu Oct 11 06:14:26 2012
@@ -43,7 +43,7 @@ $ mvn clean install assembly:assembly -P
Assuming you have installed hadoop-common/hadoop-hdfs and exported
<<$HADOOP_COMMON_HOME>>/<<$HADOOP_HDFS_HOME>>, untar hadoop mapreduce
tarball and set environment variable <<$HADOOP_MAPRED_HOME>> to the
- untarred directory. Set <<$YARN_HOME>> the same as <<$HADOOP_MAPRED_HOME>>.
+ untarred directory. Set <<$HADOOP_YARN_HOME>> the same as <<$HADOOP_MAPRED_HOME>>.
<<NOTE:>> The following instructions assume you have hdfs running.
@@ -174,7 +174,7 @@ Add the following configs to your <<<yar
* Running daemons.
Assuming that the environment variables <<$HADOOP_COMMON_HOME>>, <<$HADOOP_HDFS_HOME>>, <<$HADOO_MAPRED_HOME>>,
- <<$YARN_HOME>>, <<$JAVA_HOME>> and <<$HADOOP_CONF_DIR>> have been set appropriately.
+ <<$HADOOP_YARN_HOME>>, <<$JAVA_HOME>> and <<$HADOOP_CONF_DIR>> have been set appropriately.
Set $<<$YARN_CONF_DIR>> the same as $<<HADOOP_CONF_DIR>>
Run ResourceManager and NodeManager as:
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm Thu Oct 11 06:14:26 2012
@@ -52,6 +52,40 @@ Usage: yarn [--config confdir] COMMAND
Usage: yarn jar <jar> [mainClass] args...
-------
+** application
+
+ Prints application(s) report/kill application
+
+-------
+ Usage: yarn application <options>
+-------
+
+*---------------+--------------+
+|| COMMAND_OPTIONS || Description |
+*---------------+--------------+
+| -status ApplicationId | Specify an application id |
+*---------------+--------------+
+| -list | Lists all the Applications from RM |
+*---------------+--------------+
+| -kill ApplicationId | Specify an application id |
+*---------------+--------------+
+
+** node
+
+ Prints node report(s)
+
+-------
+ Usage: yarn node <options>
+-------
+
+*---------------+--------------+
+|| COMMAND_OPTIONS || Description |
+*---------------+--------------+
+| -status NodeId | Specify a node id |
+*---------------+--------------+
+| -list | Lists all the Nodes |
+*---------------+--------------+
+
** logs
Dump the container logs
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/pom.xml?rev=1396918&r1=1396917&r2=1396918&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/pom.xml Thu Oct 11 06:14:26 2012
@@ -148,7 +148,6 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
<groupId>org.jboss.netty</groupId>