You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cu...@apache.org on 2014/08/20 03:34:59 UTC
svn commit: r1619019 [8/10] - in
/hadoop/common/branches/YARN-1051/hadoop-yarn-project: ./ hadoop-yarn/bin/
hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/had...
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java Wed Aug 20 01:34:29 2014
@@ -171,7 +171,6 @@ public class TestApplicationMasterServic
RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
am1.registerAppAttempt();
- am1.setAMRMProtocol(rm.getApplicationMasterService());
AllocateRequestPBImpl allocateRequest = new AllocateRequestPBImpl();
List<ContainerId> release = new ArrayList<ContainerId>();
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java Wed Aug 20 01:34:29 2014
@@ -331,6 +331,10 @@ public class TestRMHA {
rm.adminService.transitionToStandby(requestInfo);
rm.adminService.transitionToActive(requestInfo);
rm.adminService.transitionToStandby(requestInfo);
+
+ MyCountingDispatcher dispatcher =
+ (MyCountingDispatcher) rm.getRMContext().getDispatcher();
+ assertTrue(!dispatcher.isStopped());
rm.adminService.transitionToActive(requestInfo);
assertEquals(errorMessageForEventHandler, expectedEventHandlerCount,
@@ -339,6 +343,11 @@ public class TestRMHA {
assertEquals(errorMessageForService, expectedServiceCount,
rm.getServices().size());
+
+ // Keep the dispatcher reference before transitioning to standby
+ dispatcher = (MyCountingDispatcher) rm.getRMContext().getDispatcher();
+
+
rm.adminService.transitionToStandby(requestInfo);
assertEquals(errorMessageForEventHandler, expectedEventHandlerCount,
((MyCountingDispatcher) rm.getRMContext().getDispatcher())
@@ -346,6 +355,8 @@ public class TestRMHA {
assertEquals(errorMessageForService, expectedServiceCount,
rm.getServices().size());
+ assertTrue(dispatcher.isStopped());
+
rm.stop();
}
@@ -380,7 +391,19 @@ public class TestRMHA {
}
@Test
- public void testHAWithRMHostName() {
+ public void testHAWithRMHostName() throws Exception {
+ innerTestHAWithRMHostName(false);
+ configuration.clear();
+ setUp();
+ innerTestHAWithRMHostName(true);
+ }
+
+ public void innerTestHAWithRMHostName(boolean includeBindHost) {
+ //this is run two times, with and without a bind host configured
+ if (includeBindHost) {
+ configuration.set(YarnConfiguration.RM_BIND_HOST, "9.9.9.9");
+ }
+
//test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set
//We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration
configuration.set(HAUtil.addSuffix(YarnConfiguration.RM_HOSTNAME,
@@ -400,6 +423,15 @@ public class TestRMHA {
RM2_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM2_NODE_ID)));
assertEquals("RPC address not set for " + confKey,
RM3_ADDRESS, conf.get(HAUtil.addSuffix(confKey, RM3_NODE_ID)));
+ if (includeBindHost) {
+ assertEquals("Web address misconfigured WITH bind-host",
+ rm.webAppAddress.substring(0, 7), "9.9.9.9");
+ } else {
+ //YarnConfiguration tries to figure out which rm host it's on by binding to it,
+ //which doesn't happen for any of these fake addresses, so we end up with 0.0.0.0
+ assertEquals("Web address misconfigured WITHOUT bind-host",
+ rm.webAppAddress.substring(0, 7), "0.0.0.0");
+ }
}
} catch (YarnRuntimeException e) {
fail("Should not throw any exceptions.");
@@ -471,6 +503,8 @@ public class TestRMHA {
private int eventHandlerCount;
+ private volatile boolean stopped = false;
+
public MyCountingDispatcher() {
super("MyCountingDispatcher");
this.eventHandlerCount = 0;
@@ -489,5 +523,15 @@ public class TestRMHA {
public int getEventHandlerCount() {
return this.eventHandlerCount;
}
+
+ @Override
+ protected void serviceStop() throws Exception {
+ this.stopped = true;
+ super.serviceStop();
+ }
+
+ public boolean isStopped() {
+ return this.stopped;
+ }
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java Wed Aug 20 01:34:29 2014
@@ -520,7 +520,7 @@ public class TestRMNodeTransitions {
int initialUnhealthy = cm.getUnhealthyNMs();
int initialDecommissioned = cm.getNumDecommisionedNMs();
int initialRebooted = cm.getNumRebootedNMs();
- node.handle(new RMNodeReconnectEvent(node.getNodeID(), node));
+ node.handle(new RMNodeReconnectEvent(node.getNodeID(), node, null));
Assert.assertEquals("Active Nodes", initialActive, cm.getNumActiveNMs());
Assert.assertEquals("Lost Nodes", initialLost, cm.getNumLostNMs());
Assert.assertEquals("Unhealthy Nodes",
@@ -542,7 +542,8 @@ public class TestRMNodeTransitions {
RMNodeImpl node = getRunningNode(nmVersion1);
Assert.assertEquals(nmVersion1, node.getNodeManagerVersion());
RMNodeImpl reconnectingNode = getRunningNode(nmVersion2);
- node.handle(new RMNodeReconnectEvent(node.getNodeID(), reconnectingNode));
+ node.handle(new RMNodeReconnectEvent(node.getNodeID(), reconnectingNode,
+ null));
Assert.assertEquals(nmVersion2, node.getNodeManagerVersion());
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Wed Aug 20 01:34:29 2014
@@ -289,7 +289,7 @@ public class TestRMRestart {
// verify old AM is not accepted
// change running AM to talk to new RM
- am1.setAMRMProtocol(rm2.getApplicationMasterService());
+ am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
AllocateResponse allocResponse = am1.allocate(
new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>());
@@ -1208,18 +1208,13 @@ public class TestRMRestart {
Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
attemptState.getMasterContainer().getId());
- // the appToken and clientTokenMasterKey that are generated when
+ // the clientTokenMasterKey that are generated when
// RMAppAttempt is created,
- HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
- tokenSet.add(attempt1.getAMRMToken());
byte[] clientTokenMasterKey =
attempt1.getClientTokenMasterKey().getEncoded();
// assert application credentials are saved
Credentials savedCredentials = attemptState.getAppAttemptCredentials();
- HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
- savedTokens.addAll(savedCredentials.getAllTokens());
- Assert.assertEquals(tokenSet, savedTokens);
Assert.assertArrayEquals("client token master key not saved",
clientTokenMasterKey, savedCredentials.getSecretKey(
RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
@@ -1232,11 +1227,8 @@ public class TestRMRestart {
rm2.getRMContext().getRMApps().get(app1.getApplicationId());
RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);
- // assert loaded attempt recovered attempt tokens
+ // assert loaded attempt recovered
Assert.assertNotNull(loadedAttempt1);
- savedTokens.clear();
- savedTokens.add(loadedAttempt1.getAMRMToken());
- Assert.assertEquals(tokenSet, savedTokens);
// assert client token master key is recovered back to api-versioned
// client token master key
@@ -1250,11 +1242,10 @@ public class TestRMRestart {
.getEncoded());
// assert AMRMTokenSecretManager also knows about the AMRMToken password
- // TODO: fix this on YARN-2211
-// Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
-// Assert.assertArrayEquals(amrmToken.getPassword(),
-// rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
-// amrmToken.decodeIdentifier()));
+ Token<AMRMTokenIdentifier> amrmToken = loadedAttempt1.getAMRMToken();
+ Assert.assertArrayEquals(amrmToken.getPassword(),
+ rm2.getRMContext().getAMRMTokenSecretManager().retrievePassword(
+ amrmToken.decodeIdentifier()));
rm1.stop();
rm2.stop();
}
@@ -1672,7 +1663,7 @@ public class TestRMRestart {
nm1.setResourceTrackerService(rm2.getResourceTrackerService());
// recover app
RMApp loadedApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId());
- am1.setAMRMProtocol(rm2.getApplicationMasterService());
+ am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>());
nm1.nodeHeartbeat(true);
nm1 = new MockNM("127.0.0.1:1234", 15120, rm2.getResourceTrackerService());
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java Wed Aug 20 01:34:29 2014
@@ -27,7 +27,10 @@ import java.util.Collection;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
import org.apache.hadoop.net.NetworkTopology;
+import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -39,8 +42,10 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -235,4 +240,75 @@ public class TestResourceManager {
}
}
+ @Test(timeout = 50000)
+ public void testFilterOverrides() throws Exception {
+ String filterInitializerConfKey = "hadoop.http.filter.initializers";
+ String[] filterInitializers =
+ {
+ AuthenticationFilterInitializer.class.getName(),
+ RMAuthenticationFilterInitializer.class.getName(),
+ AuthenticationFilterInitializer.class.getName() + ","
+ + RMAuthenticationFilterInitializer.class.getName(),
+ AuthenticationFilterInitializer.class.getName() + ", "
+ + RMAuthenticationFilterInitializer.class.getName(),
+ AuthenticationFilterInitializer.class.getName() + ", "
+ + this.getClass().getName() };
+ for (String filterInitializer : filterInitializers) {
+ resourceManager = new ResourceManager();
+ Configuration conf = new YarnConfiguration();
+ conf.set(filterInitializerConfKey, filterInitializer);
+ conf.set("hadoop.security.authentication", "kerberos");
+ conf.set("hadoop.http.authentication.type", "kerberos");
+ try {
+ try {
+ UserGroupInformation.setConfiguration(conf);
+ } catch (Exception e) {
+ // ignore we just care about getting true for
+ // isSecurityEnabled()
+ LOG.info("Got expected exception");
+ }
+ resourceManager.init(conf);
+ resourceManager.startWepApp();
+ } catch (RuntimeException e) {
+ // Exceptions are expected because we didn't setup everything
+ // just want to test filter settings
+ String tmp = resourceManager.getConfig().get(filterInitializerConfKey);
+ if (filterInitializer.contains(this.getClass().getName())) {
+ Assert.assertEquals(RMAuthenticationFilterInitializer.class.getName()
+ + "," + this.getClass().getName(), tmp);
+ } else {
+ Assert.assertEquals(
+ RMAuthenticationFilterInitializer.class.getName(), tmp);
+ }
+ resourceManager.stop();
+ }
+ }
+
+ // simple mode overrides
+ String[] simpleFilterInitializers =
+ { "", StaticUserWebFilter.class.getName() };
+ for (String filterInitializer : simpleFilterInitializers) {
+ resourceManager = new ResourceManager();
+ Configuration conf = new YarnConfiguration();
+ conf.set(filterInitializerConfKey, filterInitializer);
+ try {
+ UserGroupInformation.setConfiguration(conf);
+ resourceManager.init(conf);
+ resourceManager.startWepApp();
+ } catch (RuntimeException e) {
+ // Exceptions are expected because we didn't setup everything
+ // just want to test filter settings
+ String tmp = resourceManager.getConfig().get(filterInitializerConfKey);
+ if (filterInitializer.equals(StaticUserWebFilter.class.getName())) {
+ Assert.assertEquals(RMAuthenticationFilterInitializer.class.getName()
+ + "," + StaticUserWebFilter.class.getName(), tmp);
+ } else {
+ Assert.assertEquals(
+ RMAuthenticationFilterInitializer.class.getName(), tmp);
+ }
+ resourceManager.stop();
+ }
+ }
+ }
+
}
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Wed Aug 20 01:34:29 2014
@@ -595,7 +595,7 @@ public class TestResourceTrackerService
// reconnect of node with changed capability
nm1 = rm.registerNode("host2:5678", 10240);
dispatcher.await();
- response = nm2.nodeHeartbeat(true);
+ response = nm1.nodeHeartbeat(true);
dispatcher.await();
Assert.assertTrue(NodeAction.NORMAL.equals(response.getNodeAction()));
Assert.assertEquals(5120 + 10240, metrics.getAvailableMB());
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java Wed Aug 20 01:34:29 2014
@@ -33,10 +33,13 @@ import java.util.Set;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
@@ -57,6 +60,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -71,6 +75,9 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import com.google.common.base.Supplier;
+
+
@SuppressWarnings({"rawtypes", "unchecked"})
@RunWith(value = Parameterized.class)
public class TestWorkPreservingRMRestart {
@@ -107,7 +114,7 @@ public class TestWorkPreservingRMRestart
@Parameterized.Parameters
public static Collection<Object[]> getTestParameters() {
return Arrays.asList(new Object[][] { { CapacityScheduler.class },
- { FifoScheduler.class } });
+ { FifoScheduler.class }, {FairScheduler.class } });
}
public TestWorkPreservingRMRestart(Class<?> schedulerClass) {
@@ -224,7 +231,11 @@ public class TestWorkPreservingRMRestart
assertTrue(schedulerAttempt.getLiveContainers().contains(
scheduler.getRMContainer(runningContainer.getContainerId())));
assertEquals(schedulerAttempt.getCurrentConsumption(), usedResources);
- assertEquals(availableResources, schedulerAttempt.getHeadroom());
+
+ // Until YARN-1959 is resolved
+ if (scheduler.getClass() != FairScheduler.class) {
+ assertEquals(availableResources, schedulerAttempt.getHeadroom());
+ }
// *********** check appSchedulingInfo state ***********
assertEquals((1 << 22) + 1, schedulerAttempt.getNewContainerId());
@@ -567,8 +578,8 @@ public class TestWorkPreservingRMRestart
rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED);
rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED);
- am0.setAMRMProtocol(rm2.getApplicationMasterService());
- am0.registerAppAttempt(false);
+ am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
+ am0.registerAppAttempt(true);
rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING);
rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
@@ -641,6 +652,69 @@ public class TestWorkPreservingRMRestart
waitForNumContainersToRecover(2, rm2, am0.getApplicationAttemptId());
}
+ // Test if RM on recovery receives the container release request from AM
+ // before it receives the container status reported by NM for recovery. this
+ // container should not be recovered.
+ @Test (timeout = 30000)
+ public void testReleasedContainerNotRecovered() throws Exception {
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+ rm1 = new MockRM(conf, memStore);
+ MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+ rm1.start();
+
+ RMApp app1 = rm1.submitApp(1024);
+ final MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+
+ // Re-start RM
+ conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 8000);
+ rm2 = new MockRM(conf, memStore);
+ rm2.start();
+ nm1.setResourceTrackerService(rm2.getResourceTrackerService());
+ rm2.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
+ am1.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
+ am1.registerAppAttempt(true);
+
+ // try to release a container before the container is actually recovered.
+ final ContainerId runningContainer =
+ ContainerId.newInstance(am1.getApplicationAttemptId(), 2);
+ am1.allocate(null, Arrays.asList(runningContainer));
+
+ // send container statuses to recover the containers
+ List<NMContainerStatus> containerStatuses =
+ createNMContainerStatusForApp(am1);
+ nm1.registerNode(containerStatuses, null);
+
+ // only the am container should be recovered.
+ waitForNumContainersToRecover(1, rm2, am1.getApplicationAttemptId());
+
+ final AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler) rm2.getResourceScheduler();
+ // cached release request is cleaned.
+ // assertFalse(scheduler.getPendingRelease().contains(runningContainer));
+
+ AllocateResponse response = am1.allocate(null, null);
+ // AM gets notified of the completed container.
+ boolean receivedCompletedContainer = false;
+ for (ContainerStatus status : response.getCompletedContainersStatuses()) {
+ if (status.getContainerId().equals(runningContainer)) {
+ receivedCompletedContainer = true;
+ }
+ }
+ assertTrue(receivedCompletedContainer);
+
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ // release cache is cleaned up and previous running container is not
+ // recovered
+ return scheduler.getApplicationAttempt(am1.getApplicationAttemptId())
+ .getPendingRelease().isEmpty()
+ && scheduler.getRMContainer(runningContainer) == null;
+ }
+ }, 1000, 20000);
+ }
+
private void asserteMetrics(QueueMetrics qm, int appsSubmitted,
int appsPending, int appsRunning, int appsCompleted,
int allocatedContainers, int availableMB, int availableVirtualCores,
@@ -656,7 +730,7 @@ public class TestWorkPreservingRMRestart
assertEquals(allocatedVirtualCores, qm.getAllocatedVirtualCores());
}
- private void waitForNumContainersToRecover(int num, MockRM rm,
+ public static void waitForNumContainersToRecover(int num, MockRM rm,
ApplicationAttemptId attemptId) throws Exception {
AbstractYarnScheduler scheduler =
(AbstractYarnScheduler) rm.getResourceScheduler();
@@ -669,7 +743,9 @@ public class TestWorkPreservingRMRestart
attempt = scheduler.getApplicationAttempt(attemptId);
}
while (attempt.getLiveContainers().size() < num) {
- System.out.println("Wait for " + num + " containers to recover.");
+ System.out.println("Wait for " + num
+ + " containers to recover. currently: "
+ + attempt.getLiveContainers().size());
Thread.sleep(200);
}
}
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Wed Aug 20 01:34:29 2014
@@ -386,7 +386,8 @@ public class TestAMRestart {
ApplicationState appState =
memStore.getState().getApplicationState().get(app1.getApplicationId());
// AM should be restarted even though max-am-attempt is 1.
- MockAM am2 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ MockAM am2 =
+ rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
RMAppAttempt attempt2 = app1.getCurrentAppAttempt();
Assert.assertTrue(((RMAppAttemptImpl) attempt2).mayBeLastAttempt());
@@ -398,7 +399,8 @@ public class TestAMRestart {
am2.waitForState(RMAppAttemptState.FAILED);
Assert.assertTrue(! attempt2.shouldCountTowardsMaxAttemptRetry());
rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
- MockAM am3 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ MockAM am3 =
+ rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 3, nm1);
RMAppAttempt attempt3 = app1.getCurrentAppAttempt();
Assert.assertTrue(((RMAppAttemptImpl) attempt3).mayBeLastAttempt());
@@ -421,7 +423,8 @@ public class TestAMRestart {
.getAMContainerExitStatus());
rm1.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
- MockAM am4 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ MockAM am4 =
+ rm1.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 4, nm1);
RMAppAttempt attempt4 = app1.getCurrentAppAttempt();
Assert.assertTrue(((RMAppAttemptImpl) attempt4).mayBeLastAttempt());
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java Wed Aug 20 01:34:29 2014
@@ -55,16 +55,18 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMDTSecretManagerState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.AMRMTokenSecretManagerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
@@ -75,10 +77,9 @@ public class RMStateStoreTestBase extend
public static final Log LOG = LogFactory.getLog(RMStateStoreTestBase.class);
static class TestDispatcher implements
- Dispatcher, EventHandler<RMAppAttemptNewSavedEvent> {
+ Dispatcher, EventHandler<RMAppAttemptEvent> {
ApplicationAttemptId attemptId;
- Exception storedException;
boolean notified = false;
@@ -89,9 +90,8 @@ public class RMStateStoreTestBase extend
}
@Override
- public void handle(RMAppAttemptNewSavedEvent event) {
+ public void handle(RMAppAttemptEvent event) {
assertEquals(attemptId, event.getApplicationAttemptId());
- assertEquals(storedException, event.getStoredException());
notified = true;
synchronized (this) {
notifyAll();
@@ -109,8 +109,8 @@ public class RMStateStoreTestBase extend
interface RMStateStoreHelper {
RMStateStore getRMStateStore() throws Exception;
boolean isFinalStateValid() throws Exception;
- void writeVersion(RMStateVersion version) throws Exception;
- RMStateVersion getCurrentVersion() throws Exception;
+ void writeVersion(Version version) throws Exception;
+ Version getCurrentVersion() throws Exception;
boolean appExists(RMApp app) throws Exception;
}
@@ -161,7 +161,6 @@ public class RMStateStoreTestBase extend
when(mockAttempt.getClientTokenMasterKey())
.thenReturn(clientTokenMasterKey);
dispatcher.attemptId = attemptId;
- dispatcher.storedException = null;
store.storeNewApplicationAttempt(mockAttempt);
waitNotify(dispatcher);
return container.getId();
@@ -176,8 +175,12 @@ public class RMStateStoreTestBase extend
TestDispatcher dispatcher = new TestDispatcher();
store.setRMDispatcher(dispatcher);
- AMRMTokenSecretManager appTokenMgr = spy(
- new AMRMTokenSecretManager(conf));
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getStateStore()).thenReturn(store);
+
+ AMRMTokenSecretManager appTokenMgr =
+ spy(new AMRMTokenSecretManager(conf, rmContext));
+
MasterKeyData masterKeyData = appTokenMgr.createNewMasterKey();
when(appTokenMgr.getMasterKey()).thenReturn(masterKeyData);
@@ -192,8 +195,6 @@ public class RMStateStoreTestBase extend
// create application token and client token key for attempt1
Token<AMRMTokenIdentifier> appAttemptToken1 =
generateAMRMToken(attemptId1, appTokenMgr);
- HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
- attemptTokenSet1.add(appAttemptToken1);
SecretKey clientTokenKey1 =
clientToAMTokenMgr.createMasterKey(attemptId1);
@@ -208,8 +209,6 @@ public class RMStateStoreTestBase extend
// create application token and client token key for attempt2
Token<AMRMTokenIdentifier> appAttemptToken2 =
generateAMRMToken(attemptId2, appTokenMgr);
- HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
- attemptTokenSet2.add(appAttemptToken2);
SecretKey clientTokenKey2 =
clientToAMTokenMgr.createMasterKey(attemptId2);
@@ -274,10 +273,6 @@ public class RMStateStoreTestBase extend
assertEquals(-1000, attemptState.getAMContainerExitStatus());
// attempt1 container is loaded correctly
assertEquals(containerId1, attemptState.getMasterContainer().getId());
- // attempt1 applicationToken is loaded correctly
- HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
- savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
- assertEquals(attemptTokenSet1, savedTokens);
// attempt1 client token master key is loaded correctly
assertArrayEquals(clientTokenKey1.getEncoded(),
attemptState.getAppAttemptCredentials()
@@ -289,10 +284,6 @@ public class RMStateStoreTestBase extend
assertEquals(attemptId2, attemptState.getAttemptId());
// attempt2 container is loaded correctly
assertEquals(containerId2, attemptState.getMasterContainer().getId());
- // attempt2 applicationToken is loaded correctly
- savedTokens.clear();
- savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
- assertEquals(attemptTokenSet2, savedTokens);
// attempt2 client token master key is loaded correctly
assertArrayEquals(clientTokenKey2.getEncoded(),
attemptState.getAppAttemptCredentials()
@@ -471,13 +462,13 @@ public class RMStateStoreTestBase extend
store.setRMDispatcher(new TestDispatcher());
// default version
- RMStateVersion defaultVersion = stateStoreHelper.getCurrentVersion();
+ Version defaultVersion = stateStoreHelper.getCurrentVersion();
store.checkVersion();
Assert.assertEquals(defaultVersion, store.loadVersion());
// compatible version
- RMStateVersion compatibleVersion =
- RMStateVersion.newInstance(defaultVersion.getMajorVersion(),
+ Version compatibleVersion =
+ Version.newInstance(defaultVersion.getMajorVersion(),
defaultVersion.getMinorVersion() + 2);
stateStoreHelper.writeVersion(compatibleVersion);
Assert.assertEquals(compatibleVersion, store.loadVersion());
@@ -486,8 +477,8 @@ public class RMStateStoreTestBase extend
Assert.assertEquals(defaultVersion, store.loadVersion());
// incompatible version
- RMStateVersion incompatibleVersion =
- RMStateVersion.newInstance(defaultVersion.getMajorVersion() + 2,
+ Version incompatibleVersion =
+ Version.newInstance(defaultVersion.getMajorVersion() + 2,
defaultVersion.getMinorVersion());
stateStoreHelper.writeVersion(incompatibleVersion);
try {
@@ -576,4 +567,65 @@ public class RMStateStoreTestBase extend
}
+ public void testAMRMTokenSecretManagerStateStore(
+ RMStateStoreHelper stateStoreHelper) throws Exception {
+ System.out.println("Start testing");
+ RMStateStore store = stateStoreHelper.getRMStateStore();
+ TestDispatcher dispatcher = new TestDispatcher();
+ store.setRMDispatcher(dispatcher);
+
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getStateStore()).thenReturn(store);
+ Configuration conf = new YarnConfiguration();
+ AMRMTokenSecretManager appTokenMgr =
+ new AMRMTokenSecretManager(conf, rmContext);
+
+ //create and save the first masterkey
+ MasterKeyData firstMasterKeyData = appTokenMgr.createNewMasterKey();
+
+ AMRMTokenSecretManagerState state1 =
+ AMRMTokenSecretManagerState.newInstance(
+ firstMasterKeyData.getMasterKey(), null);
+ rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state1,
+ false);
+
+ // load state
+ store = stateStoreHelper.getRMStateStore();
+ store.setRMDispatcher(dispatcher);
+ RMState state = store.loadState();
+ Assert.assertNotNull(state.getAMRMTokenSecretManagerState());
+ Assert.assertEquals(firstMasterKeyData.getMasterKey(), state
+ .getAMRMTokenSecretManagerState().getCurrentMasterKey());
+ Assert.assertNull(state
+ .getAMRMTokenSecretManagerState().getNextMasterKey());
+
+ //create and save the second masterkey
+ MasterKeyData secondMasterKeyData = appTokenMgr.createNewMasterKey();
+ AMRMTokenSecretManagerState state2 =
+ AMRMTokenSecretManagerState
+ .newInstance(firstMasterKeyData.getMasterKey(),
+ secondMasterKeyData.getMasterKey());
+ rmContext.getStateStore().storeOrUpdateAMRMTokenSecretManagerState(state2,
+ true);
+
+ // load state
+ store = stateStoreHelper.getRMStateStore();
+ store.setRMDispatcher(dispatcher);
+ RMState state_2 = store.loadState();
+ Assert.assertNotNull(state_2.getAMRMTokenSecretManagerState());
+ Assert.assertEquals(firstMasterKeyData.getMasterKey(), state_2
+ .getAMRMTokenSecretManagerState().getCurrentMasterKey());
+ Assert.assertEquals(secondMasterKeyData.getMasterKey(), state_2
+ .getAMRMTokenSecretManagerState().getNextMasterKey());
+
+ // re-create the masterKeyData based on the recovered masterkey
+ // should have the same secretKey
+ appTokenMgr.recover(state_2);
+ Assert.assertEquals(appTokenMgr.getCurrnetMasterKeyData().getSecretKey(),
+ firstMasterKeyData.getSecretKey());
+ Assert.assertEquals(appTokenMgr.getNextMasterKeyData().getSecretKey(),
+ secondMasterKeyData.getSecretKey());
+
+ store.close();
+ }
}
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestFSRMStateStore.java Wed Aug 20 01:34:29 2014
@@ -36,10 +36,9 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
+import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -71,7 +70,7 @@ public class TestFSRMStateStore extends
return new Path(new Path(workingDirPathURI, ROOT_DIR_NAME), VERSION_NODE);
}
- public RMStateVersion getCurrentVersion() {
+ public Version getCurrentVersion() {
return CURRENT_VERSION_INFO;
}
@@ -112,13 +111,13 @@ public class TestFSRMStateStore extends
}
@Override
- public void writeVersion(RMStateVersion version) throws Exception {
- store.updateFile(store.getVersionNode(), ((RMStateVersionPBImpl) version)
+ public void writeVersion(Version version) throws Exception {
+ store.updateFile(store.getVersionNode(), ((VersionPBImpl) version)
.getProto().toByteArray());
}
@Override
- public RMStateVersion getCurrentVersion() throws Exception {
+ public Version getCurrentVersion() throws Exception {
return store.getCurrentVersion();
}
@@ -161,6 +160,7 @@ public class TestFSRMStateStore extends
testEpoch(fsTester);
testAppDeletion(fsTester);
testDeleteStore(fsTester);
+ testAMRMTokenSecretManagerStateStore(fsTester);
} finally {
cluster.shutdown();
}
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStore.java Wed Aug 20 01:34:29 2014
@@ -32,9 +32,9 @@ import org.apache.hadoop.ha.HAServicePro
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.RMStateVersionPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.zookeeper.ZooKeeper;
import org.apache.zookeeper.data.Stat;
@@ -69,7 +69,7 @@ public class TestZKRMStateStore extends
return znodeWorkingPath + "/" + ROOT_ZNODE_NAME + "/" + VERSION_NODE;
}
- public RMStateVersion getCurrentVersion() {
+ public Version getCurrentVersion() {
return CURRENT_VERSION_INFO;
}
@@ -96,13 +96,13 @@ public class TestZKRMStateStore extends
}
@Override
- public void writeVersion(RMStateVersion version) throws Exception {
- client.setData(store.getVersionNode(), ((RMStateVersionPBImpl) version)
+ public void writeVersion(Version version) throws Exception {
+ client.setData(store.getVersionNode(), ((VersionPBImpl) version)
.getProto().toByteArray(), -1);
}
@Override
- public RMStateVersion getCurrentVersion() throws Exception {
+ public Version getCurrentVersion() throws Exception {
return store.getCurrentVersion();
}
@@ -123,6 +123,7 @@ public class TestZKRMStateStore extends
testEpoch(zkTester);
testAppDeletion(zkTester);
testDeleteStore(zkTester);
+ testAMRMTokenSecretManagerStateStore(zkTester);
}
private Configuration createHARMConf(
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java Wed Aug 20 01:34:29 2014
@@ -60,7 +60,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
@@ -193,7 +192,7 @@ public class TestRMAppTransitions {
this.rmContext =
new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, new AMRMTokenSecretManager(conf),
+ null, new AMRMTokenSecretManager(conf, this.rmContext),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(),
@@ -328,15 +327,15 @@ public class TestRMAppTransitions {
private void sendAppUpdateSavedEvent(RMApp application) {
RMAppEvent event =
- new RMAppUpdateSavedEvent(application.getApplicationId(), null);
+ new RMAppEvent(application.getApplicationId(), RMAppEventType.APP_UPDATE_SAVED);
application.handle(event);
rmDispatcher.await();
}
private void sendAttemptUpdateSavedEvent(RMApp application) {
application.getCurrentAppAttempt().handle(
- new RMAppAttemptUpdateSavedEvent(application.getCurrentAppAttempt()
- .getAppAttemptId(), null));
+ new RMAppAttemptEvent(application.getCurrentAppAttempt().getAppAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
}
protected RMApp testCreateAppNewSaving(
@@ -357,7 +356,7 @@ public class TestRMAppTransitions {
RMApp application = testCreateAppNewSaving(submissionContext);
// NEW_SAVING => SUBMITTED event RMAppEventType.APP_SAVED
RMAppEvent event =
- new RMAppNewSavedEvent(application.getApplicationId(), null);
+ new RMAppEvent(application.getApplicationId(), RMAppEventType.APP_NEW_SAVED);
application.handle(event);
assertStartTimeSet(application);
assertAppState(RMAppState.SUBMITTED, application);
@@ -422,7 +421,7 @@ public class TestRMAppTransitions {
RMApp application = testCreateAppFinalSaving(submissionContext);
// FINAL_SAVING => FINISHING event RMAppEventType.APP_UPDATED
RMAppEvent appUpdated =
- new RMAppUpdateSavedEvent(application.getApplicationId(), null);
+ new RMAppEvent(application.getApplicationId(), RMAppEventType.APP_UPDATE_SAVED);
application.handle(appUpdated);
assertAppState(RMAppState.FINISHING, application);
assertTimesAtFinish(application);
@@ -763,7 +762,7 @@ public class TestRMAppTransitions {
application.handle(event);
assertAppState(RMAppState.FINAL_SAVING, application);
RMAppEvent appUpdated =
- new RMAppUpdateSavedEvent(application.getApplicationId(), null);
+ new RMAppEvent(application.getApplicationId(), RMAppEventType.APP_UPDATE_SAVED);
application.handle(appUpdated);
assertAppState(RMAppState.FINISHED, application);
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1619019&r1=1619018&r2=1619019&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Wed Aug 20 01:34:29 2014
@@ -81,14 +81,13 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptNewSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUpdateSavedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
@@ -134,7 +133,8 @@ public class TestRMAppAttemptTransitions
private RMAppAttempt applicationAttempt;
private Configuration conf = new Configuration();
- private AMRMTokenSecretManager amRMTokenManager = spy(new AMRMTokenSecretManager(conf));
+ private AMRMTokenSecretManager amRMTokenManager =
+ spy(new AMRMTokenSecretManager(conf, rmContext));
private ClientToAMTokenSecretManagerInRM clientToAMTokenManager =
spy(new ClientToAMTokenSecretManagerInRM());
private NMTokenSecretManagerInRM nmTokenManager =
@@ -347,7 +347,6 @@ public class TestRMAppAttemptTransitions
assertNull(applicationAttempt.createClientToken("some client"));
}
assertNull(applicationAttempt.createClientToken(null));
- assertNotNull(applicationAttempt.getAMRMToken());
// Check events
verify(masterService).
registerAppAttempt(applicationAttempt.getAppAttemptId());
@@ -443,7 +442,6 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.ALLOCATED,
applicationAttempt.getAppAttemptState());
assertEquals(amContainer, applicationAttempt.getMasterContainer());
-
// Check events
verify(applicationMasterLauncher).handle(any(AMLauncherEvent.class));
verify(scheduler, times(2)).
@@ -570,15 +568,15 @@ public class TestRMAppAttemptTransitions
submitApplicationAttempt();
applicationAttempt.handle(
new RMAppAttemptEvent(
- applicationAttempt.getAppAttemptId(),
+ applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.ATTEMPT_ADDED));
if(unmanagedAM){
assertEquals(RMAppAttemptState.LAUNCHED_UNMANAGED_SAVING,
applicationAttempt.getAppAttemptState());
applicationAttempt.handle(
- new RMAppAttemptNewSavedEvent(
- applicationAttempt.getAppAttemptId(), null));
+ new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
}
testAppAttemptScheduledState();
@@ -616,8 +614,8 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.ALLOCATED_SAVING,
applicationAttempt.getAppAttemptState());
applicationAttempt.handle(
- new RMAppAttemptNewSavedEvent(
- applicationAttempt.getAppAttemptId(), null));
+ new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_NEW_SAVED));
testAppAttemptAllocatedState(container);
@@ -696,8 +694,8 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.FINAL_SAVING,
applicationAttempt.getAppAttemptState());
applicationAttempt.handle(
- new RMAppAttemptUpdateSavedEvent(
- applicationAttempt.getAppAttemptId(), null));
+ new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(),
+ RMAppAttemptEventType.ATTEMPT_UPDATE_SAVED));
}
@Test
@@ -782,6 +780,32 @@ public class TestRMAppAttemptTransitions
}
@Test
+ public void testAMCrashAtScheduled() {
+ // This is to test sending CONTAINER_FINISHED event at SCHEDULED state.
+ // Verify the state transition is correct.
+ scheduleApplicationAttempt();
+ ContainerStatus cs =
+ SchedulerUtils.createAbnormalContainerStatus(
+ BuilderUtils.newContainerId(
+ applicationAttempt.getAppAttemptId(), 1),
+ SchedulerUtils.LOST_CONTAINER);
+ // send CONTAINER_FINISHED event at SCHEDULED state,
+ // The state should be FINAL_SAVING with previous state SCHEDULED
+ applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
+ applicationAttempt.getAppAttemptId(), cs));
+ // createApplicationAttemptState will return previous state (SCHEDULED),
+ // if the current state is FINAL_SAVING.
+ assertEquals(YarnApplicationAttemptState.SCHEDULED,
+ applicationAttempt.createApplicationAttemptState());
+ // send ATTEMPT_UPDATE_SAVED event,
+ // verify the state is changed to state FAILED.
+ sendAttemptUpdateSavedEvent(applicationAttempt);
+ assertEquals(RMAppAttemptState.FAILED,
+ applicationAttempt.getAppAttemptState());
+ verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
+ }
+
+ @Test
public void testAllocatedToKilled() {
Container amContainer = allocateApplicationAttempt();
applicationAttempt.handle(