You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/05/08 22:20:58 UTC
svn commit: r1480440 [2/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop...
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java?rev=1480440&r1=1480439&r2=1480440&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java Wed May 8 20:20:56 2013
@@ -404,7 +404,8 @@ public class TestRMRestart {
}
@Test
- public void testTokenRestoredOnRMrestart() throws Exception {
+ public void testDelegationTokenRestoredInDelegationTokenRenewer()
+ throws Exception {
Logger rootLogger = LogManager.getRootLogger();
rootLogger.setLevel(Level.DEBUG);
ExitUtil.disableSystemExit();
@@ -423,7 +424,7 @@ public class TestRMRestart {
Map<ApplicationId, ApplicationState> rmAppState =
rmState.getApplicationState();
- MockRM rm1 = new MyMockRM(conf, memStore);
+ MockRM rm1 = new TestSecurityMockRM(conf, memStore);
rm1.start();
HashSet<Token<RMDelegationTokenIdentifier>> tokenSet =
@@ -461,21 +462,26 @@ public class TestRMRestart {
ApplicationState appState = rmAppState.get(app.getApplicationId());
Assert.assertNotNull(appState);
+ // assert delegation tokens exist in rm1 DelegationTokenRenewr
+ Assert.assertEquals(tokenSet, rm1.getRMContext()
+ .getDelegationTokenRenewer().getDelegationTokens());
+
// assert delegation tokens are saved
DataOutputBuffer dob = new DataOutputBuffer();
ts.writeTokenStorageToStream(dob);
ByteBuffer securityTokens =
ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ securityTokens.rewind();
Assert.assertEquals(securityTokens, appState
.getApplicationSubmissionContext().getAMContainerSpec()
.getContainerTokens());
// start new RM
- MockRM rm2 = new MyMockRM(conf, memStore);
+ MockRM rm2 = new TestSecurityMockRM(conf, memStore);
rm2.start();
- // verify tokens are properly populated back to DelegationTokenRenewer
- Assert.assertEquals(tokenSet, rm1.getRMContext()
+ // verify tokens are properly populated back to rm2 DelegationTokenRenewer
+ Assert.assertEquals(tokenSet, rm2.getRMContext()
.getDelegationTokenRenewer().getDelegationTokens());
// stop the RM
@@ -483,9 +489,92 @@ public class TestRMRestart {
rm2.stop();
}
- class MyMockRM extends MockRM {
+ @Test
+ public void testAppAttemptTokensRestoredOnRMRestart() throws Exception {
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+ ExitUtil.disableSystemExit();
+
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
+ conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+ "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+ RMState rmState = memStore.getState();
+
+ Map<ApplicationId, ApplicationState> rmAppState =
+ rmState.getApplicationState();
+ MockRM rm1 = new TestSecurityMockRM(conf, memStore);
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("0.0.0.0:4321", 15120, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ // submit an app
+ RMApp app1 =
+ rm1.submitApp(200, "name", "user",
+ new HashMap<ApplicationAccessType, String>(), "default");
+
+ // assert app info is saved
+ ApplicationState appState = rmAppState.get(app1.getApplicationId());
+ Assert.assertNotNull(appState);
+
+ // Allocate the AM
+ nm1.nodeHeartbeat(true);
+ RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
+ ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
+ rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
+
+ // assert attempt info is saved
+ ApplicationAttemptState attemptState = appState.getAttempt(attemptId1);
+ Assert.assertNotNull(attemptState);
+ Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
+ attemptState.getMasterContainer().getId());
+
+ // the appToken and clientToken that are generated when RMAppAttempt is created,
+ HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
+ tokenSet.add(attempt1.getApplicationToken());
+ tokenSet.add(attempt1.getClientToken());
+
+ // assert application Token is saved
+ HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
+ savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ Assert.assertEquals(tokenSet, savedTokens);
+
+ // start new RM
+ MockRM rm2 = new TestSecurityMockRM(conf, memStore);
+ rm2.start();
+
+ RMApp loadedApp1 =
+ rm2.getRMContext().getRMApps().get(app1.getApplicationId());
+ RMAppAttempt loadedAttempt1 = loadedApp1.getRMAppAttempt(attemptId1);
+
+ // assert loaded attempt recovered attempt tokens
+ Assert.assertNotNull(loadedAttempt1);
+ savedTokens.clear();
+ savedTokens.add(loadedAttempt1.getApplicationToken());
+ savedTokens.add(loadedAttempt1.getClientToken());
+ Assert.assertEquals(tokenSet, savedTokens);
+
+ // assert clientToken is recovered back to api-versioned clientToken
+ Assert.assertEquals(attempt1.getClientToken(),
+ loadedAttempt1.getClientToken());
+
+ // Not testing ApplicationTokenSecretManager has the password populated back,
+ // that is needed in work-preserving restart
+
+ rm1.stop();
+ rm2.stop();
+ }
+
+ class TestSecurityMockRM extends MockRM {
- public MyMockRM(Configuration conf, RMStateStore store) {
+ public TestSecurityMockRM(Configuration conf, RMStateStore store) {
super(conf, store);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1480440&r1=1480439&r2=1480440&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java Wed May 8 20:20:56 2013
@@ -18,14 +18,19 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
-import static org.junit.Assert.*;
-import static org.mockito.Mockito.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
-import org.junit.Test;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -34,6 +39,8 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
@@ -44,13 +51,18 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStoredEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Test;
public class TestRMStateStore {
@@ -141,7 +153,7 @@ public class TestRMStateStore {
ApplicationAttemptId attemptId = ConverterUtils.toApplicationAttemptId(
"appattempt_1352994193343_0003_000001");
storeAttempt(testStore, attemptId,
- "container_1352994193343_0003_01_000001", dispatcher);
+ "container_1352994193343_0003_01_000001", null, null, dispatcher);
}
@Override
@@ -186,14 +198,17 @@ public class TestRMStateStore {
}
ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
- String containerIdStr, TestDispatcher dispatcher)
- throws Exception {
+ String containerIdStr, Token<ApplicationTokenIdentifier> appToken,
+ Token<ClientTokenIdentifier> clientToken, TestDispatcher dispatcher)
+ throws Exception {
Container container = new ContainerPBImpl();
container.setId(ConverterUtils.toContainerId(containerIdStr));
RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
when(mockAttempt.getMasterContainer()).thenReturn(container);
+ when(mockAttempt.getApplicationToken()).thenReturn(appToken);
+ when(mockAttempt.getClientToken()).thenReturn(clientToken);
dispatcher.attemptId = attemptId;
dispatcher.storedException = null;
store.storeApplicationAttempt(mockAttempt);
@@ -201,30 +216,58 @@ public class TestRMStateStore {
return container.getId();
}
+ @SuppressWarnings("unchecked")
void testRMStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
long submitTime = System.currentTimeMillis();
+ Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.getRMStateStore();
TestDispatcher dispatcher = new TestDispatcher();
store.setDispatcher(dispatcher);
+ ApplicationTokenSecretManager appTokenMgr =
+ new ApplicationTokenSecretManager(conf);
+ ClientToAMTokenSecretManagerInRM clientTokenMgr =
+ new ClientToAMTokenSecretManagerInRM();
+
ApplicationAttemptId attemptId1 = ConverterUtils
.toApplicationAttemptId("appattempt_1352994193343_0001_000001");
ApplicationId appId1 = attemptId1.getApplicationId();
storeApp(store, appId1, submitTime);
+
+ // create application token1 for attempt1
+ List<Token<?>> appAttemptToken1 =
+ generateTokens(attemptId1, appTokenMgr, clientTokenMgr, conf);
+ HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
+ attemptTokenSet1.addAll(appAttemptToken1);
+
ContainerId containerId1 = storeAttempt(store, attemptId1,
- "container_1352994193343_0001_01_000001", dispatcher);
+ "container_1352994193343_0001_01_000001",
+ (Token<ApplicationTokenIdentifier>) (appAttemptToken1.get(0)),
+ (Token<ClientTokenIdentifier>)(appAttemptToken1.get(1)),
+ dispatcher);
+
String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
ApplicationAttemptId attemptId2 =
- ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
+ ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
+
+ // create application token2 for attempt2
+ List<Token<?>> appAttemptToken2 =
+ generateTokens(attemptId2, appTokenMgr, clientTokenMgr, conf);
+ HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
+ attemptTokenSet2.addAll(appAttemptToken2);
+
ContainerId containerId2 = storeAttempt(store, attemptId2,
- "container_1352994193343_0001_02_000001", dispatcher);
+ "container_1352994193343_0001_02_000001",
+ (Token<ApplicationTokenIdentifier>) (appAttemptToken2.get(0)),
+ (Token<ClientTokenIdentifier>)(appAttemptToken2.get(1)),
+ dispatcher);
ApplicationAttemptId attemptIdRemoved = ConverterUtils
.toApplicationAttemptId("appattempt_1352994193343_0002_000001");
ApplicationId appIdRemoved = attemptIdRemoved.getApplicationId();
storeApp(store, appIdRemoved, submitTime);
storeAttempt(store, attemptIdRemoved,
- "container_1352994193343_0002_01_000001", dispatcher);
+ "container_1352994193343_0002_01_000001", null, null, dispatcher);
RMApp mockRemovedApp = mock(RMApp.class);
HashMap<ApplicationAttemptId, RMAppAttempt> attempts =
@@ -268,12 +311,21 @@ public class TestRMStateStore {
assertEquals(attemptId1, attemptState.getAttemptId());
// attempt1 container is loaded correctly
assertEquals(containerId1, attemptState.getMasterContainer().getId());
+ // attempt1 applicationToken is loaded correctly
+ HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
+ savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ assertEquals(attemptTokenSet1, savedTokens);
+
attemptState = appState.getAttempt(attemptId2);
// attempt2 is loaded correctly
assertNotNull(attemptState);
assertEquals(attemptId2, attemptState.getAttemptId());
// attempt2 container is loaded correctly
assertEquals(containerId2, attemptState.getMasterContainer().getId());
+ // attempt2 applicationToken is loaded correctly
+ savedTokens.clear();
+ savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+ assertEquals(attemptTokenSet2, savedTokens);
// assert store is in expected state after everything is cleaned
assertTrue(stateStoreHelper.isFinalStateValid());
@@ -281,4 +333,23 @@ public class TestRMStateStore {
store.close();
}
+ private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
+ ApplicationTokenSecretManager appTokenMgr,
+ ClientToAMTokenSecretManagerInRM clientTokenMgr, Configuration conf) {
+ ApplicationTokenIdentifier appTokenId =
+ new ApplicationTokenIdentifier(attemptId);
+ Token<ApplicationTokenIdentifier> appToken =
+ new Token<ApplicationTokenIdentifier>(appTokenId, appTokenMgr);
+ appToken.setService(new Text("appToken service"));
+
+ ClientTokenIdentifier clientTokenId = new ClientTokenIdentifier(attemptId);
+ clientTokenMgr.registerApplication(attemptId);
+ Token<ClientTokenIdentifier> clientToken =
+ new Token<ClientTokenIdentifier>(clientTokenId, clientTokenMgr);
+ clientToken.setService(new Text("clientToken service"));
+ List<Token<?>> tokenPair = new ArrayList<Token<?>>();
+ tokenPair.add(0, appToken);
+ tokenPair.add(1, clientToken);
+ return tokenPair;
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java?rev=1480440&r1=1480439&r2=1480440&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java Wed May 8 20:20:56 2013
@@ -537,6 +537,9 @@ public class TestContainerManagerSecurit
"Unauthorized request to start container. "
+ "\nExpected containerId: " + tokenId.getContainerID()
+ " Found: " + newContainerId.toString()));
+ } catch (IOException e) {
+ LOG.info("Got IOException: ",e);
+ fail("IOException is not expected.");
}
}
@@ -563,6 +566,9 @@ public class TestContainerManagerSecurit
Assert.assertTrue(e.getMessage().contains(
"\nExpected resource " + tokenId.getResource().toString()
+ " but found " + container.getResource().toString()));
+ } catch (IOException e) {
+ LOG.info("Got IOException: ",e);
+ fail("IOException is not expected.");
}
}
@@ -591,6 +597,9 @@ public class TestContainerManagerSecurit
Assert.assertTrue(e.getMessage().contains(
"Expected user-name " + tokenId.getApplicationSubmitter()
+ " but found " + context.getUser()));
+ } catch (IOException e) {
+ LOG.info("Got IOException: ",e);
+ fail("IOException is not expected.");
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1480440&r1=1480439&r2=1480440&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Wed May 8 20:20:56 2013
@@ -124,7 +124,7 @@ Hadoop MapReduce Next Generation - Fair
* The smallest container size the scheduler can allocate, in MB of memory.
- * <<<yarn.scheduler.fair.minimum-allocation-mb>>>
+ * <<<yarn.scheduler.fair.maximum-allocation-mb>>>
* The largest container the scheduler can allocate, in MB of memory.
@@ -177,7 +177,7 @@ Hadoop MapReduce Next Generation - Fair
Allocation file format
- The allocation file must be in XML format. The format contains three types of
+ The allocation file must be in XML format. The format contains four types of
elements:
* <<Queue elements>>, which represent queues. Each may contain the following