You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/06/19 01:19:51 UTC
svn commit: r1494369 [2/3] - in /hadoop/common/trunk/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/...
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Jun 18 23:19:49 2013
@@ -23,6 +23,7 @@ import static org.apache.hadoop.service.
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -38,6 +39,7 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.service.CompositeService;
@@ -65,6 +67,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
@@ -234,7 +237,7 @@ public class ContainerManagerImpl extend
server =
rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, conf,
- this.context.getContainerTokenSecretManager(),
+ this.context.getNMTokenSecretManager(),
conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
@@ -253,6 +256,8 @@ public class ContainerManagerImpl extend
NodeId nodeId = NodeId.newInstance(connectAddress.getHostName(),
connectAddress.getPort());
((NodeManager.NMContext)context).setNodeId(nodeId);
+ this.context.getNMTokenSecretManager().setNodeId(nodeId);
+ this.context.getContainerTokenSecretManager().setNodeId(nodeId);
LOG.info("ContainerManager started at " + connectAddress);
super.serviceStart();
}
@@ -274,7 +279,7 @@ public class ContainerManagerImpl extend
}
// Get the remoteUGI corresponding to the api call.
- private UserGroupInformation getRemoteUgi()
+ protected UserGroupInformation getRemoteUgi()
throws YarnException {
UserGroupInformation remoteUgi;
try {
@@ -291,91 +296,67 @@ public class ContainerManagerImpl extend
// Obtain the needed ContainerTokenIdentifier from the remote-UGI. RPC layer
// currently sets only the required id, but iterate through anyways just to
// be sure.
- private ContainerTokenIdentifier selectContainerTokenIdentifier(
+ @Private
+ @VisibleForTesting
+ protected NMTokenIdentifier selectNMTokenIdentifier(
UserGroupInformation remoteUgi) {
Set<TokenIdentifier> tokenIdentifiers = remoteUgi.getTokenIdentifiers();
- ContainerTokenIdentifier resultId = null;
+ NMTokenIdentifier resultId = null;
for (TokenIdentifier id : tokenIdentifiers) {
- if (id instanceof ContainerTokenIdentifier) {
- resultId = (ContainerTokenIdentifier) id;
+ if (id instanceof NMTokenIdentifier) {
+ resultId = (NMTokenIdentifier) id;
break;
}
}
return resultId;
}
- @Private
- @VisibleForTesting
- protected ContainerTokenIdentifier getContainerTokenIdentifier(
- UserGroupInformation remoteUgi,
- ContainerTokenIdentifier containerTokenIdentifier)
- throws YarnException {
- if (UserGroupInformation.isSecurityEnabled()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
- + remoteUgi.getTokenIdentifiers().size());
- }
- // Get the tokenId from the remote user ugi
- return selectContainerTokenIdentifier(remoteUgi);
- } else {
- return containerTokenIdentifier;
- }
- }
-
/**
- * Authorize the request.
- *
- * @param containerIDStr
- * of the container
- * @param launchContext
- * passed if verifying the startContainer, null otherwise.
- * @param remoteUgi
+ * @param containerTokenIdentifier
+ * of the container to be started
+ * @param ugi
* ugi corresponding to the remote end making the api-call
* @throws YarnException
*/
@Private
@VisibleForTesting
- protected void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext,
- UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
- throws YarnException {
+ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ UserGroupInformation ugi) throws YarnException {
+ ContainerId containerId = containerTokenIdentifier.getContainerID();
+ String containerIDStr = containerId.toString();
boolean unauthorized = false;
StringBuilder messageBuilder =
new StringBuilder("Unauthorized request to start container. ");
-
- if (!remoteUgi.getUserName().equals(containerIDStr)) {
+ if (!nmTokenIdentifier.getApplicationAttemptId().equals(
+ containerId.getApplicationAttemptId())) {
unauthorized = true;
- messageBuilder.append("\nExpected containerId: "
- + remoteUgi.getUserName() + " Found: " + containerIDStr);
- } else if (launchContext != null) {
- // Verify other things also for startContainer() request.
-
-
- if (tokenId == null) {
- unauthorized = true;
- messageBuilder
- .append("\nNo ContainerToken found for " + containerIDStr);
- } else {
-
- // Is the container being relaunched? Or RPC layer let startCall with
- // tokens generated off old-secret through?
- if (!this.context.getContainerTokenSecretManager()
- .isValidStartContainerRequest(tokenId.getContainerID())) {
- unauthorized = true;
- messageBuilder.append("\n Attempt to relaunch the same "
- + "container with id " + containerIDStr + ".");
- }
-
- // Ensure the token is not expired.
- // Token expiry is not checked for stopContainer/getContainerStatus
- if (tokenId.getExpiryTimeStamp() < System.currentTimeMillis()) {
- unauthorized = true;
- messageBuilder.append("\nThis token is expired. current time is "
- + System.currentTimeMillis() + " found "
- + tokenId.getExpiryTimeStamp());
- }
- }
+ messageBuilder.append("\nNMToken for application attempt : ")
+ .append(nmTokenIdentifier.getApplicationAttemptId())
+ .append(" was used for starting container with container token")
+ .append(" issued for application attempt : ")
+ .append(containerId.getApplicationAttemptId());
+ } else if (!ugi.getUserName().equals(
+ nmTokenIdentifier.getApplicationAttemptId().toString())) {
+ unauthorized = true;
+ messageBuilder.append("\nExpected applicationAttemptId: ")
+ .append(ugi.getUserName()).append(" Found: ")
+ .append(nmTokenIdentifier.getApplicationAttemptId().toString());
+ } else if (!this.context.getContainerTokenSecretManager()
+ .isValidStartContainerRequest(containerId)) {
+ // Is the container being relaunched? Or RPC layer let startCall with
+ // tokens generated off old-secret through?
+ unauthorized = true;
+ messageBuilder.append("\n Attempt to relaunch the same ")
+ .append("container with id ").append(containerIDStr).append(".");
+ } else if (containerTokenIdentifier.getExpiryTimeStamp() < System
+ .currentTimeMillis()) {
+ // Ensure the token is not expired.
+ unauthorized = true;
+ messageBuilder.append("\nThis token is expired. current time is ")
+ .append(System.currentTimeMillis()).append(" found ")
+ .append(containerTokenIdentifier.getExpiryTimeStamp());
}
if (unauthorized) {
@@ -384,7 +365,7 @@ public class ContainerManagerImpl extend
throw RPCUtil.getRemoteException(msg);
}
}
-
+
/**
* Start a container on this NodeManager.
*/
@@ -395,44 +376,133 @@ public class ContainerManagerImpl extend
if (blockNewContainerRequests.get()) {
throw new NMNotYetReadyException(
- "Rejecting new containers as NodeManager has not" +
- " yet connected with ResourceManager");
+ "Rejecting new containers as NodeManager has not"
+ + " yet connected with ResourceManager");
}
+ /*
+ * 1) It should save the NMToken into NMTokenSecretManager. This is done
+ * here instead of RPC layer because at the time of opening/authenticating
+ * the connection it doesn't know what all RPC calls user will make on it.
+ * Also new NMToken is issued only at startContainer (once it gets renewed).
+ *
+ * 2) It should validate containerToken. Need to check below things. a) It
+ * is signed by correct master key (part of retrieve password). b) It
+ * belongs to correct Node Manager (part of retrieve password). c) It has
+ * correct RMIdentifier. d) It is not expired.
+ */
+ // update NMToken
+
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier nmTokenIdentifier = selectNMTokenIdentifier(remoteUgi);
+
+ // Validate containerToken
+ ContainerTokenIdentifier containerTokenIdentifier =
+ verifyAndGetContainerTokenIdentifier(request.getContainerToken());
+
+ authorizeStartRequest(nmTokenIdentifier, containerTokenIdentifier,
+ remoteUgi);
+
+ if (containerTokenIdentifier.getRMIdentifer() != nodeStatusUpdater
+ .getRMIdentifier()) {
+ // Is the container coming from unknown RM
+ StringBuilder sb = new StringBuilder("\nContainer ");
+ sb.append(containerTokenIdentifier.getContainerID().toString())
+ .append(" rejected as it is allocated by a previous RM");
+ throw new InvalidContainerException(sb.toString());
+ }
+
+ updateNMTokenIdentifier(nmTokenIdentifier);
+
+ ContainerId containerId = containerTokenIdentifier.getContainerID();
+ String containerIdStr = containerId.toString();
+ String user = containerTokenIdentifier.getApplicationSubmitter();
+
+ LOG.info("Start request for " + containerIdStr + " by user " + user);
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
- org.apache.hadoop.yarn.api.records.Token token = request.getContainerToken();
- ContainerTokenIdentifier tokenIdentifier = null;
- try {
- tokenIdentifier = BuilderUtils.newContainerTokenIdentifier(token);
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
+ Credentials credentials = parseCredentials(launchContext);
+
+ Container container =
+ new ContainerImpl(getConfig(), this.dispatcher, launchContext,
+ credentials, metrics, containerTokenIdentifier);
+ ApplicationId applicationID =
+ containerId.getApplicationAttemptId().getApplicationId();
+ if (context.getContainers().putIfAbsent(containerId, container) != null) {
+ NMAuditLogger.logFailure(user, AuditConstants.START_CONTAINER,
+ "ContainerManagerImpl", "Container already running on this node!",
+ applicationID, containerId);
+ throw RPCUtil.getRemoteException("Container " + containerIdStr
+ + " already is running on this node!!");
}
- UserGroupInformation remoteUgi = getRemoteUgi();
- ContainerTokenIdentifier tokenId =
- getContainerTokenIdentifier(remoteUgi, tokenIdentifier);
+ // Create the application
+ Application application =
+ new ApplicationImpl(dispatcher, this.aclsManager, user, applicationID,
+ credentials, context);
+ if (null == context.getApplications().putIfAbsent(applicationID,
+ application)) {
+ LOG.info("Creating a new application reference for app " + applicationID);
- ContainerId containerID = tokenId.getContainerID();
- String containerIDStr = containerID.toString();
+ dispatcher.getEventHandler().handle(
+ new ApplicationInitEvent(applicationID, container.getLaunchContext()
+ .getApplicationACLs()));
+ }
+
+ dispatcher.getEventHandler().handle(
+ new ApplicationContainerInitEvent(container));
- authorizeRequest(containerIDStr, launchContext, remoteUgi, tokenId);
+ this.context.getContainerTokenSecretManager().startContainerSuccessful(
+ containerTokenIdentifier);
+ NMAuditLogger.logSuccess(user, AuditConstants.START_CONTAINER,
+ "ContainerManageImpl", applicationID, containerId);
+ StartContainerResponse response =
+ recordFactory.newRecordInstance(StartContainerResponse.class);
+ response.setAllServicesMetaData(auxiliaryServices.getMetaData());
+ // TODO launchedContainer misplaced -> doesn't necessarily mean a container
+ // launch. A finished Application will not launch containers.
+ metrics.launchedContainer();
+ metrics.allocateContainer(containerTokenIdentifier.getResource());
+ return response;
+ }
- // Is the container coming from unknown RM
- if (tokenId.getRMIdentifer() != nodeStatusUpdater
- .getRMIdentifier()) {
- String msg = "\nContainer "+ containerIDStr
- + " rejected as it is allocated by a previous RM";
- LOG.error(msg);
- throw new InvalidContainerException(msg);
+ protected ContainerTokenIdentifier verifyAndGetContainerTokenIdentifier(
+ org.apache.hadoop.yarn.api.records.Token token) throws YarnException,
+ InvalidToken {
+ ContainerTokenIdentifier containerTokenIdentifier = null;
+ try {
+ containerTokenIdentifier =
+ BuilderUtils.newContainerTokenIdentifier(token);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
}
+ byte[] password =
+ context.getContainerTokenSecretManager().retrievePassword(
+ containerTokenIdentifier);
+ byte[] tokenPass = token.getPassword().array();
+ if (password == null || tokenPass == null
+ || !Arrays.equals(password, tokenPass)) {
+ throw new InvalidToken(
+ "Invalid container token used for starting container on : "
+ + context.getNodeId().toString());
+ }
+ return containerTokenIdentifier;
+ }
- LOG.info("Start request for " + containerIDStr + " by user "
- + tokenId.getApplicationSubmitter());
+ @Private
+ @VisibleForTesting
+ protected void updateNMTokenIdentifier(NMTokenIdentifier nmTokenIdentifier)
+ throws InvalidToken {
+ context.getNMTokenSecretManager().appAttemptStartContainer(
+ nmTokenIdentifier);
+ }
+ private Credentials parseCredentials(ContainerLaunchContext launchContext)
+ throws YarnException {
+ Credentials credentials = new Credentials();
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getTokens();
- Credentials credentials = new Credentials();
+
if (tokens != null) {
DataInputByteBuffer buf = new DataInputByteBuffer();
tokens.rewind();
@@ -440,8 +510,7 @@ public class ContainerManagerImpl extend
try {
credentials.readTokenStorageStream(buf);
if (LOG.isDebugEnabled()) {
- for (Token<? extends TokenIdentifier> tk : credentials
- .getAllTokens()) {
+ for (Token<? extends TokenIdentifier> tk : credentials.getAllTokens()) {
LOG.debug(tk.getService() + " = " + tk.toString());
}
}
@@ -450,53 +519,7 @@ public class ContainerManagerImpl extend
}
}
// //////////// End of parsing credentials
- String user = tokenId.getApplicationSubmitter();
-
- Container container =
- new ContainerImpl(getConfig(), this.dispatcher, launchContext,
- credentials, metrics, tokenId);
- ApplicationId applicationID =
- containerID.getApplicationAttemptId().getApplicationId();
- if (context.getContainers().putIfAbsent(containerID, container) != null) {
- NMAuditLogger.logFailure(user,
- AuditConstants.START_CONTAINER, "ContainerManagerImpl",
- "Container already running on this node!",
- applicationID, containerID);
- throw RPCUtil.getRemoteException("Container " + containerIDStr
- + " already is running on this node!!");
- }
-
- // Create the application
- Application application =
- new ApplicationImpl(dispatcher, this.aclsManager,
- user, applicationID, credentials,
- context);
- if (null ==
- context.getApplications().putIfAbsent(applicationID, application)) {
- LOG.info("Creating a new application reference for app "
- + applicationID);
- dispatcher.getEventHandler().handle(
- new ApplicationInitEvent(applicationID, container
- .getLaunchContext().getApplicationACLs()));
- }
-
- // TODO: Validate the request
- dispatcher.getEventHandler().handle(
- new ApplicationContainerInitEvent(container));
-
- this.context.getContainerTokenSecretManager().startContainerSuccessful(
- tokenId);
- NMAuditLogger.logSuccess(user,
- AuditConstants.START_CONTAINER, "ContainerManageImpl",
- applicationID, containerID);
-
- StartContainerResponse response =
- StartContainerResponse.newInstance(auxiliaryServices.getMetaData());
- // TODO launchedContainer misplaced -> doesn't necessarily mean a container
- // launch. A finished Application will not launch containers.
- metrics.launchedContainer();
- metrics.allocateContainer(tokenId.getResource());
- return response;
+ return credentials;
}
/**
@@ -509,34 +532,20 @@ public class ContainerManagerImpl extend
ContainerId containerID = request.getContainerId();
String containerIDStr = containerID.toString();
-
- // TODO: Only the container's owner can kill containers today.
-
- UserGroupInformation remoteUgi = getRemoteUgi();
Container container = this.context.getContainers().get(containerID);
+ LOG.info("Getting container-status for " + containerIDStr);
+ authorizeGetAndStopContainerRequest(containerID, container, true);
+
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
- if (container == null) {
- LOG.warn("Trying to stop unknown container " + containerID);
- NMAuditLogger.logFailure("UnknownUser",
- AuditConstants.STOP_CONTAINER, "ContainerManagerImpl",
- "Trying to stop unknown container!",
- containerID.getApplicationAttemptId().getApplicationId(),
- containerID);
- return response; // Return immediately.
- }
- authorizeRequest(containerIDStr, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
-
dispatcher.getEventHandler().handle(
- new ContainerKillEvent(containerID,
- "Container killed by the ApplicationMaster."));
-
- NMAuditLogger.logSuccess(container.getUser(),
- AuditConstants.STOP_CONTAINER, "ContainerManageImpl",
- containerID.getApplicationAttemptId().getApplicationId(),
- containerID);
+ new ContainerKillEvent(containerID,
+ "Container killed by the ApplicationMaster."));
+
+ NMAuditLogger.logSuccess(container.getUser(),
+ AuditConstants.STOP_CONTAINER, "ContainerManageImpl", containerID
+ .getApplicationAttemptId().getApplicationId(), containerID);
// TODO: Move this code to appropriate place once kill_container is
// implemented.
@@ -547,23 +556,14 @@ public class ContainerManagerImpl extend
@Override
public GetContainerStatusResponse getContainerStatus(
- GetContainerStatusRequest request) throws YarnException,
- IOException {
+ GetContainerStatusRequest request) throws YarnException, IOException {
ContainerId containerID = request.getContainerId();
String containerIDStr = containerID.toString();
+ Container container = this.context.getContainers().get(containerID);
- // TODO: Only the container's owner can get containers' status today.
-
- UserGroupInformation remoteUgi = getRemoteUgi();
LOG.info("Getting container-status for " + containerIDStr);
- Container container = this.context.getContainers().get(containerID);
- if (container == null) {
- throw RPCUtil.getRemoteException("Container " + containerIDStr
- + " is not handled by this NodeManager");
- }
- authorizeRequest(containerIDStr, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
+ authorizeGetAndStopContainerRequest(containerID, container, false);
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
LOG.info("Returning " + containerStatus);
@@ -573,6 +573,48 @@ public class ContainerManagerImpl extend
return response;
}
+ @Private
+ @VisibleForTesting
+ protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
+ Container container, boolean stopRequest) throws YarnException {
+
+ UserGroupInformation remoteUgi = getRemoteUgi();
+ NMTokenIdentifier identifier = selectNMTokenIdentifier(remoteUgi);
+
+ /*
+ * For get/stop container status; we need to verify that 1) User (NMToken)
+ * application attempt only has started container. 2) Requested containerId
+ * belongs to the same application attempt (NMToken) which was used. (Note:-
+ * This will prevent user in knowing another application's containers).
+ */
+
+ if ((!identifier.getApplicationAttemptId().equals(
+ containerId.getApplicationAttemptId()))
+ || (container != null && !identifier.getApplicationAttemptId().equals(
+ container.getContainerId().getApplicationAttemptId()))) {
+ if (stopRequest) {
+ LOG.warn(identifier.getApplicationAttemptId()
+ + " attempted to stop non-application container : "
+ + container.getContainerId().toString());
+ NMAuditLogger.logFailure("UnknownUser", AuditConstants.STOP_CONTAINER,
+ "ContainerManagerImpl", "Trying to stop unknown container!",
+ identifier.getApplicationAttemptId().getApplicationId(),
+ container.getContainerId());
+ } else {
+ LOG.warn(identifier.getApplicationAttemptId()
+ + " attempted to get get status for non-application container : "
+ + container.getContainerId().toString());
+ }
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " is not started by this application attempt.");
+ }
+
+ if (container == null) {
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " is not handled by this NodeManager");
+ }
+ }
+
class ContainerEventDispatcher implements EventHandler<ContainerEvent> {
@Override
public void handle(ContainerEvent event) {
@@ -643,9 +685,19 @@ public class ContainerManagerImpl extend
this.blockNewContainerRequests.set(blockNewContainerRequests);
}
+ @Private
+ @VisibleForTesting
+ public boolean getBlockNewContainerRequestsStatus() {
+ return this.blockNewContainerRequests.get();
+ }
+
@Override
public void stateChanged(Service service) {
// TODO Auto-generated method stub
}
+
+ public Context getContext() {
+ return this.context;
+ }
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSecretManager.java Tue Jun 18 23:19:49 2013
@@ -18,26 +18,32 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security;
+import javax.crypto.SecretKey;
+
import org.apache.hadoop.security.token.SecretManager;
public class LocalizerTokenSecretManager extends
SecretManager<LocalizerTokenIdentifier> {
+ private final SecretKey secretKey;
+
+ public LocalizerTokenSecretManager() {
+ this.secretKey = generateSecret();
+ }
+
@Override
protected byte[] createPassword(LocalizerTokenIdentifier identifier) {
- return "testing".getBytes();
+ return createPassword(identifier.getBytes(), secretKey);
}
@Override
public byte[] retrievePassword(LocalizerTokenIdentifier identifier)
throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
- // TODO Auto-generated method stub
- return "testing".getBytes();
+ return createPassword(identifier.getBytes(), secretKey);
}
@Override
public LocalizerTokenIdentifier createIdentifier() {
- // TODO Auto-generated method stub
return new LocalizerTokenIdentifier();
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java Tue Jun 18 23:19:49 2013
@@ -30,13 +30,12 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
-import com.google.common.annotations.VisibleForTesting;
-
/**
* The NM maintains only two master-keys. The current key that RM knows and the
* key from the previous rolling-interval.
@@ -51,6 +50,7 @@ public class NMContainerTokenSecretManag
private MasterKeyData previousMasterKey;
private final Map<ApplicationId, ConcurrentMap<ContainerId, MasterKeyData>> oldMasterKeys;
+ private String nodeHostAddr;
public NMContainerTokenSecretManager(Configuration conf) {
super(conf);
@@ -122,6 +122,15 @@ public class NMContainerTokenSecretManag
masterKeyToUse = this.oldMasterKeys.get(appId).get(containerId);
}
+ if (nodeHostAddr != null
+ && !identifier.getNmHostAddress().equals(nodeHostAddr)) {
+ // Valid container token used for incorrect node.
+ throw new SecretManager.InvalidToken("Given Container "
+ + identifier.getContainerID().toString()
+ + " identifier is not valid for current Node manager. Expected : "
+ + nodeHostAddr + " Found : " + identifier.getNmHostAddress());
+ }
+
if (masterKeyToUse != null) {
return retrievePasswordInternal(identifier, masterKeyToUse);
}
@@ -186,4 +195,9 @@ public class NMContainerTokenSecretManag
public synchronized void appFinished(ApplicationId appId) {
this.oldMasterKeys.remove(appId);
}
+
+ public synchronized void setNodeId(NodeId nodeId) {
+ nodeHostAddr = nodeId.toString();
+ LOG.info("Updating node address : " + nodeHostAddr);
+ }
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java Tue Jun 18 23:19:49 2013
@@ -18,18 +18,24 @@
package org.apache.hadoop.yarn.server.nodemanager.security;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager;
import org.apache.hadoop.yarn.server.security.MasterKeyData;
+import com.google.common.annotations.VisibleForTesting;
+
public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
private static final Log LOG = LogFactory
@@ -38,10 +44,15 @@ public class NMTokenSecretManagerInNM ex
private MasterKeyData previousMasterKey;
private final Map<ApplicationAttemptId, MasterKeyData> oldMasterKeys;
+ private final Map<ApplicationId, List<ApplicationAttemptId>> appToAppAttemptMap;
+ private NodeId nodeId;
+
public NMTokenSecretManagerInNM() {
this.oldMasterKeys =
new HashMap<ApplicationAttemptId, MasterKeyData>();
+ appToAppAttemptMap =
+ new HashMap<ApplicationId, List<ApplicationAttemptId>>();
}
/**
@@ -69,46 +80,117 @@ public class NMTokenSecretManagerInNM ex
}
/**
- * This method will be used to verify NMTokens generated by different
- * master keys.
+ * This method will be used to verify NMTokens generated by different master
+ * keys.
*/
@Override
- public synchronized byte[] retrievePassword(
- NMTokenIdentifier identifier) throws InvalidToken {
- int keyId = identifier.getMastKeyId();
+ public synchronized byte[] retrievePassword(NMTokenIdentifier identifier)
+ throws InvalidToken {
+ int keyId = identifier.getKeyId();
ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();
-
+
/*
- * MasterKey used for retrieving password will be as follows.
- * 1) By default older saved master key will be used.
- * 2) If identifier's master key id matches that of previous master key
- * id then previous key will be used.
- * 3) If identifier's master key id matches that of current master key
- * id then current key will be used.
+ * MasterKey used for retrieving password will be as follows. 1) By default
+ * older saved master key will be used. 2) If identifier's master key id
+ * matches that of previous master key id then previous key will be used. 3)
+ * If identifier's master key id matches that of current master key id then
+ * current key will be used.
*/
MasterKeyData oldMasterKey = oldMasterKeys.get(appAttemptId);
MasterKeyData masterKeyToUse = oldMasterKey;
if (previousMasterKey != null
&& keyId == previousMasterKey.getMasterKey().getKeyId()) {
masterKeyToUse = previousMasterKey;
- } else if ( keyId == currentMasterKey.getMasterKey().getKeyId()) {
+ } else if (keyId == currentMasterKey.getMasterKey().getKeyId()) {
masterKeyToUse = currentMasterKey;
}
+ if (nodeId != null && !identifier.getNodeId().equals(nodeId)) {
+ throw new InvalidToken("Given NMToken for application : "
+ + appAttemptId.toString() + " is not valid for current node manager."
+ + "expected : " + nodeId.toString() + " found : "
+ + identifier.getNodeId().toString());
+ }
+
if (masterKeyToUse != null) {
byte[] password = retrivePasswordInternal(identifier, masterKeyToUse);
- if (masterKeyToUse.getMasterKey().getKeyId() != oldMasterKey
- .getMasterKey().getKeyId()) {
- oldMasterKeys.put(appAttemptId, masterKeyToUse);
- }
+ LOG.debug("NMToken password retrieved successfully!!");
return password;
}
-
+
throw new InvalidToken("Given NMToken for application : "
+ appAttemptId.toString() + " seems to have been generated illegally.");
}
+
+ public synchronized void appFinished(ApplicationId appId) {
+ List<ApplicationAttemptId> appAttemptList = appToAppAttemptMap.get(appId);
+ if (appAttemptList != null) {
+ LOG.debug("Removing application attempts NMToken keys for application "
+ + appId);
+ for (ApplicationAttemptId appAttemptId : appAttemptList) {
+ this.oldMasterKeys.remove(appAttemptId);
+ }
+ appToAppAttemptMap.remove(appId);
+ } else {
+ LOG.error("No application Attempt for application : " + appId
+ + " started on this NM.");
+ }
+ }
+
+ /**
+ * This will be called by startContainer. It will add the master key into
+ * the cache used for starting this container. This should be called before
+ * validating the startContainer request.
+ */
+ public synchronized void appAttemptStartContainer(
+ NMTokenIdentifier identifier)
+ throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
+ ApplicationAttemptId appAttemptId = identifier.getApplicationAttemptId();
+ if (!appToAppAttemptMap.containsKey(appAttemptId.getApplicationId())) {
+ // First application attempt for the given application
+ appToAppAttemptMap.put(appAttemptId.getApplicationId(),
+ new ArrayList<ApplicationAttemptId>());
+ }
+ MasterKeyData oldKey = oldMasterKeys.get(appAttemptId);
+
+ if (oldKey == null) {
+ // This is a new application attempt.
+ appToAppAttemptMap.get(appAttemptId.getApplicationId()).add(appAttemptId);
+ }
+ if (oldKey == null
+ || oldKey.getMasterKey().getKeyId() != identifier.getKeyId()) {
+ // Update key only if it is modified.
+ LOG.debug("NMToken key updated for application attempt : "
+ + identifier.getApplicationAttemptId().toString());
+ if (identifier.getKeyId() == currentMasterKey.getMasterKey()
+ .getKeyId()) {
+ oldMasterKeys.put(appAttemptId, currentMasterKey);
+ } else if (previousMasterKey != null
+ && identifier.getKeyId() == previousMasterKey.getMasterKey()
+ .getKeyId()) {
+ oldMasterKeys.put(appAttemptId, previousMasterKey);
+ } else {
+ throw new InvalidToken(
+ "Older NMToken should not be used while starting the container.");
+ }
+ }
+ }
+
+ public synchronized void setNodeId(NodeId nodeId) {
+ LOG.debug("updating nodeId : " + nodeId);
+ this.nodeId = nodeId;
+ }
- public synchronized void appFinished(ApplicationAttemptId appAttemptId) {
- this.oldMasterKeys.remove(appAttemptId);
+ @Private
+ @VisibleForTesting
+ public synchronized boolean
+ isAppAttemptNMTokenKeyPresent(ApplicationAttemptId appAttemptId) {
+ return oldMasterKeys.containsKey(appAttemptId);
+ }
+
+ @Private
+ @VisibleForTesting
+ public synchronized NodeId getNodeId() {
+ return this.nodeId;
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Jun 18 23:19:49 2013
@@ -27,10 +27,12 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -126,6 +128,19 @@ public class DummyContainerManager exten
}
@Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, getContext()
+ .getNodeId(), "testuser", getContext().getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+
+ @Override
@SuppressWarnings("unchecked")
protected ContainersLauncher createContainersLauncher(Context context,
ContainerExecutor exec) {
@@ -179,17 +194,16 @@ public class DummyContainerManager exten
}
@Override
- protected void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext,
- UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
- throws YarnException {
- // do Nothing
+ protected void authorizeStartRequest(NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ UserGroupInformation ugi) throws YarnException {
+ // do nothing
}
-
+
@Override
- protected ContainerTokenIdentifier
- getContainerTokenIdentifier(UserGroupInformation remoteUgi,
- ContainerTokenIdentifier containerTokenId) throws YarnException {
- return containerTokenId;
+ protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
+ Container container, boolean stopRequest) throws YarnException {
+ // do nothing
}
+
}
\ No newline at end of file
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java Tue Jun 18 23:19:49 2013
@@ -74,7 +74,7 @@ public class TestContainerManagerWithLCE
}
@Override
- public void testContainerSetup() throws IOException, InterruptedException,
+ public void testContainerSetup() throws Exception, InterruptedException,
YarnException {
// Don't run the test if the binary is not available.
if (!shouldRunTest()) {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Jun 18 23:19:49 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -133,18 +134,13 @@ public class TestEventFlow {
ApplicationAttemptId.newInstance(applicationId, 0);
ContainerId cID = ContainerId.newInstance(applicationAttemptId, 0);
- Resource r = BuilderUtils.newResource(1024, 1);
String user = "testing";
- String host = "127.0.0.1";
- int port = 1234;
- Token containerToken =
- BuilderUtils.newContainerToken(cID, host, port, user, r,
- System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
- SIMULATED_RM_IDENTIFIER);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
- request.setContainerToken(containerToken);
+ request.setContainerToken(TestContainerManager.createContainerToken(cID,
+ SIMULATED_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java Tue Jun 18 23:19:49 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager;
import static org.mockito.Matchers.argThat;
-import static org.mockito.Matchers.eq;
import static org.mockito.Matchers.isNull;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
@@ -50,17 +49,16 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
-import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
@@ -131,24 +129,23 @@ public class TestNodeManagerReboot {
containerLaunchContext.setLocalResources(localResources);
List<String> commands = new ArrayList<String>();
containerLaunchContext.setCommands(commands);
- Resource resource = Records.newRecord(Resource.class);
- resource.setMemory(1024);
- NodeId nodeId = BuilderUtils.newNodeId("127.0.0.1", 12345);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(),
- user, resource, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), 0);
final StartContainerRequest startRequest =
Records.newRecord(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ NodeId nodeId = nm.getNMContext().getNodeId();
+ startRequest.setContainerToken(TestContainerManager.createContainerToken(
+ cId, 0, nodeId, destinationFile, nm.getNMContext()
+ .getContainerTokenSecretManager()));
final UserGroupInformation currentUser = UserGroupInformation
- .createRemoteUser(cId.toString());
+ .createRemoteUser(cId.getApplicationAttemptId().toString());
+ NMTokenIdentifier nmIdentifier =
+ new NMTokenIdentifier(cId.getApplicationAttemptId(), nodeId, user, 123);
+ currentUser.addTokenIdentifier(nmIdentifier);
currentUser.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws YarnException, IOException {
- containerManager.startContainer(startRequest);
+ nm.getContainerManager().startContainer(startRequest);
return null;
}
});
@@ -208,8 +205,6 @@ public class TestNodeManagerReboot {
ContainerLocalizer.FILECACHE) == 0 && numOfLocalDirs(nmLocalDir
.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR)
== 0);
- verify(delService, times(1)).delete(eq(user),
- argThat(new PathInclude(user)));
verify(delService, times(1)).delete(
(String) isNull(),
argThat(new PathInclude(ResourceLocalizationService.NM_PRIVATE_DIR
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java Tue Jun 18 23:19:49 2013
@@ -184,6 +184,7 @@ public class TestNodeManagerResync {
} catch (InterruptedException e) {
} catch (BrokenBarrierException e) {
} catch (AssertionError ae) {
+ ae.printStackTrace();
assertionFailedInThread.set(true);
}
}
@@ -228,6 +229,7 @@ public class TestNodeManagerResync {
.setStopThreadFlag(false);
super.setBlockNewContainerRequests(blockNewContainerRequests);
} catch (InterruptedException e) {
+ e.printStackTrace();
}
}
}
@@ -258,6 +260,7 @@ public class TestNodeManagerResync {
} catch (InterruptedException e) {
} catch (BrokenBarrierException e) {
} catch (AssertionError ae) {
+ ae.printStackTrace();
assertionFailedInThread.set(true);
}
}
@@ -296,6 +299,7 @@ public class TestNodeManagerResync {
Assert.assertEquals(NMNotYetReadyException.class.getName(), e
.getClass().getName());
} catch (IOException e) {
+ e.printStackTrace();
assertionFailedInThread.set(true);
}
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Tue Jun 18 23:19:49 2013
@@ -52,16 +52,17 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
@@ -161,7 +162,7 @@ public class TestNodeManagerShutdown {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234);
+ NodeId nodeId = BuilderUtils.newNodeId("localhost", 12345);
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
@@ -180,17 +181,22 @@ public class TestNodeManagerShutdown {
containerLaunchContext.setLocalResources(localResources);
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource resource = BuilderUtils.newResource(1024, 1);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, nodeId.getHost(), nodeId.getPort(),
- user, resource, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), 0);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest
+ .setContainerToken(TestContainerManager.createContainerToken(cId, 0,
+ nodeId, user, nm.getNMContext().getContainerTokenSecretManager()));
+ final InetSocketAddress containerManagerBindAddress =
+ NetUtils.createSocketAddrForHost("127.0.0.1", 12345);
UserGroupInformation currentUser = UserGroupInformation
.createRemoteUser(cId.toString());
+ org.apache.hadoop.security.token.Token<NMTokenIdentifier> nmToken =
+ ConverterUtils.convertFromYarn(
+ nm.getNMContext().getNMTokenSecretManager()
+ .createNMToken(cId.getApplicationAttemptId(), nodeId, user),
+ containerManagerBindAddress);
+ currentUser.addToken(nmToken);
ContainerManagementProtocol containerManager =
currentUser.doAs(new PrivilegedAction<ContainerManagementProtocol>() {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Jun 18 23:19:49 2013
@@ -201,7 +201,7 @@ public class TestNodeStatusUpdater {
String user = "testUser";
ContainerTokenIdentifier containerToken =
BuilderUtils.newContainerTokenIdentifier(BuilderUtils
- .newContainerToken(firstContainerID, "127.0.0.1", 1234, user,
+ .newContainerToken(firstContainerID, "localhost", 1234, user,
resource, currentTime + 10000, 123, "password".getBytes(),
currentTime));
Container container =
@@ -232,7 +232,7 @@ public class TestNodeStatusUpdater {
Resource resource = BuilderUtils.newResource(3, 1);
ContainerTokenIdentifier containerToken =
BuilderUtils.newContainerTokenIdentifier(BuilderUtils
- .newContainerToken(secondContainerID, "127.0.0.1", 1234, user,
+ .newContainerToken(secondContainerID, "localhost", 1234, user,
resource, currentTime + 10000, 123,
"password".getBytes(), currentTime));
Container container =
@@ -1168,8 +1168,8 @@ public class TestNodeStatusUpdater {
private YarnConfiguration createNMConfig() {
YarnConfiguration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.NM_PMEM_MB, 5*1024); // 5GB
- conf.set(YarnConfiguration.NM_ADDRESS, "127.0.0.1:12345");
- conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "127.0.0.1:12346");
+ conf.set(YarnConfiguration.NM_ADDRESS, "localhost:12345");
+ conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "localhost:12346");
conf.set(YarnConfiguration.NM_LOG_DIRS, new Path(basedir, "logs").toUri()
.getPath());
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, new Path(basedir,
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Jun 18 23:19:49 2013
@@ -30,21 +30,20 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -58,6 +57,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
@@ -150,7 +150,7 @@ public abstract class BaseContainerManag
LOG.info("Created localDir in " + localDir.getAbsolutePath());
LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
- String bindAddress = "0.0.0.0:5555";
+ String bindAddress = "127.0.0.1:12345";
conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
@@ -173,6 +173,7 @@ public abstract class BaseContainerManag
protected ContainerManagerImpl
createContainerManager(DeletionService delSrvc) {
+
return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
metrics, new ApplicationACLsManager(conf), dirsHandler) {
@Override
@@ -182,11 +183,24 @@ public abstract class BaseContainerManag
}
@Override
- protected void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext, UserGroupInformation remoteUgi,
- ContainerTokenIdentifier tokenId) throws YarnException {
- // do nothing
- }
+ protected void authorizeGetAndStopContainerRequest(ContainerId containerId,
+ Container container, boolean stopRequest) throws YarnException {
+ // do nothing
+ }
+
+ @Override
+ protected void authorizeStartRequest(
+ NMTokenIdentifier nmTokenIdentifier,
+ ContainerTokenIdentifier containerTokenIdentifier,
+ UserGroupInformation ugi) throws YarnException {
+ // do nothing
+ }
+
+ @Override
+ protected void updateNMTokenIdentifier(
+ NMTokenIdentifier nmTokenIdentifier) throws InvalidToken {
+ // Do nothing
+ }
};
}
@@ -242,7 +256,7 @@ public abstract class BaseContainerManag
throws InterruptedException {
// Wait for app-finish
Application app =
- containerManager.context.getApplications().get(appID);
+ containerManager.getContext().getApplications().get(appID);
int timeout = 0;
while (!(app.getApplicationState().equals(finalState))
&& timeout++ < 15) {
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Jun 18 23:19:49 2013
@@ -34,6 +34,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -47,10 +48,13 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.security.NMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.ResourceManagerConstants;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
@@ -59,8 +63,11 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Before;
import org.junit.Test;
public class TestContainerManager extends BaseContainerManagerTest {
@@ -72,6 +79,12 @@ public class TestContainerManager extend
static {
LOG = LogFactory.getLog(TestContainerManager.class);
}
+
+ @Override
+ @Before
+ public void setup() throws IOException {
+ super.setup();
+ }
private ContainerId createContainerId() {
ApplicationId appId = ApplicationId.newInstance(0, 0);
@@ -81,6 +94,32 @@ public class TestContainerManager extend
return containerId;
}
+ @Override
+ protected ContainerManagerImpl
+ createContainerManager(DeletionService delSrvc) {
+ return new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler) {
+ @Override
+ public void
+ setBlockNewContainerRequests(boolean blockNewContainerRequests) {
+ // do nothing
+ }
+
+ @Override
+ protected UserGroupInformation getRemoteUgi() throws YarnException {
+ ApplicationId appId = ApplicationId.newInstance(0, 0);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser(appAttemptId.toString());
+ ugi.addTokenIdentifier(new NMTokenIdentifier(appAttemptId, context
+ .getNodeId(), user, context.getNMTokenSecretManager().getCurrentKey()
+ .getKeyId()));
+ return ugi;
+ }
+ };
+ }
+
@Test
public void testContainerManagerInitialization() throws IOException {
@@ -101,8 +140,7 @@ public class TestContainerManager extend
}
@Test
- public void testContainerSetup() throws IOException, InterruptedException,
- YarnException {
+ public void testContainerSetup() throws Exception {
containerManager.start();
@@ -134,16 +172,12 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- Resource r = BuilderUtils.newResource(512, 1);
- int port = 12345;
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(startRequest);
@@ -227,16 +261,12 @@ public class TestContainerManager extend
containerLaunchContext.setLocalResources(localResources);
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(100, 1);
- int port = 12345;
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -335,15 +365,12 @@ public class TestContainerManager extend
containerLaunchContext.setLocalResources(localResources);
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(100, 1);
- int port = 12345;
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
+
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -423,16 +450,10 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- Resource r = BuilderUtils.newResource(100, 1);
- int port = 12345;
-
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
- request.setContainerToken(containerToken);
+ request.setContainerToken(createContainerToken(cId, DUMMY_RM_IDENTIFIER,
+ context.getNodeId(), user, context.getContainerTokenSecretManager()));
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -503,24 +524,19 @@ public class TestContainerManager extend
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- String host = "127.0.0.1";
- int port = 1234;
ContainerId cId1 = createContainerId();
ContainerId cId2 = createContainerId();
containerLaunchContext
.setLocalResources(new HashMap<String, LocalResource>());
- Resource mockResource = BuilderUtils.newResource(1024, 1);
// Construct the Container with Invalid RMIdentifier
StartContainerRequest startRequest1 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest1.setContainerLaunchContext(containerLaunchContext);
- Token containerToken1 =
- BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
- System.currentTimeMillis() + 10000, 123, "password".getBytes(),
- (long) ResourceManagerConstants.RM_INVALID_IDENTIFIER);
- startRequest1.setContainerToken(containerToken1);
+ startRequest1.setContainerToken(createContainerToken(cId1,
+ ResourceManagerConstants.RM_INVALID_IDENTIFIER, context.getNodeId(),
+ user, context.getContainerTokenSecretManager()));
boolean catchException = false;
try {
containerManager.startContainer(startRequest1);
@@ -528,8 +544,8 @@ public class TestContainerManager extend
catchException = true;
Assert.assertTrue(e.getMessage().contains(
"Container " + cId1 + " rejected as it is allocated by a previous RM"));
- Assert.assertEquals(InvalidContainerException.class.getName(), e
- .getClass().getName());
+ Assert.assertTrue(e.getClass().getName()
+ .equalsIgnoreCase(InvalidContainerException.class.getName()));
}
// Verify that startContainer fail because of invalid container request
@@ -539,11 +555,9 @@ public class TestContainerManager extend
StartContainerRequest startRequest2 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest2.setContainerLaunchContext(containerLaunchContext);
- Token containerToken2 =
- BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
- System.currentTimeMillis() + 10000, 123, "password".getBytes(),
- super.DUMMY_RM_IDENTIFIER);
- startRequest2.setContainerToken(containerToken2);
+ startRequest2.setContainerToken(createContainerToken(cId2,
+ DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
boolean noException = true;
try {
containerManager.startContainer(startRequest2);
@@ -553,4 +567,20 @@ public class TestContainerManager extend
// Verify that startContainer get no YarnException
Assert.assertTrue(noException);
}
+
+ public static Token createContainerToken(ContainerId cId, long rmIdentifier,
+ NodeId nodeId, String user,
+ NMContainerTokenSecretManager containerTokenSecretManager)
+ throws IOException {
+ Resource r = BuilderUtils.newResource(1024, 1);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(cId, nodeId.toString(), user, r,
+ System.currentTimeMillis() + 100000L, 123, rmIdentifier);
+ Token containerToken =
+ BuilderUtils
+ .newContainerToken(nodeId, containerTokenSecretManager
+ .retrievePassword(containerTokenIdentifier),
+ containerTokenIdentifier);
+ return containerToken;
+ }
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Tue Jun 18 23:19:49 2013
@@ -37,6 +37,7 @@ import junit.framework.Assert;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
@@ -56,6 +57,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
@@ -229,14 +231,9 @@ public class TestContainerLaunch extends
// set up the rest of the container
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(1024, 1);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 1234,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(createContainerToken(cId));
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -378,12 +375,9 @@ public class TestContainerLaunch extends
// set up the rest of the container
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(1024, 1);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
+ Token containerToken = createContainerToken(cId);
+ StartContainerRequest startRequest =
+ recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
@@ -441,4 +435,17 @@ public class TestContainerLaunch extends
}
}
+ protected Token createContainerToken(ContainerId cId) throws InvalidToken {
+ Resource r = BuilderUtils.newResource(1024, 1);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
+ r, System.currentTimeMillis() + 10000L, 123, DUMMY_RM_IDENTIFIER);
+ Token containerToken =
+ BuilderUtils.newContainerToken(
+ context.getNodeId(),
+ context.getContainerTokenSecretManager().retrievePassword(
+ containerTokenIdentifier), containerTokenIdentifier);
+ return containerToken;
+ }
+
}
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Tue Jun 18 23:19:49 2013
@@ -23,9 +23,9 @@ import static junit.framework.Assert.ass
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyMap;
import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
@@ -73,8 +73,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -94,6 +92,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestContainerManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
@@ -810,15 +809,12 @@ public class TestLogAggregationService e
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1);
- Token containerToken =
- BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r,
- System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
- super.DUMMY_RM_IDENTIFIER);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
- startRequest.setContainerToken(containerToken);
+ startRequest.setContainerToken(TestContainerManager.createContainerToken(
+ cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
+ context.getContainerTokenSecretManager()));
this.containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(this.containerManager,
Modified: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1494369&r1=1494368&r2=1494369&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Tue Jun 18 23:19:49 2013
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
@@ -229,13 +230,16 @@ public class TestContainersMonitor exten
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ ContainerTokenIdentifier containerIdentifier =
+ new ContainerTokenIdentifier(cId, context.getNodeId().toString(), user,
+ r, System.currentTimeMillis() + 120000, 123, DUMMY_RM_IDENTIFIER);
Token containerToken =
- BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
- port, user, r, System.currentTimeMillis() + 10000L, 123,
- "password".getBytes(), super.DUMMY_RM_IDENTIFIER);
+ BuilderUtils.newContainerToken(context.getNodeId(),
+ containerManager.getContext().getContainerTokenSecretManager()
+ .createPassword(containerIdentifier), containerIdentifier);
startRequest.setContainerToken(containerToken);
containerManager.startContainer(startRequest);
-
+
int timeoutSecs = 0;
while (!processStartFile.exists() && timeoutSecs++ < 20) {
Thread.sleep(1000);
Added: hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/core-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/core-site.xml?rev=1494369&view=auto
==============================================================================
--- hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/core-site.xml (added)
+++ hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/core-site.xml Tue Jun 18 23:19:49 2013
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- Put site-specific property overrides in this file. -->
+
+<configuration>
+ <property>
+ <name>hadoop.security.token.service.use_ip</name>
+ <value>false</value>
+ </property>
+
+</configuration>