You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/05/25 01:11:26 UTC
svn commit: r1486251 [1/2] - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/...
Author: vinodkv
Date: Fri May 24 23:11:25 2013
New Revision: 1486251
URL: http://svn.apache.org/r1486251
Log:
YARN-571. Remove user from ContainerLaunchContext. Contributed by Omkar Vinit Joshi.
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/resources/core-site.xml
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MemoryRMStateStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/ApplicationStateData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/records/impl/pb/ApplicationStateDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri May 24 23:11:25 2013
@@ -49,6 +49,9 @@ Release 2.0.5-beta - UNRELEASED
YARN-615. Rename ContainerLaunchContext.containerTokens to tokens.
(Vinod Kumar Vavilapalli via sseth)
+ YARN-571. Remove user from ContainerLaunchContext. (Omkar Vinit Joshi via
+ vinodkv)
+
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java Fri May 24 23:11:25 2013
@@ -56,13 +56,12 @@ public abstract class ContainerLaunchCon
@Public
@Stable
public static ContainerLaunchContext newInstance(
- String user, Map<String, LocalResource> localResources,
+ Map<String, LocalResource> localResources,
Map<String, String> environment, List<String> commands,
Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<ApplicationAccessType, String> acls) {
ContainerLaunchContext container =
Records.newRecord(ContainerLaunchContext.class);
- container.setUser(user);
container.setLocalResources(localResources);
container.setEnvironment(environment);
container.setCommands(commands);
@@ -73,22 +72,6 @@ public abstract class ContainerLaunchCon
}
/**
- * Get the <em>user</em> to whom the container has been allocated.
- * @return the <em>user</em> to whom the container has been allocated
- */
- @Public
- @Stable
- public abstract String getUser();
-
- /**
- * Set the <em>user</em> to whom the container has been allocated
- * @param user <em>user</em> to whom the container has been allocated
- */
- @Public
- @Stable
- public abstract void setUser(String user);
-
- /**
* Get all the tokens needed by this container. It may include file-system
* tokens, ApplicationMaster related tokens if this container is an
* ApplicationMaster or framework level tokens needed by this container to
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java Fri May 24 23:11:25 2013
@@ -170,25 +170,6 @@ extends ContainerLaunchContext {
}
@Override
- public String getUser() {
- ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasUser()) {
- return null;
- }
- return (p.getUser());
- }
-
- @Override
- public void setUser(String user) {
- maybeInitBuilder();
- if (user == null) {
- builder.clearUser();
- return;
- }
- builder.setUser((user));
- }
-
- @Override
public Map<String, LocalResource> getLocalResources() {
initLocalResources();
return this.localResources;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Fri May 24 23:11:25 2013
@@ -267,13 +267,12 @@ message QueueUserACLInfoProto {
////////////////////////////////////////////////////////////////////////
message ContainerLaunchContextProto {
- optional string user = 1;
- repeated StringLocalResourceMapProto localResources = 2;
- optional bytes tokens = 3;
- repeated StringBytesMapProto service_data = 4;
- repeated StringStringMapProto environment = 5;
- repeated string command = 6;
- repeated ApplicationACLMapProto application_ACLs = 7;
+ repeated StringLocalResourceMapProto localResources = 1;
+ optional bytes tokens = 2;
+ repeated StringBytesMapProto service_data = 3;
+ repeated StringStringMapProto environment = 4;
+ repeated string command = 5;
+ repeated ApplicationACLMapProto application_ACLs = 6;
}
message ContainerStatusProto {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_server_resourcemanager_service_protos.proto Fri May 24 23:11:25 2013
@@ -67,6 +67,7 @@ message GetGroupsForUserResponseProto {
message ApplicationStateDataProto {
optional int64 submit_time = 1;
optional ApplicationSubmissionContextProto application_submission_context = 2;
+ optional string user = 3;
}
message ApplicationAttemptStateDataProto {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri May 24 23:11:25 2013
@@ -698,11 +698,6 @@ public class ApplicationMaster {
ContainerLaunchContext ctx = Records
.newRecord(ContainerLaunchContext.class);
- String jobUserName = System.getenv(ApplicationConstants.Environment.USER
- .key());
- ctx.setUser(jobUserName);
- LOG.info("Setting user in ContainerLaunchContext to: " + jobUserName);
-
// Set the environment
ctx.setEnvironment(shellEnv);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Fri May 24 23:11:25 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.util;
+import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.URI;
import java.nio.ByteBuffer;
@@ -25,6 +26,7 @@ import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -188,6 +190,16 @@ public class BuilderUtils {
return cId;
}
+ public static ContainerToken newContainerToken(ContainerId cId, String host,
+ int port, String user, Resource r, long expiryTime, int masterKeyId,
+ byte[] password) throws IOException {
+ ContainerTokenIdentifier identifier =
+ new ContainerTokenIdentifier(cId, host, user, r, expiryTime,
+ masterKeyId);
+ return newContainerToken(BuilderUtils.newNodeId(host, port), password,
+ identifier);
+ }
+
public static ContainerId newContainerId(RecordFactory recordFactory,
ApplicationId appId, ApplicationAttemptId appAttemptId,
int containerId) {
@@ -286,14 +298,24 @@ public class BuilderUtils {
return containerToken;
}
+ public static ContainerTokenIdentifier newContainerTokenIdentifier(
+ ContainerToken containerToken) throws IOException {
+ org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token =
+ new org.apache.hadoop.security.token.Token<ContainerTokenIdentifier>(
+ containerToken.getIdentifier()
+ .array(), containerToken.getPassword().array(), new Text(
+ containerToken.getKind()),
+ new Text(containerToken.getService()));
+ return token.decodeIdentifier();
+ }
+
public static ContainerLaunchContext newContainerLaunchContext(
- String user, Map<String, LocalResource> localResources,
+ Map<String, LocalResource> localResources,
Map<String, String> environment, List<String> commands,
- Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
+ Map<String, ByteBuffer> serviceData, ByteBuffer tokens,
Map<ApplicationAccessType, String> acls) {
ContainerLaunchContext container = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- container.setUser(user);
container.setLocalResources(localResources);
container.setEnvironment(environment);
container.setCommands(commands);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java Fri May 24 23:11:25 2013
@@ -89,7 +89,6 @@ public class TestContainerLaunchRPC {
server.getListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- containerLaunchContext.setUser("dummy-user");
ContainerId containerId = recordFactory
.newRecordInstance(ContainerId.class);
ApplicationId applicationId = recordFactory
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Fri May 24 23:11:25 2013
@@ -111,7 +111,6 @@ public class TestRPC {
NetUtils.getConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- containerLaunchContext.setUser("dummy-user");
ContainerId containerId =
recordFactory.newRecordInstance(ContainerId.class);
ApplicationId applicationId =
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Fri May 24 23:11:25 2013
@@ -20,8 +20,6 @@ package org.apache.hadoop.yarn.server.no
import static org.apache.hadoop.yarn.service.Service.STATE.STARTED;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
@@ -105,9 +103,9 @@ import org.apache.hadoop.yarn.server.sec
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.RpcUtil;
public class ContainerManagerImpl extends CompositeService implements
ServiceStateChangeListener, ContainerManager,
@@ -321,13 +319,9 @@ public class ContainerManagerImpl extend
// Get the tokenId from the remote user ugi
return selectContainerTokenIdentifier(remoteUgi);
} else {
- ContainerToken containerToken = container.getContainerToken();
- Token<ContainerTokenIdentifier> token =
- new Token<ContainerTokenIdentifier>(containerToken.getIdentifier()
- .array(), containerToken.getPassword().array(), new Text(
- containerToken.getKind()), new Text(containerToken.getService()));
try {
- return token.decodeIdentifier();
+ return BuilderUtils.newContainerTokenIdentifier(container
+ .getContainerToken());
} catch (IOException e) {
throw RPCUtil.getRemoteException(e);
}
@@ -370,15 +364,6 @@ public class ContainerManagerImpl extend
messageBuilder
.append("\nNo ContainerToken found for " + containerIDStr);
} else {
-
- // Is the container coming in with correct user-name?
- if (!launchContext.getUser().equals(tokenId.getApplicationSubmitter())) {
- unauthorized = true;
- messageBuilder.append("\n Expected user-name "
- + tokenId.getApplicationSubmitter() + " but found "
- + launchContext.getUser());
- }
-
// Is the container being relaunched? Or RPC layer let startCall with
// tokens generated off old-secret through?
@@ -451,7 +436,7 @@ public class ContainerManagerImpl extend
}
LOG.info("Start request for " + containerIDStr + " by user "
- + launchContext.getUser());
+ + tokenId.getApplicationSubmitter());
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getTokens();
@@ -473,13 +458,14 @@ public class ContainerManagerImpl extend
}
}
// //////////// End of parsing credentials
+ String user = tokenId.getApplicationSubmitter();
Container container = new ContainerImpl(getConfig(), this.dispatcher,
- launchContext, lauchContainer, credentials, metrics);
+ launchContext, lauchContainer, credentials, metrics, tokenId);
ApplicationId applicationID =
containerID.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerID, container) != null) {
- NMAuditLogger.logFailure(launchContext.getUser(),
+ NMAuditLogger.logFailure(user,
AuditConstants.START_CONTAINER, "ContainerManagerImpl",
"Container already running on this node!",
applicationID, containerID);
@@ -490,7 +476,8 @@ public class ContainerManagerImpl extend
// Create the application
Application application =
new ApplicationImpl(dispatcher, this.aclsManager,
- launchContext.getUser(), applicationID, credentials, context);
+ user, applicationID, credentials,
+ context);
if (null ==
context.getApplications().putIfAbsent(applicationID, application)) {
LOG.info("Creating a new application reference for app "
@@ -506,7 +493,7 @@ public class ContainerManagerImpl extend
this.context.getContainerTokenSecretManager().startContainerSuccessful(
tokenId);
- NMAuditLogger.logSuccess(launchContext.getUser(),
+ NMAuditLogger.logSuccess(user,
AuditConstants.START_CONTAINER, "ContainerManageImpl",
applicationID, containerID);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Fri May 24 23:11:25 2013
@@ -77,7 +77,7 @@ public class ApplicationImpl implements
ApplicationACLsManager aclsManager, String user, ApplicationId appId,
Credentials credentials, Context context) {
this.dispatcher = dispatcher;
- this.user = user.toString();
+ this.user = user;
this.appId = appId;
this.credentials = credentials;
this.aclsManager = aclsManager;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Fri May 24 23:11:25 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -44,6 +45,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -75,6 +77,7 @@ public class ContainerImpl implements Co
private final NodeManagerMetrics metrics;
private final ContainerLaunchContext launchContext;
private final org.apache.hadoop.yarn.api.records.Container container;
+ private final String user;
private int exitCode = ContainerExitStatus.INVALID;
private final StringBuilder diagnostics;
@@ -93,10 +96,11 @@ public class ContainerImpl implements Co
private final List<LocalResourceRequest> appRsrcs =
new ArrayList<LocalResourceRequest>();
- public ContainerImpl(Configuration conf,
- Dispatcher dispatcher, ContainerLaunchContext launchContext,
+ public ContainerImpl(Configuration conf, Dispatcher dispatcher,
+ ContainerLaunchContext launchContext,
org.apache.hadoop.yarn.api.records.Container container,
- Credentials creds, NodeManagerMetrics metrics) {
+ Credentials creds, NodeManagerMetrics metrics,
+ ContainerTokenIdentifier identifier) throws IOException {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.launchContext = launchContext;
@@ -104,7 +108,7 @@ public class ContainerImpl implements Co
this.diagnostics = new StringBuilder();
this.credentials = creds;
this.metrics = metrics;
-
+ user = identifier.getApplicationSubmitter();
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.readLock();
this.writeLock = readWriteLock.writeLock();
@@ -311,7 +315,7 @@ public class ContainerImpl implements Co
public String getUser() {
this.readLock.lock();
try {
- return this.launchContext.getUser();
+ return this.user;
} finally {
this.readLock.unlock();
}
@@ -382,7 +386,6 @@ public class ContainerImpl implements Co
@SuppressWarnings({"fallthrough", "unchecked"})
private void finished() {
ContainerId containerID = this.container.getId();
- String user = this.launchContext.getUser();
switch (getContainerState()) {
case EXITED_WITH_SUCCESS:
metrics.endRunningContainer();
@@ -486,7 +489,7 @@ public class ContainerImpl implements Co
for (Map.Entry<String,ByteBuffer> service : csd.entrySet()) {
container.dispatcher.getEventHandler().handle(
new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT,
- ctxt.getUser(), container.container.getId()
+ container.user, container.container.getId()
.getApplicationAttemptId().getApplicationId(),
service.getKey().toString(), service.getValue()));
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Fri May 24 23:11:25 2013
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
@@ -67,6 +68,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.yarn.util.Apps;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -120,11 +122,11 @@ public class ContainerLaunch implements
container.getLocalizedResources();
ContainerId containerID = container.getContainer().getId();
String containerIdStr = ConverterUtils.toString(containerID);
- final String user = launchContext.getUser();
final List<String> command = launchContext.getCommands();
int ret = -1;
try {
+ final String user = container.getUser();
// /////////////////////////// Variable expansion
// Before the container script gets written out.
List<String> newCmds = new ArrayList<String>(command.size());
@@ -334,7 +336,7 @@ public class ContainerLaunch implements
// kill process
if (processId != null) {
- String user = container.getLaunchContext().getUser();
+ String user = container.getUser();
LOG.debug("Sending signal to pid " + processId
+ " as user " + user
+ " for container " + containerIdStr);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Fri May 24 23:11:25 2013
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -47,6 +48,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
@@ -135,10 +137,16 @@ public class TestEventFlow {
cID.setApplicationAttemptId(applicationAttemptId);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cID);
- when(mockContainer.getResource()).thenReturn(recordFactory
- .newRecordInstance(Resource.class));
+ Resource r = BuilderUtils.newResource(1024, 1);
+ when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getRMIdentifer()).thenReturn(SIMULATED_RM_IDENTIFIER);
- launchContext.setUser("testing");
+ String user = "testing";
+ String host = "127.0.0.1";
+ int port = 1234;
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cID, host, port, user, r,
+ System.currentTimeMillis() + 10000L, 123, "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java Fri May 24 23:11:25 2013
@@ -115,8 +115,6 @@ public class TestNodeManagerReboot {
Records.newRecord(org.apache.hadoop.yarn.api.records.Container.class);
mockContainer.setId(cId);
- containerLaunchContext.setUser(user);
-
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(localResourceDir.getAbsolutePath())));
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Fri May 24 23:11:25 2013
@@ -21,12 +21,10 @@ package org.apache.hadoop.yarn.server.no
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
-import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.PrivilegedAction;
-import java.io.PrintWriter;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -40,8 +38,8 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -66,9 +64,6 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
-import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
-import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
@@ -169,8 +164,7 @@ public class TestNodeManagerShutdown {
NodeId nodeId = BuilderUtils.newNodeId("localhost", 1234);
mockContainer.setNodeId(nodeId);
mockContainer.setNodeHttpAddress("localhost:12345");
- containerLaunchContext.setUser(cId.toString());
-
+
URL localResourceUri =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@@ -186,7 +180,6 @@ public class TestNodeManagerShutdown {
new HashMap<String, LocalResource>();
localResources.put(destinationFile, localResource);
containerLaunchContext.setLocalResources(localResources);
- containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Resource resource = BuilderUtils.newResource(1024, 1);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri May 24 23:11:25 2013
@@ -206,7 +206,7 @@ public class TestNodeStatusUpdater {
when(mockContainer.getResource()).thenReturn(resource);
Container container =
new ContainerImpl(conf, mockDispatcher, launchContext,
- mockContainer, null, mockMetrics);
+ mockContainer, null, mockMetrics, null);
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@@ -232,7 +232,7 @@ public class TestNodeStatusUpdater {
when(mockContainer.getResource()).thenReturn(resource);
Container container =
new ContainerImpl(conf, mockDispatcher, launchContext,
- mockContainer, null, mockMetrics);
+ mockContainer, null, mockMetrics, null);
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
@@ -665,8 +665,8 @@ public class TestNodeStatusUpdater {
}
waitCount = 0;
- while (heartBeatID <= 3 && waitCount++ != 20) {
- Thread.sleep(500);
+ while (heartBeatID <= 3 && waitCount++ != 200) {
+ Thread.sleep(1000);
}
Assert.assertFalse(heartBeatID <= 3);
Assert.assertEquals("Number of registered NMs is wrong!!", 1,
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Fri May 24 23:11:25 2013
@@ -182,16 +182,6 @@ public abstract class BaseContainerManag
throws YarnRemoteException {
// do nothing
}
-
- @Override
- protected ContainerTokenIdentifier getContainerTokenIdentifier(
- UserGroupInformation remoteUgi,
- org.apache.hadoop.yarn.api.records.Container container)
- throws YarnRemoteException {
- return new ContainerTokenIdentifier(container.getId(),
- container.getNodeHttpAddress(), remoteUgi.getUserName(),
- container.getResource(), System.currentTimeMillis(), 123);
- }
};
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Fri May 24 23:11:25 2013
@@ -26,7 +26,6 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
@@ -49,6 +48,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -58,7 +58,6 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.api.ResourceManagerConstants;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
-import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
@@ -130,8 +129,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
- container.setUser(user);
-
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
@@ -149,17 +146,21 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- containerLaunchContext.setUser(container.getUser());
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(512, 1));
+ Resource r = BuilderUtils.newResource(512, 1);
+ when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":12345");
+ context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
-
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
+ port, user, r, System.currentTimeMillis() + 10000L, 123,
+ "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
@@ -230,8 +231,6 @@ public class TestContainerManager extend
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- containerLaunchContext.setUser(user);
-
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@@ -247,17 +246,22 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(100, 1)); // MB
+ Resource r = BuilderUtils.newResource(100, 1);
+ when(mockContainer.getResource()).thenReturn(r); // MB
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":12345");
+ context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
+ port, user, r, System.currentTimeMillis() + 10000L, 123,
+ "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
@@ -343,8 +347,6 @@ public class TestContainerManager extend
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- containerLaunchContext.setUser(user);
-
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@@ -360,19 +362,22 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(100, 1)); // MB
-
+ Resource r = BuilderUtils.newResource(100, 1);
+ when(mockContainer.getResource()).thenReturn(r); // MB
+ int port = 12345;
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":12345");
+ context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
-
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
+ port, user, r, System.currentTimeMillis() + 10000L, 123,
+ "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);
@@ -439,8 +444,6 @@ public class TestContainerManager extend
ContainerId cId = createContainerId();
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
- container.setUser(user);
-
// ////// Construct the container-spec.
ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
// containerLaunchContext.resources =
@@ -459,18 +462,22 @@ public class TestContainerManager extend
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- containerLaunchContext.setUser(container.getUser());
Container mockContainer = mock(Container.class);
when(mockContainer.getId()).thenReturn(cId);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(100, 1));
+ Resource r = BuilderUtils.newResource(100, 1);
+ when(mockContainer.getResource()).thenReturn(r);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":12345");
+ context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
// containerLaunchContext.command = new ArrayList<CharSequence>();
-
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
+ port, user, r, System.currentTimeMillis() + 10000L, 123,
+ "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
request.setContainer(mockContainer);
@@ -544,13 +551,13 @@ public class TestContainerManager extend
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ String host = "127.0.0.1";
+ int port = 1234;
ContainerId cId1 = createContainerId();
ContainerId cId2 = createContainerId();
- containerLaunchContext.setUser(user);
containerLaunchContext
.setLocalResources(new HashMap<String, LocalResource>());
- containerLaunchContext.setUser(containerLaunchContext.getUser());
- Resource mockResource = mock(Resource.class);
+ Resource mockResource = BuilderUtils.newResource(1024, 1);
Container mockContainer1 = mock(Container.class);
when(mockContainer1.getId()).thenReturn(cId1);
@@ -560,6 +567,11 @@ public class TestContainerManager extend
StartContainerRequest startRequest1 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest1.setContainerLaunchContext(containerLaunchContext);
+
+ ContainerToken containerToken1 =
+ BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
+ System.currentTimeMillis() + 10000, 123, "password".getBytes());
+ when(mockContainer1.getContainerToken()).thenReturn(containerToken1);
startRequest1.setContainer(mockContainer1);
boolean catchException = false;
try {
@@ -586,6 +598,11 @@ public class TestContainerManager extend
StartContainerRequest startRequest2 =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest2.setContainerLaunchContext(containerLaunchContext);
+ ContainerToken containerToken2 =
+ BuilderUtils.newContainerToken(cId1, host, port, user, mockResource,
+ System.currentTimeMillis() + 10000, 123, "password".getBytes());
+ when(mockContainer2.getContainerToken()).thenReturn(containerToken2);
+
startRequest2.setContainer(mockContainer2);
boolean noException = true;
try {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Fri May 24 23:11:25 2013
@@ -25,6 +25,7 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
import java.util.AbstractMap.SimpleEntry;
@@ -44,6 +45,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -53,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEventType;
@@ -524,8 +527,10 @@ public class TestContainer {
}
private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx,
- org.apache.hadoop.yarn.api.records.Container container) {
- return new ContainerImpl(conf, disp, ctx, container, null, metrics);
+ org.apache.hadoop.yarn.api.records.Container container,
+ ContainerTokenIdentifier identifier) throws IOException {
+ return new ContainerImpl(conf, disp, ctx, container, null, metrics,
+ identifier);
}
@SuppressWarnings("unchecked")
@@ -545,12 +550,13 @@ public class TestContainer {
final Map<String, ByteBuffer> serviceData;
final String user;
- WrappedContainer(int appId, long timestamp, int id, String user) {
+ WrappedContainer(int appId, long timestamp, int id, String user)
+ throws IOException {
this(appId, timestamp, id, user, true, false);
}
WrappedContainer(int appId, long timestamp, int id, String user,
- boolean withLocalRes, boolean withServiceData) {
+ boolean withLocalRes, boolean withServiceData) throws IOException {
dispatcher = new DrainDispatcher();
dispatcher.init(new Configuration());
@@ -572,12 +578,19 @@ public class TestContainer {
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
cId = BuilderUtils.newContainerId(appId, 1, timestamp, id);
- when(ctxt.getUser()).thenReturn(this.user);
when(mockContainer.getId()).thenReturn(cId);
Resource resource = BuilderUtils.newResource(1024, 1);
when(mockContainer.getResource()).thenReturn(resource);
-
+ String host = "127.0.0.1";
+ int port = 1234;
+ ContainerTokenIdentifier identifier =
+ new ContainerTokenIdentifier(cId, "127.0.0.1", user, resource,
+ System.currentTimeMillis() + 10000L, 123);
+ ContainerToken token =
+ BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port),
+ "password".getBytes(), identifier);
+ when(mockContainer.getContainerToken()).thenReturn(token);
if (withLocalRes) {
Random r = new Random();
long seed = r.nextLong();
@@ -600,7 +613,7 @@ public class TestContainer {
}
when(ctxt.getServiceData()).thenReturn(serviceData);
- c = newContainer(dispatcher, ctxt, mockContainer);
+ c = newContainer(dispatcher, ctxt, mockContainer, identifier);
dispatcher.start();
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Fri May 24 23:11:25 2013
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.BufferedReader;
import java.io.File;
@@ -26,14 +28,14 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.PrintWriter;
-import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import junit.framework.Assert;
+
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -50,9 +52,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
@@ -67,9 +71,6 @@ import org.apache.hadoop.yarn.util.Linux
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.junit.Before;
import org.junit.Test;
-import static org.mockito.Mockito.*;
-
-import junit.framework.Assert;
public class TestContainerLaunch extends BaseContainerManagerTest {
@@ -157,7 +158,7 @@ public class TestContainerLaunch extends
* See if environment variable is forwarded using sanitizeEnv.
* @throws Exception
*/
- @Test (timeout = 5000)
+ @Test (timeout = 60000)
public void testContainerEnvVariables() throws Exception {
containerManager.start();
@@ -175,12 +176,13 @@ public class TestContainerLaunch extends
appAttemptId.setAttemptId(1);
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
+ int port = 12345;
cId.setApplicationAttemptId(appAttemptId);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":12345");
+ context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
Map<String, String> userSetEnv = new HashMap<String, String>();
@@ -189,7 +191,6 @@ public class TestContainerLaunch extends
userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
- containerLaunchContext.setUser(user);
containerLaunchContext.setEnvironment(userSetEnv);
File scriptFile = new File(tmpDir, "scriptFile.sh");
@@ -243,13 +244,17 @@ public class TestContainerLaunch extends
containerLaunchContext.setLocalResources(localResources);
// set up the rest of the container
- containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(1024, 1));
+ Resource r = BuilderUtils.newResource(1024, 1);
+ when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
+ port, user, r, System.currentTimeMillis() + 10000L, 1234,
+ "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
@@ -376,12 +381,11 @@ public class TestContainerLaunch extends
recordFactory.newRecordInstance(ContainerLaunchContext.class);
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":12345");
+ context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
- containerLaunchContext.setUser(user);
-
// upload the script file so that the container can run it
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@@ -400,11 +404,15 @@ public class TestContainerLaunch extends
containerLaunchContext.setLocalResources(localResources);
// set up the rest of the container
- containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
containerLaunchContext.setCommands(commands);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(1024, 1));
+ Resource r = BuilderUtils.newResource(1024, 1);
+ when(mockContainer.getResource()).thenReturn(r);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
+ port, user, r, System.currentTimeMillis() + 10000L, 123,
+ "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
startRequest.setContainer(mockContainer);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Fri May 24 23:11:25 2013
@@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -75,6 +76,7 @@ import org.apache.hadoop.yarn.logaggrega
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogKey;
import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat.LogReader;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
@@ -694,8 +696,6 @@ public class TestLogAggregationService e
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getRMIdentifer()).thenReturn(super.DUMMY_RM_IDENTIFIER);
- containerLaunchContext.setUser(this.user);
-
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
.makeQualified(new Path(scriptFile.getAbsolutePath())));
@@ -711,13 +711,15 @@ public class TestLogAggregationService e
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(100 * 1024 * 1024, 1));
+ Resource r = BuilderUtils.newResource(100 * 1024 * 1024, 1);
+ when(mockContainer.getResource()).thenReturn(r);
+ when(mockContainer.getContainerToken()).thenReturn(
+ BuilderUtils.newContainerToken(cId, "127.0.0.1", 1234, user, r,
+ System.currentTimeMillis() + 10000L, 123, "password".getBytes()));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Fri May 24 23:11:25 2013
@@ -49,9 +49,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -215,11 +217,11 @@ public class TestContainersMonitor exten
when(mockContainer.getId()).thenReturn(cId);
when(mockContainer.getNodeId()).thenReturn(context.getNodeId());
+ int port = 12345;
when(mockContainer.getNodeHttpAddress()).thenReturn(
- context.getNodeId().getHost() + ":12345");
+ context.getNodeId().getHost() + ":" + port);
when(mockContainer.getRMIdentifer()).thenReturn(
super.DUMMY_RM_IDENTIFIER);
- containerLaunchContext.setUser(user);
URL resource_alpha =
ConverterUtils.getYarnUrlFromPath(localFS
@@ -236,16 +238,20 @@ public class TestContainersMonitor exten
new HashMap<String, LocalResource>();
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
- containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = new ArrayList<String>();
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- when(mockContainer.getResource()).thenReturn(
- BuilderUtils.newResource(8 * 1024 * 1024, 1));
+ Resource r = BuilderUtils.newResource(8 * 1024 * 1024, 1);
+ when(mockContainer.getResource()).thenReturn(r);
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(cId, context.getNodeId().getHost(),
+ port, user, r, System.currentTimeMillis() + 10000L, 123,
+ "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java Fri May 24 23:11:25 2013
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -30,7 +34,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -38,7 +42,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.util.BuilderUtils;
-import static org.mockito.Mockito.*;
public class MockContainer implements Container {
@@ -53,7 +56,7 @@ public class MockContainer implements Co
public MockContainer(ApplicationAttemptId appAttemptId,
Dispatcher dispatcher, Configuration conf, String user,
- ApplicationId appId, int uniqId) {
+ ApplicationId appId, int uniqId) throws IOException{
this.user = user;
this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
@@ -61,10 +64,14 @@ public class MockContainer implements Co
uniqId);
this.launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- launchContext.setUser(user);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(id, "127.0.0.1", 1234, user,
+ BuilderUtils.newResource(1024, 1),
+ System.currentTimeMillis() + 10000, 123, "password".getBytes());
this.state = ContainerState.NEW;
mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
when(mockContainer.getId()).thenReturn(id);
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Fri May 24 23:11:25 2013
@@ -33,6 +33,8 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -181,11 +183,16 @@ public class TestNMWebServer {
recordFactory.newRecordInstance(ContainerLaunchContext.class);
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
+ BuilderUtils.newResource(1024, 1),
+ System.currentTimeMillis() + 10000L, 123, "password".getBytes());
+ when(mockContainer.getContainerToken()).thenReturn(containerToken);
when(mockContainer.getId()).thenReturn(containerId);
- launchContext.setUser(user);
Container container =
new ContainerImpl(conf, dispatcher, launchContext, mockContainer,
- null, metrics) {
+ null, metrics,
+ BuilderUtils.newContainerTokenIdentifier(containerToken)) {
@Override
public ContainerState getContainerState() {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java?rev=1486251&r1=1486250&r2=1486251&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java Fri May 24 23:11:25 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.File;
+import java.io.IOException;
import java.io.StringReader;
import java.util.HashMap;
@@ -177,7 +178,8 @@ public class TestNMWebServicesApps exten
assertEquals("apps isn't NULL", JSONObject.NULL, json.get("apps"));
}
- private HashMap<String, String> addAppContainers(Application app) {
+ private HashMap<String, String> addAppContainers(Application app)
+ throws IOException {
Dispatcher dispatcher = new AsyncDispatcher();
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app.getAppId(), 1);