You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ss...@apache.org on 2013/05/31 06:14:29 UTC
svn commit: r1488086 [1/3] - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocol...
Author: sseth
Date: Fri May 31 04:14:27 2013
New Revision: 1488086
URL: http://svn.apache.org/r1488086
Log:
YARN-684. ContainerManager.startContainer should use ContainerTokenIdentifier instead of the entire Container. Contributed by Vinod Kumar Vavilapalli.
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri May 31 04:14:27 2013
@@ -54,6 +54,10 @@ Release 2.0.5-beta - UNRELEASED
YARN-716. Making ApplicationID immutable. (Siddharth Seth via vinodkv)
+ YARN-684. ContainerManager.startContainer should use
+ ContainerTokenIdentifier instead of the entire Container.
+ (Vinod Kumar Vavilapalli via sseth)
+
NEW FEATURES
YARN-482. FS: Extend SchedulingMode to intermediate queues.
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/StartContainerRequest.java Fri May 31 04:14:27 2013
@@ -21,8 +21,8 @@ package org.apache.hadoop.yarn.api.proto
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ContainerManager;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
/**
* <p>The request sent by the <code>ApplicationMaster</code> to the
@@ -62,9 +62,9 @@ public interface StartContainerRequest {
@Public
@Stable
- public Container getContainer();
+ public ContainerToken getContainerToken();
@Public
@Stable
- public void setContainer(Container container);
+ public void setContainerToken(ContainerToken container);
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/StartContainerRequestPBImpl.java Fri May 31 04:14:27 2013
@@ -19,14 +19,14 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.ProtoBase;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerLaunchContextPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerTokenPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerLaunchContextProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StartContainerRequestProtoOrBuilder;
@@ -39,7 +39,7 @@ public class StartContainerRequestPBImpl
private ContainerLaunchContext containerLaunchContext = null;
- private Container container = null;
+ private ContainerToken containerToken = null;
public StartContainerRequestPBImpl() {
builder = StartContainerRequestProto.newBuilder();
@@ -61,8 +61,8 @@ public class StartContainerRequestPBImpl
if (this.containerLaunchContext != null) {
builder.setContainerLaunchContext(convertToProtoFormat(this.containerLaunchContext));
}
- if(this.container != null) {
- builder.setContainer(convertToProtoFormat(this.container));
+ if(this.containerToken != null) {
+ builder.setContainerToken(convertToProtoFormat(this.containerToken));
}
}
@@ -104,25 +104,25 @@ public class StartContainerRequestPBImpl
}
@Override
- public Container getContainer() {
+ public ContainerToken getContainerToken() {
StartContainerRequestProtoOrBuilder p = viaProto ? proto : builder;
- if (this.container != null) {
- return this.container;
+ if (this.containerToken != null) {
+ return this.containerToken;
}
- if (!p.hasContainer()) {
+ if (!p.hasContainerToken()) {
return null;
}
- this.container = convertFromProtoFormat(p.getContainer());
- return this.container;
+ this.containerToken = convertFromProtoFormat(p.getContainerToken());
+ return this.containerToken;
}
@Override
- public void setContainer(Container container) {
+ public void setContainerToken(ContainerToken containerToken) {
maybeInitBuilder();
- if(container == null) {
- builder.clearContainer();
+ if(containerToken == null) {
+ builder.clearContainerToken();
}
- this.container = container;
+ this.containerToken = containerToken;
}
private ContainerLaunchContextPBImpl convertFromProtoFormat(ContainerLaunchContextProto p) {
@@ -135,11 +135,11 @@ public class StartContainerRequestPBImpl
- private ContainerPBImpl convertFromProtoFormat(ContainerProto containerProto) {
- return new ContainerPBImpl(containerProto);
+ private ContainerTokenPBImpl convertFromProtoFormat(TokenProto containerProto) {
+ return new ContainerTokenPBImpl(containerProto);
}
- private ContainerProto convertToProtoFormat(Container container) {
- return ((ContainerPBImpl)container).getProto();
+ private TokenProto convertToProtoFormat(ContainerToken container) {
+ return ((ContainerTokenPBImpl)container).getProto();
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto Fri May 31 04:14:27 2013
@@ -173,7 +173,7 @@ message GetQueueUserAclsInfoResponseProt
message StartContainerRequestProto {
optional ContainerLaunchContextProto container_launch_context = 1;
- optional ContainerProto container = 2;
+ optional hadoop.common.TokenProto container_token = 2;
}
message StartContainerResponseProto {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java Fri May 31 04:14:27 2013
@@ -763,7 +763,7 @@ public class ApplicationMaster {
StartContainerRequest startReq = Records
.newRecord(StartContainerRequest.class);
startReq.setContainerLaunchContext(ctx);
- startReq.setContainer(container);
+ startReq.setContainerToken(container.getContainerToken());
try {
cm.startContainer(startReq);
} catch (YarnRemoteException e) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/NMClientImpl.java Fri May 31 04:14:27 2013
@@ -212,7 +212,7 @@ public class NMClientImpl extends Abstra
try {
StartContainerRequest startRequest =
Records.newRecord(StartContainerRequest.class);
- startRequest.setContainer(container);
+ startRequest.setContainerToken(container.getContainerToken());
startRequest.setContainerLaunchContext(containerLaunchContext);
startResponse = containerManager.startContainer(startRequest);
if (LOG.isDebugEnabled()) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java Fri May 31 04:14:27 2013
@@ -39,11 +39,11 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
@@ -89,23 +90,25 @@ public class TestContainerLaunchRPC {
server.getListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- ContainerId containerId = recordFactory
- .newRecordInstance(ContainerId.class);
+
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
- ApplicationAttemptId applicationAttemptId = recordFactory
- .newRecordInstance(ApplicationAttemptId.class);
- applicationAttemptId.setApplicationId(applicationId);
- applicationAttemptId.setAttemptId(0);
- containerId.setApplicationAttemptId(applicationAttemptId);
- containerId.setId(100);
- Container container =
- BuilderUtils.newContainer(containerId, null, null, recordFactory
- .newRecordInstance(Resource.class), null, null);
+ ApplicationAttemptId applicationAttemptId =
+ ApplicationAttemptId.newInstance(applicationId, 0);
+ ContainerId containerId =
+ ContainerId.newInstance(applicationAttemptId, 100);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ Resource resource = Resource.newInstance(1234, 2);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(containerId, "localhost", "user",
+ resource, System.currentTimeMillis() + 10000, 42, 42);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(nodeId, "password".getBytes(),
+ containerTokenIdentifier);
StartContainerRequest scRequest = recordFactory
.newRecordInstance(StartContainerRequest.class);
scRequest.setContainerLaunchContext(containerLaunchContext);
- scRequest.setContainer(container);
+ scRequest.setContainerToken(containerToken);
try {
proxy.startContainer(scRequest);
} catch (Exception e) {
@@ -138,9 +141,6 @@ public class TestContainerLaunchRPC {
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException, IOException {
- StartContainerResponse response = recordFactory
- .newRecordInstance(StartContainerResponse.class);
- status = recordFactory.newRecordInstance(ContainerStatus.class);
try {
// make the thread sleep to look like its not going to respond
Thread.sleep(10000);
@@ -148,10 +148,7 @@ public class TestContainerLaunchRPC {
LOG.error(e);
throw new YarnRemoteException(e);
}
- status.setState(ContainerState.RUNNING);
- status.setContainerId(request.getContainer().getId());
- status.setExitStatus(0);
- return response;
+ throw new YarnRemoteException("Shouldn't happen!!");
}
@Override
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Fri May 31 04:14:27 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn;
+import java.io.IOException;
import java.net.InetSocketAddress;
import junit.framework.Assert;
@@ -39,18 +40,21 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
@@ -111,30 +115,29 @@ public class TestRPC {
NetUtils.getConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- ContainerId containerId =
- recordFactory.newRecordInstance(ContainerId.class);
+
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
ApplicationAttemptId applicationAttemptId =
- recordFactory.newRecordInstance(ApplicationAttemptId.class);
- applicationAttemptId.setApplicationId(applicationId);
- applicationAttemptId.setAttemptId(0);
- containerId.setApplicationAttemptId(applicationAttemptId);
- containerId.setId(100);
- Container mockContainer =
- BuilderUtils.newContainer(containerId, null, null, recordFactory
- .newRecordInstance(Resource.class), null, null);
-// containerLaunchContext.env = new HashMap<CharSequence, CharSequence>();
-// containerLaunchContext.command = new ArrayList<CharSequence>();
-
- StartContainerRequest scRequest =
+ ApplicationAttemptId.newInstance(applicationId, 0);
+ ContainerId containerId =
+ ContainerId.newInstance(applicationAttemptId, 100);
+ StartContainerRequest scRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
scRequest.setContainerLaunchContext(containerLaunchContext);
- scRequest.setContainer(mockContainer);
+ NodeId nodeId = NodeId.newInstance("localhost", 1234);
+ Resource resource = Resource.newInstance(1234, 2);
+ ContainerTokenIdentifier containerTokenIdentifier =
+ new ContainerTokenIdentifier(containerId, "localhost", "user",
+ resource, System.currentTimeMillis() + 10000, 42, 42);
+ ContainerToken containerToken =
+ BuilderUtils.newContainerToken(nodeId, "password".getBytes(),
+ containerTokenIdentifier);
+ scRequest.setContainerToken(containerToken);
proxy.startContainer(scRequest);
GetContainerStatusRequest gcsRequest =
recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(mockContainer.getId());
+ gcsRequest.setContainerId(containerId);
GetContainerStatusResponse response = proxy.getContainerStatus(gcsRequest);
ContainerStatus status = response.getStatus();
@@ -142,7 +145,7 @@ public class TestRPC {
boolean exception = false;
try {
StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(mockContainer.getId());
+ stopRequest.setContainerId(containerId);
proxy.stopContainer(stopRequest);
} catch (YarnRemoteException e) {
exception = true;
@@ -176,11 +179,19 @@ public class TestRPC {
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException {
+ ContainerToken containerToken = request.getContainerToken();
+ ContainerTokenIdentifier tokenId = null;
+
+ try {
+ tokenId = BuilderUtils.newContainerTokenIdentifier(containerToken);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
StartContainerResponse response =
recordFactory.newRecordInstance(StartContainerResponse.class);
status = recordFactory.newRecordInstance(ContainerStatus.class);
status.setState(ContainerState.RUNNING);
- status.setContainerId(request.getContainer().getId());
+ status.setContainerId(tokenId.getContainerID());
status.setExitStatus(0);
return response;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Fri May 31 04:14:27 2013
@@ -42,6 +42,13 @@ public interface Context {
*/
NodeId getNodeId();
+ /**
+ * Return the node http-address. Usable only after the Webserver is started.
+ *
+ * @return the http-port
+ */
+ int getHttpPort();
+
ConcurrentMap<ApplicationId, Application> getApplications();
ConcurrentMap<ContainerId, Container> getContainers();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Fri May 31 04:14:27 2013
@@ -113,7 +113,7 @@ public class DefaultContainerExecutor ex
List<String> localDirs, List<String> logDirs) throws IOException {
FsPermission dirPerm = new FsPermission(APPDIR_PERM);
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
// create container dirs on all disks
String containerIdStr = ConverterUtils.toString(containerId);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Fri May 31 04:14:27 2013
@@ -217,11 +217,11 @@ public class LinuxContainerExecutor exte
String user, String appId, Path containerWorkDir,
List<String> localDirs, List<String> logDirs) throws IOException {
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
String containerIdStr = ConverterUtils.toString(containerId);
resourcesHandler.preExecute(containerId,
- container.getContainer().getResource());
+ container.getResource());
String resourcesOptions = resourcesHandler.getResourcesOption(
containerId);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri May 31 04:14:27 2013
@@ -32,7 +32,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
@@ -56,7 +55,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.CompositeService;
-import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.util.Records;
import com.google.common.annotations.VisibleForTesting;
@@ -173,9 +171,10 @@ public class NodeManager extends Composi
addService(containerManager);
((NMContext) context).setContainerManager(containerManager);
- Service webServer = createWebServer(context, containerManager
+ WebServer webServer = createWebServer(context, containerManager
.getContainersMonitor(), this.aclsManager, dirsHandler);
addService(webServer);
+ ((NMContext) context).setWebServer(webServer);
dispatcher.register(ContainerManagerEventType.class, containerManager);
dispatcher.register(NodeManagerEventType.class, this);
@@ -297,6 +296,7 @@ public class NodeManager extends Composi
private final NMContainerTokenSecretManager containerTokenSecretManager;
private ContainerManager containerManager;
+ private WebServer webServer;
private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
.getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
@@ -316,6 +316,11 @@ public class NodeManager extends Composi
}
@Override
+ public int getHttpPort() {
+ return this.webServer.getPort();
+ }
+
+ @Override
public ConcurrentMap<ApplicationId, Application> getApplications() {
return this.applications;
}
@@ -342,6 +347,10 @@ public class NodeManager extends Composi
public void setContainerManager(ContainerManager containerManager) {
this.containerManager = containerManager;
}
+
+ public void setWebServer(WebServer webServer) {
+ this.webServer = webServer;
+ }
}
@@ -352,12 +361,6 @@ public class NodeManager extends Composi
return nodeHealthChecker;
}
- private void reboot() {
- LOG.info("Rebooting the node manager.");
- NodeManager nodeManager = createNewNodeManager();
- nodeManager.initAndStartNodeManager(this.getConfig(), true);
- }
-
private void initAndStartNodeManager(Configuration conf, boolean hasToReboot) {
try {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri May 31 04:14:27 2013
@@ -155,13 +155,8 @@ public class NodeStatusUpdaterImpl exten
// NodeManager is the last service to start, so NodeId is available.
this.nodeId = this.context.getNodeId();
-
- InetSocketAddress httpBindAddress = getConfig().getSocketAddr(
- YarnConfiguration.NM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_NM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_NM_WEBAPP_PORT);
+ this.httpPort = this.context.getHttpPort();
try {
- this.httpPort = httpBindAddress.getPort();
// Registration has to be in start so that ContainerManager can get the
// perNM tokens needed to authenticate ContainerTokens.
registerWithRM();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Fri May 31 04:14:27 2013
@@ -52,7 +52,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -272,14 +272,13 @@ public class ContainerManagerImpl extend
}
// Get the remoteUGI corresponding to the api call.
- private UserGroupInformation getRemoteUgi(String containerIDStr)
+ private UserGroupInformation getRemoteUgi()
throws YarnRemoteException {
UserGroupInformation remoteUgi;
try {
remoteUgi = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
- String msg = "Cannot obtain the user-name for containerId: "
- + containerIDStr + ". Got exception: "
+ String msg = "Cannot obtain the user-name. Got exception: "
+ StringUtils.stringifyException(e);
LOG.warn(msg);
throw RPCUtil.getRemoteException(msg);
@@ -307,7 +306,7 @@ public class ContainerManagerImpl extend
@VisibleForTesting
protected ContainerTokenIdentifier getContainerTokenIdentifier(
UserGroupInformation remoteUgi,
- org.apache.hadoop.yarn.api.records.Container container)
+ ContainerTokenIdentifier containerTokenIdentifier)
throws YarnRemoteException {
if (UserGroupInformation.isSecurityEnabled()) {
if (LOG.isDebugEnabled()) {
@@ -317,12 +316,7 @@ public class ContainerManagerImpl extend
// Get the tokenId from the remote user ugi
return selectContainerTokenIdentifier(remoteUgi);
} else {
- try {
- return BuilderUtils.newContainerTokenIdentifier(container
- .getContainerToken());
- } catch (IOException e) {
- throw RPCUtil.getRemoteException(e);
- }
+ return containerTokenIdentifier;
}
}
@@ -341,7 +335,6 @@ public class ContainerManagerImpl extend
@VisibleForTesting
protected void authorizeRequest(String containerIDStr,
ContainerLaunchContext launchContext,
- org.apache.hadoop.yarn.api.records.Container container,
UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
throws YarnRemoteException {
@@ -380,13 +373,6 @@ public class ContainerManagerImpl extend
+ System.currentTimeMillis() + " found "
+ tokenId.getExpiryTimeStamp());
}
-
- Resource resource = tokenId.getResource();
- if (resource == null || !resource.equals(container.getResource())) {
- unauthorized = true;
- messageBuilder.append("\nExpected resource " + resource
- + " but found " + container.getResource());
- }
}
}
@@ -412,16 +398,23 @@ public class ContainerManagerImpl extend
}
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
- org.apache.hadoop.yarn.api.records.Container lauchContainer =
- request.getContainer();
- ContainerId containerID = lauchContainer.getId();
- String containerIDStr = containerID.toString();
+ ContainerToken token = request.getContainerToken();
- UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ ContainerTokenIdentifier tokenIdentifier = null;
+ try {
+ tokenIdentifier = BuilderUtils.newContainerTokenIdentifier(token);
+ } catch (IOException e) {
+ throw RPCUtil.getRemoteException(e);
+ }
+
+ UserGroupInformation remoteUgi = getRemoteUgi();
ContainerTokenIdentifier tokenId =
- getContainerTokenIdentifier(remoteUgi, lauchContainer);
- authorizeRequest(containerIDStr, launchContext, lauchContainer, remoteUgi,
- tokenId);
+ getContainerTokenIdentifier(remoteUgi, tokenIdentifier);
+
+ ContainerId containerID = tokenId.getContainerID();
+ String containerIDStr = containerID.toString();
+
+ authorizeRequest(containerIDStr, launchContext, remoteUgi, tokenId);
// Is the container coming from unknown RM
if (tokenId.getRMIdentifer() != nodeStatusUpdater
@@ -458,8 +451,9 @@ public class ContainerManagerImpl extend
// //////////// End of parsing credentials
String user = tokenId.getApplicationSubmitter();
- Container container = new ContainerImpl(getConfig(), this.dispatcher,
- launchContext, lauchContainer, credentials, metrics, tokenId);
+ Container container =
+ new ContainerImpl(getConfig(), this.dispatcher, launchContext,
+ credentials, metrics, tokenId);
ApplicationId applicationID =
containerID.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerID, container) != null) {
@@ -501,7 +495,7 @@ public class ContainerManagerImpl extend
// TODO launchedContainer misplaced -> doesn't necessarily mean a container
// launch. A finished Application will not launch containers.
metrics.launchedContainer();
- metrics.allocateContainer(lauchContainer.getResource());
+ metrics.allocateContainer(tokenId.getResource());
return response;
}
@@ -518,7 +512,7 @@ public class ContainerManagerImpl extend
// TODO: Only the container's owner can kill containers today.
- UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ UserGroupInformation remoteUgi = getRemoteUgi();
Container container = this.context.getContainers().get(containerID);
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
@@ -532,8 +526,8 @@ public class ContainerManagerImpl extend
containerID);
return response; // Return immediately.
}
- authorizeRequest(containerIDStr, null, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainer()));
+ authorizeRequest(containerIDStr, null, remoteUgi,
+ getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
dispatcher.getEventHandler().handle(
new ContainerKillEvent(containerID,
@@ -561,15 +555,15 @@ public class ContainerManagerImpl extend
// TODO: Only the container's owner can get containers' status today.
- UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ UserGroupInformation remoteUgi = getRemoteUgi();
LOG.info("Getting container-status for " + containerIDStr);
Container container = this.context.getContainers().get(containerID);
if (container == null) {
throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " is not handled by this NodeManager");
}
- authorizeRequest(containerIDStr, null, null, remoteUgi,
- getContainerTokenIdentifier(remoteUgi, container.getContainer()));
+ authorizeRequest(containerIDStr, null, remoteUgi,
+ getContainerTokenIdentifier(remoteUgi, container.getContainerTokenIdentifier()));
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
LOG.info("Returning " + containerStatus);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerInitEvent.java Fri May 31 04:14:27 2013
@@ -36,7 +36,7 @@ public class ApplicationContainerInitEve
final Container container;
public ApplicationContainerInitEvent(Container container) {
- super(container.getContainer().getId().getApplicationAttemptId()
+ super(container.getContainerId().getApplicationAttemptId()
.getApplicationId(), ApplicationEventType.INIT_CONTAINER);
this.container = container;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Fri May 31 04:14:27 2013
@@ -273,14 +273,14 @@ public class ApplicationImpl implements
ApplicationContainerInitEvent initEvent =
(ApplicationContainerInitEvent) event;
Container container = initEvent.getContainer();
- app.containers.put(container.getContainer().getId(), container);
- LOG.info("Adding " + container.getContainer().getId()
+ app.containers.put(container.getContainerId(), container);
+ LOG.info("Adding " + container.getContainerId()
+ " to application " + app.toString());
switch (app.getApplicationState()) {
case RUNNING:
app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
- container.getContainer().getId()));
+ container.getContainerId()));
break;
case INITING:
case NEW:
@@ -301,7 +301,7 @@ public class ApplicationImpl implements
// Start all the containers waiting for ApplicationInit
for (Container container : app.containers.values()) {
app.dispatcher.getEventHandler().handle(new ContainerInitEvent(
- container.getContainer().getId()));
+ container.getContainerId()));
}
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java Fri May 31 04:14:27 2013
@@ -23,13 +23,20 @@ import java.util.Map;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
public interface Container extends EventHandler<ContainerEvent> {
- org.apache.hadoop.yarn.api.records.Container getContainer();
+ ContainerId getContainerId();
+
+ Resource getResource();
+
+ ContainerTokenIdentifier getContainerTokenIdentifier();
String getUser();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Fri May 31 04:14:27 2013
@@ -37,11 +37,13 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.ContainerExitStatus;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -76,7 +78,9 @@ public class ContainerImpl implements Co
private final Credentials credentials;
private final NodeManagerMetrics metrics;
private final ContainerLaunchContext launchContext;
- private final org.apache.hadoop.yarn.api.records.Container container;
+ private final ContainerTokenIdentifier containerTokenIdentifier;
+ private final ContainerId containerId;
+ private final Resource resource;
private final String user;
private int exitCode = ContainerExitStatus.INVALID;
private final StringBuilder diagnostics;
@@ -97,18 +101,19 @@ public class ContainerImpl implements Co
new ArrayList<LocalResourceRequest>();
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
- ContainerLaunchContext launchContext,
- org.apache.hadoop.yarn.api.records.Container container,
- Credentials creds, NodeManagerMetrics metrics,
- ContainerTokenIdentifier identifier) throws IOException {
+ ContainerLaunchContext launchContext, Credentials creds,
+ NodeManagerMetrics metrics,
+ ContainerTokenIdentifier containerTokenIdentifier) throws IOException {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.launchContext = launchContext;
- this.container = container;
+ this.containerTokenIdentifier = containerTokenIdentifier;
+ this.containerId = containerTokenIdentifier.getContainerID();
+ this.resource = containerTokenIdentifier.getResource();
this.diagnostics = new StringBuilder();
this.credentials = creds;
this.metrics = metrics;
- user = identifier.getApplicationSubmitter();
+ user = containerTokenIdentifier.getApplicationSubmitter();
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.readLock();
this.writeLock = readWriteLock.writeLock();
@@ -366,7 +371,7 @@ public class ContainerImpl implements Co
public ContainerStatus cloneAndGetContainerStatus() {
this.readLock.lock();
try {
- return BuilderUtils.newContainerStatus(this.container.getId(),
+ return BuilderUtils.newContainerStatus(this.containerId,
getCurrentState(), diagnostics.toString(), exitCode);
} finally {
this.readLock.unlock();
@@ -374,10 +379,20 @@ public class ContainerImpl implements Co
}
@Override
- public org.apache.hadoop.yarn.api.records.Container getContainer() {
+ public ContainerId getContainerId() {
+ return this.containerId;
+ }
+
+ @Override
+ public Resource getResource() {
+ return this.resource;
+ }
+
+ @Override
+ public ContainerTokenIdentifier getContainerTokenIdentifier() {
this.readLock.lock();
try {
- return this.container;
+ return this.containerTokenIdentifier;
} finally {
this.readLock.unlock();
}
@@ -385,15 +400,15 @@ public class ContainerImpl implements Co
@SuppressWarnings({"fallthrough", "unchecked"})
private void finished() {
- ContainerId containerID = this.container.getId();
+ ApplicationId applicationId =
+ containerId.getApplicationAttemptId().getApplicationId();
switch (getContainerState()) {
case EXITED_WITH_SUCCESS:
metrics.endRunningContainer();
metrics.completedContainer();
NMAuditLogger.logSuccess(user,
AuditConstants.FINISH_SUCCESS_CONTAINER, "ContainerImpl",
- containerID.getApplicationAttemptId()
- .getApplicationId(), containerID);
+ applicationId, containerId);
break;
case EXITED_WITH_FAILURE:
metrics.endRunningContainer();
@@ -403,8 +418,7 @@ public class ContainerImpl implements Co
NMAuditLogger.logFailure(user,
AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl",
"Container failed with state: " + getContainerState(),
- containerID.getApplicationAttemptId()
- .getApplicationId(), containerID);
+ applicationId, containerId);
break;
case CONTAINER_CLEANEDUP_AFTER_KILL:
metrics.endRunningContainer();
@@ -413,21 +427,21 @@ public class ContainerImpl implements Co
metrics.killedContainer();
NMAuditLogger.logSuccess(user,
AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
- containerID.getApplicationAttemptId().getApplicationId(),
- containerID);
+ applicationId,
+ containerId);
}
- metrics.releaseContainer(this.container.getResource());
+ metrics.releaseContainer(this.resource);
// Inform the application
@SuppressWarnings("rawtypes")
EventHandler eventHandler = dispatcher.getEventHandler();
- eventHandler.handle(new ApplicationContainerFinishedEvent(containerID));
+ eventHandler.handle(new ApplicationContainerFinishedEvent(containerId));
// Remove the container from the resource-monitor
- eventHandler.handle(new ContainerStopMonitoringEvent(containerID));
+ eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
// Tell the logService too
eventHandler.handle(new LogHandlerContainerFinishedEvent(
- containerID, exitCode));
+ containerId, exitCode));
}
@SuppressWarnings("unchecked") // dispatcher not typed
@@ -489,7 +503,7 @@ public class ContainerImpl implements Co
for (Map.Entry<String,ByteBuffer> service : csd.entrySet()) {
container.dispatcher.getEventHandler().handle(
new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT,
- container.user, container.container.getId()
+ container.user, container.containerId
.getApplicationAttemptId().getApplicationId(),
service.getKey().toString(), service.getValue()));
}
@@ -574,7 +588,7 @@ public class ContainerImpl implements Co
container.pendingResources.remove(rsrcEvent.getResource());
if (null == syms) {
LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.container.getId());
+ " for container " + container.containerId);
assert false;
// fail container?
return ContainerState.LOCALIZING;
@@ -602,14 +616,14 @@ public class ContainerImpl implements Co
// Inform the ContainersMonitor to start monitoring the container's
// resource usage.
long pmemBytes =
- container.container.getResource().getMemory() * 1024 * 1024L;
+ container.getResource().getMemory() * 1024 * 1024L;
float pmemRatio = container.daemonConf.getFloat(
YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
long vmemBytes = (long) (pmemRatio * pmemBytes);
container.dispatcher.getEventHandler().handle(
- new ContainerStartMonitoringEvent(container.container.getId(),
+ new ContainerStartMonitoringEvent(container.containerId,
vmemBytes, pmemBytes));
container.metrics.runningContainer();
}
@@ -743,7 +757,7 @@ public class ContainerImpl implements Co
container.pendingResources.remove(rsrcEvent.getResource());
if (null == syms) {
LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.container.getId());
+ " for container " + container.containerId);
assert false;
// fail container?
return;
@@ -848,7 +862,7 @@ public class ContainerImpl implements Co
public String toString() {
this.readLock.lock();
try {
- return ConverterUtils.toString(container.getId());
+ return ConverterUtils.toString(this.containerId);
} finally {
this.readLock.unlock();
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Fri May 31 04:14:27 2013
@@ -22,8 +22,8 @@ import static org.apache.hadoop.fs.Creat
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
import java.io.DataOutputStream;
-import java.io.IOException;
import java.io.File;
+import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.ArrayList;
@@ -53,12 +53,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.DelayedProcessKiller;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
@@ -68,7 +68,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader;
import org.apache.hadoop.yarn.util.Apps;
-import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -86,6 +85,7 @@ public class ContainerLaunch implements
private final Application app;
private final Container container;
private final Configuration conf;
+ private final Context context;
private volatile AtomicBoolean shouldLaunchContainer = new AtomicBoolean(false);
private volatile AtomicBoolean completed = new AtomicBoolean(false);
@@ -97,9 +97,10 @@ public class ContainerLaunch implements
private final LocalDirsHandlerService dirsHandler;
- public ContainerLaunch(Configuration configuration, Dispatcher dispatcher,
- ContainerExecutor exec, Application app, Container container,
- LocalDirsHandlerService dirsHandler) {
+ public ContainerLaunch(Context context, Configuration configuration,
+ Dispatcher dispatcher, ContainerExecutor exec, Application app,
+ Container container, LocalDirsHandlerService dirsHandler) {
+ this.context = context;
this.conf = configuration;
this.app = app;
this.exec = exec;
@@ -120,7 +121,7 @@ public class ContainerLaunch implements
final ContainerLaunchContext launchContext = container.getLaunchContext();
final Map<Path,List<String>> localResources =
container.getLocalizedResources();
- ContainerId containerID = container.getContainer().getId();
+ ContainerId containerID = container.getContainerId();
String containerIdStr = ConverterUtils.toString(containerID);
final List<String> command = launchContext.getCommands();
int ret = -1;
@@ -301,7 +302,7 @@ public class ContainerLaunch implements
* @throws IOException
*/
public void cleanupContainer() throws IOException {
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
String containerIdStr = ConverterUtils.toString(containerId);
LOG.info("Cleaning up container " + containerIdStr);
@@ -372,7 +373,7 @@ public class ContainerLaunch implements
*/
private String getContainerPid(Path pidFilePath) throws Exception {
String containerIdStr =
- ConverterUtils.toString(container.getContainer().getId());
+ ConverterUtils.toString(container.getContainerId());
String processId = null;
LOG.debug("Accessing pid for container " + containerIdStr
+ " from pid file " + pidFilePath);
@@ -550,16 +551,16 @@ public class ContainerLaunch implements
*/
environment.put(Environment.CONTAINER_ID.name(), container
- .getContainer().getId().toString());
+ .getContainerId().toString());
environment.put(Environment.NM_PORT.name(),
- String.valueOf(container.getContainer().getNodeId().getPort()));
+ String.valueOf(this.context.getNodeId().getPort()));
- environment.put(Environment.NM_HOST.name(), container.getContainer()
- .getNodeId().getHost());
+ environment.put(Environment.NM_HOST.name(), this.context.getNodeId()
+ .getHost());
- environment.put(Environment.NM_HTTP_PORT.name(), container.getContainer()
- .getNodeHttpAddress().split(":")[1]);
+ environment.put(Environment.NM_HTTP_PORT.name(),
+ String.valueOf(this.context.getHttpPort()));
environment.put(Environment.LOCAL_DIRS.name(),
StringUtils.join(",", appDirs));
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Fri May 31 04:14:27 2013
@@ -36,8 +36,8 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
@@ -111,15 +111,16 @@ public class ContainersLauncher extends
public void handle(ContainersLauncherEvent event) {
// TODO: ContainersLauncher launches containers one by one!!
Container container = event.getContainer();
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
switch (event.getType()) {
case LAUNCH_CONTAINER:
Application app =
context.getApplications().get(
containerId.getApplicationAttemptId().getApplicationId());
- ContainerLaunch launch = new ContainerLaunch(getConfig(), dispatcher,
- exec, app, event.getContainer(), dirsHandler);
+ ContainerLaunch launch =
+ new ContainerLaunch(context, getConfig(), dispatcher, exec, app,
+ event.getContainer(), dirsHandler);
running.put(containerId,
new RunningContainer(containerLauncher.submit(launch),
launch));
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Fri May 31 04:14:27 2013
@@ -359,14 +359,14 @@ public class ResourceLocalizationService
ContainerLocalizationRequestEvent rsrcReqs) {
Container c = rsrcReqs.getContainer();
LocalizerContext ctxt = new LocalizerContext(
- c.getUser(), c.getContainer().getId(), c.getCredentials());
+ c.getUser(), c.getContainerId(), c.getCredentials());
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
rsrcReqs.getRequestedResources();
for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
rsrcs.entrySet()) {
LocalResourcesTracker tracker =
getLocalResourcesTracker(e.getKey(), c.getUser(),
- c.getContainer().getId().getApplicationAttemptId()
+ c.getContainerId().getApplicationAttemptId()
.getApplicationId());
for (LocalResourceRequest req : e.getValue()) {
tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
@@ -396,21 +396,21 @@ public class ResourceLocalizationService
for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
rsrcs.entrySet()) {
LocalResourcesTracker tracker = getLocalResourcesTracker(e.getKey(), c.getUser(),
- c.getContainer().getId().getApplicationAttemptId()
+ c.getContainerId().getApplicationAttemptId()
.getApplicationId());
for (LocalResourceRequest req : e.getValue()) {
tracker.handle(new ResourceReleaseEvent(req,
- c.getContainer().getId()));
+ c.getContainerId()));
}
}
- String locId = ConverterUtils.toString(c.getContainer().getId());
+ String locId = ConverterUtils.toString(c.getContainerId());
localizerTracker.cleanupPrivLocalizers(locId);
// Delete the container directories
String userName = c.getUser();
String containerIDStr = c.toString();
String appIDStr = ConverterUtils.toString(
- c.getContainer().getId().getApplicationAttemptId().getApplicationId());
+ c.getContainerId().getApplicationAttemptId().getApplicationId());
for (String localDir : dirsHandler.getLocalDirs()) {
// Delete the user-owned container-dir
@@ -430,7 +430,7 @@ public class ResourceLocalizationService
}
dispatcher.getEventHandler().handle(
- new ContainerEvent(c.getContainer().getId(),
+ new ContainerEvent(c.getContainerId(),
ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Fri May 31 04:14:27 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.no
import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -43,6 +42,7 @@ public class WebServer extends AbstractS
private final Context nmContext;
private final NMWebApp nmWebApp;
private WebApp webApp;
+ private int port;
public WebServer(Context nmContext, ResourceView resourceView,
ApplicationACLsManager aclsManager,
@@ -66,9 +66,7 @@ public class WebServer extends AbstractS
this.webApp =
WebApps.$for("node", Context.class, this.nmContext, "ws")
.at(bindAddress).with(getConfig()).start(this.nmWebApp);
- int port = this.webApp.httpServer().getPort();
- String webAddress = StringUtils.split(bindAddress, ':')[0] + ":" + port;
- getConfig().set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddress);
+ this.port = this.webApp.httpServer().getPort();
} catch (Exception e) {
String msg = "NMWebapps failed to start.";
LOG.error(msg, e);
@@ -77,6 +75,10 @@ public class WebServer extends AbstractS
super.start();
}
+ public int getPort() {
+ return this.port;
+ }
+
@Override
public synchronized void stop() {
if (this.webApp != null) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java Fri May 31 04:14:27 2013
@@ -29,7 +29,6 @@ import javax.xml.bind.annotation.XmlTran
import org.apache.hadoop.yarn.api.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -60,7 +59,7 @@ public class ContainerInfo {
public ContainerInfo(final Context nmContext, final Container container,
String requestUri, String pathPrefix) {
- this.id = container.getContainer().getId().toString();
+ this.id = container.getContainerId().toString();
this.nodeId = nmContext.getNodeId().toString();
ContainerStatus containerData = container.cloneAndGetContainerStatus();
this.exitCode = containerData.getExitStatus();
@@ -74,7 +73,7 @@ public class ContainerInfo {
}
this.user = container.getUser();
- Resource res = container.getContainer().getResource();
+ Resource res = container.getResource();
if (res != null) {
this.totalMemoryNeededMB = res.getMemory();
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Fri May 31 04:14:27 2013
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.no
import static org.junit.Assert.fail;
-import java.io.IOException;
import java.util.Collection;
import org.apache.commons.logging.Log;
@@ -54,7 +53,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.util.BuilderUtils;
public class DummyContainerManager extends ContainerManagerImpl {
@@ -94,10 +92,10 @@ public class DummyContainerManager exten
.getRequestedResources().values()) {
for (LocalResourceRequest req : rc) {
LOG.info("DEBUG: " + req + ":"
- + rsrcReqs.getContainer().getContainer().getId());
+ + rsrcReqs.getContainer().getContainerId());
dispatcher.getEventHandler().handle(
new ContainerResourceLocalizedEvent(rsrcReqs.getContainer()
- .getContainer().getId(), req, new Path("file:///local"
+ .getContainerId(), req, new Path("file:///local"
+ req.getPath().toUri().getPath())));
}
}
@@ -107,7 +105,7 @@ public class DummyContainerManager exten
((ContainerLocalizationEvent) event).getContainer();
// TODO: delete the container dir
this.dispatcher.getEventHandler().handle(
- new ContainerEvent(container.getContainer().getId(),
+ new ContainerEvent(container.getContainerId(),
ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
break;
case DESTROY_APPLICATION_RESOURCES:
@@ -136,7 +134,7 @@ public class DummyContainerManager exten
@Override
public void handle(ContainersLauncherEvent event) {
Container container = event.getContainer();
- ContainerId containerId = container.getContainer().getId();
+ ContainerId containerId = container.getContainerId();
switch (event.getType()) {
case LAUNCH_CONTAINER:
dispatcher.getEventHandler().handle(
@@ -183,25 +181,15 @@ public class DummyContainerManager exten
@Override
protected void authorizeRequest(String containerIDStr,
ContainerLaunchContext launchContext,
- org.apache.hadoop.yarn.api.records.Container container,
UserGroupInformation remoteUgi, ContainerTokenIdentifier tokenId)
throws YarnRemoteException {
// do Nothing
}
@Override
- protected ContainerTokenIdentifier getContainerTokenIdentifier(
- UserGroupInformation remoteUgi,
- org.apache.hadoop.yarn.api.records.Container container)
- throws YarnRemoteException {
- try {
- return new ContainerTokenIdentifier(container.getId(),
- container.getNodeHttpAddress(), remoteUgi.getUserName(),
- container.getResource(), System.currentTimeMillis() + 100000l, 123,
- BuilderUtils.newContainerTokenIdentifier(
- container.getContainerToken()).getRMIdentifer());
- } catch (IOException e) {
- throw new YarnRemoteException(e);
- }
+ protected ContainerTokenIdentifier
+ getContainerTokenIdentifier(UserGroupInformation remoteUgi,
+ ContainerTokenIdentifier containerTokenId) throws YarnRemoteException {
+ return containerTokenId;
}
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1488086&r1=1488085&r2=1488086&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Fri May 31 04:14:27 2013
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
import java.io.File;
import java.io.IOException;
@@ -30,7 +27,6 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -80,7 +76,12 @@ public class TestEventFlow {
YarnConfiguration conf = new YarnConfiguration();
- Context context = new NMContext(new NMContainerTokenSecretManager(conf));
+ Context context = new NMContext(new NMContainerTokenSecretManager(conf)) {
+ @Override
+ public int getHttpPort() {
+ return 1234;
+ }
+ };
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
@@ -132,10 +133,7 @@ public class TestEventFlow {
applicationAttemptId.setApplicationId(applicationId);
applicationAttemptId.setAttemptId(0);
cID.setApplicationAttemptId(applicationAttemptId);
- Container mockContainer = mock(Container.class);
- when(mockContainer.getId()).thenReturn(cID);
Resource r = BuilderUtils.newResource(1024, 1);
- when(mockContainer.getResource()).thenReturn(r);
String user = "testing";
String host = "127.0.0.1";
int port = 1234;
@@ -143,11 +141,10 @@ public class TestEventFlow {
BuilderUtils.newContainerToken(cID, host, port, user, r,
System.currentTimeMillis() + 10000L, 123, "password".getBytes(),
SIMULATED_RM_IDENTIFIER);
- when(mockContainer.getContainerToken()).thenReturn(containerToken);
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
- request.setContainer(mockContainer);
+ request.setContainerToken(containerToken);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,