You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by to...@apache.org on 2012/09/05 06:58:31 UTC
svn commit: r1380990 [2/4] - in
/hadoop/common/branches/HDFS-3077/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/conf/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-ap...
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Wed Sep 5 04:57:47 2012
@@ -37,15 +37,21 @@ message NodeStatusProto {
repeated ApplicationIdProto keep_alive_applications = 5;
}
+message MasterKeyProto {
+ optional int32 key_id = 1;
+ optional bytes bytes = 2;
+}
+
message RegistrationResponseProto {
- optional bytes secret_key = 1;
+ optional MasterKeyProto master_key = 1;
optional NodeActionProto nodeAction = 2;
}
message HeartbeatResponseProto {
optional int32 response_id = 1;
- optional NodeActionProto nodeAction = 2;
- repeated ContainerIdProto containers_to_cleanup = 3;
- repeated ApplicationIdProto applications_to_cleanup = 4;
+ optional MasterKeyProto master_key = 2;
+ optional NodeActionProto nodeAction = 3;
+ repeated ContainerIdProto containers_to_cleanup = 4;
+ repeated ApplicationIdProto applications_to_cleanup = 5;
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto Wed Sep 5 04:57:47 2012
@@ -35,6 +35,7 @@ message RegisterNodeManagerResponseProto
message NodeHeartbeatRequestProto {
optional NodeStatusProto node_status = 1;
+ optional MasterKeyProto last_known_master_key = 2;
}
message NodeHeartbeatResponseProto {
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Wed Sep 5 04:57:47 2012
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
/**
* Context interface for sharing information across components in the
@@ -44,5 +45,7 @@ public interface Context {
ConcurrentMap<ContainerId, Container> getContainers();
+ NMContainerTokenSecretManager getContainerTokenSecretManager();
+
NodeHealthStatus getNodeHealthStatus();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Wed Sep 5 04:57:47 2012
@@ -46,9 +46,9 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
@@ -64,7 +64,6 @@ public class NodeManager extends Composi
private static final Log LOG = LogFactory.getLog(NodeManager.class);
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
- protected ContainerTokenSecretManager containerTokenSecretManager;
private ApplicationACLsManager aclsManager;
private NodeHealthCheckerService nodeHealthChecker;
private LocalDirsHandlerService dirsHandler;
@@ -75,10 +74,9 @@ public class NodeManager extends Composi
}
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker,
- metrics, containerTokenSecretManager);
+ metrics);
}
protected NodeResourceMonitor createNodeResourceMonitor() {
@@ -87,11 +85,10 @@ public class NodeManager extends Composi
protected ContainerManagerImpl createContainerManager(Context context,
ContainerExecutor exec, DeletionService del,
- NodeStatusUpdater nodeStatusUpdater, ContainerTokenSecretManager
- containerTokenSecretManager, ApplicationACLsManager aclsManager,
+ NodeStatusUpdater nodeStatusUpdater, ApplicationACLsManager aclsManager,
LocalDirsHandlerService dirsHandler) {
return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
- metrics, containerTokenSecretManager, aclsManager, dirsHandler);
+ metrics, aclsManager, dirsHandler);
}
protected WebServer createWebServer(Context nmContext,
@@ -110,15 +107,16 @@ public class NodeManager extends Composi
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
- Context context = new NMContext();
-
// Create the secretManager if need be.
+ NMContainerTokenSecretManager containerTokenSecretManager = null;
if (UserGroupInformation.isSecurityEnabled()) {
LOG.info("Security is enabled on NodeManager. "
+ "Creating ContainerTokenSecretManager");
- this.containerTokenSecretManager = new ContainerTokenSecretManager(conf);
+ containerTokenSecretManager = new NMContainerTokenSecretManager(conf);
}
+ Context context = new NMContext(containerTokenSecretManager);
+
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.newInstance(
@@ -139,8 +137,8 @@ public class NodeManager extends Composi
addService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.getDiskHandler();
- NodeStatusUpdater nodeStatusUpdater = createNodeStatusUpdater(context,
- dispatcher, nodeHealthChecker, this.containerTokenSecretManager);
+ NodeStatusUpdater nodeStatusUpdater =
+ createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
nodeStatusUpdater.register(this);
NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
@@ -148,7 +146,7 @@ public class NodeManager extends Composi
ContainerManagerImpl containerManager =
createContainerManager(context, exec, del, nodeStatusUpdater,
- this.containerTokenSecretManager, this.aclsManager, dirsHandler);
+ this.aclsManager, dirsHandler);
addService(containerManager);
Service webServer = createWebServer(context, containerManager
@@ -192,10 +190,13 @@ public class NodeManager extends Composi
private final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
+ private final NMContainerTokenSecretManager containerTokenSecretManager;
+
private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
.getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
- public NMContext() {
+ public NMContext(NMContainerTokenSecretManager containerTokenSecretManager) {
+ this.containerTokenSecretManager = containerTokenSecretManager;
this.nodeHealthStatus.setIsNodeHealthy(true);
this.nodeHealthStatus.setHealthReport("Healthy");
this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
@@ -220,6 +221,10 @@ public class NodeManager extends Composi
}
@Override
+ public NMContainerTokenSecretManager getContainerTokenSecretManager() {
+ return this.containerTokenSecretManager;
+ }
+ @Override
public NodeHealthStatus getNodeHealthStatus() {
return this.nodeHealthStatus;
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java Wed Sep 5 04:57:47 2012
@@ -22,7 +22,5 @@ import org.apache.hadoop.yarn.service.Se
public interface NodeStatusUpdater extends Service {
- byte[] getRMNMSharedSecret();
-
void sendOutofBandHeartBeat();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Wed Sep 5 04:57:47 2012
@@ -25,12 +25,13 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Map.Entry;
+import java.util.Random;
import org.apache.avro.AvroRuntimeException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.YarnException;
@@ -51,15 +52,14 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.AbstractService;
-
public class NodeStatusUpdaterImpl extends AbstractService implements
NodeStatusUpdater {
@@ -71,13 +71,11 @@ public class NodeStatusUpdaterImpl exten
private final Dispatcher dispatcher;
private NodeId nodeId;
- private ContainerTokenSecretManager containerTokenSecretManager;
private long heartBeatInterval;
private ResourceTracker resourceTracker;
private InetSocketAddress rmAddress;
private Resource totalResource;
private int httpPort;
- private byte[] secretKeyBytes = new byte[0];
private boolean isStopped;
private RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private boolean tokenKeepAliveEnabled;
@@ -93,14 +91,12 @@ public class NodeStatusUpdaterImpl exten
private boolean hasToRebootNode;
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
super(NodeStatusUpdaterImpl.class.getName());
this.healthChecker = healthChecker;
this.context = context;
this.dispatcher = dispatcher;
this.metrics = metrics;
- this.containerTokenSecretManager = containerTokenSecretManager;
}
@Override
@@ -116,10 +112,7 @@ public class NodeStatusUpdaterImpl exten
this.totalResource = recordFactory.newRecordInstance(Resource.class);
this.totalResource.setMemory(memoryMb);
metrics.addResource(totalResource);
- this.tokenKeepAliveEnabled =
- conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
- YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)
- && isSecurityEnabled();
+ this.tokenKeepAliveEnabled = isTokenKeepAliveEnabled(conf);
this.tokenRemovalDelayMs =
conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
@@ -168,10 +161,17 @@ public class NodeStatusUpdaterImpl exten
return this.hasToRebootNode;
}
- protected boolean isSecurityEnabled() {
+ private boolean isSecurityEnabled() {
return UserGroupInformation.isSecurityEnabled();
}
+ @Private
+ protected boolean isTokenKeepAliveEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
+ YarnConfiguration.DEFAULT_LOG_AGGREGATION_ENABLED)
+ && isSecurityEnabled();
+ }
+
protected ResourceTracker getRMClient() {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
@@ -194,30 +194,24 @@ public class NodeStatusUpdaterImpl exten
throw new YarnException(
"Recieved SHUTDOWN signal from Resourcemanager ,Registration of NodeManager failed");
}
-
- if (UserGroupInformation.isSecurityEnabled()) {
- this.secretKeyBytes = regResponse.getSecretKey().array();
- }
- // do this now so that its set before we start heartbeating to RM
if (UserGroupInformation.isSecurityEnabled()) {
+ MasterKey masterKey = regResponse.getMasterKey();
+ // do this now so that its set before we start heartbeating to RM
LOG.info("Security enabled - updating secret keys now");
// It is expected that status updater is started by this point and
- // RM gives the shared secret in registration during StatusUpdater#start().
- this.containerTokenSecretManager.setSecretKey(
- this.nodeId.toString(),
- this.getRMNMSharedSecret());
+ // RM gives the shared secret in registration during
+ // StatusUpdater#start().
+ if (masterKey != null) {
+ this.context.getContainerTokenSecretManager().setMasterKey(masterKey);
+ }
}
+
LOG.info("Registered with ResourceManager as " + this.nodeId
+ " with total resource of " + this.totalResource);
}
- @Override
- public byte[] getRMNMSharedSecret() {
- return this.secretKeyBytes.clone();
- }
-
private List<ApplicationId> createKeepAliveApplicationList() {
if (!tokenKeepAliveEnabled) {
return Collections.emptyList();
@@ -332,9 +326,24 @@ public class NodeStatusUpdaterImpl exten
NodeHeartbeatRequest request = recordFactory
.newRecordInstance(NodeHeartbeatRequest.class);
- request.setNodeStatus(nodeStatus);
+ request.setNodeStatus(nodeStatus);
+ if (isSecurityEnabled()) {
+ request.setLastKnownMasterKey(NodeStatusUpdaterImpl.this.context
+ .getContainerTokenSecretManager().getCurrentKey());
+ }
HeartbeatResponse response =
resourceTracker.nodeHeartbeat(request).getHeartbeatResponse();
+
+ // See if the master-key has rolled over
+ if (isSecurityEnabled()) {
+ MasterKey updatedMasterKey = response.getMasterKey();
+ if (updatedMasterKey != null) {
+ // Will be non-null only on roll-over on RM side
+ context.getContainerTokenSecretManager().setMasterKey(
+ updatedMasterKey);
+ }
+ }
+
if (response.getNodeAction() == NodeAction.SHUTDOWN) {
LOG
.info("Recieved SHUTDOWN signal from Resourcemanager as part of heartbeat," +
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Wed Sep 5 04:57:47 2012
@@ -21,10 +21,10 @@ package org.apache.hadoop.yarn.server.no
import static org.apache.hadoop.yarn.service.Service.STATE.STARTED;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -96,7 +96,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
@@ -110,14 +109,12 @@ public class ContainerManagerImpl extend
final Context context;
private final ContainersMonitor containersMonitor;
private Server server;
- private InetAddress resolvedAddress = null;
private final ResourceLocalizationService rsrcLocalizationSrvc;
private final ContainersLauncher containersLauncher;
private final AuxServices auxiliaryServices;
private final NodeManagerMetrics metrics;
private final NodeStatusUpdater nodeStatusUpdater;
- private ContainerTokenSecretManager containerTokenSecretManager;
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
@@ -129,8 +126,7 @@ public class ContainerManagerImpl extend
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
- NodeManagerMetrics metrics, ContainerTokenSecretManager
- containerTokenSecretManager, ApplicationACLsManager aclsManager,
+ NodeManagerMetrics metrics, ApplicationACLsManager aclsManager,
LocalDirsHandlerService dirsHandler) {
super(ContainerManagerImpl.class.getName());
this.context = context;
@@ -149,7 +145,6 @@ public class ContainerManagerImpl extend
addService(containersLauncher);
this.nodeStatusUpdater = nodeStatusUpdater;
- this.containerTokenSecretManager = containerTokenSecretManager;
this.aclsManager = aclsManager;
// Start configurable services
@@ -232,7 +227,7 @@ public class ContainerManagerImpl extend
server =
rpc.getServer(ContainerManager.class, this, initialAddress, conf,
- this.containerTokenSecretManager,
+ this.context.getContainerTokenSecretManager(),
conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
@@ -267,56 +262,78 @@ public class ContainerManagerImpl extend
super.stop();
}
+ // Get the remoteUGI corresponding to the api call.
+ private UserGroupInformation getRemoteUgi(String containerIDStr)
+ throws YarnRemoteException {
+ UserGroupInformation remoteUgi;
+ try {
+ remoteUgi = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ String msg = "Cannot obtain the user-name for containerId: "
+ + containerIDStr + ". Got exception: "
+ + StringUtils.stringifyException(e);
+ LOG.warn(msg);
+ throw RPCUtil.getRemoteException(msg);
+ }
+ return remoteUgi;
+ }
+
+ // Obtain the needed ContainerTokenIdentifier from the remote-UGI. RPC layer
+ // currently sets only the required id, but iterate through anyways just to
+ // be sure.
+ private ContainerTokenIdentifier selectContainerTokenIdentifier(
+ UserGroupInformation remoteUgi) {
+ Set<TokenIdentifier> tokenIdentifiers = remoteUgi.getTokenIdentifiers();
+ ContainerTokenIdentifier resultId = null;
+ for (TokenIdentifier id : tokenIdentifiers) {
+ if (id instanceof ContainerTokenIdentifier) {
+ resultId = (ContainerTokenIdentifier) id;
+ break;
+ }
+ }
+ return resultId;
+ }
+
/**
* Authorize the request.
*
- * @param containerID
+ * @param containerIDStr
* of the container
* @param launchContext
* passed if verifying the startContainer, null otherwise.
+ * @param remoteUgi
+ * ugi corresponding to the remote end making the api-call
* @throws YarnRemoteException
*/
- private void authorizeRequest(ContainerId containerID,
- ContainerLaunchContext launchContext) throws YarnRemoteException {
+ private void authorizeRequest(String containerIDStr,
+ ContainerLaunchContext launchContext, UserGroupInformation remoteUgi)
+ throws YarnRemoteException {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
}
- String containerIDStr = containerID.toString();
-
- UserGroupInformation remoteUgi;
- try {
- remoteUgi = UserGroupInformation.getCurrentUser();
- } catch (IOException e) {
- String msg = "Cannot obtain the user-name for containerId: "
- + containerIDStr + ". Got exception: "
- + StringUtils.stringifyException(e);
- LOG.warn(msg);
- throw RPCUtil.getRemoteException(msg);
- }
-
boolean unauthorized = false;
- StringBuilder messageBuilder = new StringBuilder(
- "Unauthorized request to start container. ");
+ StringBuilder messageBuilder =
+ new StringBuilder("Unauthorized request to start container. ");
if (!remoteUgi.getUserName().equals(containerIDStr)) {
unauthorized = true;
messageBuilder.append("\nExpected containerId: "
+ remoteUgi.getUserName() + " Found: " + containerIDStr);
- }
-
- if (launchContext != null) {
-
- // Verify other things for startContainer() request.
+ } else if (launchContext != null) {
+ // Verify other things also for startContainer() request.
if (LOG.isDebugEnabled()) {
- LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
- + remoteUgi.getTokenIdentifiers().size());
+ LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
+ + remoteUgi.getTokenIdentifiers().size());
}
- // We must and should get only one TokenIdentifier from the RPC.
- ContainerTokenIdentifier tokenId = (ContainerTokenIdentifier) remoteUgi
- .getTokenIdentifiers().iterator().next();
+
+
+ // Get the tokenId from the remote user ugi
+ ContainerTokenIdentifier tokenId =
+ selectContainerTokenIdentifier(remoteUgi);
+
if (tokenId == null) {
unauthorized = true;
messageBuilder
@@ -324,6 +341,15 @@ public class ContainerManagerImpl extend
+ containerIDStr);
} else {
+ // Is the container being relaunched? Or RPC layer let startCall with
+ // tokens generated off old-secret through
+ if (!this.context.getContainerTokenSecretManager()
+ .isValidStartContainerRequest(tokenId)) {
+ unauthorized = true;
+ messageBuilder.append("\n Attempt to relaunch the same " +
+ "container with id " + containerIDStr + ".");
+ }
+
// Ensure the token is not expired.
// Token expiry is not checked for stopContainer/getContainerStatus
if (tokenId.getExpiryTimeStamp() < System.currentTimeMillis()) {
@@ -348,7 +374,7 @@ public class ContainerManagerImpl extend
throw RPCUtil.getRemoteException(msg);
}
}
-
+
/**
* Start a container on this NodeManager.
*/
@@ -359,10 +385,13 @@ public class ContainerManagerImpl extend
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
ContainerId containerID = launchContext.getContainerId();
- authorizeRequest(containerID, launchContext);
+ String containerIDStr = containerID.toString();
+
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, launchContext, remoteUgi);
- LOG.info("Start request for " + launchContext.getContainerId()
- + " by user " + launchContext.getUser());
+ LOG.info("Start request for " + containerIDStr + " by user "
+ + launchContext.getUser());
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getContainerTokens();
@@ -394,14 +423,14 @@ public class ContainerManagerImpl extend
AuditConstants.START_CONTAINER, "ContainerManagerImpl",
"Container already running on this node!",
applicationID, containerID);
- throw RPCUtil.getRemoteException("Container " + containerID
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " already is running on this node!!");
}
// Create the application
Application application =
new ApplicationImpl(dispatcher, this.aclsManager,
- launchContext.getUser(), applicationID, credentials, context);
+ launchContext.getUser(), applicationID, credentials, context);
if (null ==
context.getApplications().putIfAbsent(applicationID, application)) {
LOG.info("Creating a new application reference for app "
@@ -414,6 +443,12 @@ public class ContainerManagerImpl extend
// TODO: Validate the request
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
+ if (UserGroupInformation.isSecurityEnabled()) {
+ ContainerTokenIdentifier tokenId =
+ selectContainerTokenIdentifier(remoteUgi);
+ this.context.getContainerTokenSecretManager().startContainerSuccessful(
+ tokenId);
+ }
NMAuditLogger.logSuccess(launchContext.getUser(),
AuditConstants.START_CONTAINER, "ContainerManageImpl",
@@ -438,8 +473,12 @@ public class ContainerManagerImpl extend
throws YarnRemoteException {
ContainerId containerID = request.getContainerId();
+ String containerIDStr = containerID.toString();
+
// TODO: Only the container's owner can kill containers today.
- authorizeRequest(containerID, null);
+
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, null, remoteUgi);
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
@@ -476,10 +515,14 @@ public class ContainerManagerImpl extend
GetContainerStatusRequest request) throws YarnRemoteException {
ContainerId containerID = request.getContainerId();
+ String containerIDStr = containerID.toString();
+
// TODO: Only the container's owner can get containers' status today.
- authorizeRequest(containerID, null);
- LOG.info("Getting container-status for " + containerID);
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, null, remoteUgi);
+
+ LOG.info("Getting container-status for " + containerIDStr);
Container container = this.context.getContainers().get(containerID);
if (container != null) {
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
@@ -490,7 +533,7 @@ public class ContainerManagerImpl extend
return response;
}
- throw RPCUtil.getRemoteException("Container " + containerID
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " is not handled by this NodeManager");
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Wed Sep 5 04:57:47 2012
@@ -28,8 +28,9 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
@@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -365,6 +367,10 @@ public class ApplicationImpl implements
@Override
public void transition(ApplicationImpl app, ApplicationEvent event) {
+ // Inform the ContainerTokenSecretManager
+ if (UserGroupInformation.isSecurityEnabled()) {
+ app.context.getContainerTokenSecretManager().appFinished(app.appId);
+ }
// Inform the logService
app.dispatcher.getEventHandler().handle(
new LogHandlerAppFinishedEvent(app.appId));
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java Wed Sep 5 04:57:47 2012
@@ -79,16 +79,18 @@ public class NonAggregatingLogHandler ex
@Override
public void stop() {
- sched.shutdown();
- boolean isShutdown = false;
- try {
- isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- sched.shutdownNow();
- isShutdown = true;
- }
- if (!isShutdown) {
- sched.shutdownNow();
+ if (sched != null) {
+ sched.shutdown();
+ boolean isShutdown = false;
+ try {
+ isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ sched.shutdownNow();
+ isShutdown = true;
+ }
+ if (!isShutdown) {
+ sched.shutdownNow();
+ }
}
super.stop();
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Wed Sep 5 04:57:47 2012
@@ -27,8 +27,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
public class DummyContainerManager extends ContainerManagerImpl {
@@ -59,11 +58,10 @@ public class DummyContainerManager exten
public DummyContainerManager(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager,
ApplicationACLsManager applicationACLsManager,
LocalDirsHandlerService dirsHandler) {
super(context, exec, deletionContext, nodeStatusUpdater, metrics,
- containerTokenSecretManager, applicationACLsManager, dirsHandler);
+ applicationACLsManager, dirsHandler);
}
@Override
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Wed Sep 5 04:57:47 2012
@@ -36,12 +36,12 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.junit.Test;
public class TestEventFlow {
@@ -69,9 +69,9 @@ public class TestEventFlow {
localLogDir.mkdir();
remoteLogDir.mkdir();
- Context context = new NMContext();
-
YarnConfiguration conf = new YarnConfiguration();
+ Context context = new NMContext(new NMContainerTokenSecretManager(conf));
+
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
@@ -86,10 +86,8 @@ public class TestEventFlow {
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
NodeManagerMetrics metrics = NodeManagerMetrics.create();
- ContainerTokenSecretManager containerTokenSecretManager =
- new ContainerTokenSecretManager(conf);
NodeStatusUpdater nodeStatusUpdater =
- new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, containerTokenSecretManager) {
+ new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {
@Override
protected ResourceTracker getRMClient() {
return new LocalRMInterface();
@@ -101,10 +99,9 @@ public class TestEventFlow {
}
};
- DummyContainerManager containerManager = new DummyContainerManager(
- context, exec, del, nodeStatusUpdater, metrics,
- containerTokenSecretManager, new ApplicationACLsManager(conf),
- dirsHandler);
+ DummyContainerManager containerManager =
+ new DummyContainerManager(context, exec, del, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler);
containerManager.init(conf);
containerManager.start();
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java Wed Sep 5 04:57:47 2012
@@ -17,37 +17,29 @@
*/
package org.apache.hadoop.yarn.server.nodemanager;
+import static junit.framework.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.TestRPC.TestImpl;
import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
-import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
-
-import org.apache.hadoop.net.NetUtils;
-
-import static org.mockito.Mockito.*;
-import static junit.framework.Assert.*;
-import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-
/**
* Tests {@link NMAuditLogger}.
*/
public class TestNMAuditLogger {
- private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class);
private static final String USER = "test";
private static final String OPERATION = "oper";
private static final String TARGET = "tgt";
@@ -212,8 +204,10 @@ public class TestNMAuditLogger {
public void testNMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
// start the IPC server
- Server server = RPC.getServer(TestProtocol.class,
- new MyTestRPCServer(), "0.0.0.0", 0, 5, true, conf, null);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
+ .setPort(0).setNumHandlers(5).setVerbose(true).build();
+
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Wed Sep 5 04:57:47 2012
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -234,10 +233,8 @@ public class TestNodeStatusUpdater {
private Context context;
public MyNodeStatusUpdater(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager) {
- super(context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
this.context = context;
}
@@ -252,10 +249,8 @@ public class TestNodeStatusUpdater {
private Context context;
public MyNodeStatusUpdater3(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager) {
- super(context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
this.context = context;
this.resourceTracker = new MyResourceTracker3(this.context);
}
@@ -266,7 +261,7 @@ public class TestNodeStatusUpdater {
}
@Override
- protected boolean isSecurityEnabled() {
+ protected boolean isTokenKeepAliveEnabled(Configuration conf) {
return true;
}
}
@@ -276,11 +271,9 @@ public class TestNodeStatusUpdater {
private MyNodeStatusUpdater3 nodeStatusUpdater;
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
this.nodeStatusUpdater =
- new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics);
return this.nodeStatusUpdater;
}
@@ -398,10 +391,9 @@ public class TestNodeStatusUpdater {
nm = new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
- metrics, containerTokenSecretManager);
+ metrics);
}
};
@@ -528,11 +520,9 @@ public class TestNodeStatusUpdater {
nm = new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
MyNodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater(
- context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ context, dispatcher, healthChecker, metrics);
MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
myResourceTracker2.registerNodeAction = NodeAction.SHUTDOWN;
nodeStatusUpdater.resourceTracker = myResourceTracker2;
@@ -556,22 +546,19 @@ public class TestNodeStatusUpdater {
nm = new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
- metrics, containerTokenSecretManager);
+ metrics);
}
@Override
protected ContainerManagerImpl createContainerManager(Context context,
ContainerExecutor exec, DeletionService del,
NodeStatusUpdater nodeStatusUpdater,
- ContainerTokenSecretManager containerTokenSecretManager,
ApplicationACLsManager aclsManager,
LocalDirsHandlerService diskhandler) {
- return new ContainerManagerImpl(context, exec, del,
- nodeStatusUpdater, metrics, containerTokenSecretManager,
- aclsManager, diskhandler) {
+ return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
+ metrics, aclsManager, diskhandler) {
@Override
public void start() {
// Simulating failure of starting RPC server
@@ -654,11 +641,9 @@ public class TestNodeStatusUpdater {
return new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
- context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ context, dispatcher, healthChecker, metrics);
MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
myResourceTracker2.heartBeatNodeAction = nodeHeartBeatAction;
myNodeStatusUpdater.resourceTracker = myResourceTracker2;
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Wed Sep 5 04:57:47 2012
@@ -54,8 +54,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.junit.After;
import org.junit.Before;
@@ -70,8 +70,6 @@ public abstract class BaseContainerManag
protected static File localLogDir;
protected static File remoteLogDir;
protected static File tmpDir;
- protected ContainerTokenSecretManager containerTokenSecretManager =
- new ContainerTokenSecretManager(new Configuration());
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
@@ -93,7 +91,8 @@ public abstract class BaseContainerManag
.getLog(BaseContainerManagerTest.class);
protected Configuration conf = new YarnConfiguration();
- protected Context context = new NMContext();
+ protected Context context = new NMContext(new NMContainerTokenSecretManager(
+ conf));
protected ContainerExecutor exec;
protected DeletionService delSrvc;
protected String user = "nobody";
@@ -101,7 +100,7 @@ public abstract class BaseContainerManag
protected LocalDirsHandlerService dirsHandler;
protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(
- context, new AsyncDispatcher(), null, metrics, this.containerTokenSecretManager) {
+ context, new AsyncDispatcher(), null, metrics) {
@Override
protected ResourceTracker getRMClient() {
return new LocalRMInterface();
@@ -155,9 +154,9 @@ public abstract class BaseContainerManag
nodeHealthChecker = new NodeHealthCheckerService();
nodeHealthChecker.init(conf);
dirsHandler = nodeHealthChecker.getDiskHandler();
- containerManager = new ContainerManagerImpl(context, exec, delSrvc,
- nodeStatusUpdater, metrics, this.containerTokenSecretManager,
- new ApplicationACLsManager(conf), dirsHandler);
+ containerManager =
+ new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler);
containerManager.init(conf);
}
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Wed Sep 5 04:57:47 2012
@@ -51,14 +51,13 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
@@ -384,11 +383,9 @@ public class TestContainerManager extend
delSrvc = new DeletionService(exec);
delSrvc.init(conf);
- ContainerTokenSecretManager containerTokenSecretManager = new
- ContainerTokenSecretManager(conf);
- containerManager = new ContainerManagerImpl(context, exec, delSrvc,
- nodeStatusUpdater, metrics, containerTokenSecretManager,
- new ApplicationACLsManager(conf), dirsHandler);
+ containerManager =
+ new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler);
containerManager.init(conf);
containerManager.start();
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Wed Sep 5 04:57:47 2012
@@ -30,9 +30,9 @@ import java.util.HashMap;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -50,8 +50,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/TestNonAggregatingLogHandler.java Wed Sep 5 04:57:47 2012
@@ -183,6 +183,24 @@ public class TestNonAggregatingLogHandle
verify(mockSched).schedule(any(Runnable.class), eq(10800l),
eq(TimeUnit.SECONDS));
}
+
+ @Test
+ public void testStop() throws Exception {
+ NonAggregatingLogHandler aggregatingLogHandler =
+ new NonAggregatingLogHandler(null, null, null);
+
+ // It should not throw NullPointerException
+ aggregatingLogHandler.stop();
+
+ NonAggregatingLogHandlerWithMockExecutor logHandler =
+ new NonAggregatingLogHandlerWithMockExecutor(null, null, null);
+ logHandler.init(new Configuration());
+ logHandler.stop();
+ verify(logHandler.mockSched).shutdown();
+ verify(logHandler.mockSched)
+ .awaitTermination(eq(10l), eq(TimeUnit.SECONDS));
+ verify(logHandler.mockSched).shutdownNow();
+ }
private class NonAggregatingLogHandlerWithMockExecutor extends
NonAggregatingLogHandler {
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Wed Sep 5 04:57:47 2012
@@ -76,7 +76,7 @@ public class TestNMWebServer {
}
private String startNMWebAppServer(String webAddr) {
- Context nmContext = new NodeManager.NMContext();
+ Context nmContext = new NodeManager.NMContext(null);
ResourceView resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
@@ -116,7 +116,7 @@ public class TestNMWebServer {
@Test
public void testNMWebApp() throws IOException {
- Context nmContext = new NodeManager.NMContext();
+ Context nmContext = new NodeManager.NMContext(null);
ResourceView resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java Wed Sep 5 04:57:47 2012
@@ -85,7 +85,7 @@ public class TestNMWebServices extends J
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
- nmContext = new NodeManager.NMContext();
+ nmContext = new NodeManager.NMContext(null);
nmContext.getNodeId().setHost("testhost.foo.com");
nmContext.getNodeId().setPort(8042);
resourceView = new ResourceView() {
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java Wed Sep 5 04:57:47 2012
@@ -91,7 +91,7 @@ public class TestNMWebServicesApps exten
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
- nmContext = new NodeManager.NMContext();
+ nmContext = new NodeManager.NMContext(null);
nmContext.getNodeId().setHost("testhost.foo.com");
nmContext.getNodeId().setPort(9999);
resourceView = new ResourceView() {
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java Wed Sep 5 04:57:47 2012
@@ -91,7 +91,7 @@ public class TestNMWebServicesContainers
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
- nmContext = new NodeManager.NMContext();
+ nmContext = new NodeManager.NMContext(null);
nmContext.getNodeId().setHost("testhost.foo.com");
nmContext.getNodeId().setPort(8042);
resourceView = new ResourceView() {
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Wed Sep 5 04:57:47 2012
@@ -21,11 +21,11 @@ package org.apache.hadoop.yarn.server.re
import java.io.IOException;
import java.net.InetSocketAddress;
-import org.apache.hadoop.ipc.Server;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
import org.apache.hadoop.security.UserGroupInformation;
@@ -171,8 +171,7 @@ public class AdminService extends Abstra
throws YarnRemoteException {
UserGroupInformation user = checkAcls("refreshQueues");
try {
- scheduler.reinitialize(conf, null, null); // ContainerTokenSecretManager can't
- // be 'refreshed'
+ scheduler.reinitialize(conf, this.rmContext);
RMAuditLogger.logSuccess(user.getShortUserName(), "refreshQueues",
"AdminService");
return recordFactory.newRecordInstance(RefreshQueuesResponse.class);
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java Wed Sep 5 04:57:47 2012
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
/**
* Context of the ResourceManager.
@@ -58,4 +59,6 @@ public interface RMContext {
DelegationTokenRenewer getDelegationTokenRenewer();
ApplicationTokenSecretManager getApplicationTokenSecretManager();
+
+ RMContainerTokenSecretManager getContainerTokenSecretManager();
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java Wed Sep 5 04:57:47 2012
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
public class RMContextImpl implements RMContext {
@@ -53,13 +54,15 @@ public class RMContextImpl implements RM
private ContainerAllocationExpirer containerAllocationExpirer;
private final DelegationTokenRenewer tokenRenewer;
private final ApplicationTokenSecretManager appTokenSecretManager;
+ private final RMContainerTokenSecretManager containerTokenSecretManager;
public RMContextImpl(Store store, Dispatcher rmDispatcher,
ContainerAllocationExpirer containerAllocationExpirer,
AMLivelinessMonitor amLivelinessMonitor,
AMLivelinessMonitor amFinishingMonitor,
DelegationTokenRenewer tokenRenewer,
- ApplicationTokenSecretManager appTokenSecretManager) {
+ ApplicationTokenSecretManager appTokenSecretManager,
+ RMContainerTokenSecretManager containerTokenSecretManager) {
this.store = store;
this.rmDispatcher = rmDispatcher;
this.containerAllocationExpirer = containerAllocationExpirer;
@@ -67,6 +70,7 @@ public class RMContextImpl implements RM
this.amFinishingMonitor = amFinishingMonitor;
this.tokenRenewer = tokenRenewer;
this.appTokenSecretManager = appTokenSecretManager;
+ this.containerTokenSecretManager = containerTokenSecretManager;
}
@Override
@@ -123,4 +127,9 @@ public class RMContextImpl implements RM
public ApplicationTokenSecretManager getApplicationTokenSecretManager() {
return this.appTokenSecretManager;
}
+
+ @Override
+ public RMContainerTokenSecretManager getContainerTokenSecretManager() {
+ return this.containerTokenSecretManager;
+ }
}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1380990&r1=1380989&r2=1380990&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-3077/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Wed Sep 5 04:57:47 2012
@@ -48,8 +48,8 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -67,9 +67,9 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
+import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxy;
@@ -100,7 +100,7 @@ public class ResourceManager extends Com
protected ClientToAMSecretManager clientToAMSecretManager =
new ClientToAMSecretManager();
- protected ContainerTokenSecretManager containerTokenSecretManager;
+ protected RMContainerTokenSecretManager containerTokenSecretManager;
protected ApplicationTokenSecretManager appTokenSecretManager;
@@ -150,8 +150,6 @@ public class ResourceManager extends Com
this.rmDispatcher);
addService(this.containerAllocationExpirer);
- this.containerTokenSecretManager = new ContainerTokenSecretManager(conf);
-
AMLivelinessMonitor amLivelinessMonitor = createAMLivelinessMonitor();
addService(amLivelinessMonitor);
@@ -160,11 +158,14 @@ public class ResourceManager extends Com
DelegationTokenRenewer tokenRenewer = createDelegationTokenRenewer();
addService(tokenRenewer);
+
+ this.containerTokenSecretManager = createContainerTokenSecretManager(conf);
- this.rmContext = new RMContextImpl(this.store, this.rmDispatcher,
- this.containerAllocationExpirer,
- amLivelinessMonitor, amFinishingMonitor,
- tokenRenewer, this.appTokenSecretManager);
+ this.rmContext =
+ new RMContextImpl(this.store, this.rmDispatcher,
+ this.containerAllocationExpirer, amLivelinessMonitor,
+ amFinishingMonitor, tokenRenewer, this.appTokenSecretManager,
+ this.containerTokenSecretManager);
// Register event handler for NodesListManager
this.nodesListManager = new NodesListManager(this.rmContext);
@@ -198,8 +199,7 @@ public class ResourceManager extends Com
addService(resourceTracker);
try {
- this.scheduler.reinitialize(conf,
- this.containerTokenSecretManager, this.rmContext);
+ this.scheduler.reinitialize(conf, this.rmContext);
} catch (IOException ioe) {
throw new RuntimeException("Failed to initialize scheduler", ioe);
}
@@ -231,6 +231,11 @@ public class ResourceManager extends Com
super.init(conf);
}
+ protected RMContainerTokenSecretManager createContainerTokenSecretManager(
+ Configuration conf) {
+ return new RMContainerTokenSecretManager(conf);
+ }
+
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
return new SchedulerEventDispatcher(this.scheduler);
}
@@ -486,6 +491,7 @@ public class ResourceManager extends Com
}
this.appTokenSecretManager.start();
+ this.containerTokenSecretManager.start();
startWepApp();
DefaultMetricsSystem.initialize("ResourceManager");
@@ -531,6 +537,7 @@ public class ResourceManager extends Com
rmDTSecretManager.stopThreads();
this.appTokenSecretManager.stop();
+ this.containerTokenSecretManager.stop();
/*synchronized(shutdown) {
shutdown.set(true);
@@ -616,7 +623,7 @@ public class ResourceManager extends Com
}
@Private
- public ContainerTokenSecretManager getContainerTokenSecretManager() {
+ public RMContainerTokenSecretManager getRMContainerTokenSecretManager() {
return this.containerTokenSecretManager;
}