You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ss...@apache.org on 2012/10/16 02:04:13 UTC
svn commit: r1398581 [4/9] - in
/hadoop/common/branches/MR-3902/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoo...
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Oct 16 00:02:55 2012
@@ -21,10 +21,10 @@ package org.apache.hadoop.yarn.server.no
import static org.apache.hadoop.yarn.service.Service.STATE.STARTED;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -96,7 +96,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
@@ -110,14 +109,12 @@ public class ContainerManagerImpl extend
final Context context;
private final ContainersMonitor containersMonitor;
private Server server;
- private InetAddress resolvedAddress = null;
private final ResourceLocalizationService rsrcLocalizationSrvc;
private final ContainersLauncher containersLauncher;
private final AuxServices auxiliaryServices;
private final NodeManagerMetrics metrics;
private final NodeStatusUpdater nodeStatusUpdater;
- private ContainerTokenSecretManager containerTokenSecretManager;
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
@@ -129,8 +126,7 @@ public class ContainerManagerImpl extend
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
- NodeManagerMetrics metrics, ContainerTokenSecretManager
- containerTokenSecretManager, ApplicationACLsManager aclsManager,
+ NodeManagerMetrics metrics, ApplicationACLsManager aclsManager,
LocalDirsHandlerService dirsHandler) {
super(ContainerManagerImpl.class.getName());
this.context = context;
@@ -149,7 +145,6 @@ public class ContainerManagerImpl extend
addService(containersLauncher);
this.nodeStatusUpdater = nodeStatusUpdater;
- this.containerTokenSecretManager = containerTokenSecretManager;
this.aclsManager = aclsManager;
// Start configurable services
@@ -232,7 +227,7 @@ public class ContainerManagerImpl extend
server =
rpc.getServer(ContainerManager.class, this, initialAddress, conf,
- this.containerTokenSecretManager,
+ this.context.getContainerTokenSecretManager(),
conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
@@ -267,56 +262,77 @@ public class ContainerManagerImpl extend
super.stop();
}
+ // Get the remoteUGI corresponding to the api call.
+ private UserGroupInformation getRemoteUgi(String containerIDStr)
+ throws YarnRemoteException {
+ UserGroupInformation remoteUgi;
+ try {
+ remoteUgi = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ String msg = "Cannot obtain the user-name for containerId: "
+ + containerIDStr + ". Got exception: "
+ + StringUtils.stringifyException(e);
+ LOG.warn(msg);
+ throw RPCUtil.getRemoteException(msg);
+ }
+ return remoteUgi;
+ }
+
+ // Obtain the needed ContainerTokenIdentifier from the remote-UGI. RPC layer
+ // currently sets only the required id, but iterate through anyways just to
+ // be sure.
+ private ContainerTokenIdentifier selectContainerTokenIdentifier(
+ UserGroupInformation remoteUgi) {
+ Set<TokenIdentifier> tokenIdentifiers = remoteUgi.getTokenIdentifiers();
+ ContainerTokenIdentifier resultId = null;
+ for (TokenIdentifier id : tokenIdentifiers) {
+ if (id instanceof ContainerTokenIdentifier) {
+ resultId = (ContainerTokenIdentifier) id;
+ break;
+ }
+ }
+ return resultId;
+ }
+
/**
* Authorize the request.
*
- * @param containerID
+ * @param containerIDStr
* of the container
* @param launchContext
* passed if verifying the startContainer, null otherwise.
+ * @param remoteUgi
+ * ugi corresponding to the remote end making the api-call
* @throws YarnRemoteException
*/
- private void authorizeRequest(ContainerId containerID,
- ContainerLaunchContext launchContext) throws YarnRemoteException {
+ private void authorizeRequest(String containerIDStr,
+ ContainerLaunchContext launchContext, UserGroupInformation remoteUgi)
+ throws YarnRemoteException {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
}
- String containerIDStr = containerID.toString();
-
- UserGroupInformation remoteUgi;
- try {
- remoteUgi = UserGroupInformation.getCurrentUser();
- } catch (IOException e) {
- String msg = "Cannot obtain the user-name for containerId: "
- + containerIDStr + ". Got exception: "
- + StringUtils.stringifyException(e);
- LOG.warn(msg);
- throw RPCUtil.getRemoteException(msg);
- }
-
boolean unauthorized = false;
- StringBuilder messageBuilder = new StringBuilder(
- "Unauthorized request to start container. ");
+ StringBuilder messageBuilder =
+ new StringBuilder("Unauthorized request to start container. ");
if (!remoteUgi.getUserName().equals(containerIDStr)) {
unauthorized = true;
messageBuilder.append("\nExpected containerId: "
+ remoteUgi.getUserName() + " Found: " + containerIDStr);
- }
-
- if (launchContext != null) {
-
- // Verify other things for startContainer() request.
+ } else if (launchContext != null) {
+ // Verify other things also for startContainer() request.
if (LOG.isDebugEnabled()) {
- LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
- + remoteUgi.getTokenIdentifiers().size());
+ LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
+ + remoteUgi.getTokenIdentifiers().size());
}
- // We must and should get only one TokenIdentifier from the RPC.
- ContainerTokenIdentifier tokenId = (ContainerTokenIdentifier) remoteUgi
- .getTokenIdentifiers().iterator().next();
+
+ // Get the tokenId from the remote user ugi
+ ContainerTokenIdentifier tokenId =
+ selectContainerTokenIdentifier(remoteUgi);
+
if (tokenId == null) {
unauthorized = true;
messageBuilder
@@ -324,6 +340,23 @@ public class ContainerManagerImpl extend
+ containerIDStr);
} else {
+ // Is the container coming in with correct user-name?
+ if (!tokenId.getApplicationSubmitter().equals(launchContext.getUser())) {
+ unauthorized = true;
+ messageBuilder.append("\n Expected user-name "
+ + tokenId.getApplicationSubmitter() + " but found "
+ + launchContext.getUser());
+ }
+
+ // Is the container being relaunched? Or RPC layer let startCall with
+ // tokens generated off old-secret through?
+ if (!this.context.getContainerTokenSecretManager()
+ .isValidStartContainerRequest(tokenId)) {
+ unauthorized = true;
+ messageBuilder.append("\n Attempt to relaunch the same " +
+ "container with id " + containerIDStr + ".");
+ }
+
// Ensure the token is not expired.
// Token expiry is not checked for stopContainer/getContainerStatus
if (tokenId.getExpiryTimeStamp() < System.currentTimeMillis()) {
@@ -348,7 +381,7 @@ public class ContainerManagerImpl extend
throw RPCUtil.getRemoteException(msg);
}
}
-
+
/**
* Start a container on this NodeManager.
*/
@@ -359,10 +392,13 @@ public class ContainerManagerImpl extend
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
ContainerId containerID = launchContext.getContainerId();
- authorizeRequest(containerID, launchContext);
+ String containerIDStr = containerID.toString();
+
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, launchContext, remoteUgi);
- LOG.info("Start request for " + launchContext.getContainerId()
- + " by user " + launchContext.getUser());
+ LOG.info("Start request for " + containerIDStr + " by user "
+ + launchContext.getUser());
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getContainerTokens();
@@ -394,14 +430,14 @@ public class ContainerManagerImpl extend
AuditConstants.START_CONTAINER, "ContainerManagerImpl",
"Container already running on this node!",
applicationID, containerID);
- throw RPCUtil.getRemoteException("Container " + containerID
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " already is running on this node!!");
}
// Create the application
Application application =
new ApplicationImpl(dispatcher, this.aclsManager,
- launchContext.getUser(), applicationID, credentials, context);
+ launchContext.getUser(), applicationID, credentials, context);
if (null ==
context.getApplications().putIfAbsent(applicationID, application)) {
LOG.info("Creating a new application reference for app "
@@ -414,6 +450,12 @@ public class ContainerManagerImpl extend
// TODO: Validate the request
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
+ if (UserGroupInformation.isSecurityEnabled()) {
+ ContainerTokenIdentifier tokenId =
+ selectContainerTokenIdentifier(remoteUgi);
+ this.context.getContainerTokenSecretManager().startContainerSuccessful(
+ tokenId);
+ }
NMAuditLogger.logSuccess(launchContext.getUser(),
AuditConstants.START_CONTAINER, "ContainerManageImpl",
@@ -438,8 +480,12 @@ public class ContainerManagerImpl extend
throws YarnRemoteException {
ContainerId containerID = request.getContainerId();
+ String containerIDStr = containerID.toString();
+
// TODO: Only the container's owner can kill containers today.
- authorizeRequest(containerID, null);
+
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, null, remoteUgi);
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
@@ -476,10 +522,14 @@ public class ContainerManagerImpl extend
GetContainerStatusRequest request) throws YarnRemoteException {
ContainerId containerID = request.getContainerId();
+ String containerIDStr = containerID.toString();
+
// TODO: Only the container's owner can get containers' status today.
- authorizeRequest(containerID, null);
- LOG.info("Getting container-status for " + containerID);
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, null, remoteUgi);
+
+ LOG.info("Getting container-status for " + containerIDStr);
Container container = this.context.getContainers().get(containerID);
if (container != null) {
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
@@ -490,7 +540,7 @@ public class ContainerManagerImpl extend
return response;
}
- throw RPCUtil.getRemoteException("Container " + containerID
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " is not handled by this NodeManager");
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Tue Oct 16 00:02:55 2012
@@ -28,8 +28,9 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
@@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -365,6 +367,10 @@ public class ApplicationImpl implements
@Override
public void transition(ApplicationImpl app, ApplicationEvent event) {
+ // Inform the ContainerTokenSecretManager
+ if (UserGroupInformation.isSecurityEnabled()) {
+ app.context.getContainerTokenSecretManager().appFinished(app.appId);
+ }
// Inform the logService
app.dispatcher.getEventHandler().handle(
new LogHandlerAppFinishedEvent(app.appId));
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Tue Oct 16 00:02:55 2012
@@ -209,6 +209,7 @@ public class ContainerLocalizer {
}
switch (rsrc.getType()) {
case ARCHIVE:
+ case PATTERN:
return 5 * rsrc.getSize();
case FILE:
default:
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java Tue Oct 16 00:02:55 2012
@@ -34,6 +34,7 @@ public class LocalResourceRequest
private final long timestamp;
private final LocalResourceType type;
private final LocalResourceVisibility visibility;
+ private final String pattern;
/**
* Wrap API resource to match against cache of localized resources.
@@ -45,22 +46,28 @@ public class LocalResourceRequest
this(ConverterUtils.getPathFromYarnURL(resource.getResource()),
resource.getTimestamp(),
resource.getType(),
- resource.getVisibility());
+ resource.getVisibility(),
+ resource.getPattern());
}
LocalResourceRequest(Path loc, long timestamp, LocalResourceType type,
- LocalResourceVisibility visibility) {
+ LocalResourceVisibility visibility, String pattern) {
this.loc = loc;
this.timestamp = timestamp;
this.type = type;
this.visibility = visibility;
+ this.pattern = pattern;
}
@Override
public int hashCode() {
- return loc.hashCode() ^
+ int hash = loc.hashCode() ^
(int)((timestamp >>> 32) ^ timestamp) *
type.hashCode();
+ if(pattern != null) {
+ hash = hash ^ pattern.hashCode();
+ }
+ return hash;
}
@Override
@@ -72,9 +79,14 @@ public class LocalResourceRequest
return false;
}
final LocalResourceRequest other = (LocalResourceRequest) o;
+ String pattern = getPattern();
+ String otherPattern = other.getPattern();
+ boolean patternEquals = (pattern == null && otherPattern == null) ||
+ (pattern != null && otherPattern != null && pattern.equals(otherPattern));
return getPath().equals(other.getPath()) &&
getTimestamp() == other.getTimestamp() &&
- getType() == other.getType();
+ getType() == other.getType() &&
+ patternEquals;
}
@Override
@@ -87,6 +99,19 @@ public class LocalResourceRequest
ret = (int)(getTimestamp() - other.getTimestamp());
if (0 == ret) {
ret = getType().ordinal() - other.getType().ordinal();
+ if (0 == ret) {
+ String pattern = getPattern();
+ String otherPattern = other.getPattern();
+ if (pattern == null && otherPattern == null) {
+ ret = 0;
+ } else if (pattern == null) {
+ ret = -1;
+ } else if (otherPattern == null) {
+ ret = 1;
+ } else {
+ ret = pattern.compareTo(otherPattern);
+ }
+ }
}
}
return ret;
@@ -122,6 +147,11 @@ public class LocalResourceRequest
}
@Override
+ public String getPattern() {
+ return pattern;
+ }
+
+ @Override
public void setResource(URL resource) {
throw new UnsupportedOperationException();
}
@@ -145,14 +175,20 @@ public class LocalResourceRequest
public void setVisibility(LocalResourceVisibility visibility) {
throw new UnsupportedOperationException();
}
-
+
+ @Override
+ public void setPattern(String pattern) {
+ throw new UnsupportedOperationException();
+ }
+
@Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("{ ");
sb.append(getPath().toString()).append(", ");
sb.append(getTimestamp()).append(", ");
- sb.append(getType()).append(" }");
+ sb.append(getType()).append(", ");
+ sb.append(getPattern()).append(" }");
return sb.toString();
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Tue Oct 16 00:02:55 2012
@@ -219,7 +219,8 @@ public class LocalizedResource implement
ContainerId container = ctxt.getContainerId();
rsrc.ref.add(container);
rsrc.dispatcher.getEventHandler().handle(
- new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt));
+ new LocalizerResourceRequestEvent(rsrc, req.getVisibility(), ctxt,
+ req.getLocalResourceRequest().getPattern()));
}
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Tue Oct 16 00:02:55 2012
@@ -181,6 +181,7 @@ public class ResourceLocalizationService
try {
// TODO queue deletions here, rather than NM init?
FileContext lfs = getLocalFileContext(conf);
+ lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
List<String> localDirs = dirsHandler.getLocalDirs();
for (String localDir : localDirs) {
// $local/usercache
@@ -751,6 +752,7 @@ public class ResourceLocalizationService
next.setTimestamp(nextRsrc.getTimestamp());
next.setType(nextRsrc.getType());
next.setVisibility(evt.getVisibility());
+ next.setPattern(evt.getPattern());
scheduled.put(nextRsrc, evt);
return next;
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java Tue Oct 16 00:02:55 2012
@@ -32,14 +32,16 @@ public class LocalizerResourceRequestEve
private final LocalizerContext context;
private final LocalizedResource resource;
private final LocalResourceVisibility vis;
+ private final String pattern;
public LocalizerResourceRequestEvent(LocalizedResource resource,
- LocalResourceVisibility vis, LocalizerContext context) {
+ LocalResourceVisibility vis, LocalizerContext context, String pattern) {
super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
ConverterUtils.toString(context.getContainerId()));
this.vis = vis;
this.context = context;
this.resource = resource;
+ this.pattern = pattern;
}
public LocalizedResource getResource() {
@@ -54,4 +56,8 @@ public class LocalizerResourceRequestEve
return vis;
}
+ public String getPattern() {
+ return pattern;
+ }
+
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregator.java Tue Oct 16 00:02:55 2012
@@ -26,7 +26,4 @@ public interface AppLogAggregator extend
boolean wasContainerSuccessful);
void finishLogAggregation();
-
- void join();
-
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java Tue Oct 16 00:02:55 2012
@@ -137,6 +137,9 @@ public class AppLogAggregatorImpl implem
try {
doAppLogAggregation();
} finally {
+ if (!this.appAggregationFinished.get()) {
+ LOG.warn("Aggregation did not complete for application " + appId);
+ }
this.appAggregationFinished.set(true);
}
}
@@ -155,6 +158,7 @@ public class AppLogAggregatorImpl implem
}
} catch (InterruptedException e) {
LOG.warn("PendingContainers queue is interrupted");
+ this.appFinishing.set(true);
}
}
@@ -197,6 +201,7 @@ public class AppLogAggregatorImpl implem
this.dispatcher.getEventHandler().handle(
new ApplicationEvent(this.appId,
ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED));
+ this.appAggregationFinished.set(true);
}
private Path getRemoteNodeTmpLogFileForApp() {
@@ -250,21 +255,4 @@ public class AppLogAggregatorImpl implem
LOG.info("Application just finished : " + this.applicationId);
this.appFinishing.set(true);
}
-
- @Override
- public void join() {
- // Aggregation service is finishing
- this.finishLogAggregation();
-
- while (!this.appAggregationFinished.get()) {
- LOG.info("Waiting for aggregation to complete for "
- + this.applicationId);
- try {
- Thread.sleep(THREAD_SLEEP_TIME);
- } catch (InterruptedException e) {
- LOG.warn("Join interrupted. Some logs may not have been aggregated!!");
- break;
- }
- }
- }
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java Tue Oct 16 00:02:55 2012
@@ -25,6 +25,7 @@ import java.util.concurrent.ConcurrentHa
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,8 +36,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
@@ -137,11 +136,33 @@ public class LogAggregationService exten
@Override
public synchronized void stop() {
LOG.info(this.getName() + " waiting for pending aggregation during exit");
- for (AppLogAggregator appLogAggregator : this.appLogAggregators.values()) {
- appLogAggregator.join();
- }
+ stopAggregators();
super.stop();
}
+
+ private void stopAggregators() {
+ threadPool.shutdown();
+ // politely ask to finish
+ for (AppLogAggregator aggregator : appLogAggregators.values()) {
+ aggregator.finishLogAggregation();
+ }
+ while (!threadPool.isTerminated()) { // wait for all threads to finish
+ for (ApplicationId appId : appLogAggregators.keySet()) {
+ LOG.info("Waiting for aggregation to complete for " + appId);
+ }
+ try {
+ if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) {
+ threadPool.shutdownNow(); // send interrupt to hurry them along
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Aggregation stop interrupted!");
+ break;
+ }
+ }
+ for (ApplicationId appId : appLogAggregators.keySet()) {
+ LOG.warn("Some logs may not have been aggregated for " + appId);
+ }
+ }
private void verifyAndCreateRemoteLogDir(Configuration conf) {
// Checking the existance of the TLD
@@ -293,10 +314,7 @@ public class LogAggregationService exten
final UserGroupInformation userUgi =
UserGroupInformation.createRemoteUser(user);
if (credentials != null) {
- for (Token<? extends TokenIdentifier> token : credentials
- .getAllTokens()) {
- userUgi.addToken(token);
- }
+ userUgi.addCredentials(credentials);
}
// New application
@@ -312,9 +330,13 @@ public class LogAggregationService exten
try {
// Create the app dir
createAppDir(user, appId, userUgi);
- } catch (YarnException e) {
+ } catch (Exception e) {
+ appLogAggregators.remove(appId);
closeFileSystems(userUgi);
- throw e;
+ if (!(e instanceof YarnException)) {
+ e = new YarnException(e);
+ }
+ throw (YarnException)e;
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java Tue Oct 16 00:02:55 2012
@@ -79,16 +79,18 @@ public class NonAggregatingLogHandler ex
@Override
public void stop() {
- sched.shutdown();
- boolean isShutdown = false;
- try {
- isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS);
- } catch (InterruptedException e) {
- sched.shutdownNow();
- isShutdown = true;
- }
- if (!isShutdown) {
- sched.shutdownNow();
+ if (sched != null) {
+ sched.shutdown();
+ boolean isShutdown = false;
+ try {
+ isShutdown = sched.awaitTermination(10, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ sched.shutdownNow();
+ isShutdown = true;
+ }
+ if (!isShutdown) {
+ sched.shutdownNow();
+ }
}
super.stop();
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java Tue Oct 16 00:02:55 2012
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.service.AbstractService;
-import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import com.google.common.base.Preconditions;
@@ -60,6 +60,8 @@ public class ContainersMonitorImpl exten
private final Dispatcher eventDispatcher;
private final Context context;
private ResourceCalculatorPlugin resourceCalculatorPlugin;
+ private Configuration conf;
+ private Class<? extends ResourceCalculatorProcessTree> processTreeClass;
private long maxVmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
private long maxPmemAllottedForContainers = DISABLED_MEMORY_LIMIT;
@@ -96,6 +98,11 @@ public class ContainersMonitorImpl exten
ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
LOG.info(" Using ResourceCalculatorPlugin : "
+ this.resourceCalculatorPlugin);
+ processTreeClass = conf.getClass(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, null,
+ ResourceCalculatorProcessTree.class);
+ this.conf = conf;
+ LOG.info(" Using ResourceCalculatorProcessTree : "
+ + this.processTreeClass);
long totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT;
if (this.resourceCalculatorPlugin != null) {
@@ -140,7 +147,7 @@ public class ContainersMonitorImpl exten
/**
* Is the total physical memory check enabled?
- *
+ *
* @return true if total physical memory check is enabled.
*/
boolean isPhysicalMemoryCheckEnabled() {
@@ -149,7 +156,7 @@ public class ContainersMonitorImpl exten
/**
* Is the total virtual memory check enabled?
- *
+ *
* @return true if total virtual memory check is enabled.
*/
boolean isVirtualMemoryCheckEnabled() {
@@ -157,12 +164,16 @@ public class ContainersMonitorImpl exten
}
private boolean isEnabled() {
- if (!ProcfsBasedProcessTree.isAvailable()) {
- LOG.info("ProcessTree implementation is missing on this system. "
- + this.getClass().getName() + " is disabled.");
- return false;
+ if (resourceCalculatorPlugin == null) {
+ LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
+ + this.getClass().getName() + " is disabled.");
+ return false;
+ }
+ if (ResourceCalculatorProcessTree.getResourceCalculatorProcessTree("0", processTreeClass, conf) == null) {
+ LOG.info("ResourceCalculatorProcessTree is unavailable on this system. "
+ + this.getClass().getName() + " is disabled.");
+ return false;
}
-
if (!(isPhysicalMemoryCheckEnabled() || isVirtualMemoryCheckEnabled())) {
LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
"needed. Not running the monitor-thread");
@@ -196,12 +207,12 @@ public class ContainersMonitorImpl exten
private static class ProcessTreeInfo {
private ContainerId containerId;
private String pid;
- private ProcfsBasedProcessTree pTree;
+ private ResourceCalculatorProcessTree pTree;
private long vmemLimit;
private long pmemLimit;
public ProcessTreeInfo(ContainerId containerId, String pid,
- ProcfsBasedProcessTree pTree, long vmemLimit, long pmemLimit) {
+ ResourceCalculatorProcessTree pTree, long vmemLimit, long pmemLimit) {
this.containerId = containerId;
this.pid = pid;
this.pTree = pTree;
@@ -221,11 +232,11 @@ public class ContainersMonitorImpl exten
this.pid = pid;
}
- public ProcfsBasedProcessTree getProcessTree() {
+ public ResourceCalculatorProcessTree getProcessTree() {
return this.pTree;
}
- public void setProcessTree(ProcfsBasedProcessTree pTree) {
+ public void setProcessTree(ResourceCalculatorProcessTree pTree) {
this.pTree = pTree;
}
@@ -245,20 +256,20 @@ public class ContainersMonitorImpl exten
/**
* Check whether a container's process tree's current memory usage is over
* limit.
- *
+ *
* When a java process exec's a program, it could momentarily account for
* double the size of it's memory, because the JVM does a fork()+exec()
* which at fork time creates a copy of the parent's memory. If the
* monitoring thread detects the memory used by the container tree at the
* same instance, it could assume it is over limit and kill the tree, for no
* fault of the process itself.
- *
+ *
* We counter this problem by employing a heuristic check: - if a process
* tree exceeds the memory limit by more than twice, it is killed
* immediately - if a process tree has processes older than the monitoring
* interval exceeding the memory limit by even 1 time, it is killed. Else it
* is given the benefit of doubt to lie around for one more iteration.
- *
+ *
* @param containerId
* Container Id for the container tree
* @param currentMemUsage
@@ -295,7 +306,7 @@ public class ContainersMonitorImpl exten
}
// method provided just for easy testing purposes
- boolean isProcessTreeOverLimit(ProcfsBasedProcessTree pTree,
+ boolean isProcessTreeOverLimit(ResourceCalculatorProcessTree pTree,
String containerId, long limit) {
long currentMemUsage = pTree.getCumulativeVmem();
// as processes begin with an age 1, we want to see if there are processes
@@ -370,9 +381,8 @@ public class ContainersMonitorImpl exten
LOG.debug("Tracking ProcessTree " + pId
+ " for the first time");
- ProcfsBasedProcessTree pt =
- new ProcfsBasedProcessTree(pId,
- ContainerExecutor.isSetsidAvailable);
+ ResourceCalculatorProcessTree pt =
+ ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pId, processTreeClass, conf);
ptInfo.setPid(pId);
ptInfo.setProcessTree(pt);
}
@@ -385,7 +395,7 @@ public class ContainersMonitorImpl exten
LOG.debug("Constructing ProcessTree for : PID = " + pId
+ " ContainerId = " + containerId);
- ProcfsBasedProcessTree pTree = ptInfo.getProcessTree();
+ ResourceCalculatorProcessTree pTree = ptInfo.getProcessTree();
pTree = pTree.getProcessTree(); // get the updated process-tree
ptInfo.setProcessTree(pTree); // update ptInfo with proces-tree of
// updated state
@@ -471,7 +481,7 @@ public class ContainersMonitorImpl exten
private String formatErrorMessage(String memTypeExceeded,
long currentVmemUsage, long vmemLimit,
long currentPmemUsage, long pmemLimit,
- String pId, ContainerId containerId, ProcfsBasedProcessTree pTree) {
+ String pId, ContainerId containerId, ResourceCalculatorProcessTree pTree) {
return
String.format("Container [pid=%s,containerID=%s] is running beyond %s memory limits. ",
pId, containerId, memTypeExceeded) +
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Tue Oct 16 00:02:55 2012
@@ -27,8 +27,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
public class DummyContainerManager extends ContainerManagerImpl {
@@ -59,11 +58,10 @@ public class DummyContainerManager exten
public DummyContainerManager(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager,
ApplicationACLsManager applicationACLsManager,
LocalDirsHandlerService dirsHandler) {
super(context, exec, deletionContext, nodeStatusUpdater, metrics,
- containerTokenSecretManager, applicationACLsManager, dirsHandler);
+ applicationACLsManager, dirsHandler);
}
@Override
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java Tue Oct 16 00:02:55 2012
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import java.io.FileNotFoundException;
import java.io.InputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -26,8 +27,11 @@ import java.util.EnumSet;
import java.util.List;
import java.util.Random;
+import junit.framework.Assert;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
@@ -38,6 +42,7 @@ import org.apache.hadoop.fs.permission.F
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.util.Progressable;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.FakeFSDataInputStream;
import static org.apache.hadoop.fs.CreateFlag.*;
@@ -91,11 +96,16 @@ public class TestDefaultContainerExecuto
}
*/
+ private static final Path BASE_TMP_PATH = new Path("target",
+ TestDefaultContainerExecutor.class.getSimpleName());
+
@AfterClass
public static void deleteTmpFiles() throws IOException {
FileContext lfs = FileContext.getLocalFSFileContext();
- lfs.delete(new Path("target",
- TestDefaultContainerExecutor.class.getSimpleName()), true);
+ try {
+ lfs.delete(BASE_TMP_PATH, true);
+ } catch (FileNotFoundException e) {
+ }
}
byte[] createTmpFile(Path dst, Random r, int len)
@@ -116,6 +126,71 @@ public class TestDefaultContainerExecuto
return bytes;
}
+ @Test
+ public void testDirPermissions() throws Exception {
+ deleteTmpFiles();
+
+ final String user = "somebody";
+ final String appId = "app_12345_123";
+ final FsPermission userCachePerm = new FsPermission(
+ DefaultContainerExecutor.USER_PERM);
+ final FsPermission appCachePerm = new FsPermission(
+ DefaultContainerExecutor.APPCACHE_PERM);
+ final FsPermission fileCachePerm = new FsPermission(
+ DefaultContainerExecutor.FILECACHE_PERM);
+ final FsPermission appDirPerm = new FsPermission(
+ DefaultContainerExecutor.APPDIR_PERM);
+ final FsPermission logDirPerm = new FsPermission(
+ DefaultContainerExecutor.LOGDIR_PERM);
+ List<String> localDirs = new ArrayList<String>();
+ localDirs.add(new Path(BASE_TMP_PATH, "localDirA").toString());
+ localDirs.add(new Path(BASE_TMP_PATH, "localDirB").toString());
+ List<String> logDirs = new ArrayList<String>();
+ logDirs.add(new Path(BASE_TMP_PATH, "logDirA").toString());
+ logDirs.add(new Path(BASE_TMP_PATH, "logDirB").toString());
+
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+ FileContext lfs = FileContext.getLocalFSFileContext(conf);
+ DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs);
+ executor.init();
+
+ try {
+ executor.createUserLocalDirs(localDirs, user);
+ executor.createUserCacheDirs(localDirs, user);
+ executor.createAppDirs(localDirs, user, appId);
+
+ for (String dir : localDirs) {
+ FileStatus stats = lfs.getFileStatus(
+ new Path(new Path(dir, ContainerLocalizer.USERCACHE), user));
+ Assert.assertEquals(userCachePerm, stats.getPermission());
+ }
+
+ for (String dir : localDirs) {
+ Path userCachePath = new Path(
+ new Path(dir, ContainerLocalizer.USERCACHE), user);
+ Path appCachePath = new Path(userCachePath,
+ ContainerLocalizer.APPCACHE);
+ FileStatus stats = lfs.getFileStatus(appCachePath);
+ Assert.assertEquals(appCachePerm, stats.getPermission());
+ stats = lfs.getFileStatus(
+ new Path(userCachePath, ContainerLocalizer.FILECACHE));
+ Assert.assertEquals(fileCachePerm, stats.getPermission());
+ stats = lfs.getFileStatus(new Path(appCachePath, appId));
+ Assert.assertEquals(appDirPerm, stats.getPermission());
+ }
+
+ executor.createAppLogDirs(appId, logDirs);
+
+ for (String dir : logDirs) {
+ FileStatus stats = lfs.getFileStatus(new Path(dir, appId));
+ Assert.assertEquals(logDirPerm, stats.getPermission());
+ }
+ } finally {
+ deleteTmpFiles();
+ }
+ }
+
// @Test
// public void testInit() throws IOException, InterruptedException {
// Configuration conf = new Configuration();
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java Tue Oct 16 00:02:55 2012
@@ -23,7 +23,13 @@ import java.io.IOException;
import java.util.List;
import java.util.ListIterator;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -65,4 +71,37 @@ public class TestDirectoryCollection {
// Verify no ConcurrentModification is thrown
li.next();
}
+
+ @Test
+ public void testCreateDirectories() throws IOException {
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
+ FileContext localFs = FileContext.getLocalFSFileContext(conf);
+
+ String dirA = new File(testDir, "dirA").getPath();
+ String dirB = new File(dirA, "dirB").getPath();
+ String dirC = new File(testDir, "dirC").getPath();
+ Path pathC = new Path(dirC);
+ FsPermission permDirC = new FsPermission((short)0710);
+
+ localFs.mkdir(pathC, null, true);
+ localFs.setPermission(pathC, permDirC);
+
+ String[] dirs = { dirA, dirB, dirC };
+ DirectoryCollection dc = new DirectoryCollection(dirs);
+ FsPermission defaultPerm = FsPermission.getDefault()
+ .applyUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
+ boolean createResult = dc.createNonExistentDirs(localFs, defaultPerm);
+ Assert.assertTrue(createResult);
+
+ FileStatus status = localFs.getFileStatus(new Path(dirA));
+ Assert.assertEquals("local dir parent not created with proper permissions",
+ defaultPerm, status.getPermission());
+ status = localFs.getFileStatus(new Path(dirB));
+ Assert.assertEquals("local dir not created with proper permissions",
+ defaultPerm, status.getPermission());
+ status = localFs.getFileStatus(pathC);
+ Assert.assertEquals("existing local directory permissions modified",
+ permDirC, status.getPermission());
+ }
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Oct 16 00:02:55 2012
@@ -36,12 +36,12 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.junit.Test;
public class TestEventFlow {
@@ -69,9 +69,9 @@ public class TestEventFlow {
localLogDir.mkdir();
remoteLogDir.mkdir();
- Context context = new NMContext();
-
YarnConfiguration conf = new YarnConfiguration();
+ Context context = new NMContext(new NMContainerTokenSecretManager(conf));
+
conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
@@ -86,10 +86,8 @@ public class TestEventFlow {
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
NodeManagerMetrics metrics = NodeManagerMetrics.create();
- ContainerTokenSecretManager containerTokenSecretManager =
- new ContainerTokenSecretManager(conf);
NodeStatusUpdater nodeStatusUpdater =
- new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics, containerTokenSecretManager) {
+ new NodeStatusUpdaterImpl(context, dispatcher, healthChecker, metrics) {
@Override
protected ResourceTracker getRMClient() {
return new LocalRMInterface();
@@ -101,10 +99,9 @@ public class TestEventFlow {
}
};
- DummyContainerManager containerManager = new DummyContainerManager(
- context, exec, del, nodeStatusUpdater, metrics,
- containerTokenSecretManager, new ApplicationACLsManager(conf),
- dirsHandler);
+ DummyContainerManager containerManager =
+ new DummyContainerManager(context, exec, del, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler);
containerManager.init(conf);
containerManager.start();
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java Tue Oct 16 00:02:55 2012
@@ -17,37 +17,29 @@
*/
package org.apache.hadoop.yarn.server.nodemanager;
+import static junit.framework.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.TestRPC.TestImpl;
import org.apache.hadoop.ipc.TestRPC.TestProtocol;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
-import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.Keys;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
-
-import org.apache.hadoop.net.NetUtils;
-
-import static org.mockito.Mockito.*;
-import static junit.framework.Assert.*;
-import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-
/**
* Tests {@link NMAuditLogger}.
*/
public class TestNMAuditLogger {
- private static final Log LOG = LogFactory.getLog(TestNMAuditLogger.class);
private static final String USER = "test";
private static final String OPERATION = "oper";
private static final String TARGET = "tgt";
@@ -212,8 +204,10 @@ public class TestNMAuditLogger {
public void testNMAuditLoggerWithIP() throws Exception {
Configuration conf = new Configuration();
// start the IPC server
- Server server = RPC.getServer(TestProtocol.class,
- new MyTestRPCServer(), "0.0.0.0", 0, 5, true, conf, null);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new MyTestRPCServer()).setBindAddress("0.0.0.0")
+ .setPort(0).setNumHandlers(5).setVerbose(true).build();
+
server.start();
InetSocketAddress addr = NetUtils.getConnectAddress(server);
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Oct 16 00:02:55 2012
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -234,10 +233,8 @@ public class TestNodeStatusUpdater {
private Context context;
public MyNodeStatusUpdater(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager) {
- super(context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
this.context = context;
}
@@ -252,10 +249,8 @@ public class TestNodeStatusUpdater {
private Context context;
public MyNodeStatusUpdater3(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager) {
- super(context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
this.context = context;
this.resourceTracker = new MyResourceTracker3(this.context);
}
@@ -266,7 +261,7 @@ public class TestNodeStatusUpdater {
}
@Override
- protected boolean isSecurityEnabled() {
+ protected boolean isTokenKeepAliveEnabled(Configuration conf) {
return true;
}
}
@@ -276,11 +271,9 @@ public class TestNodeStatusUpdater {
private MyNodeStatusUpdater3 nodeStatusUpdater;
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
this.nodeStatusUpdater =
- new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ new MyNodeStatusUpdater3(context, dispatcher, healthChecker, metrics);
return this.nodeStatusUpdater;
}
@@ -398,10 +391,9 @@ public class TestNodeStatusUpdater {
nm = new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
- metrics, containerTokenSecretManager);
+ metrics);
}
};
@@ -528,11 +520,9 @@ public class TestNodeStatusUpdater {
nm = new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
MyNodeStatusUpdater nodeStatusUpdater = new MyNodeStatusUpdater(
- context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ context, dispatcher, healthChecker, metrics);
MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
myResourceTracker2.registerNodeAction = NodeAction.SHUTDOWN;
nodeStatusUpdater.resourceTracker = myResourceTracker2;
@@ -556,22 +546,19 @@ public class TestNodeStatusUpdater {
nm = new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new MyNodeStatusUpdater(context, dispatcher, healthChecker,
- metrics, containerTokenSecretManager);
+ metrics);
}
@Override
protected ContainerManagerImpl createContainerManager(Context context,
ContainerExecutor exec, DeletionService del,
NodeStatusUpdater nodeStatusUpdater,
- ContainerTokenSecretManager containerTokenSecretManager,
ApplicationACLsManager aclsManager,
LocalDirsHandlerService diskhandler) {
- return new ContainerManagerImpl(context, exec, del,
- nodeStatusUpdater, metrics, containerTokenSecretManager,
- aclsManager, diskhandler) {
+ return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
+ metrics, aclsManager, diskhandler) {
@Override
public void start() {
// Simulating failure of starting RPC server
@@ -654,11 +641,9 @@ public class TestNodeStatusUpdater {
return new NodeManager() {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
- context, dispatcher, healthChecker, metrics,
- containerTokenSecretManager);
+ context, dispatcher, healthChecker, metrics);
MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
myResourceTracker2.heartBeatNodeAction = nodeHeartBeatAction;
myNodeStatusUpdater.resourceTracker = myResourceTracker2;
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Oct 16 00:02:55 2012
@@ -54,8 +54,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.junit.After;
import org.junit.Before;
@@ -70,8 +70,6 @@ public abstract class BaseContainerManag
protected static File localLogDir;
protected static File remoteLogDir;
protected static File tmpDir;
- protected ContainerTokenSecretManager containerTokenSecretManager =
- new ContainerTokenSecretManager(new Configuration());
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
@@ -93,7 +91,8 @@ public abstract class BaseContainerManag
.getLog(BaseContainerManagerTest.class);
protected Configuration conf = new YarnConfiguration();
- protected Context context = new NMContext();
+ protected Context context = new NMContext(new NMContainerTokenSecretManager(
+ conf));
protected ContainerExecutor exec;
protected DeletionService delSrvc;
protected String user = "nobody";
@@ -101,7 +100,7 @@ public abstract class BaseContainerManag
protected LocalDirsHandlerService dirsHandler;
protected NodeStatusUpdater nodeStatusUpdater = new NodeStatusUpdaterImpl(
- context, new AsyncDispatcher(), null, metrics, this.containerTokenSecretManager) {
+ context, new AsyncDispatcher(), null, metrics) {
@Override
protected ResourceTracker getRMClient() {
return new LocalRMInterface();
@@ -155,9 +154,9 @@ public abstract class BaseContainerManag
nodeHealthChecker = new NodeHealthCheckerService();
nodeHealthChecker.init(conf);
dirsHandler = nodeHealthChecker.getDiskHandler();
- containerManager = new ContainerManagerImpl(context, exec, delSrvc,
- nodeStatusUpdater, metrics, this.containerTokenSecretManager,
- new ApplicationACLsManager(conf), dirsHandler);
+ containerManager =
+ new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler);
containerManager.init(conf);
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Tue Oct 16 00:02:55 2012
@@ -51,14 +51,13 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
-import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
+import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
@@ -384,11 +383,9 @@ public class TestContainerManager extend
delSrvc = new DeletionService(exec);
delSrvc.init(conf);
- ContainerTokenSecretManager containerTokenSecretManager = new
- ContainerTokenSecretManager(conf);
- containerManager = new ContainerManagerImpl(context, exec, delSrvc,
- nodeStatusUpdater, metrics, containerTokenSecretManager,
- new ApplicationACLsManager(conf), dirsHandler);
+ containerManager =
+ new ContainerManagerImpl(context, exec, delSrvc, nodeStatusUpdater,
+ metrics, new ApplicationACLsManager(conf), dirsHandler);
containerManager.init(conf);
containerManager.start();
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Tue Oct 16 00:02:55 2012
@@ -30,9 +30,9 @@ import java.util.HashMap;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -50,8 +50,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorEventType;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java Tue Oct 16 00:02:55 2012
@@ -37,7 +37,7 @@ import static org.junit.Assert.*;
public class TestLocalResource {
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
- long timestamp, LocalResourceType type, LocalResourceVisibility state)
+ long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
throws URISyntaxException {
org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
@@ -45,6 +45,7 @@ public class TestLocalResource {
ret.setTimestamp(timestamp);
ret.setType(type);
ret.setVisibility(state);
+ ret.setPattern(pattern);
return ret;
}
@@ -72,9 +73,9 @@ public class TestLocalResource {
long basetime = r.nextLong() >>> 2;
org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
- new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null);
org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
- new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, null);
final LocalResourceRequest a = new LocalResourceRequest(yA);
LocalResourceRequest b = new LocalResourceRequest(yA);
checkEqual(a, b);
@@ -83,31 +84,37 @@ public class TestLocalResource {
// ignore visibility
yB = getYarnResource(
- new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE);
+ new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PRIVATE, null);
b = new LocalResourceRequest(yB);
checkEqual(a, b);
// ignore size
yB = getYarnResource(
- new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE);
+ new Path("http://yak.org:80/foobar"), 0, basetime, FILE, PRIVATE, null);
b = new LocalResourceRequest(yB);
checkEqual(a, b);
// note path
yB = getYarnResource(
- new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC);
+ new Path("hdfs://dingo.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null);
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
// note type
yB = getYarnResource(
- new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC);
+ new Path("http://yak.org:80/foobar"), 0, basetime, ARCHIVE, PUBLIC, null);
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
// note timestamp
yB = getYarnResource(
- new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, null);
+ b = new LocalResourceRequest(yB);
+ checkNotEqual(a, b);
+
+ // note pattern
+ yB = getYarnResource(
+ new Path("http://yak.org:80/foobar"), 0, basetime + 1, FILE, PUBLIC, "^/foo/.*");
b = new LocalResourceRequest(yB);
checkNotEqual(a, b);
}
@@ -120,24 +127,35 @@ public class TestLocalResource {
System.out.println("SEED: " + seed);
long basetime = r.nextLong() >>> 2;
org.apache.hadoop.yarn.api.records.LocalResource yA = getYarnResource(
- new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar"), -1, basetime, FILE, PUBLIC, "^/foo/.*");
final LocalResourceRequest a = new LocalResourceRequest(yA);
// Path primary
org.apache.hadoop.yarn.api.records.LocalResource yB = getYarnResource(
- new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobaz"), -1, basetime, FILE, PUBLIC, "^/foo/.*");
LocalResourceRequest b = new LocalResourceRequest(yB);
assertTrue(0 > a.compareTo(b));
// timestamp secondary
yB = getYarnResource(
- new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC);
+ new Path("http://yak.org:80/foobar"), -1, basetime + 1, FILE, PUBLIC, "^/foo/.*");
b = new LocalResourceRequest(yB);
assertTrue(0 > a.compareTo(b));
// type tertiary
yB = getYarnResource(
- new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC);
+ new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/foo/.*");
+ b = new LocalResourceRequest(yB);
+ assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
+
+ // path 4th
+ yB = getYarnResource(
+ new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, "^/food/.*");
+ b = new LocalResourceRequest(yB);
+ assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
+
+ yB = getYarnResource(
+ new Path("http://yak.org:80/foobar"), -1, basetime, ARCHIVE, PUBLIC, null);
b = new LocalResourceRequest(yB);
assertTrue(0 != a.compareTo(b)); // don't care about order, just ne
}
Modified: hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1398581&r1=1398580&r2=1398581&view=diff
==============================================================================
--- hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/MR-3902/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java Tue Oct 16 00:02:55 2012
@@ -230,7 +230,7 @@ public class TestLocalResourcesTrackerIm
long ts, LocalResourceVisibility vis) {
final LocalResourceRequest req =
new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i),
- ts + i * 2000, LocalResourceType.FILE, vis);
+ ts + i * 2000, LocalResourceType.FILE, vis, null);
return req;
}