You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/14 01:06:02 UTC
svn commit: r1467713 [2/3] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apach...
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java Sat Apr 13 23:05:54 2013
@@ -40,8 +40,5 @@ interface LocalResourcesTracker
String getUser();
- // TODO: Remove this in favour of EventHandler.handle
- void localizationCompleted(LocalResourceRequest req, boolean success);
-
long nextUniqueNumber();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java Sat Apr 13 23:05:54 2013
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
/**
@@ -96,13 +97,22 @@ class LocalResourcesTrackerImpl implemen
this.conf = conf;
}
+ /*
+ * Synchronizing this method for avoiding races due to multiple ResourceEvent's
+ * coming to LocalResourcesTracker from Public/Private localizer and
+ * Resource Localization Service.
+ */
@Override
- public void handle(ResourceEvent event) {
+ public synchronized void handle(ResourceEvent event) {
LocalResourceRequest req = event.getLocalResourceRequest();
LocalizedResource rsrc = localrsrc.get(req);
switch (event.getType()) {
- case REQUEST:
case LOCALIZED:
+ if (useLocalCacheDirectoryManager) {
+ inProgressLocalResourcesMap.remove(req);
+ }
+ break;
+ case REQUEST:
if (rsrc != null && (!isResourcePresent(rsrc))) {
LOG.info("Resource " + rsrc.getLocalPath()
+ " is missing, localizing it again");
@@ -117,10 +127,24 @@ class LocalResourcesTrackerImpl implemen
break;
case RELEASE:
if (null == rsrc) {
- LOG.info("Release unknown rsrc null (discard)");
+ // The container sent a release event on a resource which
+ // 1) Failed
+ // 2) Removed for some reason (ex. disk is no longer accessible)
+ ResourceReleaseEvent relEvent = (ResourceReleaseEvent) event;
+ LOG.info("Container " + relEvent.getContainer()
+ + " sent RELEASE event on a resource request " + req
+ + " not present in cache.");
return;
}
break;
+ case LOCALIZATION_FAILED:
+ decrementFileCountForLocalCacheDirectory(req, null);
+ /*
+ * If resource localization fails then Localized resource will be
+ * removed from local cache.
+ */
+ localrsrc.remove(req);
+ break;
}
rsrc.handle(event);
}
@@ -280,18 +304,6 @@ class LocalResourcesTrackerImpl implemen
}
@Override
- public void localizationCompleted(LocalResourceRequest req,
- boolean success) {
- if (useLocalCacheDirectoryManager) {
- if (!success) {
- decrementFileCountForLocalCacheDirectory(req, null);
- } else {
- inProgressLocalResourcesMap.remove(req);
- }
- }
- }
-
- @Override
public long nextUniqueNumber() {
return uniqueNumberGenerator.incrementAndGet();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java Sat Apr 13 23:05:54 2013
@@ -32,10 +32,12 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
@@ -89,6 +91,8 @@ public class LocalizedResource implement
.addTransition(ResourceState.DOWNLOADING,
EnumSet.of(ResourceState.DOWNLOADING, ResourceState.INIT),
ResourceEventType.RELEASE, new ReleasePendingTransition())
+ .addTransition(ResourceState.DOWNLOADING, ResourceState.FAILED,
+ ResourceEventType.LOCALIZATION_FAILED, new FetchFailedTransition())
// From LOCALIZED (ref >= 0, on disk)
.addTransition(ResourceState.LOCALIZED, ResourceState.LOCALIZED,
@@ -126,12 +130,14 @@ public class LocalizedResource implement
}
private void release(ContainerId container) {
- if (!ref.remove(container)) {
- LOG.info("Attempt to release claim on " + this +
- " from unregistered container " + container);
- assert false; // TODO: FIX
+ if (ref.remove(container)) {
+ // updating the timestamp only in case of success.
+ timestamp.set(currentTime());
+ } else {
+ LOG.info("Container " + container
+ + " doesn't exist in the container list of the Resource " + this
+ + " to which it sent RELEASE event");
}
- timestamp.set(currentTime());
}
private long currentTime() {
@@ -251,6 +257,25 @@ public class LocalizedResource implement
}
/**
+ * Resource localization failed, notify waiting containers.
+ */
+ @SuppressWarnings("unchecked")
+ private static class FetchFailedTransition extends ResourceTransition {
+ @Override
+ public void transition(LocalizedResource rsrc, ResourceEvent event) {
+ ResourceFailedLocalizationEvent failedEvent =
+ (ResourceFailedLocalizationEvent) event;
+ Queue<ContainerId> containers = rsrc.ref;
+ Throwable failureCause = failedEvent.getCause();
+ for (ContainerId container : containers) {
+ rsrc.dispatcher.getEventHandler().handle(
+ new ContainerResourceFailedEvent(container, failedEvent
+ .getLocalResourceRequest(), failureCause));
+ }
+ }
+ }
+
+ /**
* Resource already localized, notify immediately.
*/
@SuppressWarnings("unchecked") // dispatcher not typed
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Sat Apr 13 23:05:54 2013
@@ -84,7 +84,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
@@ -101,6 +100,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
@@ -683,7 +683,6 @@ public class ResourceLocalizationService
}
@Override
- @SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
try {
// TODO shutdown, better error handling esp. DU
@@ -699,10 +698,8 @@ public class ResourceLocalizationService
return;
}
LocalResourceRequest key = assoc.getResource().getRequest();
- assoc.getResource().handle(
- new ResourceLocalizedEvent(key,
- local, FileUtil.getDU(new File(local.toUri()))));
- publicRsrc.localizationCompleted(key, true);
+ publicRsrc.handle(new ResourceLocalizedEvent(key, local, FileUtil
+ .getDU(new File(local.toUri()))));
synchronized (attempts) {
attempts.remove(key);
}
@@ -710,13 +707,10 @@ public class ResourceLocalizationService
LOG.info("Failed to download rsrc " + assoc.getResource(),
e.getCause());
LocalResourceRequest req = assoc.getResource().getRequest();
- dispatcher.getEventHandler().handle(
- new ContainerResourceFailedEvent(
- assoc.getContext().getContainerId(),
- req, e.getCause()));
- publicRsrc.localizationCompleted(req, false);
- List<LocalizerResourceRequestEvent> reqs;
+ publicRsrc.handle(new ResourceFailedLocalizationEvent(req, e
+ .getCause()));
synchronized (attempts) {
+ List<LocalizerResourceRequestEvent> reqs;
reqs = attempts.get(req);
if (null == reqs) {
LOG.error("Missing pending list for " + req);
@@ -724,13 +718,6 @@ public class ResourceLocalizationService
}
attempts.remove(req);
}
- // let the other containers know about the localization failure
- for (LocalizerResourceRequestEvent reqEvent : reqs) {
- dispatcher.getEventHandler().handle(
- new ContainerResourceFailedEvent(
- reqEvent.getContext().getContainerId(),
- reqEvent.getResource().getRequest(), e.getCause()));
- }
} catch (CancellationException e) {
// ignore; shutting down
}
@@ -810,13 +797,14 @@ public class ResourceLocalizationService
return null;
}
- // TODO this sucks. Fix it later
- @SuppressWarnings("unchecked") // dispatcher not typed
LocalizerHeartbeatResponse update(
List<LocalResourceStatus> remoteResourceStatuses) {
LocalizerHeartbeatResponse response =
recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
+ String user = context.getUser();
+ ApplicationId applicationId =
+ context.getContainerId().getApplicationAttemptId().getApplicationId();
// The localizer has just spawned. Start giving it resources for
// remote-fetching.
if (remoteResourceStatuses.isEmpty()) {
@@ -847,6 +835,11 @@ public class ResourceLocalizationService
}
ArrayList<ResourceLocalizationSpec> rsrcs =
new ArrayList<ResourceLocalizationSpec>();
+ /*
+ * TODO : It doesn't support multiple downloads per ContainerLocalizer
+ * at the same time. We need to think whether we should support this.
+ */
+
for (LocalResourceStatus stat : remoteResourceStatuses) {
LocalResource rsrc = stat.getResource();
LocalResourceRequest req = null;
@@ -865,11 +858,10 @@ public class ResourceLocalizationService
case FETCH_SUCCESS:
// notify resource
try {
- assoc.getResource().handle(
- new ResourceLocalizedEvent(req,
- ConverterUtils.getPathFromYarnURL(stat.getLocalPath()),
- stat.getLocalSize()));
- localizationCompleted(stat);
+ getLocalResourcesTracker(req.getVisibility(), user, applicationId)
+ .handle(
+ new ResourceLocalizedEvent(req, ConverterUtils
+ .getPathFromYarnURL(stat.getLocalPath()), stat.getLocalSize()));
} catch (URISyntaxException e) { }
if (pending.isEmpty()) {
// TODO: Synchronization
@@ -899,19 +891,16 @@ public class ResourceLocalizationService
LOG.info("DEBUG: FAILED " + req, stat.getException());
assoc.getResource().unlock();
response.setLocalizerAction(LocalizerAction.DIE);
- localizationCompleted(stat);
- // TODO: Why is this event going directly to the container. Why not
- // the resource itself? What happens to the resource? Is it removed?
- dispatcher.getEventHandler().handle(
- new ContainerResourceFailedEvent(context.getContainerId(),
- req, stat.getException()));
+ getLocalResourcesTracker(req.getVisibility(), user, applicationId)
+ .handle(
+ new ResourceFailedLocalizationEvent(req, stat.getException()));
break;
default:
LOG.info("Unknown status: " + stat.getStatus());
response.setLocalizerAction(LocalizerAction.DIE);
- dispatcher.getEventHandler().handle(
- new ContainerResourceFailedEvent(context.getContainerId(),
- req, stat.getException()));
+ getLocalResourcesTracker(req.getVisibility(), user, applicationId)
+ .handle(
+ new ResourceFailedLocalizationEvent(req, stat.getException()));
break;
}
}
@@ -919,27 +908,6 @@ public class ResourceLocalizationService
return response;
}
- private void localizationCompleted(LocalResourceStatus stat) {
- try {
- LocalResource rsrc = stat.getResource();
- LocalResourceRequest key = new LocalResourceRequest(rsrc);
- String user = context.getUser();
- ApplicationId appId =
- context.getContainerId().getApplicationAttemptId()
- .getApplicationId();
- LocalResourceVisibility vis = rsrc.getVisibility();
- LocalResourcesTracker tracker =
- getLocalResourcesTracker(vis, user, appId);
- if (stat.getStatus() == ResourceStatusType.FETCH_SUCCESS) {
- tracker.localizationCompleted(key, true);
- } else {
- tracker.localizationCompleted(key, false);
- }
- } catch (URISyntaxException e) {
- LOG.error("Invalid resource URL specified", e);
- }
- }
-
private Path getPathForLocalization(LocalResource rsrc) throws IOException,
URISyntaxException {
String user = context.getUser();
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceState.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceState.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceState.java Sat Apr 13 23:05:54 2013
@@ -20,5 +20,6 @@ package org.apache.hadoop.yarn.server.no
enum ResourceState {
INIT,
DOWNLOADING,
- LOCALIZED
+ LOCALIZED,
+ FAILED
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java Sat Apr 13 23:05:54 2013
@@ -29,5 +29,7 @@ public enum ResourceEventType {
/** See {@link ResourceLocalizedEvent} */
LOCALIZED,
/** See {@link ResourceReleaseEvent} */
- RELEASE
+ RELEASE,
+ /** See {@link ResourceFailedLocalizationEvent} */
+ LOCALIZATION_FAILED
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/ContainerInfo.java Sat Apr 13 23:05:54 2013
@@ -72,7 +72,7 @@ public class ContainerInfo {
}
this.user = container.getUser();
- Resource res = container.getLaunchContext().getResource();
+ Resource res = container.getResource();
if (res != null) {
this.totalMemoryNeededMB = res.getMemory();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Sat Apr 13 23:05:54 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -43,6 +44,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.junit.Test;
+import static org.mockito.Mockito.*;
+
public class TestEventFlow {
@@ -117,12 +120,15 @@ public class TestEventFlow {
applicationAttemptId.setApplicationId(applicationId);
applicationAttemptId.setAttemptId(0);
cID.setApplicationAttemptId(applicationAttemptId);
- launchContext.setContainerId(cID);
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cID);
+ when(mockContainer.getResource()).thenReturn(recordFactory
+ .newRecordInstance(Resource.class));
launchContext.setUser("testing");
- launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
StartContainerRequest request =
recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(launchContext);
+ request.setContainer(mockContainer);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cID,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java Sat Apr 13 23:05:54 2013
@@ -99,7 +99,9 @@ public class TestNodeManagerReboot {
Records.newRecord(ContainerLaunchContext.class);
// Construct the Container-id
ContainerId cId = createContainerId();
- containerLaunchContext.setContainerId(cId);
+ org.apache.hadoop.yarn.api.records.Container mockContainer =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -122,12 +124,13 @@ public class TestNodeManagerReboot {
containerLaunchContext.setUser(containerLaunchContext.getUser());
List<String> commands = new ArrayList<String>();
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(Records
- .newRecord(Resource.class));
- containerLaunchContext.getResource().setMemory(1024);
+ Resource resource = Records.newRecord(Resource.class);
+ resource.setMemory(1024);
+ when(mockContainer.getResource()).thenReturn(resource);
StartContainerRequest startRequest =
Records.newRecord(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
GetContainerStatusRequest request =
@@ -160,7 +163,10 @@ public class TestNodeManagerReboot {
"container is launched", numOfLocalDirs(nmLocalDir.getAbsolutePath(),
ResourceLocalizationService.NM_PRIVATE_DIR) > 0);
- nm.handle(new NodeManagerEvent(NodeManagerEventType.REBOOT));
+ // restart the NodeManager
+ nm.stop();
+ nm = new MyNodeManager();
+ nm.start();
numTries = 0;
while ((numOfLocalDirs(nmLocalDir.getAbsolutePath(), ContainerLocalizer
@@ -250,26 +256,6 @@ public class TestNodeManagerReboot {
return delService;
}
- // mimic part of reboot process
- @Override
- public void handle(NodeManagerEvent event) {
- switch (event.getType()) {
- case SHUTDOWN:
- this.stop();
- break;
- case REBOOT:
- this.stop();
- this.createNewMyNodeManager().start();
- break;
- default:
- LOG.warn("Invalid shutdown event " + event.getType() + ". Ignoring.");
- }
- }
-
- private MyNodeManager createNewMyNodeManager() {
- return new MyNodeManager();
- }
-
private YarnConfiguration createNMConfig() {
YarnConfiguration conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.NM_PMEM_MB, 5 * 1024); // 5GB
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerShutdown.java Sat Apr 13 23:05:54 2013
@@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
@@ -28,6 +31,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.BrokenBarrierException;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CyclicBarrier;
import junit.framework.Assert;
@@ -38,6 +44,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -49,9 +56,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
import org.junit.Before;
@@ -71,6 +81,7 @@ public class TestNodeManagerShutdown {
.getRecordFactory(null);
static final String user = "nobody";
private FileContext localFS;
+ private CyclicBarrier syncBarrier = new CyclicBarrier(2);
@Before
public void setup() throws UnsupportedFileSystemException {
@@ -91,16 +102,69 @@ public class TestNodeManagerShutdown {
NodeManager nm = getNodeManager();
nm.init(createNMConfig());
nm.start();
+ startContainers(nm);
+
+ final int MAX_TRIES=20;
+ int numTries = 0;
+ while (!processStartFile.exists() && numTries < MAX_TRIES) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException ex) {ex.printStackTrace();}
+ numTries++;
+ }
+
+ nm.stop();
+ // Now verify the contents of the file
+ // Script generates a message when it receives a sigterm
+ // so we look for that
+ BufferedReader reader =
+ new BufferedReader(new FileReader(processStartFile));
+
+ boolean foundSigTermMessage = false;
+ while (true) {
+ String line = reader.readLine();
+ if (line == null) {
+ break;
+ }
+ if (line.contains("SIGTERM")) {
+ foundSigTermMessage = true;
+ break;
+ }
+ }
+ Assert.assertTrue("Did not find sigterm message", foundSigTermMessage);
+ reader.close();
+ }
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testKillContainersOnResync() throws IOException, InterruptedException {
+ NodeManager nm = new TestNodeManager();
+ YarnConfiguration conf = createNMConfig();
+ nm.init(conf);
+ nm.start();
+ startContainers(nm);
+
+ assert ((TestNodeManager) nm).getNMRegistrationCount() == 1;
+ nm.getNMDispatcher().getEventHandler().
+ handle( new NodeManagerEvent(NodeManagerEventType.RESYNC));
+ try {
+ syncBarrier.await();
+ } catch (BrokenBarrierException e) {
+ }
+ assert ((TestNodeManager) nm).getNMRegistrationCount() == 2;
+ }
+
+ private void startContainers(NodeManager nm) throws IOException {
ContainerManagerImpl containerManager = nm.getContainerManager();
File scriptFile = createUnhaltingScriptFile();
- ContainerLaunchContext containerLaunchContext =
+ ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
+ Container mockContainer = mock(Container.class);
// Construct the Container-id
ContainerId cId = createContainerId();
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -124,11 +188,12 @@ public class TestNodeManagerShutdown {
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(1024);
- StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
+ Resource resource = BuilderUtils.newResource(1024, 1);
+ when(mockContainer.getResource()).thenReturn(resource);
+ StartContainerRequest startRequest =
+ recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
GetContainerStatusRequest request =
@@ -137,37 +202,6 @@ public class TestNodeManagerShutdown {
ContainerStatus containerStatus =
containerManager.getContainerStatus(request).getStatus();
Assert.assertEquals(ContainerState.RUNNING, containerStatus.getState());
-
- final int MAX_TRIES=20;
- int numTries = 0;
- while (!processStartFile.exists() && numTries < MAX_TRIES) {
- try {
- Thread.sleep(500);
- } catch (InterruptedException ex) {ex.printStackTrace();}
- numTries++;
- }
-
- nm.stop();
-
- // Now verify the contents of the file
- // Script generates a message when it receives a sigterm
- // so we look for that
- BufferedReader reader =
- new BufferedReader(new FileReader(processStartFile));
-
- boolean foundSigTermMessage = false;
- while (true) {
- String line = reader.readLine();
- if (line == null) {
- break;
- }
- if (line.contains("SIGTERM")) {
- foundSigTermMessage = true;
- break;
- }
- }
- Assert.assertTrue("Did not find sigterm message", foundSigTermMessage);
- reader.close();
}
private ContainerId createContainerId() {
@@ -226,4 +260,48 @@ public class TestNodeManagerShutdown {
}
};
}
+
+ class TestNodeManager extends NodeManager {
+
+ private int registrationCount = 0;
+
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ return new TestNodeStatusUpdaterImpl(context, dispatcher,
+ healthChecker, metrics);
+ }
+
+ public int getNMRegistrationCount() {
+ return registrationCount;
+ }
+
+ class TestNodeStatusUpdaterImpl extends MockNodeStatusUpdater {
+
+ public TestNodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
+ }
+
+ @Override
+ protected void registerWithRM() throws YarnRemoteException {
+ super.registerWithRM();
+ registrationCount++;
+ }
+
+ @Override
+ protected void rebootNodeStatusUpdater() {
+ ConcurrentMap<ContainerId, org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container> containers =
+ getNMContext().getContainers();
+ // ensure that containers are empty before restart nodeStatusUpdater
+ Assert.assertTrue(containers.isEmpty());
+ super.rebootNodeStatusUpdater();
+ try {
+ syncBarrier.await();
+ } catch (InterruptedException e) {
+ } catch (BrokenBarrierException e) {
+ }
+ }
+ }
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Sat Apr 13 23:05:54 2013
@@ -99,7 +99,6 @@ public class TestNodeStatusUpdater {
private final List<NodeId> registeredNodes = new ArrayList<NodeId>();
private final Configuration conf = createNMConfig();
private NodeManager nm;
- protected NodeManager rebootedNodeManager;
private boolean containerStatusBackupSuccessfully = true;
private List<ContainerStatus> completedContainerStatusList = new ArrayList<ContainerStatus>();
@@ -177,6 +176,8 @@ public class TestNodeStatusUpdater {
nodeStatus.setResponseId(heartBeatID++);
Map<ApplicationId, List<ContainerStatus>> appToContainers =
getAppToContainerStatusMap(nodeStatus.getContainersStatuses());
+ org.apache.hadoop.yarn.api.records.Container mockContainer =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
if (heartBeatID == 1) {
Assert.assertEquals(0, nodeStatus.getContainersStatuses().size());
@@ -187,11 +188,12 @@ public class TestNodeStatusUpdater {
firstContainerID.setId(heartBeatID);
ContainerLaunchContext launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- launchContext.setContainerId(firstContainerID);
- launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
- launchContext.getResource().setMemory(2);
- Container container = new ContainerImpl(conf , mockDispatcher,
- launchContext, null, mockMetrics);
+ when(mockContainer.getId()).thenReturn(firstContainerID);
+ Resource resource = BuilderUtils.newResource(2, 1);
+ when(mockContainer.getResource()).thenReturn(resource);
+ Container container =
+ new ContainerImpl(conf, mockDispatcher, launchContext,
+ mockContainer, null, mockMetrics);
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@@ -212,11 +214,12 @@ public class TestNodeStatusUpdater {
secondContainerID.setId(heartBeatID);
ContainerLaunchContext launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- launchContext.setContainerId(secondContainerID);
- launchContext.setResource(recordFactory.newRecordInstance(Resource.class));
- launchContext.getResource().setMemory(3);
- Container container = new ContainerImpl(conf, mockDispatcher,
- launchContext, null, mockMetrics);
+ when(mockContainer.getId()).thenReturn(secondContainerID);
+ Resource resource = BuilderUtils.newResource(3, 1);
+ when(mockContainer.getResource()).thenReturn(resource);
+ Container container =
+ new ContainerImpl(conf, mockDispatcher, launchContext,
+ mockContainer, null, mockMetrics);
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
@@ -663,8 +666,8 @@ public class TestNodeStatusUpdater {
}
@Override
- protected void cleanupContainers() {
- super.cleanupContainers();
+ protected void cleanupContainers(NodeManagerEventType eventType) {
+ super.cleanupContainers(NodeManagerEventType.SHUTDOWN);
numCleanups.incrementAndGet();
}
};
@@ -718,50 +721,6 @@ public class TestNodeStatusUpdater {
}
@Test
- public void testNodeReboot() throws Exception {
- nm = getNodeManager(NodeAction.REBOOT);
- YarnConfiguration conf = createNMConfig();
- nm.init(conf);
- Assert.assertEquals(STATE.INITED, nm.getServiceState());
- nm.start();
-
- int waitCount = 0;
- while (heartBeatID < 1 && waitCount++ != 20) {
- Thread.sleep(500);
- }
- Assert.assertFalse(heartBeatID < 1);
-
- // NM takes a while to reach the STOPPED state.
- waitCount = 0;
- while (nm.getServiceState() != STATE.STOPPED && waitCount++ != 20) {
- LOG.info("Waiting for NM to stop..");
- Thread.sleep(1000);
- }
- Assert.assertEquals(STATE.STOPPED, nm.getServiceState());
-
- waitCount = 0;
- while (null == rebootedNodeManager && waitCount++ != 20) {
- LOG.info("Waiting for NM to reinitialize..");
- Thread.sleep(1000);
- }
-
- waitCount = 0;
- while (rebootedNodeManager.getServiceState() != STATE.STARTED && waitCount++ != 20) {
- LOG.info("Waiting for NM to start..");
- Thread.sleep(1000);
- }
- Assert.assertEquals(STATE.STARTED, rebootedNodeManager.getServiceState());
-
- rebootedNodeManager.stop();
- waitCount = 0;
- while (rebootedNodeManager.getServiceState() != STATE.STOPPED && waitCount++ != 20) {
- LOG.info("Waiting for NM to stop..");
- Thread.sleep(1000);
- }
- Assert.assertEquals(STATE.STOPPED, rebootedNodeManager.getServiceState());
- }
-
- @Test
public void testNMShutdownForRegistrationFailure() {
nm = new NodeManager() {
@@ -1108,12 +1067,6 @@ public class TestNodeStatusUpdater {
myNodeStatusUpdater.resourceTracker = myResourceTracker2;
return myNodeStatusUpdater;
}
-
- @Override
- NodeManager createNewNodeManager() {
- rebootedNodeManager = getNodeManager(NodeAction.NORMAL);
- return rebootedNodeManager;
- }
};
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Sat Apr 13 23:05:54 2013
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -47,7 +48,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
@@ -58,8 +58,10 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
+import static org.mockito.Mockito.*;
public class TestContainerManager extends BaseContainerManagerTest {
@@ -121,7 +123,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
- container.setContainerId(cId);
container.setUser(user);
@@ -143,14 +144,16 @@ public class TestContainerManager extend
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
- containerLaunchContext.setContainerId(container.getContainerId());
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(512, 1));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
-
+ startRequest.setContainer(mockContainer);
+
containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -208,7 +211,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
- containerLaunchContext.setContainerId(cId);
containerLaunchContext.setUser(user);
@@ -232,11 +234,13 @@ public class TestContainerManager extend
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100 * 1024 * 1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -310,7 +314,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
- containerLaunchContext.setContainerId(cId);
containerLaunchContext.setUser(user);
@@ -334,12 +337,14 @@ public class TestContainerManager extend
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100 * 1024 * 1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -402,7 +407,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
- container.setContainerId(cId);
container.setUser(user);
@@ -425,14 +429,16 @@ public class TestContainerManager extend
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
- containerLaunchContext.setContainerId(container.getContainerId());
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100, 1));
// containerLaunchContext.command = new ArrayList<CharSequence>();
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
+ request.setContainer(mockContainer);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Sat Apr 13 23:05:54 2013
@@ -525,8 +525,9 @@ public class TestContainer {
return serviceData;
}
- private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx) {
- return new ContainerImpl(conf, disp, ctx, null, metrics);
+ private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx,
+ org.apache.hadoop.yarn.api.records.Container container) {
+ return new ContainerImpl(conf, disp, ctx, container, null, metrics);
}
@SuppressWarnings("unchecked")
@@ -570,12 +571,14 @@ public class TestContainer {
this.user = user;
ctxt = mock(ContainerLaunchContext.class);
+ org.apache.hadoop.yarn.api.records.Container mockContainer =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
cId = BuilderUtils.newContainerId(appId, 1, timestamp, id);
when(ctxt.getUser()).thenReturn(this.user);
- when(ctxt.getContainerId()).thenReturn(cId);
+ when(mockContainer.getId()).thenReturn(cId);
Resource resource = BuilderUtils.newResource(1024, 1);
- when(ctxt.getResource()).thenReturn(resource);
+ when(mockContainer.getResource()).thenReturn(resource);
if (withLocalRes) {
Random r = new Random();
@@ -599,7 +602,7 @@ public class TestContainer {
}
when(ctxt.getServiceData()).thenReturn(serviceData);
- c = newContainer(dispatcher, ctxt);
+ c = newContainer(dispatcher, ctxt, mockContainer);
dispatcher.start();
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Sat Apr 13 23:05:54 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -57,11 +58,14 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.junit.Before;
import org.junit.Test;
+import static org.mockito.Mockito.*;
+
import junit.framework.Assert;
public class TestContainerLaunch extends BaseContainerManagerTest {
@@ -184,6 +188,7 @@ public class TestContainerLaunch extends
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
appId.setClusterTimestamp(0);
@@ -195,7 +200,7 @@ public class TestContainerLaunch extends
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
cId.setApplicationAttemptId(appAttemptId);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -222,11 +227,11 @@ public class TestContainerLaunch extends
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -301,7 +306,7 @@ public class TestContainerLaunch extends
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
appId.setClusterTimestamp(1);
@@ -313,7 +318,7 @@ public class TestContainerLaunch extends
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
cId.setApplicationAttemptId(appAttemptId);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -339,11 +344,11 @@ public class TestContainerLaunch extends
List<String> commands = new ArrayList<String>();
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java Sat Apr 13 23:05:54 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import static org.mockito.Mockito.any;
+import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -41,11 +42,15 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
@@ -224,6 +229,142 @@ public class TestLocalResourcesTrackerIm
}
}
+ @Test(timeout = 1000)
+ @SuppressWarnings("unchecked")
+ public void testLocalResourceCache() {
+ String user = "testuser";
+ DrainDispatcher dispatcher = null;
+ try {
+ Configuration conf = new Configuration();
+ dispatcher = createDispatcher(conf);
+
+ EventHandler<LocalizerEvent> localizerEventHandler =
+ mock(EventHandler.class);
+ EventHandler<ContainerEvent> containerEventHandler =
+ mock(EventHandler.class);
+
+ // Registering event handlers.
+ dispatcher.register(LocalizerEventType.class, localizerEventHandler);
+ dispatcher.register(ContainerEventType.class, containerEventHandler);
+
+ ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
+ new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
+ LocalResourcesTracker tracker =
+ new LocalResourcesTrackerImpl(user, dispatcher, localrsrc, true, conf);
+
+ LocalResourceRequest lr =
+ createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
+
+ // Creating 2 containers for same application which will be requesting
+ // same local resource.
+ // Container 1 requesting local resource.
+ ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
+ LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
+ ResourceEvent reqEvent1 =
+ new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc1);
+
+ // No resource request is initially present in local cache
+ Assert.assertEquals(0, localrsrc.size());
+
+ // Container-1 requesting local resource.
+ tracker.handle(reqEvent1);
+
+ // New localized Resource should have been added to local resource map
+ // and the requesting container will be added to its waiting queue.
+ Assert.assertEquals(1, localrsrc.size());
+ Assert.assertTrue(localrsrc.containsKey(lr));
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId1));
+ Assert.assertEquals(ResourceState.DOWNLOADING, localrsrc.get(lr)
+ .getState());
+
+ // Container 2 requesting the resource
+ ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
+ LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
+ ResourceEvent reqEvent2 =
+ new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc2);
+ tracker.handle(reqEvent2);
+
+ // Container 2 should have been added to the waiting queue of the local
+ // resource
+ Assert.assertEquals(2, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));
+
+ // Failing resource localization
+ ResourceEvent resourceFailedEvent =
+ new ResourceFailedLocalizationEvent(lr, new Exception("test"));
+
+ // Backing up the resource to track its state change as it will be
+ // removed after the failed event.
+ LocalizedResource localizedResource = localrsrc.get(lr);
+
+ tracker.handle(resourceFailedEvent);
+
+ // After receiving failed resource event; all waiting containers will be
+ // notified with Container Resource Failed Event.
+ Assert.assertEquals(0, localrsrc.size());
+ verify(containerEventHandler, times(2)).handle(
+ isA(ContainerResourceFailedEvent.class));
+ Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());
+
+ // Container 1 trying to release the resource (This resource is already
+ // deleted from the cache. This call should return silently without
+ // exception.
+ ResourceReleaseEvent relEvent1 = new ResourceReleaseEvent(lr, cId1);
+ tracker.handle(relEvent1);
+
+ // Container-3 now requests for the same resource. This request call
+ // is coming prior to Container-2's release call.
+ ContainerId cId3 = BuilderUtils.newContainerId(1, 1, 1, 3);
+ LocalizerContext lc3 = new LocalizerContext(user, cId3, null);
+ ResourceEvent reqEvent3 =
+ new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc3);
+ tracker.handle(reqEvent3);
+
+ // Local resource cache now should have the requested resource and the
+ // number of waiting containers should be 1.
+ Assert.assertEquals(1, localrsrc.size());
+ Assert.assertTrue(localrsrc.containsKey(lr));
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
+
+ // Container-2 Releases the resource
+ ResourceReleaseEvent relEvent2 = new ResourceReleaseEvent(lr, cId2);
+ tracker.handle(relEvent2);
+
+ // Making sure that there is no change in the cache after the release.
+ Assert.assertEquals(1, localrsrc.size());
+ Assert.assertTrue(localrsrc.containsKey(lr));
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
+
+ // Sending ResourceLocalizedEvent to tracker. In turn resource should
+ // send Container Resource Localized Event to waiting containers.
+ Path localizedPath = new Path("/tmp/file1");
+ ResourceLocalizedEvent localizedEvent =
+ new ResourceLocalizedEvent(lr, localizedPath, 123L);
+ tracker.handle(localizedEvent);
+
+ // Verifying ContainerResourceLocalizedEvent .
+ verify(containerEventHandler, times(1)).handle(
+ isA(ContainerResourceLocalizedEvent.class));
+ Assert.assertEquals(ResourceState.LOCALIZED, localrsrc.get(lr)
+ .getState());
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+
+ // Container-3 releasing the resource.
+ ResourceReleaseEvent relEvent3 = new ResourceReleaseEvent(lr, cId3);
+ tracker.handle(relEvent3);
+
+ Assert.assertEquals(0, localrsrc.get(lr).getRefCount());
+
+ } finally {
+ if (dispatcher != null) {
+ dispatcher.stop();
+ }
+ }
+ }
+
@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testHierarchicalLocalCacheDirectories() {
@@ -266,19 +407,25 @@ public class TestLocalResourcesTrackerIm
// Simulate the process of localization of lr1
Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);
// Simulate lr1 getting localized
- ResourceLocalizedEvent rle =
+ ResourceLocalizedEvent rle1 =
new ResourceLocalizedEvent(lr1,
new Path(hierarchicalPath1.toUri().toString() +
Path.SEPARATOR + "file1"), 120);
- tracker.handle(rle);
+ tracker.handle(rle1);
// Localization successful.
- tracker.localizationCompleted(lr1, true);
LocalResourceRequest lr2 = createLocalResourceRequest(user, 3, 3,
LocalResourceVisibility.PUBLIC);
+ // Container 1 requests lr2 to be localized.
+ ResourceEvent reqEvent2 =
+ new ResourceRequestEvent(lr2, LocalResourceVisibility.PUBLIC, lc1);
+ tracker.handle(reqEvent2);
+
Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir);
// localization failed.
- tracker.localizationCompleted(lr2, false);
+ ResourceFailedLocalizationEvent rfe2 =
+ new ResourceFailedLocalizationEvent(lr2, new Exception("Test"));
+ tracker.handle(rfe2);
/*
* The path returned for two localization should be different because we
@@ -292,7 +439,11 @@ public class TestLocalResourcesTrackerIm
LocalResourceVisibility.PUBLIC, lc1);
tracker.handle(reqEvent3);
Path hierarchicalPath3 = tracker.getPathForLocalization(lr3, localDir);
- tracker.localizationCompleted(lr3, true);
+ // localization successful
+ ResourceLocalizedEvent rle3 =
+ new ResourceLocalizedEvent(lr3, new Path(hierarchicalPath3.toUri()
+ .toString() + Path.SEPARATOR + "file3"), 120);
+ tracker.handle(rle3);
// Verifying that path created is inside the subdirectory
Assert.assertEquals(hierarchicalPath3.toUri().toString(),
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Sat Apr 13 23:05:54 2013
@@ -56,13 +56,13 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -91,6 +91,7 @@ import org.mockito.Mockito;
import org.mortbay.util.MultiException;
+
//@Ignore
public class TestLogAggregationService extends BaseContainerManagerTest {
@@ -679,7 +680,7 @@ public class TestLogAggregationService e
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId =
recordFactory.newRecordInstance(ApplicationId.class);
@@ -689,7 +690,7 @@ public class TestLogAggregationService e
BuilderUtils.newApplicationAttemptId(appId, 1);
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(this.user);
@@ -713,12 +714,12 @@ public class TestLogAggregationService e
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100 * 1024 * 1024, 1));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
this.containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(this.containerManager,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Sat Apr 13 23:05:54 2013
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.no
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.*;
import java.io.BufferedReader;
import java.io.File;
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -51,7 +52,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
@@ -197,7 +198,7 @@ public class TestContainersMonitor exten
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId =
recordFactory.newRecordInstance(ApplicationId.class);
@@ -210,7 +211,7 @@ public class TestContainersMonitor exten
ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
cId.setId(0);
cId.setApplicationAttemptId(appAttemptId);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -234,12 +235,12 @@ public class TestContainersMonitor exten
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(8 * 1024 * 1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(8 * 1024 * 1024, 1));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java Sat Apr 13 23:05:54 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -58,7 +59,6 @@ public class MockContainer implements Co
uniqId);
this.launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- launchContext.setContainerId(id);
launchContext.setUser(user);
this.state = ContainerState.NEW;
@@ -104,7 +104,6 @@ public class MockContainer implements Co
.newRecordInstance(ContainerStatus.class);
containerStatus
.setState(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
- containerStatus.setContainerId(this.launchContext.getContainerId());
containerStatus.setDiagnostics("testing");
containerStatus.setExitStatus(0);
return containerStatus;
@@ -119,4 +118,9 @@ public class MockContainer implements Co
public void handle(ContainerEvent event) {
}
+ @Override
+ public Resource getResource() {
+ return null;
+ }
+
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Sat Apr 13 23:05:54 2013
@@ -179,15 +179,19 @@ public class TestNMWebServer {
// TODO: Use builder utils
ContainerLaunchContext launchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- launchContext.setContainerId(containerId);
+ org.apache.hadoop.yarn.api.records.Container mockContainer =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(mockContainer.getId()).thenReturn(containerId);
launchContext.setUser(user);
Container container =
- new ContainerImpl(conf, dispatcher, launchContext, null, metrics) {
- @Override
- public ContainerState getContainerState() {
- return ContainerState.RUNNING;
- };
- };
+ new ContainerImpl(conf, dispatcher, launchContext, mockContainer,
+ null, metrics) {
+
+ @Override
+ public ContainerState getContainerState() {
+ return ContainerState.RUNNING;
+ };
+ };
nmContext.getContainers().put(containerId, container);
//TODO: Gross hack. Fix in code.
ApplicationId applicationId =
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Sat Apr 13 23:05:54 2013
@@ -266,7 +266,7 @@ public class ClientRMService extends Abs
ApplicationSubmissionContext submissionContext = request
.getApplicationSubmissionContext();
ApplicationId applicationId = submissionContext.getApplicationId();
- String user = submissionContext.getUser();
+ String user = submissionContext.getAMContainerSpec().getUser();
try {
user = UserGroupInformation.getCurrentUser().getShortUserName();
if (rmContext.getRMApps().get(applicationId) != null) {
@@ -275,13 +275,13 @@ public class ClientRMService extends Abs
}
// Safety
- submissionContext.setUser(user);
+ submissionContext.getAMContainerSpec().setUser(user);
// Check whether AM resource requirements are within required limits
if (!submissionContext.getUnmanagedAM()) {
ResourceRequest amReq = BuilderUtils.newResourceRequest(
RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
- submissionContext.getAMContainerSpec().getResource(), 1);
+ submissionContext.getResource(), 1);
try {
SchedulerUtils.validateResourceRequest(amReq,
scheduler.getMaximumResourceCapability());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1467713&r1=1467712&r2=1467713&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Sat Apr 13 23:05:54 2013
@@ -249,10 +249,11 @@ public class RMAppManager implements Eve
// Create RMApp
application =
new RMAppImpl(applicationId, rmContext, this.conf,
- submissionContext.getApplicationName(),
- submissionContext.getUser(), submissionContext.getQueue(),
- submissionContext, this.scheduler, this.masterService,
- submitTime);
+ submissionContext.getApplicationName(),
+ submissionContext.getAMContainerSpec().getUser(),
+ submissionContext.getQueue(),
+ submissionContext, this.scheduler, this.masterService,
+ submitTime);
// Sanity check - duplicate?
if (rmContext.getRMApps().putIfAbsent(applicationId, application) !=