You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by at...@apache.org on 2013/04/13 01:05:38 UTC
svn commit: r1467511 [4/6] - in
/hadoop/common/branches/HDFS-347/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java Fri Apr 12 23:05:28 2013
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -47,7 +48,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
@@ -58,8 +58,10 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
+import static org.mockito.Mockito.*;
public class TestContainerManager extends BaseContainerManagerTest {
@@ -121,7 +123,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
- container.setContainerId(cId);
container.setUser(user);
@@ -143,14 +144,16 @@ public class TestContainerManager extend
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
- containerLaunchContext.setContainerId(container.getContainerId());
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(512, 1));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
-
+ startRequest.setContainer(mockContainer);
+
containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -208,7 +211,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
- containerLaunchContext.setContainerId(cId);
containerLaunchContext.setUser(user);
@@ -232,11 +234,13 @@ public class TestContainerManager extend
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100 * 1024 * 1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -310,7 +314,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
- containerLaunchContext.setContainerId(cId);
containerLaunchContext.setUser(user);
@@ -334,12 +337,14 @@ public class TestContainerManager extend
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100 * 1024 * 1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
@@ -402,7 +407,6 @@ public class TestContainerManager extend
// ////// Construct the Container-id
ContainerId cId = createContainerId();
ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
- container.setContainerId(cId);
container.setUser(user);
@@ -425,14 +429,16 @@ public class TestContainerManager extend
localResources.put(destinationFile, rsrc_alpha);
containerLaunchContext.setLocalResources(localResources);
containerLaunchContext.setUser(container.getUser());
- containerLaunchContext.setContainerId(container.getContainerId());
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
+ Container mockContainer = mock(Container.class);
+ when(mockContainer.getId()).thenReturn(cId);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100, 1));
// containerLaunchContext.command = new ArrayList<CharSequence>();
StartContainerRequest request = recordFactory.newRecordInstance(StartContainerRequest.class);
request.setContainerLaunchContext(containerLaunchContext);
+ request.setContainer(mockContainer);
containerManager.startContainer(request);
BaseContainerManagerTest.waitForContainerState(containerManager, cId,
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Fri Apr 12 23:05:28 2013
@@ -525,8 +525,9 @@ public class TestContainer {
return serviceData;
}
- private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx) {
- return new ContainerImpl(conf, disp, ctx, null, metrics);
+ private Container newContainer(Dispatcher disp, ContainerLaunchContext ctx,
+ org.apache.hadoop.yarn.api.records.Container container) {
+ return new ContainerImpl(conf, disp, ctx, container, null, metrics);
}
@SuppressWarnings("unchecked")
@@ -570,12 +571,14 @@ public class TestContainer {
this.user = user;
ctxt = mock(ContainerLaunchContext.class);
+ org.apache.hadoop.yarn.api.records.Container mockContainer =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
cId = BuilderUtils.newContainerId(appId, 1, timestamp, id);
when(ctxt.getUser()).thenReturn(this.user);
- when(ctxt.getContainerId()).thenReturn(cId);
+ when(mockContainer.getId()).thenReturn(cId);
Resource resource = BuilderUtils.newResource(1024, 1);
- when(ctxt.getResource()).thenReturn(resource);
+ when(mockContainer.getResource()).thenReturn(resource);
if (withLocalRes) {
Random r = new Random();
@@ -599,7 +602,7 @@ public class TestContainer {
}
when(ctxt.getServiceData()).thenReturn(serviceData);
- c = newContainer(dispatcher, ctxt);
+ c = newContainer(dispatcher, ctxt, mockContainer);
dispatcher.start();
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Fri Apr 12 23:05:28 2013
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -57,11 +58,14 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.junit.Before;
import org.junit.Test;
+import static org.mockito.Mockito.*;
+
import junit.framework.Assert;
public class TestContainerLaunch extends BaseContainerManagerTest {
@@ -184,6 +188,7 @@ public class TestContainerLaunch extends
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
appId.setClusterTimestamp(0);
@@ -195,7 +200,7 @@ public class TestContainerLaunch extends
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
cId.setApplicationAttemptId(appAttemptId);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -222,11 +227,11 @@ public class TestContainerLaunch extends
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
@@ -301,7 +306,7 @@ public class TestContainerLaunch extends
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId = recordFactory.newRecordInstance(ApplicationId.class);
appId.setClusterTimestamp(1);
@@ -313,7 +318,7 @@ public class TestContainerLaunch extends
ContainerId cId =
recordFactory.newRecordInstance(ContainerId.class);
cId.setApplicationAttemptId(appAttemptId);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -339,11 +344,11 @@ public class TestContainerLaunch extends
List<String> commands = new ArrayList<String>();
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(1024, 1));
StartContainerRequest startRequest = recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalizerHeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalizerHeartbeatResponse.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalizerHeartbeatResponse.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalizerHeartbeatResponse.java Fri Apr 12 23:05:28 2013
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.no
import java.util.ArrayList;
import java.util.List;
-import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
@@ -28,28 +28,30 @@ public class MockLocalizerHeartbeatRespo
implements LocalizerHeartbeatResponse {
LocalizerAction action;
- List<LocalResource> rsrc;
+ List<ResourceLocalizationSpec> resourceSpecs;
MockLocalizerHeartbeatResponse() {
- rsrc = new ArrayList<LocalResource>();
+ resourceSpecs = new ArrayList<ResourceLocalizationSpec>();
}
MockLocalizerHeartbeatResponse(
- LocalizerAction action, List<LocalResource> rsrc) {
+ LocalizerAction action, List<ResourceLocalizationSpec> resources) {
this.action = action;
- this.rsrc = rsrc;
+ this.resourceSpecs = resources;
}
public LocalizerAction getLocalizerAction() { return action; }
- public List<LocalResource> getAllResources() { return rsrc; }
- public LocalResource getLocalResource(int i) { return rsrc.get(i); }
public void setLocalizerAction(LocalizerAction action) {
this.action = action;
}
- public void addAllResources(List<LocalResource> resources) {
- rsrc.addAll(resources);
+
+ @Override
+ public List<ResourceLocalizationSpec> getResourceSpecs() {
+ return resourceSpecs;
+}
+
+ @Override
+ public void setResourceSpecs(List<ResourceLocalizationSpec> resourceSpecs) {
+ this.resourceSpecs = resourceSpecs;
}
- public void addResource(LocalResource resource) { rsrc.add(resource); }
- public void removeResource(int index) { rsrc.remove(index); }
- public void clearResources() { rsrc.clear(); }
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java Fri Apr 12 23:05:28 2013
@@ -50,7 +50,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
-import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.DataInputBuffer;
@@ -66,9 +65,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
+import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
import org.mockito.invocation.InvocationOnMock;
@@ -95,12 +96,33 @@ public class TestContainerLocalizer {
public void testContainerLocalizerMain() throws Exception {
ContainerLocalizer localizer = setupContainerLocalizerForTest();
+ // verify created cache
+ List<Path> privCacheList = new ArrayList<Path>();
+ List<Path> appCacheList = new ArrayList<Path>();
+ for (Path p : localDirs) {
+ Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
+ Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
+ privCacheList.add(privcache);
+ Path appDir =
+ new Path(base, new Path(ContainerLocalizer.APPCACHE, appId));
+ Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
+ appCacheList.add(appcache);
+ }
+
// mock heartbeat responses from NM
- LocalResource rsrcA = getMockRsrc(random, LocalResourceVisibility.PRIVATE);
- LocalResource rsrcB = getMockRsrc(random, LocalResourceVisibility.PRIVATE);
- LocalResource rsrcC = getMockRsrc(random,
- LocalResourceVisibility.APPLICATION);
- LocalResource rsrcD = getMockRsrc(random, LocalResourceVisibility.PRIVATE);
+ ResourceLocalizationSpec rsrcA =
+ getMockRsrc(random, LocalResourceVisibility.PRIVATE,
+ privCacheList.get(0));
+ ResourceLocalizationSpec rsrcB =
+ getMockRsrc(random, LocalResourceVisibility.PRIVATE,
+ privCacheList.get(0));
+ ResourceLocalizationSpec rsrcC =
+ getMockRsrc(random, LocalResourceVisibility.APPLICATION,
+ appCacheList.get(0));
+ ResourceLocalizationSpec rsrcD =
+ getMockRsrc(random, LocalResourceVisibility.PRIVATE,
+ privCacheList.get(0));
+
when(nmProxy.heartbeat(isA(LocalizerStatus.class)))
.thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
Collections.singletonList(rsrcA)))
@@ -111,27 +133,33 @@ public class TestContainerLocalizer {
.thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
Collections.singletonList(rsrcD)))
.thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.LIVE,
- Collections.<LocalResource>emptyList()))
+ Collections.<ResourceLocalizationSpec>emptyList()))
.thenReturn(new MockLocalizerHeartbeatResponse(LocalizerAction.DIE,
null));
- doReturn(new FakeDownload(rsrcA.getResource().getFile(), true)).when(
- localizer).download(isA(LocalDirAllocator.class), eq(rsrcA),
+ LocalResource tRsrcA = rsrcA.getResource();
+ LocalResource tRsrcB = rsrcB.getResource();
+ LocalResource tRsrcC = rsrcC.getResource();
+ LocalResource tRsrcD = rsrcD.getResource();
+ doReturn(
+ new FakeDownload(rsrcA.getResource().getResource().getFile(), true))
+ .when(localizer).download(isA(Path.class), eq(tRsrcA),
isA(UserGroupInformation.class));
- doReturn(new FakeDownload(rsrcB.getResource().getFile(), true)).when(
- localizer).download(isA(LocalDirAllocator.class), eq(rsrcB),
+ doReturn(
+ new FakeDownload(rsrcB.getResource().getResource().getFile(), true))
+ .when(localizer).download(isA(Path.class), eq(tRsrcB),
isA(UserGroupInformation.class));
- doReturn(new FakeDownload(rsrcC.getResource().getFile(), true)).when(
- localizer).download(isA(LocalDirAllocator.class), eq(rsrcC),
+ doReturn(
+ new FakeDownload(rsrcC.getResource().getResource().getFile(), true))
+ .when(localizer).download(isA(Path.class), eq(tRsrcC),
isA(UserGroupInformation.class));
- doReturn(new FakeDownload(rsrcD.getResource().getFile(), true)).when(
- localizer).download(isA(LocalDirAllocator.class), eq(rsrcD),
+ doReturn(
+ new FakeDownload(rsrcD.getResource().getResource().getFile(), true))
+ .when(localizer).download(isA(Path.class), eq(tRsrcD),
isA(UserGroupInformation.class));
// run localization
assertEquals(0, localizer.runLocalization(nmAddr));
-
- // verify created cache
for (Path p : localDirs) {
Path base = new Path(new Path(p, ContainerLocalizer.USERCACHE), appUser);
Path privcache = new Path(base, ContainerLocalizer.FILECACHE);
@@ -143,15 +171,14 @@ public class TestContainerLocalizer {
Path appcache = new Path(appDir, ContainerLocalizer.FILECACHE);
verify(spylfs).mkdir(eq(appcache), isA(FsPermission.class), eq(false));
}
-
// verify tokens read at expected location
verify(spylfs).open(tokenPath);
// verify downloaded resources reported to NM
- verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcA)));
- verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcB)));
- verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcC)));
- verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcD)));
+ verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcA.getResource())));
+ verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcB.getResource())));
+ verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcC.getResource())));
+ verify(nmProxy).heartbeat(argThat(new HBMatches(rsrcD.getResource())));
// verify all HB use localizerID provided
verify(nmProxy, never()).heartbeat(argThat(
@@ -306,10 +333,12 @@ public class TestContainerLocalizer {
return mockRF;
}
- static LocalResource getMockRsrc(Random r,
- LocalResourceVisibility vis) {
- LocalResource rsrc = mock(LocalResource.class);
+ static ResourceLocalizationSpec getMockRsrc(Random r,
+ LocalResourceVisibility vis, Path p) {
+ ResourceLocalizationSpec resourceLocalizationSpec =
+ mock(ResourceLocalizationSpec.class);
+ LocalResource rsrc = mock(LocalResource.class);
String name = Long.toHexString(r.nextLong());
URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
when(uri.getScheme()).thenReturn("file");
@@ -322,7 +351,10 @@ public class TestContainerLocalizer {
when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
when(rsrc.getVisibility()).thenReturn(vis);
- return rsrc;
+ when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
+ when(resourceLocalizationSpec.getDestinationDirectory()).
+ thenReturn(ConverterUtils.getYarnUrlFromPath(p));
+ return resourceLocalizationSpec;
}
@SuppressWarnings({ "rawtypes", "unchecked" })
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java Fri Apr 12 23:05:28 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import static org.mockito.Mockito.any;
+import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@@ -36,31 +37,36 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceLocalizedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceFailedLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceReleaseEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
-import org.mortbay.log.Log;
public class TestLocalResourcesTrackerImpl {
- @Test
+ @Test(timeout=10000)
@SuppressWarnings("unchecked")
public void test() {
String user = "testuser";
DrainDispatcher dispatcher = null;
try {
- dispatcher = createDispatcher(new Configuration());
+ Configuration conf = new Configuration();
+ dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler =
mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler =
@@ -86,7 +92,8 @@ public class TestLocalResourcesTrackerIm
localrsrc.put(req1, lr1);
localrsrc.put(req2, lr2);
LocalResourcesTracker tracker =
- new LocalResourcesTrackerImpl(user, dispatcher, localrsrc);
+ new LocalResourcesTrackerImpl(user, dispatcher, localrsrc, false,
+ conf);
ResourceEvent req11Event =
new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
@@ -152,13 +159,14 @@ public class TestLocalResourcesTrackerIm
}
}
- @Test
+ @Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
String user = "testuser";
DrainDispatcher dispatcher = null;
try {
- dispatcher = createDispatcher(new Configuration());
+ Configuration conf = new Configuration();
+ dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
dispatcher.register(LocalizerEventType.class, localizerEventHandler);
@@ -172,7 +180,7 @@ public class TestLocalResourcesTrackerIm
ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
localrsrc.put(req1, lr1);
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- dispatcher, localrsrc);
+ dispatcher, localrsrc, false, conf);
ResourceEvent req11Event = new ResourceRequestEvent(req1,
LocalResourceVisibility.PUBLIC, lc1);
@@ -221,6 +229,259 @@ public class TestLocalResourcesTrackerIm
}
}
+ @Test(timeout = 1000)
+ @SuppressWarnings("unchecked")
+ public void testLocalResourceCache() {
+ String user = "testuser";
+ DrainDispatcher dispatcher = null;
+ try {
+ Configuration conf = new Configuration();
+ dispatcher = createDispatcher(conf);
+
+ EventHandler<LocalizerEvent> localizerEventHandler =
+ mock(EventHandler.class);
+ EventHandler<ContainerEvent> containerEventHandler =
+ mock(EventHandler.class);
+
+ // Registering event handlers.
+ dispatcher.register(LocalizerEventType.class, localizerEventHandler);
+ dispatcher.register(ContainerEventType.class, containerEventHandler);
+
+ ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
+ new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
+ LocalResourcesTracker tracker =
+ new LocalResourcesTrackerImpl(user, dispatcher, localrsrc, true, conf);
+
+ LocalResourceRequest lr =
+ createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
+
+ // Creating 2 containers for same application which will be requesting
+ // same local resource.
+ // Container 1 requesting local resource.
+ ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
+ LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
+ ResourceEvent reqEvent1 =
+ new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc1);
+
+ // No resource request is initially present in local cache
+ Assert.assertEquals(0, localrsrc.size());
+
+ // Container-1 requesting local resource.
+ tracker.handle(reqEvent1);
+
+ // New localized Resource should have been added to local resource map
+ // and the requesting container will be added to its waiting queue.
+ Assert.assertEquals(1, localrsrc.size());
+ Assert.assertTrue(localrsrc.containsKey(lr));
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId1));
+ Assert.assertEquals(ResourceState.DOWNLOADING, localrsrc.get(lr)
+ .getState());
+
+ // Container 2 requesting the resource
+ ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
+ LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
+ ResourceEvent reqEvent2 =
+ new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc2);
+ tracker.handle(reqEvent2);
+
+ // Container 2 should have been added to the waiting queue of the local
+ // resource
+ Assert.assertEquals(2, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));
+
+ // Failing resource localization
+ ResourceEvent resourceFailedEvent =
+ new ResourceFailedLocalizationEvent(lr, new Exception("test"));
+
+ // Backing up the resource to track its state change as it will be
+ // removed after the failed event.
+ LocalizedResource localizedResource = localrsrc.get(lr);
+
+ tracker.handle(resourceFailedEvent);
+
+ // After receiving failed resource event; all waiting containers will be
+ // notified with Container Resource Failed Event.
+ Assert.assertEquals(0, localrsrc.size());
+ verify(containerEventHandler, times(2)).handle(
+ isA(ContainerResourceFailedEvent.class));
+ Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());
+
+ // Container 1 trying to release the resource (This resource is already
+ // deleted from the cache. This call should return silently without
+ // exception.
+ ResourceReleaseEvent relEvent1 = new ResourceReleaseEvent(lr, cId1);
+ tracker.handle(relEvent1);
+
+ // Container-3 now requests for the same resource. This request call
+ // is coming prior to Container-2's release call.
+ ContainerId cId3 = BuilderUtils.newContainerId(1, 1, 1, 3);
+ LocalizerContext lc3 = new LocalizerContext(user, cId3, null);
+ ResourceEvent reqEvent3 =
+ new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc3);
+ tracker.handle(reqEvent3);
+
+ // Local resource cache now should have the requested resource and the
+ // number of waiting containers should be 1.
+ Assert.assertEquals(1, localrsrc.size());
+ Assert.assertTrue(localrsrc.containsKey(lr));
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
+
+ // Container-2 Releases the resource
+ ResourceReleaseEvent relEvent2 = new ResourceReleaseEvent(lr, cId2);
+ tracker.handle(relEvent2);
+
+ // Making sure that there is no change in the cache after the release.
+ Assert.assertEquals(1, localrsrc.size());
+ Assert.assertTrue(localrsrc.containsKey(lr));
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+ Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
+
+ // Sending ResourceLocalizedEvent to tracker. In turn resource should
+ // send Container Resource Localized Event to waiting containers.
+ Path localizedPath = new Path("/tmp/file1");
+ ResourceLocalizedEvent localizedEvent =
+ new ResourceLocalizedEvent(lr, localizedPath, 123L);
+ tracker.handle(localizedEvent);
+
+ // Verifying ContainerResourceLocalizedEvent .
+ verify(containerEventHandler, times(1)).handle(
+ isA(ContainerResourceLocalizedEvent.class));
+ Assert.assertEquals(ResourceState.LOCALIZED, localrsrc.get(lr)
+ .getState());
+ Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
+
+ // Container-3 releasing the resource.
+ ResourceReleaseEvent relEvent3 = new ResourceReleaseEvent(lr, cId3);
+ tracker.handle(relEvent3);
+
+ Assert.assertEquals(0, localrsrc.get(lr).getRefCount());
+
+ } finally {
+ if (dispatcher != null) {
+ dispatcher.stop();
+ }
+ }
+ }
+
+ @Test(timeout = 100000)
+ @SuppressWarnings("unchecked")
+ public void testHierarchicalLocalCacheDirectories() {
+ String user = "testuser";
+ DrainDispatcher dispatcher = null;
+ try {
+ Configuration conf = new Configuration();
+ // setting per directory file limit to 1.
+ conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37");
+ dispatcher = createDispatcher(conf);
+
+ EventHandler<LocalizerEvent> localizerEventHandler =
+ mock(EventHandler.class);
+ EventHandler<LocalizerEvent> containerEventHandler =
+ mock(EventHandler.class);
+ dispatcher.register(LocalizerEventType.class, localizerEventHandler);
+ dispatcher.register(ContainerEventType.class, containerEventHandler);
+
+ DeletionService mockDelService = mock(DeletionService.class);
+
+ ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
+ new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
+ LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
+ dispatcher, localrsrc, true, conf);
+
+ // This is a random path. NO File creation will take place at this place.
+ Path localDir = new Path("/tmp");
+
+ // Container 1 needs lr1 resource
+ ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
+ LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
+ LocalResourceVisibility.PUBLIC);
+ LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
+
+ // Container 1 requests lr1 to be localized
+ ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
+ LocalResourceVisibility.PUBLIC, lc1);
+ tracker.handle(reqEvent1);
+
+ // Simulate the process of localization of lr1
+ Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);
+ // Simulate lr1 getting localized
+ ResourceLocalizedEvent rle1 =
+ new ResourceLocalizedEvent(lr1,
+ new Path(hierarchicalPath1.toUri().toString() +
+ Path.SEPARATOR + "file1"), 120);
+ tracker.handle(rle1);
+ // Localization successful.
+
+ LocalResourceRequest lr2 = createLocalResourceRequest(user, 3, 3,
+ LocalResourceVisibility.PUBLIC);
+ // Container 1 requests lr2 to be localized.
+ ResourceEvent reqEvent2 =
+ new ResourceRequestEvent(lr2, LocalResourceVisibility.PUBLIC, lc1);
+ tracker.handle(reqEvent2);
+
+ Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir);
+ // localization failed.
+ ResourceFailedLocalizationEvent rfe2 =
+ new ResourceFailedLocalizationEvent(lr2, new Exception("Test"));
+ tracker.handle(rfe2);
+
+ /*
+ * The path returned for two localization should be different because we
+ * are limiting one file per sub-directory.
+ */
+ Assert.assertNotSame(hierarchicalPath1, hierarchicalPath2);
+
+ LocalResourceRequest lr3 = createLocalResourceRequest(user, 2, 2,
+ LocalResourceVisibility.PUBLIC);
+ ResourceEvent reqEvent3 = new ResourceRequestEvent(lr3,
+ LocalResourceVisibility.PUBLIC, lc1);
+ tracker.handle(reqEvent3);
+ Path hierarchicalPath3 = tracker.getPathForLocalization(lr3, localDir);
+ // localization successful
+ ResourceLocalizedEvent rle3 =
+ new ResourceLocalizedEvent(lr3, new Path(hierarchicalPath3.toUri()
+ .toString() + Path.SEPARATOR + "file3"), 120);
+ tracker.handle(rle3);
+
+ // Verifying that path created is inside the subdirectory
+ Assert.assertEquals(hierarchicalPath3.toUri().toString(),
+ hierarchicalPath1.toUri().toString() + Path.SEPARATOR + "0");
+
+ // Container 1 releases resource lr1
+ ResourceEvent relEvent1 = new ResourceReleaseEvent(lr1, cId1);
+ tracker.handle(relEvent1);
+
+ // Validate the file counts now
+ int resources = 0;
+ Iterator<LocalizedResource> iter = tracker.iterator();
+ while (iter.hasNext()) {
+ iter.next();
+ resources++;
+ }
+ // There should be only two resources lr1 and lr3 now.
+ Assert.assertEquals(2, resources);
+
+ // Now simulate cache cleanup - removes unused resources.
+ iter = tracker.iterator();
+ while (iter.hasNext()) {
+ LocalizedResource rsrc = iter.next();
+ if (rsrc.getRefCount() == 0) {
+ Assert.assertTrue(tracker.remove(rsrc, mockDelService));
+ resources--;
+ }
+ }
+ // lr1 is not used by anyone and will be removed, only lr3 will hang
+ // around
+ Assert.assertEquals(1, resources);
+ } finally {
+ if (dispatcher != null) {
+ dispatcher.stop();
+ }
+ }
+ }
+
private boolean createdummylocalizefile(Path path) {
boolean ret = false;
File file = new File(path.toUri().getRawPath().toString());
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java Fri Apr 12 23:05:28 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyLong;
@@ -35,6 +36,7 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.times;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -375,7 +377,7 @@ public class TestResourceLocalizationSer
}
}
- @Test
+ @Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
public void testLocalizationHeartbeat() throws Exception {
Configuration conf = new YarnConfiguration();
@@ -386,12 +388,17 @@ public class TestResourceLocalizationSer
isA(Path.class), isA(FsPermission.class), anyBoolean());
List<Path> localDirs = new ArrayList<Path>();
- String[] sDirs = new String[4];
- for (int i = 0; i < 4; ++i) {
- localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
- sDirs[i] = localDirs.get(i).toString();
- }
+ String[] sDirs = new String[1];
+ // Making sure that we have only one local disk so that it will only be
+ // selected for consecutive resource localization calls. This is required
+ // to test LocalCacheDirectoryManager.
+ localDirs.add(lfs.makeQualified(new Path(basedir, 0 + "")));
+ sDirs[0] = localDirs.get(0).toString();
+
conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
+ // Adding configuration to make sure there is only one file per
+ // directory
+ conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37");
String logDir = lfs.makeQualified(new Path(basedir, "logdir " )).toString();
conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
DrainDispatcher dispatcher = new DrainDispatcher();
@@ -452,12 +459,23 @@ public class TestResourceLocalizationSer
doReturn(out).when(spylfs).createInternal(isA(Path.class),
isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(),
anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
- final LocalResource resource = getPrivateMockedResource(r);
- final LocalResourceRequest req = new LocalResourceRequest(resource);
+ final LocalResource resource1 = getPrivateMockedResource(r);
+ LocalResource resource2 = null;
+ do {
+ resource2 = getPrivateMockedResource(r);
+ } while (resource2 == null || resource2.equals(resource1));
+ // above call to make sure we don't get identical resources.
+
+ final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
+ final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
new HashMap<LocalResourceVisibility,
Collection<LocalResourceRequest>>();
- rsrcs.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(req));
+ List<LocalResourceRequest> privateResourceList =
+ new ArrayList<LocalResourceRequest>();
+ privateResourceList.add(req1);
+ privateResourceList.add(req2);
+ rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
// Sigh. Thread init of private localizer not accessible
Thread.sleep(1000);
@@ -471,33 +489,67 @@ public class TestResourceLocalizationSer
Path localizationTokenPath = tokenPathCaptor.getValue();
// heartbeat from localizer
- LocalResourceStatus rsrcStat = mock(LocalResourceStatus.class);
+ LocalResourceStatus rsrcStat1 = mock(LocalResourceStatus.class);
+ LocalResourceStatus rsrcStat2 = mock(LocalResourceStatus.class);
LocalizerStatus stat = mock(LocalizerStatus.class);
when(stat.getLocalizerId()).thenReturn(ctnrStr);
- when(rsrcStat.getResource()).thenReturn(resource);
- when(rsrcStat.getLocalSize()).thenReturn(4344L);
+ when(rsrcStat1.getResource()).thenReturn(resource1);
+ when(rsrcStat2.getResource()).thenReturn(resource2);
+ when(rsrcStat1.getLocalSize()).thenReturn(4344L);
+ when(rsrcStat2.getLocalSize()).thenReturn(2342L);
URL locPath = getPath("/cache/private/blah");
- when(rsrcStat.getLocalPath()).thenReturn(locPath);
- when(rsrcStat.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
+ when(rsrcStat1.getLocalPath()).thenReturn(locPath);
+ when(rsrcStat2.getLocalPath()).thenReturn(locPath);
+ when(rsrcStat1.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
+ when(rsrcStat2.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
when(stat.getResources())
.thenReturn(Collections.<LocalResourceStatus>emptyList())
- .thenReturn(Collections.singletonList(rsrcStat))
+ .thenReturn(Collections.singletonList(rsrcStat1))
+ .thenReturn(Collections.singletonList(rsrcStat2))
.thenReturn(Collections.<LocalResourceStatus>emptyList());
- // get rsrc
+ String localPath = Path.SEPARATOR + ContainerLocalizer.USERCACHE +
+ Path.SEPARATOR + "user0" + Path.SEPARATOR +
+ ContainerLocalizer.FILECACHE;
+
+ // get first resource
LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
- assertEquals(req, new LocalResourceRequest(response.getLocalResource(0)));
+ assertEquals(1, response.getResourceSpecs().size());
+ assertEquals(req1,
+ new LocalResourceRequest(response.getResourceSpecs().get(0).getResource()));
+ URL localizedPath =
+ response.getResourceSpecs().get(0).getDestinationDirectory();
+ // Appending to local path unique number(10) generated as a part of
+ // LocalResourcesTracker
+ assertTrue(localizedPath.getFile().endsWith(
+ localPath + Path.SEPARATOR + "10"));
+
+ // get second resource
+ response = spyService.heartbeat(stat);
+ assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
+ assertEquals(1, response.getResourceSpecs().size());
+ assertEquals(req2, new LocalResourceRequest(response.getResourceSpecs()
+ .get(0).getResource()));
+ localizedPath =
+ response.getResourceSpecs().get(0).getDestinationDirectory();
+ // Resource's destination path should be now inside sub directory 0 as
+ // LocalCacheDirectoryManager will be used and we have restricted number
+ // of files per directory to 1.
+ assertTrue(localizedPath.getFile().endsWith(
+ localPath + Path.SEPARATOR + "0" + Path.SEPARATOR + "11"));
// empty rsrc
response = spyService.heartbeat(stat);
assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
- assertEquals(0, response.getAllResources().size());
+ assertEquals(0, response.getResourceSpecs().size());
// get shutdown
response = spyService.heartbeat(stat);
assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
+
+ dispatcher.await();
// verify container notification
ArgumentMatcher<ContainerEvent> matchesContainerLoc =
new ArgumentMatcher<ContainerEvent>() {
@@ -508,9 +560,9 @@ public class TestResourceLocalizationSer
&& c.getContainerID() == evt.getContainerID();
}
};
- dispatcher.await();
- verify(containerBus).handle(argThat(matchesContainerLoc));
-
+ // total 2 resource localzation calls. one for each resource.
+ verify(containerBus, times(2)).handle(argThat(matchesContainerLoc));
+
// Verify deletion of localization token.
verify(delService).delete((String)isNull(), eq(localizationTokenPath));
} finally {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java Fri Apr 12 23:05:28 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.no
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -76,10 +77,11 @@ public class TestResourceRetention {
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
long nRsrcs, long timestamp, long tsstep) {
+ Configuration conf = new Configuration();
ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
- trackerResources));
+ trackerResources, false, conf));
for (int i = 0; i < nRsrcs; ++i) {
final LocalResourceRequest req = new LocalResourceRequest(
new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java Fri Apr 12 23:05:28 2013
@@ -56,13 +56,13 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -91,6 +91,7 @@ import org.mockito.Mockito;
import org.mortbay.util.MultiException;
+
//@Ignore
public class TestLogAggregationService extends BaseContainerManagerTest {
@@ -679,7 +680,7 @@ public class TestLogAggregationService e
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId =
recordFactory.newRecordInstance(ApplicationId.class);
@@ -689,7 +690,7 @@ public class TestLogAggregationService e
BuilderUtils.newApplicationAttemptId(appId, 1);
ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(this.user);
@@ -713,12 +714,12 @@ public class TestLogAggregationService e
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(100 * 1024 * 1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(100 * 1024 * 1024, 1));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
this.containerManager.startContainer(startRequest);
BaseContainerManagerTest.waitForContainerState(this.containerManager,
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java Fri Apr 12 23:05:28 2013
@@ -21,7 +21,7 @@ package org.apache.hadoop.yarn.server.no
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.*;
import java.io.BufferedReader;
import java.io.File;
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -51,7 +52,6 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
@@ -197,7 +198,7 @@ public class TestContainersMonitor exten
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
-
+ Container mockContainer = mock(Container.class);
// ////// Construct the Container-id
ApplicationId appId =
recordFactory.newRecordInstance(ApplicationId.class);
@@ -210,7 +211,7 @@ public class TestContainersMonitor exten
ContainerId cId = recordFactory.newRecordInstance(ContainerId.class);
cId.setId(0);
cId.setApplicationAttemptId(appAttemptId);
- containerLaunchContext.setContainerId(cId);
+ when(mockContainer.getId()).thenReturn(cId);
containerLaunchContext.setUser(user);
@@ -234,12 +235,12 @@ public class TestContainersMonitor exten
commands.add("/bin/bash");
commands.add(scriptFile.getAbsolutePath());
containerLaunchContext.setCommands(commands);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
- containerLaunchContext.getResource().setMemory(8 * 1024 * 1024);
+ when(mockContainer.getResource()).thenReturn(
+ BuilderUtils.newResource(8 * 1024 * 1024, 1));
StartContainerRequest startRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
startRequest.setContainerLaunchContext(containerLaunchContext);
+ startRequest.setContainer(mockContainer);
containerManager.startContainer(startRequest);
int timeoutSecs = 0;
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java Fri Apr 12 23:05:28 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -58,7 +59,6 @@ public class MockContainer implements Co
uniqId);
this.launchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
- launchContext.setContainerId(id);
launchContext.setUser(user);
this.state = ContainerState.NEW;
@@ -104,7 +104,6 @@ public class MockContainer implements Co
.newRecordInstance(ContainerStatus.class);
containerStatus
.setState(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
- containerStatus.setContainerId(this.launchContext.getContainerId());
containerStatus.setDiagnostics("testing");
containerStatus.setExitStatus(0);
return containerStatus;
@@ -119,4 +118,9 @@ public class MockContainer implements Co
public void handle(ContainerEvent event) {
}
+ @Override
+ public Resource getResource() {
+ return null;
+ }
+
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Fri Apr 12 23:05:28 2013
@@ -179,15 +179,19 @@ public class TestNMWebServer {
// TODO: Use builder utils
ContainerLaunchContext launchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- launchContext.setContainerId(containerId);
+ org.apache.hadoop.yarn.api.records.Container mockContainer =
+ mock(org.apache.hadoop.yarn.api.records.Container.class);
+ when(mockContainer.getId()).thenReturn(containerId);
launchContext.setUser(user);
Container container =
- new ContainerImpl(conf, dispatcher, launchContext, null, metrics) {
- @Override
- public ContainerState getContainerState() {
- return ContainerState.RUNNING;
- };
- };
+ new ContainerImpl(conf, dispatcher, launchContext, mockContainer,
+ null, metrics) {
+
+ @Override
+ public ContainerState getContainerState() {
+ return ContainerState.RUNNING;
+ };
+ };
nmContext.getContainers().put(containerId, container);
//TODO: Gross hack. Fix in code.
ApplicationId applicationId =
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Fri Apr 12 23:05:28 2013
@@ -54,15 +54,17 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptRegistrationEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptStatusupdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.service.AbstractService;
@@ -276,6 +278,14 @@ public class ApplicationMasterService ex
List<ResourceRequest> ask = request.getAskList();
List<ContainerId> release = request.getReleaseList();
+ // sanity check
+ try {
+ SchedulerUtils.validateResourceRequests(ask,
+ rScheduler.getMaximumResourceCapability());
+ } catch (InvalidResourceRequestException e) {
+ LOG.warn("Invalid resource ask by application " + appAttemptId, e);
+ throw RPCUtil.getRemoteException(e);
+ }
// Send new requests to appAttempt.
Allocation allocation =
this.rScheduler.allocate(appAttemptId, ask, release);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri Apr 12 23:05:28 2013
@@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -86,8 +87,11 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.InvalidResourceRequestException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -262,7 +266,7 @@ public class ClientRMService extends Abs
ApplicationSubmissionContext submissionContext = request
.getApplicationSubmissionContext();
ApplicationId applicationId = submissionContext.getApplicationId();
- String user = submissionContext.getUser();
+ String user = submissionContext.getAMContainerSpec().getUser();
try {
user = UserGroupInformation.getCurrentUser().getShortUserName();
if (rmContext.getRMApps().get(applicationId) != null) {
@@ -271,7 +275,22 @@ public class ClientRMService extends Abs
}
// Safety
- submissionContext.setUser(user);
+ submissionContext.getAMContainerSpec().setUser(user);
+
+ // Check whether AM resource requirements are within required limits
+ if (!submissionContext.getUnmanagedAM()) {
+ ResourceRequest amReq = BuilderUtils.newResourceRequest(
+ RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY,
+ submissionContext.getResource(), 1);
+ try {
+ SchedulerUtils.validateResourceRequest(amReq,
+ scheduler.getMaximumResourceCapability());
+ } catch (InvalidResourceRequestException e) {
+ LOG.warn("RM app submission failed in validating AM resource request"
+ + " for application " + applicationId, e);
+ throw RPCUtil.getRemoteException(e);
+ }
+ }
// This needs to be synchronous as the client can query
// immediately following the submission to get the application status.
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java Fri Apr 12 23:05:28 2013
@@ -57,6 +57,7 @@ public class RMAppManager implements Eve
private static final Log LOG = LogFactory.getLog(RMAppManager.class);
private int completedAppsMax = YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS;
+ private int globalMaxAppAttempts;
private LinkedList<ApplicationId> completedApps = new LinkedList<ApplicationId>();
private final RMContext rmContext;
@@ -76,6 +77,8 @@ public class RMAppManager implements Eve
setCompletedAppsMax(conf.getInt(
YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS,
YarnConfiguration.DEFAULT_RM_MAX_COMPLETED_APPLICATIONS));
+ globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
}
/**
@@ -246,10 +249,11 @@ public class RMAppManager implements Eve
// Create RMApp
application =
new RMAppImpl(applicationId, rmContext, this.conf,
- submissionContext.getApplicationName(),
- submissionContext.getUser(), submissionContext.getQueue(),
- submissionContext, this.scheduler, this.masterService,
- submitTime);
+ submissionContext.getApplicationName(),
+ submissionContext.getAMContainerSpec().getUser(),
+ submissionContext.getQueue(),
+ submissionContext, this.scheduler, this.masterService,
+ submitTime);
// Sanity check - duplicate?
if (rmContext.getRMApps().putIfAbsent(applicationId, application) !=
@@ -308,6 +312,7 @@ public class RMAppManager implements Eve
Map<ApplicationId, ApplicationState> appStates = state.getApplicationState();
LOG.info("Recovering " + appStates.size() + " applications");
for(ApplicationState appState : appStates.values()) {
+ boolean shouldRecover = true;
// re-submit the application
// this is going to send an app start event but since the async dispatcher
// has not started that event will be queued until we have completed re
@@ -318,16 +323,39 @@ public class RMAppManager implements Eve
// This will need to be changed in work preserving recovery in which
// RM will re-connect with the running AM's instead of restarting them
LOG.info("Not recovering unmanaged application " + appState.getAppId());
- store.removeApplication(appState);
+ shouldRecover = false;
+ }
+ int individualMaxAppAttempts = appState.getApplicationSubmissionContext()
+ .getMaxAppAttempts();
+ int maxAppAttempts;
+ if (individualMaxAppAttempts <= 0 ||
+ individualMaxAppAttempts > globalMaxAppAttempts) {
+ maxAppAttempts = globalMaxAppAttempts;
+ LOG.warn("The specific max attempts: " + individualMaxAppAttempts
+ + " for application: " + appState.getAppId()
+ + " is invalid, because it is out of the range [1, "
+ + globalMaxAppAttempts + "]. Use the global max attempts instead.");
} else {
+ maxAppAttempts = individualMaxAppAttempts;
+ }
+ if(appState.getAttemptCount() >= maxAppAttempts) {
+ LOG.info("Not recovering application " + appState.getAppId() +
+ " due to recovering attempt is beyond maxAppAttempt limit");
+ shouldRecover = false;
+ }
+
+ if(shouldRecover) {
LOG.info("Recovering application " + appState.getAppId());
submitApplication(appState.getApplicationSubmissionContext(),
- appState.getSubmitTime());
+ appState.getSubmitTime());
// re-populate attempt information in application
RMAppImpl appImpl = (RMAppImpl) rmContext.getRMApps().get(
- appState.getAppId());
+ appState.getAppId());
appImpl.recover(state);
}
+ else {
+ store.removeApplication(appState);
+ }
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Fri Apr 12 23:05:28 2013
@@ -327,12 +327,52 @@ public class ResourceManager extends Com
this.applicationACLsManager, this.conf);
}
+ // sanity check for configurations
protected static void validateConfigs(Configuration conf) {
- int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ // validate max-attempts
+ int globalMaxAppAttempts =
+ conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
if (globalMaxAppAttempts <= 0) {
- throw new YarnException(
- "The global max attempts should be a positive integer.");
+ throw new YarnException("Invalid global max attempts configuration"
+ + ", " + YarnConfiguration.RM_AM_MAX_ATTEMPTS
+ + "=" + globalMaxAppAttempts + ", it should be a positive integer.");
+ }
+
+ // validate scheduler memory allocation setting
+ int minMem = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
+ int maxMem = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
+
+ if (minMem <= 0 || minMem > maxMem) {
+ throw new YarnException("Invalid resource scheduler memory"
+ + " allocation configuration"
+ + ", " + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB
+ + "=" + minMem
+ + ", " + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB
+ + "=" + maxMem + ", min and max should be greater than 0"
+ + ", max should be no smaller than min.");
+ }
+
+ // validate scheduler vcores allocation setting
+ int minVcores = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ int maxVcores = conf.getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);
+
+ if (minVcores <= 0 || minVcores > maxVcores) {
+ throw new YarnException("Invalid resource scheduler vcores"
+ + " allocation configuration"
+ + ", " + YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES
+ + "=" + minVcores
+ + ", " + YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES
+ + "=" + maxVcores + ", min and max should be greater than 0"
+ + ", max should be no smaller than min.");
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Fri Apr 12 23:05:28 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
+import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -67,16 +69,17 @@ public class ResourceTrackerService exte
private final NMLivelinessMonitor nmLivelinessMonitor;
private final RMContainerTokenSecretManager containerTokenSecretManager;
+ private long nextHeartBeatInterval;
private Server server;
private InetSocketAddress resourceTrackerAddress;
- private static final NodeHeartbeatResponse reboot = recordFactory
+ private static final NodeHeartbeatResponse resync = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
private static final NodeHeartbeatResponse shutDown = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
static {
- reboot.setNodeAction(NodeAction.REBOOT);
+ resync.setNodeAction(NodeAction.RESYNC);
shutDown.setNodeAction(NodeAction.SHUTDOWN);
}
@@ -100,6 +103,14 @@ public class ResourceTrackerService exte
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
RackResolver.init(conf);
+ nextHeartBeatInterval =
+ conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
+ if (nextHeartBeatInterval <= 0) {
+ throw new YarnException("Invalid Configuration. "
+ + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
+ + " should be larger than 0.");
+ }
super.init(conf);
}
@@ -209,7 +220,7 @@ public class ResourceTrackerService exte
if (rmNode == null) {
/* node does not exist */
LOG.info("Node not found rebooting " + remoteNodeStatus.getNodeId());
- return reboot;
+ return resync;
}
// Send ping
@@ -223,9 +234,6 @@ public class ResourceTrackerService exte
new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
return shutDown;
}
-
- NodeHeartbeatResponse nodeHeartBeatResponse = recordFactory
- .newRecordInstance(NodeHeartbeatResponse.class);
// 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat
NodeHeartbeatResponse lastNodeHeartbeatResponse = rmNode.getLastNodeHeartBeatResponse();
@@ -242,14 +250,15 @@ public class ResourceTrackerService exte
// TODO: Just sending reboot is not enough. Think more.
this.rmContext.getDispatcher().getEventHandler().handle(
new RMNodeEvent(nodeId, RMNodeEventType.REBOOTING));
- return reboot;
+ return resync;
}
// Heartbeat response
- nodeHeartBeatResponse.setResponseId(lastNodeHeartbeatResponse.getResponseId() + 1);
+ NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
+ .newNodeHeartbeatResponse(lastNodeHeartbeatResponse.
+ getResponseId() + 1, NodeAction.NORMAL, null, null, null,
+ nextHeartBeatInterval);
rmNode.updateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse);
- nodeHeartBeatResponse.setNodeAction(NodeAction.NORMAL);
-
// Check if node's masterKey needs to be updated and if the currentKey has
// roller over, send it across
if (isSecurityEnabled()) {