You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sz...@apache.org on 2013/04/05 04:43:33 UTC
svn commit: r1464815 [2/2] - in
/hadoop/common/branches/HDFS-2802/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ap...
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri Apr 5 02:43:29 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -29,6 +30,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -43,6 +45,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -58,12 +61,15 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.Service.STATE;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -91,6 +97,8 @@ public class TestNodeStatusUpdater {
private final Configuration conf = createNMConfig();
private NodeManager nm;
protected NodeManager rebootedNodeManager;
+ private boolean containerStatusBackupSuccessfully = true;
+ private List<ContainerStatus> completedContainerStatusList = new ArrayList<ContainerStatus>();
@After
public void tearDown() {
@@ -214,9 +222,8 @@ public class TestNodeStatusUpdater {
Assert.assertEquals(2, activeContainers.size());
}
- NodeHeartbeatResponse nhResponse = recordFactory
- .newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setResponseId(heartBeatID);
+ NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils.
+ newNodeHeartbeatResponse(heartBeatID, null, null, null, null, 1000L);
return nhResponse;
}
}
@@ -237,6 +244,22 @@ public class TestNodeStatusUpdater {
}
}
+ private class MyNodeStatusUpdater2 extends NodeStatusUpdaterImpl {
+ public ResourceTracker resourceTracker;
+
+ public MyNodeStatusUpdater2(Context context, Dispatcher dispatcher,
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
+ super(context, dispatcher, healthChecker, metrics);
+ resourceTracker = new MyResourceTracker4(context);
+ }
+
+ @Override
+ protected ResourceTracker getRMClient() {
+ return resourceTracker;
+ }
+
+ }
+
private class MyNodeStatusUpdater3 extends NodeStatusUpdaterImpl {
public ResourceTracker resourceTracker;
private Context context;
@@ -325,10 +348,9 @@ public class TestNodeStatusUpdater {
NodeStatus nodeStatus = request.getNodeStatus();
nodeStatus.setResponseId(heartBeatID++);
- NodeHeartbeatResponse nhResponse = recordFactory
- .newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setResponseId(heartBeatID);
- nhResponse.setNodeAction(heartBeatNodeAction);
+ NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils.
+ newNodeHeartbeatResponse(heartBeatID, heartBeatNodeAction, null,
+ null, null, 1000L);
return nhResponse;
}
}
@@ -361,10 +383,9 @@ public class TestNodeStatusUpdater {
LOG.info("Got heartBeatId: [" + heartBeatID +"]");
NodeStatus nodeStatus = request.getNodeStatus();
nodeStatus.setResponseId(heartBeatID++);
- NodeHeartbeatResponse nhResponse =
- recordFactory.newRecordInstance(NodeHeartbeatResponse.class);
- nhResponse.setResponseId(heartBeatID);
- nhResponse.setNodeAction(heartBeatNodeAction);
+ NodeHeartbeatResponse nhResponse = YarnServerBuilderUtils.
+ newNodeHeartbeatResponse(heartBeatID, heartBeatNodeAction, null,
+ null, null, 1000L);
if (nodeStatus.getKeepAliveApplications() != null
&& nodeStatus.getKeepAliveApplications().size() > 0) {
@@ -386,6 +407,104 @@ public class TestNodeStatusUpdater {
}
}
+ private class MyResourceTracker4 implements ResourceTracker {
+
+ public NodeAction registerNodeAction = NodeAction.NORMAL;
+ public NodeAction heartBeatNodeAction = NodeAction.NORMAL;
+ private Context context;
+
+ public MyResourceTracker4(Context context) {
+ this.context = context;
+ }
+
+ @Override
+ public RegisterNodeManagerResponse registerNodeManager(
+ RegisterNodeManagerRequest request) throws YarnRemoteException {
+ RegisterNodeManagerResponse response = recordFactory
+ .newRecordInstance(RegisterNodeManagerResponse.class);
+ response.setNodeAction(registerNodeAction);
+ return response;
+ }
+
+ @Override
+ public NodeHeartbeatResponse nodeHeartbeat(NodeHeartbeatRequest request)
+ throws YarnRemoteException {
+ try {
+ if (heartBeatID == 0) {
+ Assert.assertEquals(request.getNodeStatus().getContainersStatuses()
+ .size(), 0);
+ Assert.assertEquals(context.getContainers().size(), 0);
+ } else if (heartBeatID == 1) {
+ Assert.assertEquals(request.getNodeStatus().getContainersStatuses()
+ .size(), 5);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(0).getState() == ContainerState.RUNNING
+ && request.getNodeStatus().getContainersStatuses().get(0)
+ .getContainerId().getId() == 1);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(1).getState() == ContainerState.RUNNING
+ && request.getNodeStatus().getContainersStatuses().get(1)
+ .getContainerId().getId() == 2);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(2).getState() == ContainerState.COMPLETE
+ && request.getNodeStatus().getContainersStatuses().get(2)
+ .getContainerId().getId() == 3);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(3).getState() == ContainerState.COMPLETE
+ && request.getNodeStatus().getContainersStatuses().get(3)
+ .getContainerId().getId() == 4);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(4).getState() == ContainerState.RUNNING
+ && request.getNodeStatus().getContainersStatuses().get(4)
+ .getContainerId().getId() == 5);
+ throw new YarnException("Lost the heartbeat response");
+ } else if (heartBeatID == 2) {
+ Assert.assertEquals(request.getNodeStatus().getContainersStatuses()
+ .size(), 7);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(0).getState() == ContainerState.COMPLETE
+ && request.getNodeStatus().getContainersStatuses().get(0)
+ .getContainerId().getId() == 3);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(1).getState() == ContainerState.COMPLETE
+ && request.getNodeStatus().getContainersStatuses().get(1)
+ .getContainerId().getId() == 4);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(2).getState() == ContainerState.RUNNING
+ && request.getNodeStatus().getContainersStatuses().get(2)
+ .getContainerId().getId() == 1);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(3).getState() == ContainerState.RUNNING
+ && request.getNodeStatus().getContainersStatuses().get(3)
+ .getContainerId().getId() == 2);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(4).getState() == ContainerState.RUNNING
+ && request.getNodeStatus().getContainersStatuses().get(4)
+ .getContainerId().getId() == 5);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(5).getState() == ContainerState.RUNNING
+ && request.getNodeStatus().getContainersStatuses().get(5)
+ .getContainerId().getId() == 6);
+ Assert.assertTrue(request.getNodeStatus().getContainersStatuses()
+ .get(6).getState() == ContainerState.COMPLETE
+ && request.getNodeStatus().getContainersStatuses().get(6)
+ .getContainerId().getId() == 7);
+ }
+ } catch (AssertionError error) {
+ LOG.info(error);
+ containerStatusBackupSuccessfully = false;
+ } finally {
+ heartBeatID++;
+ }
+ NodeStatus nodeStatus = request.getNodeStatus();
+ nodeStatus.setResponseId(heartBeatID);
+ NodeHeartbeatResponse nhResponse =
+ YarnServerBuilderUtils.newNodeHeartbeatResponse(heartBeatID,
+ heartBeatNodeAction, null, null, null, 1000L);
+ return nhResponse;
+ }
+ }
+
@Before
public void clearError() {
nmStartError = null;
@@ -727,6 +846,127 @@ public class TestNodeStatusUpdater {
}
}
+ /**
+ * Test completed containerStatus get back up when heart beat lost
+ */
+ @Test(timeout = 20000)
+ public void testCompletedContainerStatusBackup() throws Exception {
+ nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
+ MyNodeStatusUpdater2 myNodeStatusUpdater =
+ new MyNodeStatusUpdater2(context, dispatcher, healthChecker,
+ metrics);
+ return myNodeStatusUpdater;
+ }
+
+ @Override
+ protected NMContext createNMContext(
+ NMContainerTokenSecretManager containerTokenSecretManager) {
+ return new MyNMContext(containerTokenSecretManager);
+ }
+
+ };
+
+ YarnConfiguration conf = createNMConfig();
+ nm.init(conf);
+ nm.start();
+
+ int waitCount = 0;
+ while (heartBeatID <= 3 && waitCount++ != 20) {
+ Thread.sleep(500);
+ }
+ if(!containerStatusBackupSuccessfully) {
+ Assert.fail("ContainerStatus Backup failed");
+ }
+ nm.stop();
+ }
+
+ private class MyNMContext extends NMContext {
+ ConcurrentMap<ContainerId, Container> containers =
+ new ConcurrentSkipListMap<ContainerId, Container>();
+
+ public MyNMContext(NMContainerTokenSecretManager
+ containerTokenSecretManager) {
+ super(containerTokenSecretManager);
+ }
+
+ @Override
+ public ConcurrentMap<ContainerId, Container> getContainers() {
+ if (heartBeatID == 0) {
+ return containers;
+ } else if (heartBeatID == 1) {
+ ContainerStatus containerStatus1 =
+ createContainerStatus(1, ContainerState.RUNNING);
+ Container container1 = getMockContainer(containerStatus1);
+ containers.put(containerStatus1.getContainerId(), container1);
+
+ ContainerStatus containerStatus2 =
+ createContainerStatus(2, ContainerState.RUNNING);
+ Container container2 = getMockContainer(containerStatus2);
+ containers.put(containerStatus2.getContainerId(), container2);
+
+ ContainerStatus containerStatus3 =
+ createContainerStatus(3, ContainerState.COMPLETE);
+ Container container3 = getMockContainer(containerStatus3);
+ containers.put(containerStatus3.getContainerId(), container3);
+ completedContainerStatusList.add(containerStatus3);
+
+ ContainerStatus containerStatus4 =
+ createContainerStatus(4, ContainerState.COMPLETE);
+ Container container4 = getMockContainer(containerStatus4);
+ containers.put(containerStatus4.getContainerId(), container4);
+ completedContainerStatusList.add(containerStatus4);
+
+ ContainerStatus containerStatus5 =
+ createContainerStatus(5, ContainerState.RUNNING);
+ Container container5 = getMockContainer(containerStatus5);
+ containers.put(containerStatus5.getContainerId(), container5);
+
+ return containers;
+ } else if (heartBeatID == 2) {
+ ContainerStatus containerStatus6 =
+ createContainerStatus(6, ContainerState.RUNNING);
+ Container container6 = getMockContainer(containerStatus6);
+ containers.put(containerStatus6.getContainerId(), container6);
+
+ ContainerStatus containerStatus7 =
+ createContainerStatus(7, ContainerState.COMPLETE);
+ Container container7 = getMockContainer(containerStatus7);
+ containers.put(containerStatus7.getContainerId(), container7);
+ completedContainerStatusList.add(containerStatus7);
+
+ return containers;
+ } else {
+ containers.clear();
+
+ return containers;
+ }
+ }
+
+ private ContainerStatus createContainerStatus(int id,
+ ContainerState containerState) {
+ ApplicationId applicationId =
+ BuilderUtils.newApplicationId(System.currentTimeMillis(), id);
+ ApplicationAttemptId applicationAttemptId =
+ BuilderUtils.newApplicationAttemptId(applicationId, id);
+ ContainerId contaierId =
+ BuilderUtils.newContainerId(applicationAttemptId, id);
+ ContainerStatus containerStatus =
+ BuilderUtils.newContainerStatus(contaierId, containerState,
+ "test_containerStatus: id=" + id + ", containerState: "
+ + containerState, 0);
+ return containerStatus;
+ }
+
+ private Container getMockContainer(ContainerStatus containerStatus) {
+ Container container = mock(Container.class);
+ when(container.cloneAndGetContainerStatus()).thenReturn(containerStatus);
+ return container;
+ }
+ }
+
private void verifyNodeStartFailure(String errMessage) {
YarnConfiguration conf = createNMConfig();
nm.init(conf);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java Fri Apr 5 02:43:29 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -50,17 +51,17 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceRequestEvent;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
-import org.mortbay.log.Log;
public class TestLocalResourcesTrackerImpl {
- @Test
+ @Test(timeout=10000)
@SuppressWarnings("unchecked")
public void test() {
String user = "testuser";
DrainDispatcher dispatcher = null;
try {
- dispatcher = createDispatcher(new Configuration());
+ Configuration conf = new Configuration();
+ dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler =
mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler =
@@ -86,7 +87,8 @@ public class TestLocalResourcesTrackerIm
localrsrc.put(req1, lr1);
localrsrc.put(req2, lr2);
LocalResourcesTracker tracker =
- new LocalResourcesTrackerImpl(user, dispatcher, localrsrc);
+ new LocalResourcesTrackerImpl(user, dispatcher, localrsrc, false,
+ conf);
ResourceEvent req11Event =
new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
@@ -152,13 +154,14 @@ public class TestLocalResourcesTrackerIm
}
}
- @Test
+ @Test(timeout=10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
String user = "testuser";
DrainDispatcher dispatcher = null;
try {
- dispatcher = createDispatcher(new Configuration());
+ Configuration conf = new Configuration();
+ dispatcher = createDispatcher(conf);
EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
dispatcher.register(LocalizerEventType.class, localizerEventHandler);
@@ -172,7 +175,7 @@ public class TestLocalResourcesTrackerIm
ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
localrsrc.put(req1, lr1);
LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
- dispatcher, localrsrc);
+ dispatcher, localrsrc, false, conf);
ResourceEvent req11Event = new ResourceRequestEvent(req1,
LocalResourceVisibility.PUBLIC, lc1);
@@ -221,6 +224,113 @@ public class TestLocalResourcesTrackerIm
}
}
+ @Test(timeout = 100000)
+ @SuppressWarnings("unchecked")
+ public void testHierarchicalLocalCacheDirectories() {
+ String user = "testuser";
+ DrainDispatcher dispatcher = null;
+ try {
+ Configuration conf = new Configuration();
+ // setting per directory file limit to 1.
+ conf.set(YarnConfiguration.NM_LOCAL_CACHE_MAX_FILES_PER_DIRECTORY, "37");
+ dispatcher = createDispatcher(conf);
+
+ EventHandler<LocalizerEvent> localizerEventHandler =
+ mock(EventHandler.class);
+ EventHandler<LocalizerEvent> containerEventHandler =
+ mock(EventHandler.class);
+ dispatcher.register(LocalizerEventType.class, localizerEventHandler);
+ dispatcher.register(ContainerEventType.class, containerEventHandler);
+
+ DeletionService mockDelService = mock(DeletionService.class);
+
+ ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc =
+ new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
+ LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user,
+ dispatcher, localrsrc, true, conf);
+
+ // This is a random path. NO File creation will take place at this place.
+ Path localDir = new Path("/tmp");
+
+ // Container 1 needs lr1 resource
+ ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
+ LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1,
+ LocalResourceVisibility.PUBLIC);
+ LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
+
+ // Container 1 requests lr1 to be localized
+ ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1,
+ LocalResourceVisibility.PUBLIC, lc1);
+ tracker.handle(reqEvent1);
+
+ // Simulate the process of localization of lr1
+ Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir);
+ // Simulate lr1 getting localized
+ ResourceLocalizedEvent rle =
+ new ResourceLocalizedEvent(lr1,
+ new Path(hierarchicalPath1.toUri().toString() +
+ Path.SEPARATOR + "file1"), 120);
+ tracker.handle(rle);
+ // Localization successful.
+ tracker.localizationCompleted(lr1, true);
+
+ LocalResourceRequest lr2 = createLocalResourceRequest(user, 3, 3,
+ LocalResourceVisibility.PUBLIC);
+ Path hierarchicalPath2 = tracker.getPathForLocalization(lr2, localDir);
+ // localization failed.
+ tracker.localizationCompleted(lr2, false);
+
+ /*
+ * The path returned for two localization should be different because we
+ * are limiting one file per sub-directory.
+ */
+ Assert.assertNotSame(hierarchicalPath1, hierarchicalPath2);
+
+ LocalResourceRequest lr3 = createLocalResourceRequest(user, 2, 2,
+ LocalResourceVisibility.PUBLIC);
+ ResourceEvent reqEvent3 = new ResourceRequestEvent(lr3,
+ LocalResourceVisibility.PUBLIC, lc1);
+ tracker.handle(reqEvent3);
+ Path hierarchicalPath3 = tracker.getPathForLocalization(lr3, localDir);
+ tracker.localizationCompleted(lr3, true);
+
+ // Verifying that path created is inside the subdirectory
+ Assert.assertEquals(hierarchicalPath3.toUri().toString(),
+ hierarchicalPath1.toUri().toString() + Path.SEPARATOR + "0");
+
+ // Container 1 releases resource lr1
+ ResourceEvent relEvent1 = new ResourceReleaseEvent(lr1, cId1);
+ tracker.handle(relEvent1);
+
+ // Validate the file counts now
+ int resources = 0;
+ Iterator<LocalizedResource> iter = tracker.iterator();
+ while (iter.hasNext()) {
+ iter.next();
+ resources++;
+ }
+ // There should be only two resources lr1 and lr3 now.
+ Assert.assertEquals(2, resources);
+
+ // Now simulate cache cleanup - removes unused resources.
+ iter = tracker.iterator();
+ while (iter.hasNext()) {
+ LocalizedResource rsrc = iter.next();
+ if (rsrc.getRefCount() == 0) {
+ Assert.assertTrue(tracker.remove(rsrc, mockDelService));
+ resources--;
+ }
+ }
+ // lr1 is not used by anyone and will be removed, only lr3 will hang
+ // around
+ Assert.assertEquals(1, resources);
+ } finally {
+ if (dispatcher != null) {
+ dispatcher.stop();
+ }
+ }
+ }
+
private boolean createdummylocalizefile(Path path) {
boolean ret = false;
File file = new File(path.toUri().getRawPath().toString());
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java Fri Apr 5 02:43:29 2013
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.no
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -76,10 +77,11 @@ public class TestResourceRetention {
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
long nRsrcs, long timestamp, long tsstep) {
+ Configuration conf = new Configuration();
ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
- trackerResources));
+ trackerResources, false, conf));
for (int i = 0; i < nRsrcs; ++i) {
final LocalResourceRequest req = new LocalResourceRequest(
new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Fri Apr 5 02:43:29 2013
@@ -27,6 +27,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
+import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -67,6 +69,7 @@ public class ResourceTrackerService exte
private final NMLivelinessMonitor nmLivelinessMonitor;
private final RMContainerTokenSecretManager containerTokenSecretManager;
+ private long nextHeartBeatInterval;
private Server server;
private InetSocketAddress resourceTrackerAddress;
@@ -100,6 +103,14 @@ public class ResourceTrackerService exte
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
RackResolver.init(conf);
+ nextHeartBeatInterval =
+ conf.getLong(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS);
+ if (nextHeartBeatInterval <= 0) {
+ throw new YarnException("Invalid Configuration. "
+ + YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS
+ + " should be larger than 0.");
+ }
super.init(conf);
}
@@ -223,9 +234,6 @@ public class ResourceTrackerService exte
new RMNodeEvent(nodeId, RMNodeEventType.DECOMMISSION));
return shutDown;
}
-
- NodeHeartbeatResponse nodeHeartBeatResponse = recordFactory
- .newRecordInstance(NodeHeartbeatResponse.class);
// 3. Check if it's a 'fresh' heartbeat i.e. not duplicate heartbeat
NodeHeartbeatResponse lastNodeHeartbeatResponse = rmNode.getLastNodeHeartBeatResponse();
@@ -246,10 +254,11 @@ public class ResourceTrackerService exte
}
// Heartbeat response
- nodeHeartBeatResponse.setResponseId(lastNodeHeartbeatResponse.getResponseId() + 1);
+ NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
+ .newNodeHeartbeatResponse(lastNodeHeartbeatResponse.
+ getResponseId() + 1, NodeAction.NORMAL, null, null, null,
+ nextHeartBeatInterval);
rmNode.updateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse);
- nodeHeartBeatResponse.setNodeAction(NodeAction.NORMAL);
-
// Check if node's masterKey needs to be updated and if the currentKey has
// roller over, send it across
if (isSecurityEnabled()) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Fri Apr 5 02:43:29 2013
@@ -771,6 +771,11 @@ public class RMAppAttemptImpl implements
// Set the masterContainer
appAttempt.setMasterContainer(amContainerAllocation.getContainers().get(
0));
+ // Updating CLC's resource is no longer necessary once YARN-486 is
+ // completed, because nothing from Container to CLC will be copied into
+ // CLC then.
+ appAttempt.getSubmissionContext().getAMContainerSpec().setResource(
+ appAttempt.getMasterContainer().getResource());
RMStateStore store = appAttempt.rmContext.getStateStore();
appAttempt.storeAttempt(store);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java Fri Apr 5 02:43:29 2013
@@ -288,14 +288,9 @@ public class RMContainerImpl implements
public void transition(RMContainerImpl container, RMContainerEvent event) {
RMContainerFinishedEvent finishedEvent = (RMContainerFinishedEvent) event;
- // Update container-status for diagnostics. Today we completely
- // replace it on finish. We may just need to update diagnostics.
- container.container.setContainerStatus(finishedEvent
- .getRemoteContainerStatus());
-
// Inform AppAttempt
container.eventHandler.handle(new RMAppAttemptContainerFinishedEvent(
- container.appAttemptId, container.container.getContainerStatus()));
+ container.appAttemptId, finishedEvent.getRemoteContainerStatus()));
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Fri Apr 5 02:43:29 2013
@@ -108,8 +108,7 @@ public class SchedulerUtils {
Resource normalized =
Resources.normalize(
resourceCalculator, ask.getCapability(), minimumResource);
- ask.getCapability().setMemory(normalized.getMemory());
- ask.getCapability().setVirtualCores(normalized.getVirtualCores());
+ ask.setCapability(normalized);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Fri Apr 5 02:43:29 2013
@@ -108,7 +108,7 @@ implements ResourceScheduler, CapacitySc
new Comparator<FiCaSchedulerApp>() {
@Override
public int compare(FiCaSchedulerApp a1, FiCaSchedulerApp a2) {
- return a1.getApplicationId().getId() - a2.getApplicationId().getId();
+ return a1.getApplicationId().compareTo(a2.getApplicationId());
}
};
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java Fri Apr 5 02:43:29 2013
@@ -26,7 +26,6 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
@@ -71,11 +70,11 @@ public class MockNM {
this.resourceTracker = resourceTracker;
}
- public void containerStatus(Container container) throws Exception {
+ public void containerStatus(ContainerStatus containerStatus) throws Exception {
Map<ApplicationId, List<ContainerStatus>> conts =
new HashMap<ApplicationId, List<ContainerStatus>>();
- conts.put(container.getId().getApplicationAttemptId().getApplicationId(),
- Arrays.asList(new ContainerStatus[] { container.getContainerStatus() }));
+ conts.put(containerStatus.getContainerId().getApplicationAttemptId().getApplicationId(),
+ Arrays.asList(new ContainerStatus[] { containerStatus }));
nodeHeartbeat(conts, true);
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Fri Apr 5 02:43:29 2013
@@ -76,6 +76,9 @@ public class NodeManager implements Cont
final Map<ApplicationId, List<Container>> containers =
new HashMap<ApplicationId, List<Container>>();
+ final Map<Container, ContainerStatus> containerStatusMap =
+ new HashMap<Container, ContainerStatus>();
+
public NodeManager(String hostName, int containerManagerPort, int httpPort,
String rackName, Resource capability,
ResourceTrackerService resourceTrackerService, RMContext rmContext)
@@ -137,7 +140,7 @@ public class NodeManager implements Cont
List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
for (List<Container> appContainers : containers.values()) {
for (Container container : appContainers) {
- containerStatuses.add(container.getContainerStatus());
+ containerStatuses.add(containerStatusMap.get(container));
}
}
return containerStatuses;
@@ -189,8 +192,11 @@ public class NodeManager implements Cont
null, null // DKDC - Doesn't matter
);
+ ContainerStatus containerStatus =
+ BuilderUtils.newContainerStatus(container.getId(), ContainerState.NEW,
+ "", -1000);
applicationContainers.add(container);
-
+ containerStatusMap.put(container, containerStatus);
Resources.subtractFrom(available, containerLaunchContext.getResource());
Resources.addTo(used, containerLaunchContext.getResource());
@@ -223,7 +229,9 @@ public class NodeManager implements Cont
List<Container> applicationContainers = containers.get(applicationId);
for (Container c : applicationContainers) {
if (c.getId().compareTo(containerID) == 0) {
- c.setState(ContainerState.COMPLETE);
+ ContainerStatus containerStatus = containerStatusMap.get(c);
+ containerStatus.setState(ContainerState.COMPLETE);
+ containerStatusMap.put(c, containerStatus);
}
}
@@ -277,8 +285,8 @@ public class NodeManager implements Cont
}
GetContainerStatusResponse response =
recordFactory.newRecordInstance(GetContainerStatusResponse.class);
- if (container != null && container.getContainerStatus() != null) {
- response.setStatus(container.getContainerStatus());
+ if (container != null && containerStatusMap.get(container).getState() != null) {
+ response.setStatus(containerStatusMap.get(container));
}
return response;
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java Fri Apr 5 02:43:29 2013
@@ -139,8 +139,9 @@ public class TestFifoScheduler {
Container c1 = allocated1.get(0);
Assert.assertEquals(GB, c1.getResource().getMemory());
- c1.setState(ContainerState.COMPLETE);
- nm1.containerStatus(c1);
+ ContainerStatus containerStatus = BuilderUtils.newContainerStatus(
+ c1.getId(), ContainerState.COMPLETE, "", 0);
+ nm1.containerStatus(containerStatus);
int waitCount = 0;
while (attempt1.getJustFinishedContainers().size() < 1
&& waitCount++ != 20) {
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java Fri Apr 5 02:43:29 2013
@@ -54,6 +54,29 @@ public class TestResourceTrackerService
private MockRM rm;
/**
+ * Test RM read NM next heartBeat Interval correctly from Configuration file,
+ * and NM get next heartBeat Interval from RM correctly
+ */
+ @Test (timeout = 5000)
+ public void testGetNextHeartBeatInterval() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, "4000");
+
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:5678", 10240);
+
+ NodeHeartbeatResponse nodeHeartbeat = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(4000, nodeHeartbeat.getNextHeartBeatInterval());
+
+ NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ Assert.assertEquals(4000, nodeHeartbeat2.getNextHeartBeatInterval());
+
+ }
+
+ /**
* Decommissioning using a pre-configured include hosts file
*/
@Test
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Fri Apr 5 02:43:29 2013
@@ -202,9 +202,10 @@ public class TestRMAppAttemptTransitions
submissionContext = mock(ApplicationSubmissionContext.class);
when(submissionContext.getUser()).thenReturn(user);
when(submissionContext.getQueue()).thenReturn(queue);
- ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);
- Resource resource = mock(Resource.class);
- when(amContainerSpec.getResource()).thenReturn(resource);
+ Resource resource = BuilderUtils.newResource(1536, 1);
+ ContainerLaunchContext amContainerSpec =
+ BuilderUtils.newContainerLaunchContext(null, user, resource, null, null,
+ null, null, null, null);
when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec);
unmanagedAM = false;
@@ -469,8 +470,10 @@ public class TestRMAppAttemptTransitions
// Mock the allocation of AM container
Container container = mock(Container.class);
+ Resource resource = BuilderUtils.newResource(2048, 1);
when(container.getId()).thenReturn(
BuilderUtils.newContainerId(applicationAttempt.getAppAttemptId(), 1));
+ when(container.getResource()).thenReturn(resource);
Allocation allocation = mock(Allocation.class);
when(allocation.getContainers()).
thenReturn(Collections.singletonList(container));
@@ -491,6 +494,9 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(
new RMAppAttemptStoredEvent(
applicationAttempt.getAppAttemptId(), null));
+ assertEquals(resource,
+ applicationAttempt.getSubmissionContext()
+ .getAMContainerSpec().getResource());
testAppAttemptAllocatedState(container);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java Fri Apr 5 02:43:29 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -43,35 +42,23 @@ public class TestSchedulerUtils {
// case negative memory
ask.setCapability(Resources.createResource(-1024));
- Resource before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
- Resource after = ask.getCapability();
assertEquals(minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
// case zero memory
ask.setCapability(Resources.createResource(0));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
- after = ask.getCapability();
assertEquals(minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
// case memory is a multiple of minMemory
ask.setCapability(Resources.createResource(2 * minMemory));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
- after = ask.getCapability();
assertEquals(2 * minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
// case memory is not a multiple of minMemory
ask.setCapability(Resources.createResource(minMemory + 10));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource);
- after = ask.getCapability();
assertEquals(2 * minMemory, ask.getCapability().getMemory());
- assertTrue(before == after);
}
@@ -86,33 +73,24 @@ public class TestSchedulerUtils {
// case negative memory/vcores
ask.setCapability(Resources.createResource(-1024, -1));
- Resource before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
- Resource after = ask.getCapability();
assertEquals(minResource, ask.getCapability());
- assertTrue(before == after);
// case zero memory/vcores
ask.setCapability(Resources.createResource(0, 0));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
- after = ask.getCapability();
assertEquals(minResource, ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(1024, ask.getCapability().getMemory());
- assertTrue(before == after);
// case non-zero memory & zero cores
ask.setCapability(Resources.createResource(1536, 0));
- before = ask.getCapability();
SchedulerUtils.normalizeRequest(
ask, resourceCalculator, clusterResource, minResource);
- after = ask.getCapability();
assertEquals(Resources.createResource(2048, 1), ask.getCapability());
assertEquals(1, ask.getCapability().getVirtualCores());
assertEquals(2048, ask.getCapability().getMemory());
- assertTrue(before == after);
}
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Fri Apr 5 02:43:29 2013
@@ -19,8 +19,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Comparator;
import java.util.List;
import junit.framework.Assert;
@@ -43,13 +45,19 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
+import static org.mockito.Mockito.*;
+
public class TestCapacityScheduler {
private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class);
@@ -452,5 +460,33 @@ public class TestCapacityScheduler {
}
return result;
}
+
+
+ @Test (timeout = 5000)
+ public void testApplicationComparator()
+ {
+ CapacityScheduler cs = new CapacityScheduler();
+ Comparator<FiCaSchedulerApp> appComparator= cs.getApplicationComparator();
+ ApplicationId id1 = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class);
+ id1.setClusterTimestamp(1);
+ id1.setId(1);
+ ApplicationId id2 = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class);
+ id2.setClusterTimestamp(1);
+ id2.setId(2);
+ ApplicationId id3 = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ApplicationId.class);
+ id3.setClusterTimestamp(2);
+ id3.setId(1);
+ //same clusterId
+ FiCaSchedulerApp app1 = Mockito.mock(FiCaSchedulerApp.class);
+ when(app1.getApplicationId()).thenReturn(id1);
+ FiCaSchedulerApp app2 = Mockito.mock(FiCaSchedulerApp.class);
+ when(app2.getApplicationId()).thenReturn(id2);
+ FiCaSchedulerApp app3 = Mockito.mock(FiCaSchedulerApp.class);
+ when(app3.getApplicationId()).thenReturn(id3);
+ assertTrue(appComparator.compare(app1, app2) < 0);
+ //different clusterId
+ assertTrue(appComparator.compare(app1, app3) < 0);
+ assertTrue(appComparator.compare(app2, app3) < 0);
+ }
}
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java Fri Apr 5 02:43:29 2013
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.util.BuilderUtils;
public class TestUtils {
private static final Log LOG = LogFactory.getLog(TestUtils.class);
@@ -136,9 +137,7 @@ public class TestUtils {
public static ApplicationAttemptId
getMockApplicationAttemptId(int appId, int attemptId) {
- ApplicationId applicationId = mock(ApplicationId.class);
- when(applicationId.getClusterTimestamp()).thenReturn(0L);
- when(applicationId.getId()).thenReturn(appId);
+ ApplicationId applicationId = BuilderUtils.newApplicationId(0l, appId);
ApplicationAttemptId applicationAttemptId = mock(ApplicationAttemptId.class);
when(applicationAttemptId.getApplicationId()).thenReturn(applicationId);
when(applicationAttemptId.getAttemptId()).thenReturn(attemptId);
Modified: hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm?rev=1464815&r1=1464814&r2=1464815&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm (original)
+++ hadoop/common/branches/HDFS-2802/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm Fri Apr 5 02:43:29 2013
@@ -85,8 +85,9 @@ Hadoop MapReduce Next Generation - Fair
cause too much intermediate data to be created or too much context-switching.
Limiting the apps does not cause any subsequently submitted apps to fail,
only to wait in the scheduler's queue until some of the user's earlier apps
- finish. apps to run from each user/queue are chosen in order of priority and
- then submit time, as in the default FIFO scheduler in Hadoop.
+ finish. Apps to run from each user/queue are chosen in the same fair sharing
+ manner, but can alternatively be configured to be chosen in order of submit
+ time, as in the default FIFO scheduler in Hadoop.
Certain add-ons are not yet supported which existed in the original (MR1)
Fair Scheduler. Among them, is the use of a custom policies governing
@@ -142,7 +143,9 @@ Hadoop MapReduce Next Generation - Fair
* <<<yarn.scheduler.fair.sizebasedweight>>>
* Whether to assign shares to individual apps based on their size, rather than
- providing an equal share to all apps regardless of size. Defaults to false.
+ providing an equal share to all apps regardless of size. When set to true,
+ apps are weighted by the natural logarithm of one plus the app's total
+ requested memory, divided by the natural logarithm of 2. Defaults to false.
* <<<yarn.scheduler.fair.assignmultiple>>>
@@ -180,16 +183,29 @@ Allocation file format
* <<Queue elements>>, which represent queues. Each may contain the following
properties:
- * minResources: minimum amount of aggregate memory
+ * minResources: minimum MB of aggregate memory the queue expects. If a queue
+ demands resources, and its current allocation is below its configured minimum,
+ it will be assigned available resources before any queue that is not in this
+ situation. If multiple queues are in this situation, resources go to the
+ queue with the smallest ratio between allocation and minimum. Note that it is
+ possible that a queue that is below its minimum may not immediately get up to
+ its minimum when it submits an application, because already-running jobs may
+ be using those resources.
- * maxResources: maximum amount of aggregate memory
+ * maxResources: maximum MB of aggregate memory a queue is allowed. A queue
+ will never be assigned a container that would put it over this limit.
* maxRunningApps: limit the number of apps from the queue to run at once
- * weight: to share the cluster non-proportionally with other queues
+ * weight: to share the cluster non-proportionally with other queues. Weights
+ default to 1, and a queue with weight 2 should receive approximately twice
+ as many resources as a queue with the default weight.
* schedulingMode: either "fifo" or "fair" depending on the in-queue scheduling
- policy desired
+ policy desired. Defaults to "fair". If "fifo", apps with earlier submit
+ times are given preference for containers, but apps submitted later may
+ run concurrently if there is leftover space on the cluster after satisfying
+ the earlier app's requests.
* aclSubmitApps: a list of users that can submit apps to the queue. A (default)
value of "*" means that any users can submit apps. A queue inherits the ACL of