You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ac...@apache.org on 2013/04/12 23:18:35 UTC
svn commit: r1467469 - in
/hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/
hadoop-yarn/hadoo...
Author: acmurthy
Date: Fri Apr 12 21:18:35 2013
New Revision: 1467469
URL: http://svn.apache.org/r1467469
Log:
YARN-412. Pushing to 2.0.5-beta only.
Modified:
hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
Modified: hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/CHANGES.txt?rev=1467469&r1=1467468&r2=1467469&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/CHANGES.txt Fri Apr 12 21:18:35 2013
@@ -18,10 +18,6 @@ Release 2.0.4-alpha - UNRELEASED
YARN-470. Support a way to disable resource monitoring on the NodeManager.
(Siddharth Seth via hitesh)
- YARN-412. Fixed FifoScheduler to check hostname of a NodeManager rather
- than its host:port during scheduling which caused incorrect locality for
- containers. (Roger Hoover via acmurthy)
-
Release 2.0.3-alpha - 2013-02-06
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1467469&r1=1467468&r2=1467469&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Fri Apr 12 21:18:35 2013
@@ -453,7 +453,7 @@ public class FifoScheduler implements Re
FiCaSchedulerApp application, Priority priority) {
int assignedContainers = 0;
ResourceRequest request =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
if (request != null) {
// Don't allocate on this node if we don't need containers on this rack
ResourceRequest rackRequest =
Modified: hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1467469&r1=1467468&r2=1467469&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original)
+++ hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Fri Apr 12 21:18:35 2013
@@ -209,7 +209,6 @@ public class MockNodes {
final String rackName = "rack"+ rack;
final int nid = hostnum;
final String hostName = "host"+ nid;
- final String nodeAddr = hostName + ":" + nid;
final int port = 123;
final NodeId nodeID = newNodeID(hostName, port);
final String httpAddress = httpAddr;
@@ -219,7 +218,7 @@ public class MockNodes {
nodeHealthStatus.setIsNodeHealthy(true);
nodeHealthStatus.setHealthReport("HealthyMe");
}
- return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, rackName,
+ return new MockRMNodeImpl(nodeID, hostName, httpAddress, perNode, rackName,
nodeHealthStatus, nid, hostName, state);
}
Modified: hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1467469&r1=1467468&r2=1467469&view=diff
==============================================================================
--- hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/common/branches/branch-2.0.4-alpha/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Fri Apr 12 21:18:35 2013
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
import junit.framework.Assert;
@@ -30,17 +28,13 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.InlineDispatcher;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.Application;
-import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -49,10 +43,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.After;
@@ -64,9 +55,6 @@ public class TestFifoScheduler {
private ResourceManager resourceManager = null;
- private static final RecordFactory recordFactory =
- RecordFactoryProvider.getRecordFactory(null);
-
@Before
public void setUp() throws Exception {
resourceManager = new ResourceManager();
@@ -90,38 +78,14 @@ public class TestFifoScheduler {
.getRMContext());
}
- private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
- ApplicationAttemptId attId = recordFactory
- .newRecordInstance(ApplicationAttemptId.class);
- ApplicationId appIdImpl = recordFactory
- .newRecordInstance(ApplicationId.class);
- appIdImpl.setId(appId);
- attId.setAttemptId(attemptId);
- attId.setApplicationId(appIdImpl);
- return attId;
- }
-
- private ResourceRequest createResourceRequest(int memory, String host,
- int priority, int numContainers) {
- ResourceRequest request = recordFactory
- .newRecordInstance(ResourceRequest.class);
- request.setCapability(Resources.createResource(memory));
- request.setHostName(host);
- request.setNumContainers(numContainers);
- Priority prio = recordFactory.newRecordInstance(Priority.class);
- prio.setPriority(priority);
- request.setPriority(prio);
- return request;
- }
-
- @Test(timeout=5000)
+ @Test
public void testFifoSchedulerCapacityWhenNoNMs() {
FifoScheduler scheduler = new FifoScheduler();
QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity());
}
- @Test(timeout=5000)
+ @Test
public void testAppAttemptMetrics() throws Exception {
AsyncDispatcher dispatcher = new InlineDispatcher();
RMContext rmContext = new RMContextImpl(dispatcher, null,
@@ -147,59 +111,6 @@ public class TestFifoScheduler {
Assert.assertEquals(1, metrics.getAppsSubmitted());
}
- @Test(timeout=2000)
- public void testNodeLocalAssignment() throws Exception {
- AsyncDispatcher dispatcher = new InlineDispatcher();
- RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
- null, null, null);
-
- FifoScheduler scheduler = new FifoScheduler();
- scheduler.reinitialize(new Configuration(), rmContext);
-
- RMNode node0 = MockNodes.newNodeInfo(1,
- Resources.createResource(1024 * 64), 1234);
- NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
- scheduler.handle(nodeEvent1);
-
- int _appId = 1;
- int _appAttemptId = 1;
- ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
- _appAttemptId);
- AppAddedSchedulerEvent appEvent1 = new AppAddedSchedulerEvent(appAttemptId,
- "queue1", "user1");
- scheduler.handle(appEvent1);
-
- int memory = 64;
- int nConts = 3;
- int priority = 20;
-
- List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
- ResourceRequest nodeLocal = createResourceRequest(memory,
- node0.getHostName(), priority, nConts);
- ResourceRequest rackLocal = createResourceRequest(memory,
- node0.getRackName(), priority, nConts);
- ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
- nConts);
- ask.add(nodeLocal);
- ask.add(rackLocal);
- ask.add(any);
- scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>());
-
- NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
-
- // Before the node update event, there are 3 local requests outstanding
- Assert.assertEquals(3, nodeLocal.getNumContainers());
-
- scheduler.handle(node0Update);
-
- // After the node update event, check that there are no more local requests
- // outstanding
- Assert.assertEquals(0, nodeLocal.getNumContainers());
- //Also check that the containers were scheduled
- SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
- Assert.assertEquals(3, info.getLiveContainers().size());
- }
-
// @Test
public void testFifoScheduler() throws Exception {