You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/04/20 07:38:59 UTC
svn commit: r1095271 [2/2] - in /hadoop/mapreduce/branches/MR-279:
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/
...
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Wed Apr 20 05:38:57 2011
@@ -74,7 +74,9 @@ public class TestNodeStatusUpdater {
@Override
public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnRemoteException {
- String node = request.getNode();
+ String host = request.getHost();
+ int cmPort = request.getContainerManagerPort();
+ String node = host + ":" + cmPort;
Resource resource = request.getResource();
LOG.info("Registering " + node);
try {
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1095271&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Wed Apr 20 05:38:57 2011
@@ -0,0 +1,105 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.junit.Test;
+
+public class TestNMWebServer {
+
+ @Test
+ public void testNMWebApp() throws InterruptedException {
+ Context nmContext = new NodeManager.NMContext();
+ ResourceView resourceView = new ResourceView() {
+ @Override
+ public long getVmemAllocatedForContainers() {
+ return 0;
+ }
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return 0;
+ }
+ };
+ WebServer server = new WebServer(nmContext, resourceView);
+ Configuration conf = new Configuration();
+ server.init(conf);
+ server.start();
+
+ // Add a couple of applications and the corresponding containers
+ RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(conf);
+ Dispatcher dispatcher = new AsyncDispatcher();
+ String user = "nobody";
+ long clusterTimeStamp = 1234;
+ Map<String, String> env = new HashMap<String, String>();
+ Map<String, LocalResource> resources =
+ new HashMap<String, LocalResource>();
+ ByteBuffer containerTokens = ByteBuffer.allocate(0);
+ ApplicationId appId1 =
+ BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
+ ApplicationId appId2 =
+ BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 2);
+ for (ApplicationId appId : new ApplicationId[] { appId1, appId2 }) {
+ Application app =
+ new ApplicationImpl(dispatcher, user, appId, env, resources,
+ containerTokens);
+ nmContext.getApplications().put(appId, app);
+ }
+ ContainerId container11 =
+ BuilderUtils.newContainerId(recordFactory, appId1, 0);
+ ContainerId container12 =
+ BuilderUtils.newContainerId(recordFactory, appId1, 1);
+ ContainerId container21 =
+ BuilderUtils.newContainerId(recordFactory, appId2, 0);
+ for (ContainerId containerId : new ContainerId[] { container11,
+ container12, container21 }) {
+ // TODO: Use builder utils
+ ContainerLaunchContext launchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ launchContext.setContainerId(containerId);
+ Container container = new ContainerImpl(dispatcher, launchContext);
+ nmContext.getContainers().put(containerId, container);
+ //TODO: Gross hack. Fix in code.
+ nmContext.getApplications().get(containerId.getAppId()).getContainers()
+ .put(containerId, container);
+
+ }
+// Thread.sleep(1000000);
+ }
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Wed Apr 20 05:38:57 2011
@@ -209,7 +209,7 @@ public class ClientRMService extends Abs
private NodeManagerInfo createNodeManagerInfo(NodeInfo nodeInfo) {
NodeManagerInfo node =
recordFactory.newRecordInstance(NodeManagerInfo.class);
- node.setNodeName(nodeInfo.getHostName());
+ node.setNodeAddress(nodeInfo.getNodeAddress());
node.setRackName(nodeInfo.getRackName());
node.setCapability(nodeInfo.getTotalCapability());
node.setUsed(nodeInfo.getUsedResource());
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/ApplicationsManagerImpl.java Wed Apr 20 05:38:57 2011
@@ -148,7 +148,7 @@ public class ApplicationsManagerImpl ext
@Override
public ApplicationId getNewApplicationID() {
ApplicationId applicationId =
- org.apache.hadoop.yarn.server.resourcemanager.resource.ApplicationID.create(
+ org.apache.hadoop.yarn.util.BuilderUtils.newApplicationId(recordFactory,
ResourceManager.clusterTimeStamp, applicationCounter.incrementAndGet());
LOG.info("Allocated new applicationId: " + applicationId.getId());
return applicationId;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java Wed Apr 20 05:38:57 2011
@@ -24,19 +24,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.util.BuilderUtils;
public class Container {
- private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
+ public static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
- private static ContainerId getNewContainerId(ApplicationId applicationId,
- int containerId) {
- ContainerId id = recordFactory.newRecordInstance(ContainerId.class);
- id.setAppId(applicationId);
- id.setId(containerId);
- return id;
- }
-
public static org.apache.hadoop.yarn.api.records.Container create(
org.apache.hadoop.yarn.api.records.Container c) {
org.apache.hadoop.yarn.api.records.Container container = recordFactory.newRecordInstance(org.apache.hadoop.yarn.api.records.Container.class);
@@ -48,9 +41,11 @@ public class Container {
}
public static org.apache.hadoop.yarn.api.records.Container create(
- ApplicationId applicationId, int containerId,
- String hostName, Resource resource) {
- ContainerId containerID = getNewContainerId(applicationId, containerId);
+ RecordFactory recordFactory, ApplicationId applicationId,
+ int containerId, String hostName, Resource resource) {
+ ContainerId containerID =
+ BuilderUtils
+ .newContainerId(recordFactory, applicationId, containerId);
return create(containerID, hostName, resource);
}
@@ -65,7 +60,7 @@ public class Container {
return container;
}
- public static class Comparator
+ public static class ContainerComparator
implements java.util.Comparator<org.apache.hadoop.yarn.api.records.Container> {
@Override
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/NodeInfo.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/NodeInfo.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/NodeInfo.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/NodeInfo.java Wed Apr 20 05:38:57 2011
@@ -34,10 +34,15 @@ public interface NodeInfo {
*/
public NodeId getNodeID();
/**
- * the hostname for this node.
- * @return the hostname for this node.
+ * the ContainerManager address for this node.
+ * @return the ContainerManager address for this node.
*/
- public String getHostName();
+ public String getNodeAddress();
+ /**
+ * the http-Address for this node.
+ * @return the http-url address for this node
+ */
+ public String getHttpAddress();
/**
* the health-status for this node
* @return the health-status for this node.
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java Wed Apr 20 05:38:57 2011
@@ -162,7 +162,10 @@ ResourceTracker, ResourceContext {
@Override
public RegisterNodeManagerResponse registerNodeManager(RegisterNodeManagerRequest request) throws YarnRemoteException {
- String node = request.getNode();
+ String host = request.getHost();
+ int cmPort = request.getContainerManagerPort();
+ String node = host + ":" + cmPort;
+ String httpAddress = host + ":" + request.getHttpPort();
Resource capability = request.getResource();
NodeId nodeId = getNodeId(node);
@@ -172,7 +175,7 @@ ResourceTracker, ResourceContext {
if (!nodeManagers.containsKey(nodeId)) {
/* we do the resolving here, so that scheduler does not have to do it */
NodeManager nodeManager =
- new NodeManagerImpl(nodeId, node.toString(),
+ new NodeManagerImpl(nodeId, node.toString(), httpAddress,
resolve(node.toString()),
capability);
// Inform the scheduler
@@ -187,8 +190,9 @@ ResourceTracker, ResourceContext {
}
}
addForTracking(nodeId);
- LOG.info("NodeManager from node " + node + " registered with capability: " +
- capability.getMemory() + ", assigned nodeId " + nodeId.getId());
+ LOG.info("NodeManager from node " + node + "(web-url: " + httpAddress
+ + ") registered with capability: " + capability.getMemory()
+ + ", assigned nodeId " + nodeId.getId());
RegistrationResponse regResponse = recordFactory.newRecordInstance(RegistrationResponse.class);
regResponse.setNodeId(nodeId);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java Wed Apr 20 05:38:57 2011
@@ -197,9 +197,9 @@ public class Application {
}
synchronized public ResourceRequest getResourceRequest(Priority priority,
- String node) {
+ String nodeAddress) {
Map<String, ResourceRequest> nodeRequests = requests.get(priority);
- return (nodeRequests == null) ? null : nodeRequests.get(node);
+ return (nodeRequests == null) ? null : nodeRequests.get(nodeAddress);
}
synchronized public void completedContainer(Container container) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java Wed Apr 20 05:38:57 2011
@@ -53,7 +53,8 @@ public class NodeManagerImpl implements
private static final Log LOG = LogFactory.getLog(NodeManager.class);
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private final NodeId nodeId;
- private final String hostName;
+ private final String nodeAddress; // The containerManager address
+ private final String httpAddress;
private Resource totalCapability;
private Resource availableResource = recordFactory.newRecordInstance(Resource.class);
private Resource usedResource = recordFactory.newRecordInstance(Resource.class);
@@ -77,7 +78,7 @@ public class NodeManagerImpl implements
/* set of containers that need to be cleaned */
private final Set<Container> containersToClean =
- new TreeSet<Container>(new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.Comparator());
+ new TreeSet<Container>(new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.ContainerComparator());
/* the list of applications that have finished and need to be purged */
@@ -85,11 +86,12 @@ public class NodeManagerImpl implements
private volatile int numContainers;
- public NodeManagerImpl(NodeId nodeId, String hostname,
- Node node, Resource capability) {
+ public NodeManagerImpl(NodeId nodeId, String nodeAddress,
+ String httpAddress, Node node, Resource capability) {
this.nodeId = nodeId;
this.totalCapability = capability;
- this.hostName = hostname;
+ this.nodeAddress = nodeAddress;
+ this.httpAddress = httpAddress;
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource(
availableResource, capability);
this.node = node;
@@ -121,7 +123,7 @@ public class NodeManagerImpl implements
}
LOG.info("addContainers:" +
- " node=" + getHostName() +
+ " node=" + getNodeAddress() +
" #containers=" + containers.size() +
" available=" + getAvailableResource().getMemory() +
" used=" + getUsedResource().getMemory());
@@ -166,7 +168,7 @@ public class NodeManagerImpl implements
if (allocatedContainers.remove(container.getId()) != null) {
activeContainers.put(container.getId(), container);
LOG.info("Activated container " + container.getId() + " on node " +
- getHostName());
+ getNodeAddress());
}
if (container.getState() == ContainerState.COMPLETE) {
@@ -176,7 +178,7 @@ public class NodeManagerImpl implements
}
completedContainers.add(container);
LOG.info("Removed completed container " + container.getId() + " on node " +
- getHostName());
+ getNodeAddress());
}
else if (container.getState() != ContainerState.COMPLETE &&
(!allocatedContainers.containsKey(container.getId())) &&
@@ -198,10 +200,10 @@ public class NodeManagerImpl implements
allocatedContainers.put(container.getId(), container);
LOG.info("Allocated container " + container.getId() +
- " to node " + getHostName());
+ " to node " + getNodeAddress());
LOG.info("Assigned container " + container.getId() +
- " of capacity " + container.getResource() + " on host " + getHostName() +
+ " of capacity " + container.getResource() + " on host " + getNodeAddress() +
", which currently has " + numContainers + " containers, " +
getUsedResource() + " used and " +
getAvailableResource() + " available");
@@ -239,7 +241,7 @@ public class NodeManagerImpl implements
updateResource(container);
LOG.info("Released container " + container.getId() +
- " of capacity " + container.getResource() + " on host " + getHostName() +
+ " of capacity " + container.getResource() + " on host " + getNodeAddress() +
", which currently has " + numContainers + " containers, " +
getUsedResource() + " used and " + getAvailableResource()
+ " available" + ", release resources=" + true);
@@ -252,8 +254,13 @@ public class NodeManagerImpl implements
}
@Override
- public String getHostName() {
- return this.hostName;
+ public String getNodeAddress() {
+ return this.nodeAddress;
+ }
+
+ @Override
+ public String getHttpAddress() {
+ return this.httpAddress;
}
@Override
@@ -283,7 +290,8 @@ public class NodeManagerImpl implements
public synchronized void addAvailableResource(Resource resource) {
if (resource == null) {
- LOG.error("Invalid resource addition of null resource for " + this.hostName);
+ LOG.error("Invalid resource addition of null resource for "
+ + this.nodeAddress);
return;
}
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource(
@@ -294,7 +302,8 @@ public class NodeManagerImpl implements
public synchronized void deductAvailableResource(Resource resource) {
if (resource == null) {
- LOG.error("Invalid deduction of null resource for "+ this.hostName);
+ LOG.error("Invalid deduction of null resource for "
+ + this.nodeAddress);
}
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
availableResource, resource);
@@ -333,7 +342,7 @@ public class NodeManagerImpl implements
@Override
public String toString() {
- return "host: " + getHostName() + " #containers=" + getNumContainers() +
+ return "host: " + getNodeAddress() + " #containers=" + getNumContainers() +
" available=" + getAvailableResource().getMemory() +
" used=" + getUsedResource().getMemory();
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Wed Apr 20 05:38:57 2011
@@ -93,7 +93,7 @@ implements ResourceScheduler, CapacitySc
private Map<ApplicationId, Application> applications =
new TreeMap<ApplicationId, Application>(
- new org.apache.hadoop.yarn.server.resourcemanager.resource.ApplicationID.Comparator());
+ new org.apache.hadoop.yarn.util.BuilderUtils.ApplicationIdComparator());
private boolean initialized = false;
@@ -343,7 +343,7 @@ implements ResourceScheduler, CapacitySc
// Completed containers
processCompletedContainers(nodeResponse.getCompletedContainers());
- NodeManager nm = nodes.get(node.getHostName());
+ NodeManager nm = nodes.get(node.getNodeAddress());
// Assign new containers
root.assignContainers(clusterResource, nm);
@@ -428,17 +428,17 @@ implements ResourceScheduler, CapacitySc
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
clusterResource, nodeInfo.getTotalCapability());
//TODO inform the applications that the containers are completed/failed
- nodes.remove(nodeInfo.getHostName());
+ nodes.remove(nodeInfo.getNodeAddress());
}
public synchronized boolean isTracked(NodeInfo nodeInfo) {
- NodeManager node = nodes.get(nodeInfo.getHostName());
+ NodeManager node = nodes.get(nodeInfo.getNodeAddress());
return (node == null? false: true);
}
@Override
public synchronized void addNode(NodeManager nodeManager) {
- nodes.put(nodeManager.getHostName(), nodeManager);
+ nodes.put(nodeManager.getNodeAddress(), nodeManager);
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource(
clusterResource, nodeManager.getTotalCapability());
}
@@ -453,8 +453,8 @@ implements ResourceScheduler, CapacitySc
public synchronized NodeResponse nodeUpdateInternal(NodeInfo nodeInfo,
Map<String,List<Container>> containers) {
- NodeManager node = nodes.get(nodeInfo.getHostName());
- LOG.debug("nodeUpdate: node=" + nodeInfo.getHostName() +
+ NodeManager node = nodes.get(nodeInfo.getNodeAddress());
+ LOG.debug("nodeUpdate: node=" + nodeInfo.getNodeAddress() +
" available=" + nodeInfo.getAvailableResource().getMemory());
return node.statusUpdate(containers);
@@ -462,14 +462,14 @@ implements ResourceScheduler, CapacitySc
public synchronized void addAllocatedContainers(NodeInfo nodeInfo,
ApplicationId applicationId, List<Container> containers) {
- NodeManager node = nodes.get(nodeInfo.getHostName());
+ NodeManager node = nodes.get(nodeInfo.getNodeAddress());
node.allocateContainer(applicationId, containers);
}
public synchronized void finishedApplication(ApplicationId applicationId,
List<NodeInfo> nodesToNotify) {
for (NodeInfo node: nodesToNotify) {
- NodeManager nodeManager = nodes.get(node.getHostName());
+ NodeManager nodeManager = nodes.get(node.getNodeAddress());
nodeManager.notifyFinishedApplication(applicationId);
}
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Apr 20 05:38:57 2011
@@ -409,7 +409,7 @@ public class LeafQueue implements Queue
assignContainers(Resource clusterResource, NodeManager node) {
LOG.info("DEBUG --- assignContainers:" +
- " node=" + node.getHostName() +
+ " node=" + node.getNodeAddress() +
" #applications=" + applications.size());
// Try to assign containers to applications in fifo order
@@ -586,7 +586,7 @@ public class LeafQueue implements Queue
Resource assignNodeLocalContainers(Resource clusterResource, NodeManager node,
Application application, Priority priority) {
ResourceRequest request =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeAddress());
if (request != null) {
if (canAssign(application, priority, node, NodeType.DATA_LOCAL)) {
return assignContainer(clusterResource, node, application, priority, request,
@@ -652,7 +652,7 @@ public class LeafQueue implements Queue
if (type == NodeType.DATA_LOCAL) {
ResourceRequest nodeLocalRequest =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeAddress());
if (nodeLocalRequest != null) {
return nodeLocalRequest.getNumContainers() > 0;
}
@@ -665,7 +665,7 @@ public class LeafQueue implements Queue
Application application,
Priority priority, ResourceRequest request, NodeType type) {
LOG.info("DEBUG --- assignContainers:" +
- " node=" + node.getHostName() +
+ " node=" + node.getNodeAddress() +
" application=" + application.getApplicationId().getId() +
" priority=" + priority.getPriority() +
" request=" + request + " type=" + type);
@@ -684,9 +684,9 @@ public class LeafQueue implements Queue
new ArrayList<Container>();
Container container =
org.apache.hadoop.yarn.server.resourcemanager.resource.Container
- .create(application.getApplicationId(),
+ .create(recordFactory, application.getApplicationId(),
application.getNewContainerId(),
- node.getHostName(), capability);
+ node.getNodeAddress(), capability);
// If security is enabled, send the container-tokens too.
if (UserGroupInformation.isSecurityEnabled()) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Wed Apr 20 05:38:57 2011
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent;
@@ -58,6 +59,9 @@ import org.apache.hadoop.yarn.server.sec
public class FifoScheduler implements ResourceScheduler {
private static final Log LOG = LogFactory.getLog(FifoScheduler.class);
+
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
Configuration conf;
private ContainerTokenSecretManager containerTokenSecretManager;
@@ -73,7 +77,7 @@ public class FifoScheduler implements Re
Map<ApplicationId, Application> applications =
new TreeMap<ApplicationId, Application>(
- new org.apache.hadoop.yarn.server.resourcemanager.resource.ApplicationID.Comparator());
+ new org.apache.hadoop.yarn.util.BuilderUtils.ApplicationIdComparator());
private final Queue DEFAULT_QUEUE = new Queue() {
QueueInfo queueInfo;
@@ -209,7 +213,7 @@ public class FifoScheduler implements Re
*/
private synchronized void assignContainers(NodeInfo node) {
LOG.debug("assignContainers:" +
- " node=" + node.getHostName() +
+ " node=" + node.getNodeAddress() +
" #applications=" + applications.size());
// Try to assign containers to applications in fifo order
@@ -266,7 +270,7 @@ public class FifoScheduler implements Re
if (type == NodeType.DATA_LOCAL) {
ResourceRequest nodeLocalRequest =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeAddress());
if (nodeLocalRequest != null) {
maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
}
@@ -293,7 +297,7 @@ public class FifoScheduler implements Re
LOG.debug("assignContainersOnNode:" +
- " node=" + node.getHostName() +
+ " node=" + node.getNodeAddress() +
" application=" + application.getApplicationId().getId() +
" priority=" + priority.getPriority() +
" #assigned=" +
@@ -307,7 +311,7 @@ public class FifoScheduler implements Re
Application application, Priority priority) {
int assignedContainers = 0;
ResourceRequest request =
- application.getResourceRequest(priority, node.getHostName());
+ application.getResourceRequest(priority, node.getNodeAddress());
if (request != null) {
int assignableContainers =
Math.min(
@@ -356,7 +360,7 @@ public class FifoScheduler implements Re
Priority priority, int assignableContainers,
ResourceRequest request, NodeType type) {
LOG.debug("assignContainers:" +
- " node=" + node.getHostName() +
+ " node=" + node.getNodeAddress() +
" application=" + application.getApplicationId().getId() +
" priority=" + priority.getPriority() +
" assignableContainers=" + assignableContainers +
@@ -379,9 +383,9 @@ public class FifoScheduler implements Re
for (int i=0; i < assignedContainers; ++i) {
Container container =
org.apache.hadoop.yarn.server.resourcemanager.resource.Container
- .create(application.getApplicationId(),
+ .create(recordFactory, application.getApplicationId(),
application.getNewContainerId(),
- node.getHostName(), capability);
+ node.getNodeAddress(), capability);
// If security is enabled, send the container-tokens too.
if (UserGroupInformation.isSecurityEnabled()) {
ContainerToken containerToken = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ContainerToken.class);
@@ -473,7 +477,7 @@ public class FifoScheduler implements Re
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
clusterResource, nodeInfo.getTotalCapability());
//TODO inform the the applications that the containers are completed/failed
- nodes.remove(nodeInfo.getHostName());
+ nodes.remove(nodeInfo.getNodeAddress());
}
@@ -496,13 +500,13 @@ public class FifoScheduler implements Re
}
public synchronized boolean isTracked(NodeInfo nodeInfo) {
- NodeManager node = nodes.get(nodeInfo.getHostName());
+ NodeManager node = nodes.get(nodeInfo.getNodeAddress());
return (node == null? false: true);
}
@Override
public synchronized void addNode(NodeManager nodeManager) {
- nodes.put(nodeManager.getHostName(), nodeManager);
+ nodes.put(nodeManager.getNodeAddress(), nodeManager);
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.addResource(
clusterResource, nodeManager.getTotalCapability());
}
@@ -517,8 +521,8 @@ public class FifoScheduler implements Re
private synchronized NodeResponse nodeUpdateInternal(NodeInfo nodeInfo,
Map<String,List<Container>> containers) {
- NodeManager node = nodes.get(nodeInfo.getHostName());
- LOG.debug("nodeUpdate: node=" + nodeInfo.getHostName() +
+ NodeManager node = nodes.get(nodeInfo.getNodeAddress());
+ LOG.debug("nodeUpdate: node=" + nodeInfo.getNodeAddress() +
" available=" + nodeInfo.getAvailableResource().getMemory());
return node.statusUpdate(containers);
@@ -526,14 +530,14 @@ public class FifoScheduler implements Re
public synchronized void addAllocatedContainers(NodeInfo nodeInfo,
ApplicationId applicationId, List<Container> containers) {
- NodeManager node = nodes.get(nodeInfo.getHostName());
+ NodeManager node = nodes.get(nodeInfo.getNodeAddress());
node.allocateContainer(applicationId, containers);
}
public synchronized void finishedApplication(ApplicationId applicationId,
List<NodeInfo> nodesToNotify) {
for (NodeInfo node: nodesToNotify) {
- NodeManager nodeManager = nodes.get(node.getHostName());
+ NodeManager nodeManager = nodes.get(node.getNodeAddress());
nodeManager.notifyFinishedApplication(applicationId);
}
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Wed Apr 20 05:38:57 2011
@@ -59,7 +59,7 @@ class NodesPage extends RmView {
tbody.tr().
td(ni.getRackName()).
td(String.valueOf(ni.getNodeID().getId())).
- td(ni.getHostName()).
+ td().a("http://" + ni.getHttpAddress(), ni.getHttpAddress())._().
td(ni.getNodeHealthStatus().getIsNodeHealthy() ? "Healthy"
: "Unhealthy").
td(String.valueOf(ni.getNodeHealthStatus()
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Wed Apr 20 05:38:57 2011
@@ -80,7 +80,7 @@ public class Application {
new org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceRequest.Comparator());
final private Set<Container> release =
new TreeSet<Container>(
- new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.Comparator());
+ new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.ContainerComparator());
final private Map<String, NodeManager> nodes =
new HashMap<String, NodeManager>();
@@ -142,8 +142,9 @@ public class Application {
}
}
- public synchronized void addNodeManager(String host, NodeManager nodeManager) {
- nodes.put(host, nodeManager);
+ public synchronized void addNodeManager(String host,
+ int containerManagerPort, NodeManager nodeManager) {
+ nodes.put(host + ":" + containerManagerPort, nodeManager);
}
private synchronized NodeManager getNodeManager(String host) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Wed Apr 20 05:38:57 2011
@@ -77,6 +77,7 @@ public class MockNodes {
final int nid = NODE_ID++;
final NodeId nodeID = newNodeID(nid);
final String hostName = "host"+ nid;
+ final String httpAddress = "localhost:0";
final NodeHealthStatus nodeHealthStatus =
recordFactory.newRecordInstance(NodeHealthStatus.class);
final Resource used = newUsedResource(perNode);
@@ -89,11 +90,16 @@ public class MockNodes {
}
@Override
- public String getHostName() {
+ public String getNodeAddress() {
return hostName;
}
@Override
+ public String getHttpAddress() {
+ return httpAddress;
+ }
+
+ @Override
public Resource getTotalCapability() {
return perNode;
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Wed Apr 20 05:38:57 2011
@@ -73,8 +73,9 @@ public class NodeManager implements Cont
final Map<String, List<Container>> containers =
new HashMap<String, List<Container>>();
- public NodeManager(String hostName, String rackName, int memory,
- RMResourceTrackerImpl resourceTracker) throws IOException {
+ public NodeManager(String hostName, int containerManagerPort, int httpPort,
+ String rackName, int memory, RMResourceTrackerImpl resourceTracker)
+ throws IOException {
this.hostName = hostName;
this.rackName = rackName;
this.resourceTracker = resourceTracker;
@@ -85,7 +86,9 @@ public class NodeManager implements Cont
available, capability);
RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
- request.setNode(hostName);
+ request.setHost(hostName);
+ request.setContainerManagerPort(containerManagerPort);
+ request.setHttpPort(httpPort);
request.setResource(capability);
RegistrationResponse response =
resourceTracker.registerNodeManager(request).getRegistrationResponse();
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java Wed Apr 20 05:38:57 2011
@@ -52,11 +52,12 @@ public class TestResourceManager extends
public void tearDown() throws Exception {
}
- private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
- registerNode(String hostName, String rackName, int memory)
- throws IOException {
+ private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
+ registerNode(String hostName, int containerManagerPort, int httpPort,
+ String rackName, int memory) throws IOException {
return new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
- hostName, rackName, memory, resourceManager.getResourceTracker());
+ hostName, containerManagerPort, httpPort, rackName, memory,
+ resourceManager.getResourceTracker());
}
@Test
@@ -68,13 +69,13 @@ public class TestResourceManager extends
// Register node1
String host1 = "host1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1 =
- registerNode(host1, NetworkTopology.DEFAULT_RACK, memory);
+ registerNode(host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory);
nm1.heartbeat();
// Register node2
String host2 = "host2";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm2 =
- registerNode(host2, NetworkTopology.DEFAULT_RACK, memory);
+ registerNode(host2, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory);
nm2.heartbeat();
LOG.info("--- END: testResourceManagerInitialization ---");
@@ -89,13 +90,13 @@ public class TestResourceManager extends
// Register node1
String host1 = "host1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1 =
- registerNode(host1, NetworkTopology.DEFAULT_RACK, memory);
+ registerNode(host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory);
nm1.heartbeat();
// Register node 2
String host2 = "host1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm2 =
- registerNode(host2, NetworkTopology.DEFAULT_RACK, memory);
+ registerNode(host2, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory);
nm2.heartbeat();
// Submit an application
@@ -114,21 +115,21 @@ public class TestResourceManager extends
// Register node1
String host1 = "host1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1 =
- registerNode(host1, NetworkTopology.DEFAULT_RACK, memory);
+ registerNode(host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory);
nm1.heartbeat();
// Register node2
String host2 = "host2";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm2 =
- registerNode(host2, NetworkTopology.DEFAULT_RACK, memory/2);
+ registerNode(host2, 1234, 2345, NetworkTopology.DEFAULT_RACK, memory/2);
nm2.heartbeat();
// Submit an application
Application application = new Application("user1", resourceManager);
application.submit();
- application.addNodeManager(host1, nm1);
- application.addNodeManager(host2, nm2);
+ application.addNodeManager(host1, 1234, nm1);
+ application.addNodeManager(host2, 1234, nm2);
// Application resource requirements
final int memory1 = 1024;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java Wed Apr 20 05:38:57 2011
@@ -262,7 +262,7 @@ public class TestApplicationCleanup exte
Resource capability = recordFactory.newRecordInstance(Resource.class);
capability.setMemory(1024);
NodeManager nodeManager =
- new NodeManagerImpl(nodeId, hostName, node, capability);
+ new NodeManagerImpl(nodeId, hostName, "localhost:0", node, capability);
scheduler.addNode(nodeManager);
return nodeManager;
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java Wed Apr 20 05:38:57 2011
@@ -137,18 +137,24 @@ public class TestNMExpiry extends TestCa
@Test
public void testNMExpiry() throws Exception {
- String hostname1 = "localhost:1";
- String hostname2 = "localhost:2";
- String hostname3 = "localhost:3";
+ String hostname1 = "localhost1";
+ String hostname2 = "localhost2";
+ String hostname3 = "localhost3";
Resource capability = recordFactory.newRecordInstance(Resource.class);
RegisterNodeManagerRequest request1 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
- request1.setNode(hostname1);
+ request1.setHost(hostname1);
+ request1.setContainerManagerPort(0);
+ request1.setHttpPort(0);
request1.setResource(capability);
RegisterNodeManagerRequest request2 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
- request2.setNode(hostname2);
+ request2.setHost(hostname2);
+ request2.setContainerManagerPort(0);
+ request2.setHttpPort(0);
request2.setResource(capability);
RegisterNodeManagerRequest request3 = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
- request3.setNode(hostname3);
+ request3.setHost(hostname3);
+ request3.setContainerManagerPort(0);
+ request3.setHttpPort(0);
request3.setResource(capability);
resourceTracker.registerNodeManager(request1);
resourceTracker.registerNodeManager(request2);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java Wed Apr 20 05:38:57 2011
@@ -87,7 +87,9 @@ public class TestRMNMRPCResponseId exten
String node = "localhost";
Resource capability = recordFactory.newRecordInstance(Resource.class);
RegisterNodeManagerRequest request = recordFactory.newRecordInstance(RegisterNodeManagerRequest.class);
- request.setNode(node);
+ request.setHost(node);
+ request.setContainerManagerPort(0);
+ request.setHttpPort(0);
request.setResource(capability);
rmResourceTrackerImpl.registerNodeManager(request);
org.apache.hadoop.yarn.server.api.records.NodeStatus nodeStatus = recordFactory.newRecordInstance(org.apache.hadoop.yarn.server.api.records.NodeStatus.class);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java Wed Apr 20 05:38:57 2011
@@ -61,14 +61,14 @@ public class TestCapacityScheduler exten
public void tearDown() throws Exception {
}
- private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
- registerNode(String hostName, String rackName, int memory)
- throws IOException {
+ private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
+ registerNode(String hostName, int containerManagerPort, int httpPort,
+ String rackName, int memory)
+ throws IOException {
return new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
- hostName, rackName, memory, resourceManager.getResourceTracker());
- }
-
-
+ hostName, containerManagerPort, httpPort, rackName, memory,
+ resourceManager.getResourceTracker());
+ }
@Test
public void testCapacityScheduler() throws Exception {
@@ -80,13 +80,13 @@ public class TestCapacityScheduler exten
// Register node1
String host_0 = "host_0";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 =
- registerNode(host_0, NetworkTopology.DEFAULT_RACK, 4 * GB);
+ registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, 4 * GB);
nm_0.heartbeat();
// Register node2
String host_1 = "host_1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 =
- registerNode(host_1, NetworkTopology.DEFAULT_RACK, 2 * GB);
+ registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, 2 * GB);
nm_1.heartbeat();
// ResourceRequest priorities
@@ -99,8 +99,8 @@ public class TestCapacityScheduler exten
Application application_0 = new Application("user_0", "a1", resourceManager);
application_0.submit();
- application_0.addNodeManager(host_0, nm_0);
- application_0.addNodeManager(host_1, nm_1);
+ application_0.addNodeManager(host_0, 1234, nm_0);
+ application_0.addNodeManager(host_1, 1234, nm_1);
Resource capability_0_0 =
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.createResource(
@@ -120,8 +120,8 @@ public class TestCapacityScheduler exten
Application application_1 = new Application("user_1", "b2", resourceManager);
application_1.submit();
- application_1.addNodeManager(host_0, nm_0);
- application_1.addNodeManager(host_1, nm_1);
+ application_1.addNodeManager(host_0, 1234, nm_0);
+ application_1.addNodeManager(host_1, 1234, nm_1);
Resource capability_1_0 =
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.createResource(
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java?rev=1095271&r1=1095270&r2=1095271&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java Wed Apr 20 05:38:57 2011
@@ -52,11 +52,12 @@ public class TestFifoScheduler extends T
public void tearDown() throws Exception {
}
- private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
- registerNode(String hostName, String rackName, int memory)
- throws IOException {
+ private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
+ registerNode(String hostName, int containerManagerPort, int nmHttpPort,
+ String rackName, int memory) throws IOException {
return new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
- hostName, rackName, memory, resourceManager.getResourceTracker());
+ hostName, containerManagerPort, nmHttpPort, rackName, memory,
+ resourceManager.getResourceTracker());
}
@@ -71,13 +72,13 @@ public class TestFifoScheduler extends T
// Register node1
String host_0 = "host_0";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_0 =
- registerNode(host_0, NetworkTopology.DEFAULT_RACK, 4 * GB);
+ registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK, 4 * GB);
nm_0.heartbeat();
// Register node2
String host_1 = "host_1";
org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm_1 =
- registerNode(host_1, NetworkTopology.DEFAULT_RACK, 2 * GB);
+ registerNode(host_1, 1234, 2345, NetworkTopology.DEFAULT_RACK, 2 * GB);
nm_1.heartbeat();
// ResourceRequest priorities
@@ -90,8 +91,8 @@ public class TestFifoScheduler extends T
Application application_0 = new Application("user_0", resourceManager);
application_0.submit();
- application_0.addNodeManager(host_0, nm_0);
- application_0.addNodeManager(host_1, nm_1);
+ application_0.addNodeManager(host_0, 1234, nm_0);
+ application_0.addNodeManager(host_1, 1234, nm_1);
Resource capability_0_0 =
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.createResource(
@@ -111,8 +112,8 @@ public class TestFifoScheduler extends T
Application application_1 = new Application("user_1", resourceManager);
application_1.submit();
- application_1.addNodeManager(host_0, nm_0);
- application_1.addNodeManager(host_1, nm_1);
+ application_1.addNodeManager(host_0, 1234, nm_0);
+ application_1.addNodeManager(host_1, 1234, nm_1);
Resource capability_1_0 =
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.createResource(