You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by dd...@apache.org on 2011/06/15 01:22:18 UTC
svn commit: r1135847 - in /hadoop/common/branches/MR-279/mapreduce: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/se...
Author: ddas
Date: Tue Jun 14 23:22:18 2011
New Revision: 1135847
URL: http://svn.apache.org/viewvc?rev=1135847&view=rev
Log:
Changes to do rack resolutions in the RM and in the AM. Contributed by Devaraj Das.
Modified:
hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
Modified: hadoop/common/branches/MR-279/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/CHANGES.txt?rev=1135847&r1=1135846&r2=1135847&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/MR-279/mapreduce/CHANGES.txt Tue Jun 14 23:22:18 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
MAPREDUCE-279
+
+ Changes to do rack resolutions in the RM and in the AM (ddas)
Fix NPE in history event handling (siddharth seth via mahadev)
Modified: hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1135847&r1=1135846&r2=1135847&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Jun 14 23:22:18 2011
@@ -114,6 +114,7 @@ import org.apache.hadoop.yarn.state.Stat
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.RackResolver;
/**
* Implementation of TaskAttempt interface.
@@ -417,6 +418,7 @@ public abstract class TaskAttemptImpl im
this.resourceCapability = recordFactory.newRecordInstance(Resource.class);
this.resourceCapability.setMemory(getMemoryRequired(conf, taskId.getTaskType()));
this.dataLocalHosts = dataLocalHosts;
+ RackResolver.init(conf);
// This "this leak" is okay because the retained pointer is in an
// instance variable.
@@ -872,7 +874,6 @@ public abstract class TaskAttemptImpl im
return tauce;
}
- private static String[] racks = new String[] {NetworkTopology.DEFAULT_RACK};
private static class RequestContainerTransition implements
SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
boolean rescheduled = false;
@@ -892,6 +893,11 @@ public abstract class TaskAttemptImpl im
taskAttempt.attemptId,
taskAttempt.resourceCapability));
} else {
+ int i = 0;
+ String[] racks = new String[taskAttempt.dataLocalHosts.length];
+ for (String host : taskAttempt.dataLocalHosts) {
+ racks[i++] = RackResolver.resolve(host).getNetworkLocation();
+ }
taskAttempt.eventHandler.handle(
new ContainerRequestEvent(taskAttempt.attemptId,
taskAttempt.resourceCapability,
Modified: hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java?rev=1135847&r1=1135846&r2=1135847&view=diff
==============================================================================
--- hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java (original)
+++ hadoop/common/branches/MR-279/mapreduce/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/RMResourceTrackerImpl.java Tue Jun 14 23:22:18 2011
@@ -36,9 +36,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.net.Node;
-import org.apache.hadoop.net.NodeBase;
import org.apache.hadoop.util.HostsFileReader;
import org.apache.hadoop.yarn.Lock;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -64,6 +62,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.hadoop.yarn.util.RackResolver;
/**
* This class is responsible for the interaction with the NodeManagers.
@@ -153,6 +152,7 @@ NodeTracker, ClusterTracker {
this.hostsReader = null;
}
}
+ RackResolver.init(conf);
}
private void printConfiguredHosts() {
@@ -194,7 +194,7 @@ NodeTracker, ClusterTracker {
*/
@Lock(Lock.NoLock.class)
public static Node resolve(String hostName) {
- return new NodeBase(hostName, NetworkTopology.DEFAULT_RACK);
+ return RackResolver.resolve(hostName);
}
@Lock(Lock.NoLock.class)