You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2009/10/19 13:04:53 UTC
svn commit: r826635 - in /hadoop/mapreduce/trunk: ./
src/examples/org/apache/hadoop/examples/pi/
src/java/org/apache/hadoop/mapred/ src/java/org/apache/hadoop/mapreduce/
src/java/org/apache/hadoop/mapreduce/protocol/
src/java/org/apache/hadoop/mapreduc...
Author: sharad
Date: Mon Oct 19 11:04:53 2009
New Revision: 826635
URL: http://svn.apache.org/viewvc?rev=826635&view=rev
Log:
MAPREDUCE-1048. (Revert) Add occupied/reserved slot usage summary on jobtracker UI.
Removed:
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/TestClusterStatus.java
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java
hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java
hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java
hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Mon Oct 19 11:04:53 2009
@@ -10,9 +10,6 @@
MAPREDUCE-999. Improve Sqoop test speed and refactor tests.
(Aaron Kimball via tomwhite)
-
- MAPREDUCE-1048. Add occupied/reserved slot usage summary on jobtracker UI.
- (Amareshwari Sriramadasu via sharad)
MAPREDUCE-906. Update Sqoop documentation. (Aaron Kimball via cdouglas)
Modified: hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java (original)
+++ hadoop/mapreduce/trunk/src/examples/org/apache/hadoop/examples/pi/DistSum.java Mon Oct 19 11:04:53 2009
@@ -38,9 +38,9 @@
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobTracker;
-import org.apache.hadoop.mapreduce.Cluster;
-import org.apache.hadoop.mapreduce.ClusterMetrics;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -379,14 +379,14 @@
public static class MixMachine extends Machine {
private static final MixMachine INSTANCE = new MixMachine();
- private Cluster cluster;
+ private JobClient jobclient;
/** {@inheritDoc} */
@Override
public synchronized void init(Job job) throws IOException {
final Configuration conf = job.getConfiguration();
- if (cluster == null)
- cluster = new Cluster(JobTracker.getAddress(conf), conf);
+ if (jobclient == null)
+ jobclient = new JobClient(JobTracker.getAddress(conf), conf);
chooseMachine(conf).init(job);
}
@@ -398,11 +398,9 @@
try {
for(;; Thread.sleep(2000)) {
//get cluster status
- final ClusterMetrics status = cluster.getClusterStatus();
- final int m =
- status.getMapSlotCapacity() - status.getOccupiedMapSlots();
- final int r =
- status.getReduceSlotCapacity() - status.getOccupiedReduceSlots();
+ final ClusterStatus status = jobclient.getClusterStatus();
+ final int m = status.getMaxMapTasks() - status.getMapTasks();
+ final int r = status.getMaxReduceTasks() - status.getReduceTasks();
if (m >= parts || r >= parts) {
//favor ReduceSide machine
final Machine value = r >= parts?
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobInProgress.java Mon Oct 19 11:04:53 2009
@@ -1703,14 +1703,6 @@
return trackersReservedForReduces.size();
}
- public int getReservedMapSlots(TaskTracker taskTracker) {
- return trackersReservedForMaps.get(taskTracker).getNumSlots();
- }
-
- public int getReservedReduceSlots(TaskTracker taskTracker) {
- return trackersReservedForReduces.get(taskTracker).getNumSlots();
- }
-
private int getTrackerTaskFailures(String trackerName) {
String trackerHostName = convertTrackerNameToHostName(trackerName);
Integer failedTasks = trackerToFailuresMap.get(trackerHostName);
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/JobTracker.java Mon Oct 19 11:04:53 2009
@@ -20,6 +20,7 @@
import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
@@ -1214,10 +1215,6 @@
//
int totalMaps = 0;
int totalReduces = 0;
- private int occupiedMapSlots = 0;
- private int occupiedReduceSlots = 0;
- private int reservedMapSlots = 0;
- private int reservedReduceSlots = 0;
private HashMap<String, TaskTracker> taskTrackers =
new HashMap<String, TaskTracker>();
Map<String,Integer>uniqueHostsMap = new ConcurrentHashMap<String, Integer>();
@@ -2317,7 +2314,6 @@
HeartbeatResponse response = new HeartbeatResponse(newResponseId, null);
List<TaskTrackerAction> actions = new ArrayList<TaskTrackerAction>();
isBlacklisted = faultyTrackers.isBlacklisted(status.getHost());
- removeTrackerReservations(getTaskTracker(trackerName));
// Check for new tasks to be executed on the tasktracker
if (acceptNewTasks && !isBlacklisted) {
TaskTrackerStatus taskTrackerStatus = getTaskTrackerStatus(trackerName) ;
@@ -2337,7 +2333,6 @@
}
}
}
- addTrackerReservations(getTaskTracker(trackerName));
// Check for tasks to be killed
List<TaskTrackerAction> killTasksList = getTasksToKill(trackerName);
@@ -2428,8 +2423,6 @@
if (oldStatus != null) {
totalMaps -= oldStatus.countMapTasks();
totalReduces -= oldStatus.countReduceTasks();
- occupiedMapSlots -= oldStatus.countOccupiedMapSlots();
- occupiedReduceSlots -= oldStatus.countOccupiedReduceSlots();
if (!faultyTrackers.isBlacklisted(oldStatus.getHost())) {
int mapSlots = oldStatus.getMaxMapSlots();
totalMapTaskCapacity -= mapSlots;
@@ -2452,8 +2445,6 @@
if (status != null) {
totalMaps += status.countMapTasks();
totalReduces += status.countReduceTasks();
- occupiedMapSlots += status.countOccupiedMapSlots();
- occupiedReduceSlots += status.countOccupiedReduceSlots();
if (!faultyTrackers.isBlacklisted(status.getHost())) {
int mapSlots = status.getMaxMapSlots();
totalMapTaskCapacity += mapSlots;
@@ -2522,18 +2513,6 @@
}
- // remove the tracker reservations from statistics
- private void removeTrackerReservations(TaskTracker tt) {
- reservedMapSlots -= tt.getReservedMapSlots();
- reservedReduceSlots -= tt.getReservedReduceSlots();
- }
-
- // add the tracker reservations to statistics
- private void addTrackerReservations(TaskTracker tt) {
- reservedMapSlots += tt.getReservedMapSlots();
- reservedReduceSlots += tt.getReservedReduceSlots();
- }
-
private void updateNodeHealthStatus(TaskTrackerStatus trackerStatus) {
TaskTrackerHealthStatus status = trackerStatus.getHealthStatus();
synchronized (faultyTrackers) {
@@ -2580,10 +2559,9 @@
}
}
}
- removeTrackerReservations(getTaskTracker(trackerName));
+
updateTaskStatuses(trackerStatus);
updateNodeHealthStatus(trackerStatus);
- addTrackerReservations(getTaskTracker(trackerName));
return true;
}
@@ -2972,9 +2950,8 @@
}
public synchronized ClusterMetrics getClusterMetrics() {
- return new ClusterMetrics(occupiedMapSlots, occupiedReduceSlots,
- reservedMapSlots, reservedReduceSlots,
- totalMapTaskCapacity, totalReduceTaskCapacity, taskTrackers.size() -
+ return new ClusterMetrics(totalMaps, totalReduces, totalMapTaskCapacity,
+ totalReduceTaskCapacity, taskTrackers.size() -
getBlacklistedTrackerCount(),
getBlacklistedTrackerCount(), getExcludedNodes().size()) ;
}
@@ -3747,7 +3724,6 @@
// Cleanup
taskTracker.cancelAllReservations();
- removeTrackerReservations(taskTracker);
// Purge 'marked' tasks, needs to be done
// here to prevent hanging references!
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapred/LocalJobRunner.java Mon Oct 19 11:04:53 2009
@@ -506,7 +506,7 @@
}
public ClusterMetrics getClusterMetrics() {
- return new ClusterMetrics(map_tasks, reduce_tasks, 0, 0, 1, 1, 1, 0, 0);
+ return new ClusterMetrics(map_tasks, reduce_tasks, 1, 1, 1, 0, 0);
}
public State getJobTrackerState() throws IOException, InterruptedException {
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/ClusterMetrics.java Mon Oct 19 11:04:53 2009
@@ -35,10 +35,10 @@
* Number of blacklisted and decommissioned trackers.
* </li>
* <li>
- * Slot capacity of the cluster.
+ * Task capacity of the cluster.
* </li>
* <li>
- * The number of currently occupied/reserved map & reduce slots.
+ * The number of currently running map & reduce tasks.
* </li>
* </ol></p>
*
@@ -48,30 +48,24 @@
* @see Cluster
*/
public class ClusterMetrics implements Writable {
- private int occupiedMapSlots;
- private int occupiedReduceSlots;
- private int reservedMapSlots;
- private int reservedReduceSlots;
- private int totalMapSlots;
- private int totalReduceSlots;
- private int numTrackers;
- private int numBlacklistedTrackers;
- private int numDecommissionedTrackers;
+ int runningMaps;
+ int runningReduces;
+ int mapSlots;
+ int reduceSlots;
+ int numTrackers;
+ int numBlacklistedTrackers;
+ int numDecommissionedTrackers;
public ClusterMetrics() {
}
- public ClusterMetrics(int occupiedMapSlots, int occupiedReduceSlots,
- int reservedMapSlots, int reservedReduceSlots,
- int mapSlots, int reduceSlots,
- int numTrackers, int numBlacklistedTrackers,
- int numDecommisionedNodes) {
- this.occupiedMapSlots = occupiedMapSlots;
- this.occupiedReduceSlots = occupiedReduceSlots;
- this.reservedMapSlots = reservedMapSlots;
- this.reservedReduceSlots = reservedReduceSlots;
- this.totalMapSlots = mapSlots;
- this.totalReduceSlots = reduceSlots;
+ public ClusterMetrics(int runningMaps, int runningReduces, int mapSlots,
+ int reduceSlots, int numTrackers, int numBlacklistedTrackers,
+ int numDecommisionedNodes) {
+ this.runningMaps = runningMaps;
+ this.runningReduces = runningReduces;
+ this.mapSlots = mapSlots;
+ this.reduceSlots = reduceSlots;
this.numTrackers = numTrackers;
this.numBlacklistedTrackers = numBlacklistedTrackers;
this.numDecommissionedTrackers = numDecommisionedNodes;
@@ -83,7 +77,7 @@
* @return occupied map slot count
*/
public int getOccupiedMapSlots() {
- return occupiedMapSlots;
+ return runningMaps;
}
/**
@@ -92,34 +86,16 @@
* @return occupied reduce slot count
*/
public int getOccupiedReduceSlots() {
- return occupiedReduceSlots;
- }
-
- /**
- * Get number of reserved map slots in the cluster.
- *
- * @return reserved map slot count
- */
- public int getReservedMapSlots() {
- return reservedMapSlots;
+ return runningReduces;
}
/**
- * Get the number of reserved reduce slots in the cluster.
- *
- * @return reserved reduce slot count
- */
- public int getReservedReduceSlots() {
- return reservedReduceSlots;
- }
-
- /**
* Get the total number of map slots in the cluster.
*
* @return map slot capacity
*/
public int getMapSlotCapacity() {
- return totalMapSlots;
+ return mapSlots;
}
/**
@@ -128,7 +104,7 @@
* @return reduce slot capacity
*/
public int getReduceSlotCapacity() {
- return totalReduceSlots;
+ return reduceSlots;
}
/**
@@ -160,12 +136,10 @@
@Override
public void readFields(DataInput in) throws IOException {
- occupiedMapSlots = in.readInt();
- occupiedReduceSlots = in.readInt();
- reservedMapSlots = in.readInt();
- reservedReduceSlots = in.readInt();
- totalMapSlots = in.readInt();
- totalReduceSlots = in.readInt();
+ runningMaps = in.readInt();
+ runningReduces = in.readInt();
+ mapSlots = in.readInt();
+ reduceSlots = in.readInt();
numTrackers = in.readInt();
numBlacklistedTrackers = in.readInt();
numDecommissionedTrackers = in.readInt();
@@ -173,12 +147,10 @@
@Override
public void write(DataOutput out) throws IOException {
- out.writeInt(occupiedMapSlots);
- out.writeInt(occupiedReduceSlots);
- out.writeInt(reservedMapSlots);
- out.writeInt(reservedReduceSlots);
- out.writeInt(totalMapSlots);
- out.writeInt(totalReduceSlots);
+ out.writeInt(runningMaps);
+ out.writeInt(runningReduces);
+ out.writeInt(mapSlots);
+ out.writeInt(reduceSlots);
out.writeInt(numTrackers);
out.writeInt(numBlacklistedTrackers);
out.writeInt(numDecommissionedTrackers);
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/protocol/ClientProtocol.java Mon Oct 19 11:04:53 2009
@@ -85,9 +85,8 @@
* Version 27: Changed protocol to use new api objects. And the protocol is
* renamed from JobSubmissionProtocol to ClientProtocol.
* Version 28: Added getJobHistoryDir() as part of MAPREDUCE-975.
- * Version 29: Added reservedSlots to ClusterMetrics.
*/
- public static final long versionID = 29L;
+ public static final long versionID = 28L;
/**
* Allocate a name for the job.
Modified: hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java (original)
+++ hadoop/mapreduce/trunk/src/java/org/apache/hadoop/mapreduce/server/jobtracker/TaskTracker.java Mon Oct 19 11:04:53 2009
@@ -100,30 +100,6 @@
}
/**
- * Get the reserved map slots for the tracker.
- *
- * @return reserved map slots
- */
- public int getReservedMapSlots() {
- if (jobForFallowMapSlot != null) {
- return jobForFallowMapSlot.getReservedMapSlots(this);
- }
- return 0;
- }
-
- /**
- * Get the reserved reduce slots for the tracker.
- *
- * @return reserved reduce slots
- */
- public int getReservedReduceSlots() {
- if (jobForFallowReduceSlot != null) {
- return jobForFallowReduceSlot.getReservedReduceSlots(this);
- }
- return 0;
- }
-
- /**
* Get the {@link JobInProgress} for which the fallow slot(s) are held.
* @param taskType {@link TaskType} of the task
* @return the task for which the fallow slot(s) are held,
Modified: hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java (original)
+++ hadoop/mapreduce/trunk/src/test/mapred/org/apache/hadoop/mapred/FakeObjectUtilities.java Mon Oct 19 11:04:53 2009
@@ -223,7 +223,7 @@
}
static short sendHeartBeat(JobTracker jt, TaskTrackerStatus status,
- boolean initialContact, boolean acceptNewTasks,
+ boolean initialContact,
String tracker, short responseId)
throws IOException {
if (status == null) {
@@ -231,13 +231,13 @@
JobInProgress.convertTrackerNameToHostName(tracker));
}
- jt.heartbeat(status, false, initialContact, acceptNewTasks, responseId);
+ jt.heartbeat(status, false, initialContact, false, responseId);
return ++responseId ;
}
static void establishFirstContact(JobTracker jt, String tracker)
throws IOException {
- sendHeartBeat(jt, null, true, false, tracker, (short) 0);
+ sendHeartBeat(jt, null, true, tracker, (short) 0);
}
static class FakeTaskInProgress extends TaskInProgress {
Modified: hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp?rev=826635&r1=826634&r2=826635&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp (original)
+++ hadoop/mapreduce/trunk/src/webapps/job/jobtracker.jsp Mon Oct 19 11:04:53 2009
@@ -25,7 +25,6 @@
import="java.util.*"
import="java.text.DecimalFormat"
import="org.apache.hadoop.mapred.*"
- import="org.apache.hadoop.mapreduce.*"
import="org.apache.hadoop.util.*"
%>
<%! private static final long serialVersionUID = 1L;
@@ -45,29 +44,22 @@
public void generateSummaryTable(JspWriter out, ClusterStatus status,
JobTracker tracker) throws IOException {
- ClusterMetrics metrics = tracker.getClusterMetrics();
String tasksPerNode = status.getTaskTrackers() > 0 ?
percentFormat.format(((double)(status.getMaxMapTasks() +
status.getMaxReduceTasks())) / status.getTaskTrackers()):
"-";
out.print("<table border=\"1\" cellpadding=\"5\" cellspacing=\"0\">\n"+
"<tr><th>Queues</th>" +
- "<th>Running Map Tasks</th><th>Running Reduce Tasks</th>" +
- "<th>Occupied Map Slots</th><th>Occupied Reduce Slots</th>" +
- "<th>Reserved Map Slots</th><th>Reserved Reduce Slots</th>" +
+ "<th>Maps</th><th>Reduces</th>" +
"<th>Total Submissions</th>" +
- "<th>Nodes</th><th>Map Slot Capacity</th>" +
- "<th>Reduce Slot Capacity</th><th>Avg. Slots/Node</th>" +
+ "<th>Nodes</th><th>Map Task Capacity</th>" +
+ "<th>Reduce Task Capacity</th><th>Avg. Tasks/Node</th>" +
"<th>Blacklisted Nodes</th>" +
"<th>Excluded Nodes</th></tr>\n");
out.print("<tr><td><a href=\"queueinfo.jsp\">" +
tracker.getRootQueues().length + "</a></td><td>" +
status.getMapTasks() + "</td><td>" +
status.getReduceTasks() + "</td><td>" +
- metrics.getOccupiedMapSlots() + "</td><td>" +
- metrics.getOccupiedReduceSlots() + "</td><td>" +
- metrics.getReservedMapSlots() + "</td><td>" +
- metrics.getReservedReduceSlots() + "</td><td>" +
tracker.getTotalSubmissions() +
"</td><td><a href=\"machines.jsp?type=active\">" +
status.getTaskTrackers() +