You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2013/06/01 05:57:40 UTC
svn commit: r1488458 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoo...
Author: vinodkv
Date: Sat Jun 1 03:57:40 2013
New Revision: 1488458
URL: http://svn.apache.org/r1488458
Log:
MAPREDUCE-5280. Bring back removed constructor and a method in mapreduce ClusterMetrics for binary compatibility with 1.x APIs. Contributed by Mayank Bansal.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/ClusterMetrics.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1488458&r1=1488457&r2=1488458&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Sat Jun 1 03:57:40 2013
@@ -263,6 +263,10 @@ Release 2.0.5-beta - UNRELEASED
mapreduce CombineFileRecordReader for binary compatibility with 1.x APIs.
(Mayank Bansal via vinodkv)
+ MAPREDUCE-5280. Bring back removed constructor and a method in mapreduce
+ ClusterMetrics for binary compatibility with 1.x APIs. (Mayank Bansal via
+ vinodkv)
+
OPTIMIZATIONS
MAPREDUCE-4974. Optimising the LineRecordReader initialize() method
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java?rev=1488458&r1=1488457&r2=1488458&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/ClusterStatus.java Sat Jun 1 03:57:40 2013
@@ -188,6 +188,7 @@ public class ClusterStatus implements Wr
private JobTrackerStatus status;
private Collection<BlackListInfo> blacklistedTrackersInfo =
new ArrayList<BlackListInfo>();
+ private int grayListedTrackers;
ClusterStatus() {}
@@ -223,9 +224,30 @@ public class ClusterStatus implements Wr
* @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
* @param numDecommissionedNodes number of decommission trackers
*/
- ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
- int maps, int reduces, int maxMaps, int maxReduces,
- JobTrackerStatus status, int numDecommissionedNodes) {
+ ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
+ int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
+ int numDecommissionedNodes) {
+ this(trackers, blacklists, ttExpiryInterval, maps, reduces, maxMaps,
+ maxReduces, status, numDecommissionedNodes, 0);
+ }
+
+ /**
+ * Construct a new cluster status.
+ *
+ * @param trackers no. of tasktrackers in the cluster
+ * @param blacklists no of blacklisted task trackers in the cluster
+ * @param ttExpiryInterval the tasktracker expiry interval
+ * @param maps no. of currently running map-tasks in the cluster
+ * @param reduces no. of currently running reduce-tasks in the cluster
+ * @param maxMaps the maximum no. of map tasks in the cluster
+ * @param maxReduces the maximum no. of reduce tasks in the cluster
+ * @param status the {@link JobTrackerStatus} of the <code>JobTracker</code>
+ * @param numDecommissionedNodes number of decommission trackers
+ * @param numGrayListedTrackers number of graylisted trackers
+ */
+ ClusterStatus(int trackers, int blacklists, long ttExpiryInterval, int maps,
+ int reduces, int maxMaps, int maxReduces, JobTrackerStatus status,
+ int numDecommissionedNodes, int numGrayListedTrackers) {
numActiveTrackers = trackers;
numBlacklistedTrackers = blacklists;
this.numExcludedNodes = numDecommissionedNodes;
@@ -235,6 +257,7 @@ public class ClusterStatus implements Wr
max_map_tasks = maxMaps;
max_reduce_tasks = maxReduces;
this.status = status;
+ this.grayListedTrackers = numGrayListedTrackers;
}
/**
@@ -339,7 +362,7 @@ public class ClusterStatus implements Wr
*/
@Deprecated
public int getGraylistedTrackers() {
- return 0;
+ return grayListedTrackers;
}
/**
@@ -480,6 +503,7 @@ public class ClusterStatus implements Wr
out.writeInt(max_map_tasks);
out.writeInt(max_reduce_tasks);
WritableUtils.writeEnum(out, status);
+ out.writeInt(grayListedTrackers);
}
public void readFields(DataInput in) throws IOException {
@@ -507,5 +531,6 @@ public class ClusterStatus implements Wr
max_map_tasks = in.readInt();
max_reduce_tasks = in.readInt();
status = WritableUtils.readEnum(in, JobTrackerStatus.class);
+ grayListedTrackers = in.readInt();
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java?rev=1488458&r1=1488457&r2=1488458&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobClient.java Sat Jun 1 03:57:40 2013
@@ -711,19 +711,18 @@ public class JobClient extends CLI {
public ClusterStatus getClusterStatus() throws IOException {
try {
return clientUgi.doAs(new PrivilegedExceptionAction<ClusterStatus>() {
- public ClusterStatus run() throws IOException, InterruptedException {
+ public ClusterStatus run() throws IOException, InterruptedException {
ClusterMetrics metrics = cluster.getClusterStatus();
- return new ClusterStatus(metrics.getTaskTrackerCount(),
- metrics.getBlackListedTaskTrackerCount(), cluster.getTaskTrackerExpiryInterval(),
- metrics.getOccupiedMapSlots(),
- metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
- metrics.getReduceSlotCapacity(),
- cluster.getJobTrackerStatus(),
- metrics.getDecommissionedTaskTrackerCount());
+ return new ClusterStatus(metrics.getTaskTrackerCount(), metrics
+ .getBlackListedTaskTrackerCount(), cluster
+ .getTaskTrackerExpiryInterval(), metrics.getOccupiedMapSlots(),
+ metrics.getOccupiedReduceSlots(), metrics.getMapSlotCapacity(),
+ metrics.getReduceSlotCapacity(), cluster.getJobTrackerStatus(),
+ metrics.getDecommissionedTaskTrackerCount(), metrics
+ .getGrayListedTaskTrackerCount());
}
});
- }
- catch (InterruptedException ie) {
+ } catch (InterruptedException ie) {
throw new IOException(ie);
}
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/ClusterMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/ClusterMetrics.java?rev=1488458&r1=1488457&r2=1488458&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/ClusterMetrics.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/ClusterMetrics.java Sat Jun 1 03:57:40 2013
@@ -69,18 +69,28 @@ public class ClusterMetrics implements W
private int totalJobSubmissions;
private int numTrackers;
private int numBlacklistedTrackers;
+ private int numGraylistedTrackers;
private int numDecommissionedTrackers;
public ClusterMetrics() {
}
public ClusterMetrics(int runningMaps, int runningReduces,
- int occupiedMapSlots, int occupiedReduceSlots,
- int reservedMapSlots, int reservedReduceSlots,
- int mapSlots, int reduceSlots,
- int totalJobSubmissions,
- int numTrackers, int numBlacklistedTrackers,
+ int occupiedMapSlots, int occupiedReduceSlots, int reservedMapSlots,
+ int reservedReduceSlots, int mapSlots, int reduceSlots,
+ int totalJobSubmissions, int numTrackers, int numBlacklistedTrackers,
int numDecommissionedNodes) {
+ this(runningMaps, runningReduces, occupiedMapSlots, occupiedReduceSlots,
+ reservedMapSlots, reservedReduceSlots, mapSlots, reduceSlots,
+ totalJobSubmissions, numTrackers, numBlacklistedTrackers, 0,
+ numDecommissionedNodes);
+ }
+
+ public ClusterMetrics(int runningMaps, int runningReduces,
+ int occupiedMapSlots, int occupiedReduceSlots, int reservedMapSlots,
+ int reservedReduceSlots, int mapSlots, int reduceSlots,
+ int totalJobSubmissions, int numTrackers, int numBlacklistedTrackers,
+ int numGraylistedTrackers, int numDecommissionedNodes) {
this.runningMaps = runningMaps;
this.runningReduces = runningReduces;
this.occupiedMapSlots = occupiedMapSlots;
@@ -92,6 +102,7 @@ public class ClusterMetrics implements W
this.totalJobSubmissions = totalJobSubmissions;
this.numTrackers = numTrackers;
this.numBlacklistedTrackers = numBlacklistedTrackers;
+ this.numGraylistedTrackers = numGraylistedTrackers;
this.numDecommissionedTrackers = numDecommissionedNodes;
}
@@ -195,6 +206,15 @@ public class ClusterMetrics implements W
}
/**
+ * Get the number of graylisted trackers in the cluster.
+ *
+ * @return graylisted tracker count
+ */
+ public int getGrayListedTaskTrackerCount() {
+ return numGraylistedTrackers;
+ }
+
+ /**
* Get the number of decommissioned trackers in the cluster.
*
* @return decommissioned tracker count
@@ -216,6 +236,7 @@ public class ClusterMetrics implements W
totalJobSubmissions = in.readInt();
numTrackers = in.readInt();
numBlacklistedTrackers = in.readInt();
+ numGraylistedTrackers = in.readInt();
numDecommissionedTrackers = in.readInt();
}
@@ -232,6 +253,7 @@ public class ClusterMetrics implements W
out.writeInt(totalJobSubmissions);
out.writeInt(numTrackers);
out.writeInt(numBlacklistedTrackers);
+ out.writeInt(numGraylistedTrackers);
out.writeInt(numDecommissionedTrackers);
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java?rev=1488458&r1=1488457&r2=1488458&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestNetworkedJob.java Sat Jun 1 03:57:40 2013
@@ -206,6 +206,7 @@ public class TestNetworkedJob {
assertEquals(status.getTaskTrackers(), 2);
assertEquals(status.getTTExpiryInterval(), 0);
assertEquals(status.getJobTrackerStatus(), JobTrackerStatus.RUNNING);
+ assertEquals(status.getGraylistedTrackers(), 0);
// test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();