You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:10:07 UTC
svn commit: r1077389 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
mapred/org/apache/hadoop/mapred/ test/org/apache/hadoop/mapred/
Author: omalley
Date: Fri Mar 4 04:10:06 2011
New Revision: 1077389
URL: http://svn.apache.org/viewvc?rev=1077389&view=rev
Log:
commit 964dce520c0544c1bdd06abadb8f91a3009d10c6
Author: Arun C Murthy <ac...@apache.org>
Date: Mon Apr 12 16:48:37 2010 -0700
MAPREDUCE-1683. Removes JNI calls to get jvm current/max heap usage in ClusterStatus by default.
From https://issues.apache.org/jira/secure/attachment/12441563/MAPREDUCE-1683_yhadoop_20_S.patch
+++ b/YAHOO-CHANGES.txt
+ MAPREDUCE-1683. Removes JNI calls to get jvm current/max heap usage in
+ ClusterStatus by default. (acmurthy)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestClusterStatus.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java?rev=1077389&r1=1077388&r2=1077389&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/ClusterStatus.java Fri Mar 4 04:10:06 2011
@@ -68,8 +68,10 @@ public class ClusterStatus implements Wr
private int max_map_tasks;
private int max_reduce_tasks;
private JobTracker.State state;
- private long used_memory;
- private long max_memory;
+
+ public static final long UNINITIALIZED_MEMORY_VALUE = -1;
+ private long used_memory = UNINITIALIZED_MEMORY_VALUE;
+ private long max_memory = UNINITIALIZED_MEMORY_VALUE;
ClusterStatus() {}
@@ -125,17 +127,9 @@ public class ClusterStatus implements Wr
ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
int maps, int reduces, int maxMaps, int maxReduces,
JobTracker.State state, int numDecommissionedNodes) {
- numActiveTrackers = trackers;
- numBlacklistedTrackers = blacklists;
- this.numExcludedNodes = numDecommissionedNodes;
- this.ttExpiryInterval = ttExpiryInterval;
- map_tasks = maps;
- reduce_tasks = reduces;
- max_map_tasks = maxMaps;
- max_reduce_tasks = maxReduces;
- this.state = state;
- used_memory = Runtime.getRuntime().totalMemory();
- max_memory = Runtime.getRuntime().maxMemory();
+ this(trackers, blacklists, ttExpiryInterval, maps, reduces,
+ maxMaps, maxReduces, state, numDecommissionedNodes,
+ UNINITIALIZED_MEMORY_VALUE, UNINITIALIZED_MEMORY_VALUE);
}
/**
@@ -159,6 +153,23 @@ public class ClusterStatus implements Wr
maxMaps, maxReduces, state, 0);
}
+ ClusterStatus(int trackers, int blacklists, long ttExpiryInterval,
+ int maps, int reduces, int maxMaps, int maxReduces,
+ JobTracker.State state, int numDecommissionedNodes,
+ long used_memory, long max_memory) {
+ numActiveTrackers = trackers;
+ numBlacklistedTrackers = blacklists;
+ this.numExcludedNodes = numDecommissionedNodes;
+ this.ttExpiryInterval = ttExpiryInterval;
+ map_tasks = maps;
+ reduce_tasks = reduces;
+ max_map_tasks = maxMaps;
+ max_reduce_tasks = maxReduces;
+ this.state = state;
+ this.used_memory = used_memory;
+ this.max_memory = max_memory;
+ }
+
/**
* Construct a new cluster status.
* @param activeTrackers active tasktrackers in the cluster
@@ -176,12 +187,12 @@ public class ClusterStatus implements Wr
int maps, int reduces, int maxMaps, int maxReduces,
JobTracker.State state, int numDecommissionNodes) {
this(activeTrackers.size(), blacklistedTrackers.size(), ttExpiryInterval,
- maps, reduces, maxMaps, maxReduces, state, numDecommissionNodes);
+ maps, reduces, maxMaps, maxReduces, state, numDecommissionNodes,
+ Runtime.getRuntime().totalMemory(), Runtime.getRuntime().maxMemory());
this.activeTrackers = activeTrackers;
this.blacklistedTrackers = blacklistedTrackers;
}
-
/**
* Get the number of task trackers in the cluster.
*
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java?rev=1077389&r1=1077388&r2=1077389&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobClient.java Fri Mar 4 04:10:06 2011
@@ -1075,7 +1075,7 @@ public class JobClient extends Configure
* Get status information about the Map-Reduce cluster.
*
* @param detailed if true then get a detailed status including the
- * tracker names
+ * tracker names and memory usage of the JobTracker
* @return the status information about the Map-Reduce cluster as an object
* of {@link ClusterStatus}.
* @throws IOException
Modified: hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java?rev=1077389&r1=1077388&r2=1077389&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/mapred/org/apache/hadoop/mapred/JobSubmissionProtocol.java Fri Mar 4 04:10:06 2011
@@ -96,7 +96,7 @@ interface JobSubmissionProtocol extends
/**
* Get the current status of the cluster
- * @param detailed if true then report tracker names as well
+ * @param detailed if true then report tracker names and memory usage
* @return summary of the state of the cluster
*/
public ClusterStatus getClusterStatus(boolean detailed) throws IOException;
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestClusterStatus.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestClusterStatus.java?rev=1077389&r1=1077388&r2=1077389&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestClusterStatus.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/mapred/TestClusterStatus.java Fri Mar 4 04:10:06 2011
@@ -307,4 +307,32 @@ public class TestClusterStatus extends T
assertEquals("reduce slots should have been unreserved",
0, metrics.getReservedReduceSlots());
}
+
+ public void testClusterStatus() throws Exception {
+ ClusterStatus clusterStatus = client.getClusterStatus();
+ assertEquals("JobTracker used-memory is " + clusterStatus.getUsedMemory() +
+ ", expected " + ClusterStatus.UNINITIALIZED_MEMORY_VALUE,
+ ClusterStatus.UNINITIALIZED_MEMORY_VALUE, clusterStatus.getUsedMemory());
+ assertEquals("JobTracker max-memory is " + clusterStatus.getMaxMemory() +
+ ", expected " + ClusterStatus.UNINITIALIZED_MEMORY_VALUE,
+ ClusterStatus.UNINITIALIZED_MEMORY_VALUE, clusterStatus.getMaxMemory());
+
+ clusterStatus = client.getClusterStatus(false);
+ assertEquals("JobTracker used-memory is " + clusterStatus.getUsedMemory() +
+ ", expected " + ClusterStatus.UNINITIALIZED_MEMORY_VALUE,
+ ClusterStatus.UNINITIALIZED_MEMORY_VALUE, clusterStatus.getUsedMemory());
+ assertEquals("JobTracker max-memory is " + clusterStatus.getMaxMemory() +
+ ", expected " + ClusterStatus.UNINITIALIZED_MEMORY_VALUE,
+ ClusterStatus.UNINITIALIZED_MEMORY_VALUE, clusterStatus.getMaxMemory());
+
+ clusterStatus = client.getClusterStatus(true);
+ if (ClusterStatus.UNINITIALIZED_MEMORY_VALUE == clusterStatus.getUsedMemory()) {
+ assertEquals("JobTracker used-memory is " + clusterStatus.getUsedMemory(),
+ true, false);
+ }
+ if (ClusterStatus.UNINITIALIZED_MEMORY_VALUE == clusterStatus.getMaxMemory()) {
+ assertEquals("JobTracker max-memory is " + clusterStatus.getMaxMemory(),
+ true, false);
+ }
+ }
}