You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2012/01/31 09:31:16 UTC
svn commit: r1238256 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/
hadoop-yarn/hadoop-yarn-...
Author: acmurthy
Date: Tue Jan 31 08:31:16 2012
New Revision: 1238256
URL: http://svn.apache.org/viewvc?rev=1238256&view=rev
Log:
Merge -c 1238255 from trunk to branch-0.23 to fix MAPREDUCE-3764. Fixed resource usage metrics for queues and users.
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1238256&r1=1238255&r2=1238256&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jan 31 08:31:16 2012
@@ -576,6 +576,9 @@ Release 0.23.1 - Unreleased
MAPREDUCE-3748. Changed a log in CapacityScheduler.nodeUpdate to debug.
(ramya via acmurthy)
+ MAPREDUCE-3764. Fixed resource usage metrics for queues and users.
+ (acmurthy)
+
Release 0.23.0 - 2011-11-01
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java?rev=1238256&r1=1238255&r2=1238256&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java Tue Jan 31 08:31:16 2012
@@ -51,20 +51,19 @@ public class QueueMetrics {
@Metric("# of apps killed") MutableCounterInt appsKilled;
@Metric("# of apps failed") MutableCounterInt appsFailed;
- @Metric("Allocated memory in GiB") MutableGaugeInt allocatedGB;
+ @Metric("Allocated memory in MB") MutableGaugeInt allocatedMB;
@Metric("# of allocated containers") MutableGaugeInt allocatedContainers;
@Metric("Aggregate # of allocated containers") MutableCounterLong aggregateContainersAllocated;
@Metric("Aggregate # of released containers") MutableCounterLong aggregateContainersReleased;
- @Metric("Available memory in GiB") MutableGaugeInt availableGB;
- @Metric("Pending memory allocation in GiB") MutableGaugeInt pendingGB;
+ @Metric("Available memory in MB") MutableGaugeInt availableMB;
+ @Metric("Pending memory allocation in MB") MutableGaugeInt pendingMB;
@Metric("# of pending containers") MutableGaugeInt pendingContainers;
- @Metric("# of reserved memory in GiB") MutableGaugeInt reservedGB;
+ @Metric("# of reserved memory in MB") MutableGaugeInt reservedMB;
@Metric("# of reserved containers") MutableGaugeInt reservedContainers;
@Metric("# of active users") MutableGaugeInt activeUsers;
@Metric("# of active users") MutableGaugeInt activeApplications;
static final Logger LOG = LoggerFactory.getLogger(QueueMetrics.class);
- static final int GB = 1024; // resource.memory is in MB
static final MetricsInfo RECORD_INFO = info("QueueMetrics",
"Metrics for the resource scheduler");
static final MetricsInfo QUEUE_INFO = info("Queue", "Metrics by queue");
@@ -183,7 +182,7 @@ public class QueueMetrics {
* @param limit resource limit
*/
public void setAvailableResourcesToQueue(Resource limit) {
- availableGB.set(limit.getMemory()/GB);
+ availableMB.set(limit.getMemory());
}
/**
@@ -219,7 +218,7 @@ public class QueueMetrics {
private void _incrPendingResources(int containers, Resource res) {
pendingContainers.incr(containers);
- pendingGB.incr(res.getMemory()/GB);
+ pendingMB.incr(res.getMemory());
}
public void decrPendingResources(String user, int containers, Resource res) {
@@ -235,13 +234,13 @@ public class QueueMetrics {
private void _decrPendingResources(int containers, Resource res) {
pendingContainers.decr(containers);
- pendingGB.decr(res.getMemory()/GB);
+ pendingMB.decr(res.getMemory());
}
public void allocateResources(String user, int containers, Resource res) {
allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
- allocatedGB.incr(res.getMemory()/GB * containers);
+ allocatedMB.incr(res.getMemory() * containers);
_decrPendingResources(containers, multiply(res, containers));
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
@@ -255,7 +254,7 @@ public class QueueMetrics {
public void releaseResources(String user, int containers, Resource res) {
allocatedContainers.decr(containers);
aggregateContainersReleased.incr(containers);
- allocatedGB.decr(res.getMemory()/GB * containers);
+ allocatedMB.decr(res.getMemory() * containers);
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.releaseResources(user, containers, res);
@@ -267,7 +266,7 @@ public class QueueMetrics {
public void reserveResource(String user, Resource res) {
reservedContainers.incr();
- reservedGB.incr(res.getMemory()/GB);
+ reservedMB.incr(res.getMemory());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.reserveResource(user, res);
@@ -279,7 +278,7 @@ public class QueueMetrics {
public void unreserveResource(String user, Resource res) {
reservedContainers.decr();
- reservedGB.decr(res.getMemory()/GB);
+ reservedMB.decr(res.getMemory());
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.unreserveResource(user, res);
@@ -343,28 +342,28 @@ public class QueueMetrics {
return appsFailed.value();
}
- public int getAllocatedGB() {
- return allocatedGB.value();
+ public int getAllocatedMB() {
+ return allocatedMB.value();
}
public int getAllocatedContainers() {
return allocatedContainers.value();
}
- public int getAvailableGB() {
- return availableGB.value();
+ public int getAvailableMB() {
+ return availableMB.value();
}
- public int getPendingGB() {
- return pendingGB.value();
+ public int getPendingMB() {
+ return pendingMB.value();
}
public int getPendingContainers() {
return pendingContainers.value();
}
- public int getReservedGB() {
- return reservedGB.value();
+ public int getReservedMB() {
+ return reservedMB.value();
}
public int getReservedContainers() {
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java?rev=1238256&r1=1238255&r2=1238256&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java Tue Jan 31 08:31:16 2012
@@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.server.res
@XmlAccessorType(XmlAccessType.FIELD)
public class ClusterMetricsInfo {
- private static final long MB_IN_GB = 1024;
-
protected int appsSubmitted;
protected long reservedMB;
protected long availableMB;
@@ -55,9 +53,9 @@ public class ClusterMetricsInfo {
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
this.appsSubmitted = metrics.getAppsSubmitted();
- this.reservedMB = metrics.getReservedGB() * MB_IN_GB;
- this.availableMB = metrics.getAvailableGB() * MB_IN_GB;
- this.allocatedMB = metrics.getAllocatedGB() * MB_IN_GB;
+ this.reservedMB = metrics.getReservedMB();
+ this.availableMB = metrics.getAvailableMB();
+ this.allocatedMB = metrics.getAllocatedMB();
this.containersAllocated = metrics.getAllocatedContainers();
this.totalMB = availableMB + reservedMB + allocatedMB;
this.activeNodes = clusterMetrics.getNumActiveNMs();
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java?rev=1238256&r1=1238255&r2=1238256&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/UserMetricsInfo.java Tue Jan 31 08:31:16 2012
@@ -31,8 +31,6 @@ import org.apache.hadoop.yarn.server.res
@XmlAccessorType(XmlAccessType.FIELD)
public class UserMetricsInfo {
- private static final long MB_IN_GB = 1024;
-
protected int appsSubmitted;
protected int runningContainers;
protected int pendingContainers;
@@ -60,9 +58,9 @@ public class UserMetricsInfo {
this.runningContainers = userMetrics.getAllocatedContainers();
this.pendingContainers = userMetrics.getPendingContainers();
this.reservedContainers = userMetrics.getReservedContainers();
- this.reservedMB = userMetrics.getReservedGB() * MB_IN_GB;
- this.pendingMB = userMetrics.getPendingGB() * MB_IN_GB;
- this.allocatedMB = userMetrics.getAllocatedGB() * MB_IN_GB;
+ this.reservedMB = userMetrics.getReservedMB();
+ this.pendingMB = userMetrics.getPendingMB();
+ this.allocatedMB = userMetrics.getAllocatedMB();
}
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java?rev=1238256&r1=1238255&r2=1238256&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java Tue Jan 31 08:31:16 2012
@@ -57,16 +57,16 @@ public class TestQueueMetrics {
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
- checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
+ checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2*GB));
- checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0);
+ checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB));
- checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
+ checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@@ -92,20 +92,20 @@ public class TestQueueMetrics {
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
- checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
- checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
+ checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
+ checkResources(userSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
checkApps(userSource, 1, 0, 1, 0, 0, 0);
metrics.allocateResources(user, 3, Resources.createResource(2*GB));
- checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 0, 0);
- checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 0, 0);
+ checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 0, 0);
+ checkResources(userSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 0, 0);
metrics.releaseResources(user, 1, Resources.createResource(2*GB));
- checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
- checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
+ checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
+ checkResources(userSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@@ -141,10 +141,10 @@ public class TestQueueMetrics {
parentMetrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
metrics.setAvailableResourcesToUser(user, Resources.createResource(10*GB));
metrics.incrPendingResources(user, 5, Resources.createResource(15*GB));
- checkResources(queueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
- checkResources(parentQueueSource, 0, 0, 0, 0, 100, 15, 5, 0, 0);
- checkResources(userSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
- checkResources(parentUserSource, 0, 0, 0, 0, 10, 15, 5, 0, 0);
+ checkResources(queueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
+ checkResources(parentQueueSource, 0, 0, 0, 0, 100*GB, 15*GB, 5, 0, 0);
+ checkResources(userSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
+ checkResources(parentUserSource, 0, 0, 0, 0, 10*GB, 15*GB, 5, 0, 0);
metrics.incrAppsRunning(user);
checkApps(queueSource, 1, 0, 1, 0, 0, 0);
@@ -154,17 +154,17 @@ public class TestQueueMetrics {
metrics.reserveResource(user, Resources.createResource(3*GB));
// Available resources is set externally, as it depends on dynamic
// configurable cluster/queue resources
- checkResources(queueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1);
- checkResources(parentQueueSource, 6, 3, 3, 0, 100, 9, 2, 3, 1);
- checkResources(userSource, 6, 3, 3, 0, 10, 9, 2, 3, 1);
- checkResources(parentUserSource, 6, 3, 3, 0, 10, 9, 2, 3, 1);
+ checkResources(queueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 3*GB, 1);
+ checkResources(parentQueueSource, 6*GB, 3, 3, 0, 100*GB, 9*GB, 2, 3*GB, 1);
+ checkResources(userSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 3*GB, 1);
+ checkResources(parentUserSource, 6*GB, 3, 3, 0, 10*GB, 9*GB, 2, 3*GB, 1);
metrics.releaseResources(user, 1, Resources.createResource(2*GB));
metrics.unreserveResource(user, Resources.createResource(3*GB));
- checkResources(queueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
- checkResources(parentQueueSource, 4, 2, 3, 1, 100, 9, 2, 0, 0);
- checkResources(userSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
- checkResources(parentUserSource, 4, 2, 3, 1, 10, 9, 2, 0, 0);
+ checkResources(queueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
+ checkResources(parentQueueSource, 4*GB, 2, 3, 1, 100*GB, 9*GB, 2, 0, 0);
+ checkResources(userSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
+ checkResources(parentUserSource, 4*GB, 2, 3, 1, 10*GB, 9*GB, 2, 0, 0);
metrics.finishApp(app, RMAppAttemptState.FINISHED);
checkApps(queueSource, 1, 0, 0, 1, 0, 0);
@@ -184,18 +184,19 @@ public class TestQueueMetrics {
assertCounter("AppsKilled", killed, rb);
}
- public static void checkResources(MetricsSource source, int allocGB,
- int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs, int availGB, int pendingGB, int pendingCtnrs,
- int reservedGB, int reservedCtnrs) {
+ public static void checkResources(MetricsSource source, int allocatedMB,
+ int allocCtnrs, long aggreAllocCtnrs, long aggreReleasedCtnrs,
+ int availableMB, int pendingMB, int pendingCtnrs,
+ int reservedMB, int reservedCtnrs) {
MetricsRecordBuilder rb = getMetrics(source);
- assertGauge("AllocatedGB", allocGB, rb);
+ assertGauge("AllocatedMB", allocatedMB, rb);
assertGauge("AllocatedContainers", allocCtnrs, rb);
assertCounter("AggregateContainersAllocated", aggreAllocCtnrs, rb);
assertCounter("AggregateContainersReleased", aggreReleasedCtnrs, rb);
- assertGauge("AvailableGB", availGB, rb);
- assertGauge("PendingGB", pendingGB, rb);
+ assertGauge("AvailableMB", availableMB, rb);
+ assertGauge("PendingMB", pendingMB, rb);
assertGauge("PendingContainers", pendingCtnrs, rb);
- assertGauge("ReservedGB", reservedGB, rb);
+ assertGauge("ReservedMB", reservedMB, rb);
assertGauge("ReservedContainers", reservedCtnrs, rb);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1238256&r1=1238255&r2=1238256&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Tue Jan 31 08:31:16 2012
@@ -251,7 +251,7 @@ public class TestLeafQueue {
// Only 1 container
a.assignContainers(clusterResource, node_0);
- assertEquals(7, a.getMetrics().getAvailableGB());
+ assertEquals(7*GB, a.getMetrics().getAvailableMB());
}
@@ -307,9 +307,9 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(1, a.getMetrics().getAllocatedGB());
- assertEquals(0, a.getMetrics().getAvailableGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(1*GB, a.getMetrics().getAllocatedMB());
+ assertEquals(0*GB, a.getMetrics().getAvailableMB());
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit
@@ -317,16 +317,16 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(2, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Can't allocate 3rd due to user-limit
a.assignContainers(clusterResource, node_0);
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(2, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Bump up user-limit-factor, now allocate should work
a.setUserLimitFactor(10);
@@ -334,16 +334,16 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(3, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(3*GB, a.getMetrics().getAllocatedMB());
// One more should work, for app_1, due to user-limit-factor
a.assignContainers(clusterResource, node_0);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(4, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(4*GB, a.getMetrics().getAllocatedMB());
// Test max-capacity
// Now - no more allocs since we are at max-cap
@@ -352,8 +352,8 @@ public class TestLeafQueue {
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(4, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(4*GB, a.getMetrics().getAllocatedMB());
// Release each container from app_0
for (RMContainer rmContainer : app_0.getLiveContainers()) {
@@ -363,8 +363,8 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(1, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(1*GB, a.getMetrics().getAllocatedMB());
// Release each container from app_1
for (RMContainer rmContainer : app_1.getLiveContainers()) {
@@ -374,9 +374,9 @@ public class TestLeafQueue {
assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(0, a.getMetrics().getAllocatedGB());
- assertEquals(1, a.getMetrics().getAvailableGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(0*GB, a.getMetrics().getAllocatedMB());
+ assertEquals(1*GB, a.getMetrics().getAvailableMB());
}
@Test
@@ -700,9 +700,9 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(1, a.getMetrics().getAllocatedGB());
- assertEquals(0, a.getMetrics().getAvailableGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(1*GB, a.getMetrics().getAllocatedMB());
+ assertEquals(0*GB, a.getMetrics().getAvailableMB());
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit
@@ -710,8 +710,8 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(2, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Now, reservation should kick in for app_1
a.assignContainers(clusterResource, node_0);
@@ -720,8 +720,8 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(2*GB, node_0.getUsedResource().getMemory());
- assertEquals(4, a.getMetrics().getReservedGB());
- assertEquals(2, a.getMetrics().getAllocatedGB());
+ assertEquals(4*GB, a.getMetrics().getReservedMB());
+ assertEquals(2*GB, a.getMetrics().getAllocatedMB());
// Now free 1 container from app_0 i.e. 1G
a.completedContainer(clusterResource, app_0, node_0,
@@ -732,8 +732,8 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(1*GB, node_0.getUsedResource().getMemory());
- assertEquals(4, a.getMetrics().getReservedGB());
- assertEquals(1, a.getMetrics().getAllocatedGB());
+ assertEquals(4*GB, a.getMetrics().getReservedMB());
+ assertEquals(1*GB, a.getMetrics().getAllocatedMB());
// Now finish another container from app_0 and fulfill the reservation
a.completedContainer(clusterResource, app_0, node_0,
@@ -744,8 +744,8 @@ public class TestLeafQueue {
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
assertEquals(4*GB, node_0.getUsedResource().getMemory());
- assertEquals(0, a.getMetrics().getReservedGB());
- assertEquals(4, a.getMetrics().getAllocatedGB());
+ assertEquals(0*GB, a.getMetrics().getReservedMB());
+ assertEquals(4*GB, a.getMetrics().getAllocatedMB());
}
@Test
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java?rev=1238256&r1=1238255&r2=1238256&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java Tue Jan 31 08:31:16 2012
@@ -398,19 +398,19 @@ public class TestRMWebServices extends J
ResourceScheduler rs = rm.getResourceScheduler();
QueueMetrics metrics = rs.getRootQueueMetrics();
ClusterMetrics clusterMetrics = ClusterMetrics.getMetrics();
- final long MB_IN_GB = 1024;
- long totalMBExpect = (metrics.getReservedGB() * MB_IN_GB)
- + (metrics.getAvailableGB() * MB_IN_GB)
- + (metrics.getAllocatedGB() * MB_IN_GB);
+ long totalMBExpect =
+ metrics.getReservedMB()+ metrics.getAvailableMB()
+ + metrics.getAllocatedMB();
- assertEquals("appsSubmitted doesn't match", metrics.getAppsSubmitted(), sub);
+ assertEquals("appsSubmitted doesn't match",
+ metrics.getAppsSubmitted(), sub);
assertEquals("reservedMB doesn't match",
- metrics.getReservedGB() * MB_IN_GB, reservedMB);
- assertEquals("availableMB doesn't match", metrics.getAvailableGB()
- * MB_IN_GB, availableMB);
- assertEquals("allocatedMB doesn't match", metrics.getAllocatedGB()
- * MB_IN_GB, allocMB);
+ metrics.getReservedMB(), reservedMB);
+ assertEquals("availableMB doesn't match",
+ metrics.getAvailableMB(), availableMB);
+ assertEquals("allocatedMB doesn't match",
+ metrics.getAllocatedMB(), allocMB);
assertEquals("containersAllocated doesn't match", 0, containersAlloc);
assertEquals("totalMB doesn't match", totalMBExpect, totalMB);
assertEquals(