You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/09/21 03:14:21 UTC
svn commit: r1173453 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/
hadoop-yarn/hadoop-yarn-server/hadoop-y...
Author: acmurthy
Date: Wed Sep 21 01:14:20 2011
New Revision: 1173453
URL: http://svn.apache.org/viewvc?rev=1173453&view=rev
Log:
MAPREDUCE-3036. Fixed metrics for reserved resources in CS. Contributed by Robert Evans.
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1173453&r1=1173452&r2=1173453&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Sep 21 01:14:20 2011
@@ -1372,6 +1372,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3018. Fixed -file option for streaming. (mahadev via acmurthy)
+ MAPREDUCE-3036. Fixed metrics for reserved resources in CS. (Robert Evans
+ via acmurthy)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java?rev=1173453&r1=1173452&r2=1173453&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java Wed Sep 21 01:14:20 2011
@@ -32,10 +32,8 @@ import static org.apache.hadoop.metrics2
import org.apache.hadoop.metrics2.lib.MetricsRegistry;
import org.apache.hadoop.metrics2.lib.MutableCounterInt;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
-import org.apache.hadoop.yarn.api.records.ApplicationState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
-import org.apache.hadoop.yarn.util.Self;
import static org.apache.hadoop.yarn.server.resourcemanager.resource.Resources.*;
import org.slf4j.LoggerFactory;
@@ -282,4 +280,56 @@ public class QueueMetrics {
parent.unreserveResource(user, res);
}
}
+
+ public int getAppsSubmitted() {
+ return appsSubmitted.value();
+ }
+
+ public int getAppsRunning() {
+ return appsRunning.value();
+ }
+
+ public int getAppsPending() {
+ return appsPending.value();
+ }
+
+ public int getAppsCompleted() {
+ return appsCompleted.value();
+ }
+
+ public int getAppsKilled() {
+ return appsKilled.value();
+ }
+
+ public int getAppsFailed() {
+ return appsFailed.value();
+ }
+
+ public int getAllocatedGB() {
+ return allocatedGB.value();
+ }
+
+ public int getAllocatedContainers() {
+ return allocatedContainers.value();
+ }
+
+ public int getAvailableGB() {
+ return availableGB.value();
+ }
+
+ public int getPendingGB() {
+ return pendingGB.value();
+ }
+
+ public int getPendingContainers() {
+ return pendingContainers.value();
+ }
+
+ public int getReservedGB() {
+ return reservedGB.value();
+ }
+
+ public int getReservedContainers() {
+ return reservedContainers.value();
+ }
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1173453&r1=1173452&r2=1173453&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed Sep 21 01:14:20 2011
@@ -1152,14 +1152,17 @@ public class LeafQueue implements CSQueu
private void reserve(SchedulerApp application, Priority priority,
SchedulerNode node, RMContainer rmContainer, Container container) {
- rmContainer = application.reserve(node, priority, rmContainer, container);
- node.reserveResource(application, priority, rmContainer);
-
// Update reserved metrics if this is the first reservation
if (rmContainer == null) {
getMetrics().reserveResource(
application.getUser(), container.getResource());
}
+
+ // Inform the application
+ rmContainer = application.reserve(node, priority, rmContainer, container);
+
+ // Update the node
+ node.reserveResource(application, priority, rmContainer);
}
private void unreserve(SchedulerApp application, Priority priority,
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java?rev=1173453&r1=1173452&r2=1173453&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java Wed Sep 21 01:14:20 2011
@@ -202,6 +202,8 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(1, a.getMetrics().getAllocatedGB());
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit
@@ -209,12 +211,16 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(2, a.getMetrics().getAllocatedGB());
// Can't allocate 3rd due to user-limit
a.assignContainers(clusterResource, node_0);
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(2, a.getMetrics().getAllocatedGB());
// Bump up user-limit-factor, now allocate should work
a.setUserLimitFactor(10);
@@ -222,12 +228,16 @@ public class TestLeafQueue {
assertEquals(3*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(3, a.getMetrics().getAllocatedGB());
// One more should work, for app_1, due to user-limit-factor
a.assignContainers(clusterResource, node_0);
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(4, a.getMetrics().getAllocatedGB());
// Test max-capacity
// Now - no more allocs since we are at max-cap
@@ -236,6 +246,8 @@ public class TestLeafQueue {
assertEquals(4*GB, a.getUsedResources().getMemory());
assertEquals(3*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(4, a.getMetrics().getAllocatedGB());
// Release each container from app_0
for (RMContainer rmContainer : app_0.getLiveContainers()) {
@@ -245,6 +257,8 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(1, a.getMetrics().getAllocatedGB());
// Release each container from app_1
for (RMContainer rmContainer : app_1.getLiveContainers()) {
@@ -254,6 +268,8 @@ public class TestLeafQueue {
assertEquals(0*GB, a.getUsedResources().getMemory());
assertEquals(0*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(0, a.getMetrics().getAllocatedGB());
}
@Test
@@ -473,6 +489,8 @@ public class TestLeafQueue {
assertEquals(1*GB, a.getUsedResources().getMemory());
assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(1, a.getMetrics().getAllocatedGB());
// Also 2nd -> minCapacity = 1024 since (.1 * 8G) < minAlloc, also
// you can get one container more than user-limit
@@ -480,6 +498,8 @@ public class TestLeafQueue {
assertEquals(2*GB, a.getUsedResources().getMemory());
assertEquals(2*GB, app_0.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(2, a.getMetrics().getAllocatedGB());
// Now, reservation should kick in for app_1
a.assignContainers(clusterResource, node_0);
@@ -488,6 +508,8 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(2*GB, node_0.getUsedResource().getMemory());
+ assertEquals(4, a.getMetrics().getReservedGB());
+ assertEquals(2, a.getMetrics().getAllocatedGB());
// Now free 1 container from app_0 i.e. 1G
a.completedContainer(clusterResource, app_0, node_0,
@@ -498,6 +520,8 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(4*GB, app_1.getCurrentReservation().getMemory());
assertEquals(1*GB, node_0.getUsedResource().getMemory());
+ assertEquals(4, a.getMetrics().getReservedGB());
+ assertEquals(1, a.getMetrics().getAllocatedGB());
// Now finish another container from app_0 and fulfill the reservation
a.completedContainer(clusterResource, app_0, node_0,
@@ -508,6 +532,8 @@ public class TestLeafQueue {
assertEquals(4*GB, app_1.getCurrentConsumption().getMemory());
assertEquals(0*GB, app_1.getCurrentReservation().getMemory());
assertEquals(4*GB, node_0.getUsedResource().getMemory());
+ assertEquals(0, a.getMetrics().getReservedGB());
+ assertEquals(4, a.getMetrics().getAllocatedGB());
}
@Test