You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ac...@apache.org on 2011/10/19 22:37:51 UTC
svn commit: r1186468 - in
/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/
hadoop-yarn/hadoop-yarn-...
Author: acmurthy
Date: Wed Oct 19 20:37:51 2011
New Revision: 1186468
URL: http://svn.apache.org/viewvc?rev=1186468&view=rev
Log:
Merge -c 1186467 from trunk to branch-0.23 to complete fix for MAPREDUCE-2788.
Added:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
- copied unchanged from r1186467, hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1186468&r1=1186467&r2=1186468&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed Oct 19 20:37:51 2011
@@ -1638,6 +1638,9 @@ Release 0.23.0 - Unreleased
MAPREDUCE-3181. Fixed MapReduce runtime to load yarn-default.xml and
yarn-site.xml. (acmurthy)
+ MAPREDUCE-2788. Normalize resource requests in FifoScheduler
+ appropriately. (Ahmed Radwan via acmurthy)
+
Release 0.22.0 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1186468&r1=1186467&r2=1186468&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Wed Oct 19 20:37:51 2011
@@ -17,11 +17,14 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import java.util.List;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -72,5 +75,35 @@ public class SchedulerUtils {
return containerStatus;
}
+ /**
+ * Utility method to normalize a list of resource requests, by insuring that
+ * the memory for each request is a multiple of minMemory and is not zero.
+ *
+ * @param asks
+ * a list of resource requests.
+ * @param minMemory
+ * the configured minimum memory allocation.
+ */
+ public static void normalizeRequests(List<ResourceRequest> asks,
+ int minMemory) {
+ for (ResourceRequest ask : asks) {
+ normalizeRequest(ask, minMemory);
+ }
+ }
+
+ /**
+ * Utility method to normalize a resource request, by insuring that the
+ * requested memory is a multiple of minMemory and is not zero.
+ *
+ * @param ask
+ * the resource request.
+ * @param minMemory
+ * the configured minimum memory allocation.
+ */
+ public static void normalizeRequest(ResourceRequest ask, int minMemory) {
+ int memory = Math.max(ask.getCapability().getMemory(), minMemory);
+ ask.getCapability().setMemory(
+ minMemory * ((memory / minMemory) + (memory % minMemory > 0 ? 1 : 0)));
+ }
}
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1186468&r1=1186467&r2=1186468&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Wed Oct 19 20:37:51 2011
@@ -441,7 +441,7 @@ implements ResourceScheduler, CapacitySc
}
// Sanity check
- normalizeRequests(ask);
+ SchedulerUtils.normalizeRequests(ask, minimumAllocation.getMemory());
// Release containers
for (ContainerId releasedContainerId : release) {
@@ -521,21 +521,6 @@ implements ResourceScheduler, CapacitySc
return root.getQueueUserAclInfo(user);
}
- @Lock(Lock.NoLock.class)
- private void normalizeRequests(List<ResourceRequest> asks) {
- for (ResourceRequest ask : asks) {
- normalizeRequest(ask);
- }
- }
-
- @Lock(Lock.NoLock.class)
- private void normalizeRequest(ResourceRequest ask) {
- int minMemory = minimumAllocation.getMemory();
- int memory = Math.max(ask.getCapability().getMemory(), minMemory);
- ask.getCapability().setMemory (
- minMemory * ((memory/minMemory) + (memory%minMemory > 0 ? 1 : 0)));
- }
-
private synchronized void nodeUpdate(RMNode nm,
List<ContainerStatus> newlyLaunchedContainers,
List<ContainerStatus> completedContainers) {
Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1186468&r1=1186467&r2=1186468&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Wed Oct 19 20:37:51 2011
@@ -217,7 +217,7 @@ public class FifoScheduler implements Re
}
// Sanity check
- normalizeRequests(ask);
+ SchedulerUtils.normalizeRequests(ask, MINIMUM_MEMORY);
// Release containers
for (ContainerId releasedContainer : release) {
@@ -260,21 +260,6 @@ public class FifoScheduler implements Re
application.getHeadroom());
}
- private void normalizeRequests(List<ResourceRequest> asks) {
- for (ResourceRequest ask : asks) {
- normalizeRequest(ask);
- }
- }
-
- private void normalizeRequest(ResourceRequest ask) {
- int memory = ask.getCapability().getMemory();
- // FIXME: TestApplicationCleanup is relying on unnormalized behavior.
- memory =
- MINIMUM_MEMORY *
- ((memory/MINIMUM_MEMORY) + (memory%MINIMUM_MEMORY > 0 ? 1 : 0));
- ask.setCapability(Resources.createResource(memory));
- }
-
private SchedulerApp getApplication(
ApplicationAttemptId applicationAttemptId) {
return applications.get(applicationAttemptId);