You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by sa...@apache.org on 2014/02/24 23:36:03 UTC
svn commit: r1571469 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/
Author: sandy
Date: Mon Feb 24 22:36:03 2014
New Revision: 1571469
URL: http://svn.apache.org/r1571469
Log:
YARN-1678. Fair scheduler gabs incessantly about reservations (Sandy Ryza)
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1571469&r1=1571468&r2=1571469&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Mon Feb 24 22:36:03 2014
@@ -13,6 +13,8 @@ Release 2.5.0 - UNRELEASED
YARN-1736. FS: AppSchedulable.assignContainer's priority argument is
redundant. (Naren Koneru via kasha)
+ YARN-1678. Fair scheduler gabs incessantly about reservations (Sandy Ryza)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java?rev=1571469&r1=1571468&r2=1571469&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AppSchedulable.java Mon Feb 24 22:36:03 2014
@@ -201,6 +201,22 @@ public class AppSchedulable extends Sche
* Assign a container to this node to facilitate {@code request}. If node does
* not have enough memory, create a reservation. This is called once we are
* sure the particular request should be facilitated by this node.
+ *
+ * @param node
+ * The node to try placing the container on.
+ * @param priority
+ * The requested priority for the container.
+ * @param request
+ * The ResourceRequest we're trying to satisfy.
+ * @param type
+ * The locality of the assignment.
+ * @param reserved
+ * Whether there's already a container reserved for this app on the node.
+ * @return
+ * If an assignment was made, returns the resources allocated to the
+ * container. If a reservation was made, returns
+ * FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was
+ * made, returns an empty resource.
*/
private Resource assignContainer(FSSchedulerNode node,
ResourceRequest request, NodeType type,
@@ -255,17 +271,6 @@ public class AppSchedulable extends Sche
LOG.debug("Node offered to app: " + getName() + " reserved: " + reserved);
}
- if (reserved) {
- RMContainer rmContainer = node.getReservedContainer();
- Priority priority = rmContainer.getReservedPriority();
-
- // Make sure the application still needs requests at this priority
- if (app.getTotalRequiredResources(priority) == 0) {
- unreserve(priority, node);
- return Resources.none();
- }
- }
-
Collection<Priority> prioritiesToTry = (reserved) ?
Arrays.asList(node.getReservedContainer().getReservedPriority()) :
app.getPriorities();
@@ -338,7 +343,33 @@ public class AppSchedulable extends Sche
return Resources.none();
}
+ /**
+ * Called when this application already has an existing reservation on the
+ * given node. Sees whether we can turn the reservation into an allocation.
+ * Also checks whether the application needs the reservation anymore, and
+ * releases it if not.
+ *
+ * @param node
+ * Node that the application has an existing reservation on
+ */
public Resource assignReservedContainer(FSSchedulerNode node) {
+ RMContainer rmContainer = node.getReservedContainer();
+ Priority priority = rmContainer.getReservedPriority();
+
+ // Make sure the application still needs requests at this priority
+ if (app.getTotalRequiredResources(priority) == 0) {
+ unreserve(priority, node);
+ return Resources.none();
+ }
+
+ // Fail early if the reserved container won't fit.
+ // Note that we have an assumption here that there's only one container size
+ // per priority.
+ if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(),
+ node.getAvailableResource())) {
+ return Resources.none();
+ }
+
return assignContainer(node, true);
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java?rev=1571469&r1=1571468&r2=1571469&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java Mon Feb 24 22:36:03 2014
@@ -1046,10 +1046,12 @@ public class FairScheduler extends Abstr
reservedAppSchedulable = null;
} else {
// Reservation exists; try to fulfill the reservation
- LOG.info("Trying to fulfill reservation for application "
- + reservedAppSchedulable.getApp().getApplicationAttemptId()
- + " on node: " + node);
-
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to fulfill reservation for application "
+ + reservedAppSchedulable.getApp().getApplicationAttemptId()
+ + " on node: " + node);
+ }
+
node.getReservedAppSchedulable().assignReservedContainer(node);
}
}