You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by br...@apache.org on 2013/08/02 03:31:40 UTC
git commit: Allowed specification of mininum slot count.
Updated Branches:
refs/heads/master 503a98fad -> 9c2f669a5
Allowed specification of mininum slot count.
To ensure Hadoop jobs begin promptly, we can specify a minimum number of
'hot slots' to be available for use. This addresses the TaskTracker
spin up delay that exists with Hadoop on Mesos. This can be a nuisance
with lower latency applications, such as ad-hoc Hive queries.
Review: https://reviews.apache.org/r/12921
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/9c2f669a
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/9c2f669a
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/9c2f669a
Branch: refs/heads/master
Commit: 9c2f669a582a708d433937ec1d06687b3bb86609
Parents: 503a98f
Author: Brenden Matthews <br...@airbnb.com>
Authored: Mon Jun 17 09:38:43 2013 -0700
Committer: Brenden Matthews <br...@airbnb.com>
Committed: Thu Aug 1 18:31:32 2013 -0700
----------------------------------------------------------------------
.../apache/hadoop/mapred/MesosScheduler.java | 21 ++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/9c2f669a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
index 1923d49..027389d 100644
--- a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
+++ b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
@@ -375,9 +375,22 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
}
}
+ // To ensure Hadoop jobs begin promptly, we can specify a minimum number
+ // of 'hot slots' to be available for use. This addresses the
+ // TaskTracker spin up delay that exists with Hadoop on Mesos. This can
+ // be a nuisance with lower latency applications, such as ad-hoc Hive
+ // queries.
+ int minimumMapSlots = conf.getInt("mapred.mesos.total.map.slots.minimum", 0);
+ int minimumReduceSlots =
+ conf.getInt("mapred.mesos.total.reduce.slots.minimum", 0);
+
// Compute how many slots we need to allocate.
- int neededMapSlots = Math.max(0, pendingMaps - (idleMapSlots + inactiveMapSlots));
- int neededReduceSlots = Math.max(0, pendingReduces - (idleReduceSlots + inactiveReduceSlots));
+ int neededMapSlots = Math.max(
+ minimumMapSlots - (idleMapSlots + inactiveMapSlots),
+ pendingMaps - (idleMapSlots + inactiveMapSlots));
+ int neededReduceSlots = Math.max(
+ minimumReduceSlots - (idleReduceSlots + inactiveReduceSlots),
+ pendingReduces - (idleReduceSlots + inactiveReduceSlots));
LOG.info(join("\n", Arrays.asList(
"JobTracker Status",
@@ -403,6 +416,10 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
continue;
}
+ // Ensure these values aren't < 0.
+ neededMapSlots = Math.max(0, neededMapSlots);
+ neededReduceSlots = Math.max(0, neededReduceSlots);
+
double cpus = -1.0;
double mem = -1.0;
double disk = -1.0;