You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by br...@apache.org on 2013/07/30 21:48:52 UTC
[4/4] git commit: Allowed some JVM memory vars to be configured.
Allowed some JVM memory vars to be configured.
Also modified the slot CPU/mem values to be more sane defaults.
Review: https://reviews.apache.org/r/11130
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo
Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/0605e904
Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/0605e904
Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/0605e904
Branch: refs/heads/master
Commit: 0605e90445beb34f621b369063d9bc452ff8cd39
Parents: aac37e8
Author: Brenden Matthews <br...@airbnb.com>
Authored: Fri May 10 14:34:22 2013 -0700
Committer: Brenden Matthews <br...@airbnb.com>
Committed: Tue Jul 30 12:48:29 2013 -0700
----------------------------------------------------------------------
hadoop/mapred-site.xml.patch | 6 ++---
.../apache/hadoop/mapred/MesosScheduler.java | 27 ++++++++++++--------
2 files changed, 19 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/mesos/blob/0605e904/hadoop/mapred-site.xml.patch
----------------------------------------------------------------------
diff --git a/hadoop/mapred-site.xml.patch b/hadoop/mapred-site.xml.patch
index 8b39979..d5a99f6 100644
--- a/hadoop/mapred-site.xml.patch
+++ b/hadoop/mapred-site.xml.patch
@@ -37,7 +37,7 @@ index 970c8fe..f9f272d 100644
+# that are allocated to a Hadoop slot (i.e., map/reduce task) by Mesos.
+ <property>
+ <name>mapred.mesos.slot.cpus</name>
-+ <value>0.2</value>
++ <value>1</value>
+ </property>
+ <property>
+ <name>mapred.mesos.slot.disk</name>
@@ -47,8 +47,8 @@ index 970c8fe..f9f272d 100644
+ <property>
+ <name>mapred.mesos.slot.mem</name>
+ <!-- Note that this is the total memory required for
-+ JVM overhead (256 MB) and the heap (-Xmx) of the task.
++ JVM overhead (10% of this value) and the heap (-Xmx) of the task.
+ The value is in MB. -->
-+ <value>512</value>
++ <value>1024</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/mesos/blob/0605e904/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
index e4fbb80..69d4655 100644
--- a/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
+++ b/hadoop/mesos/src/java/org/apache/hadoop/mapred/MesosScheduler.java
@@ -50,7 +50,7 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
// This is the memory overhead for a jvm process. This needs to be added
// to a jvm process's resource requirement, in addition to its heap size.
- private static final int JVM_MEM_OVERHEAD = 256; // 256 MB.
+ private static final double JVM_MEM_OVERHEAD_PERCENT_DEFAULT = 0.1; // 10%.
// TODO(vinod): Consider parsing the slot memory from the configuration jvm
// heap options (e.g: mapred.child.java.opts).
@@ -62,12 +62,10 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
// 1 GB of disk space.
private static final double SLOT_CPUS_DEFAULT = 0.2; // 0.2 cores.
private static final int SLOT_DISK_DEFAULT = 1024; // 1 GB.
- private static final int SLOT_JVM_HEAP_DEFAULT = 256; // MB.
+ private static final int SLOT_JVM_HEAP_DEFAULT = 256; // 256MB.
private static final double TASKTRACKER_CPUS = 1.0; // 1 core.
- private static final int TASKTRACKER_JVM_HEAP = 1024; // 1 GB.
- private static final int TASKTRACKER_MEM =
- TASKTRACKER_JVM_HEAP + JVM_MEM_OVERHEAD;
+ private static final int TASKTRACKER_MEM_DEFAULT = 1024; // 1 GB.
// The default behavior in Hadoop is to use 4 slots per TaskTracker:
private static final int MAP_SLOTS_DEFAULT = 2;
@@ -445,14 +443,21 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
(float) SLOT_CPUS_DEFAULT);
double slotDisk = conf.getInt("mapred.mesos.slot.disk",
SLOT_DISK_DEFAULT);
- double slotMem = conf.getInt("mapred.mesos.slot.mem",
- SLOT_JVM_HEAP_DEFAULT + JVM_MEM_OVERHEAD);
- double slotJVMHeap = slotMem - JVM_MEM_OVERHEAD;
+
+ int slotMem = conf.getInt("mapred.mesos.slot.mem",
+ SLOT_JVM_HEAP_DEFAULT);
+ long slotJVMHeap = Math.round((double)slotMem -
+ (JVM_MEM_OVERHEAD_PERCENT_DEFAULT * slotMem));
+
+ int tasktrackerMem = conf.getInt("mapred.mesos.tasktracker.mem",
+ TASKTRACKER_MEM_DEFAULT);
+ long tasktrackerJVMHeap = Math.round((double)tasktrackerMem -
+ (JVM_MEM_OVERHEAD_PERCENT_DEFAULT * tasktrackerMem));
// Minimum resource requirements for the container (TaskTracker + map/red
// tasks).
double containerCpus = TASKTRACKER_CPUS;
- double containerMem = TASKTRACKER_MEM;
+ double containerMem = tasktrackerMem;
double containerDisk = 0;
// Determine how many slots we can allocate.
@@ -557,7 +562,7 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("HADOOP_HEAPSIZE")
- .setValue("" + TASKTRACKER_JVM_HEAP))
+ .setValue("" + tasktrackerJVMHeap))
.addVariables(
Protos.Environment.Variable.newBuilder()
.setName("mapred.tasktracker.map.tasks.maximum")
@@ -663,7 +668,7 @@ public class MesosScheduler extends TaskScheduler implements Scheduler {
.setName("mem")
.setType(Value.Type.SCALAR)
.setScalar(Value.Scalar.newBuilder().setValue(
- (TASKTRACKER_MEM)))).setCommand(commandInfo))
+ (tasktrackerMem)))).setCommand(commandInfo))
.build();
driver.launchTasks(offer.getId(), Arrays.asList(info));