You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2011/10/27 22:53:37 UTC
svn commit: r1190020 - in /hadoop/common/branches/branch-0.20-security-205:
CHANGES.txt src/packages/hadoop-setup-conf.sh
src/packages/templates/conf/mapred-site.xml
Author: ramya
Date: Thu Oct 27 20:53:37 2011
New Revision: 1190020
URL: http://svn.apache.org/viewvc?rev=1190020&view=rev
Log:
HADOOP-7728. Enable task memory management to be configurable in hadoop config setup script. (ramya)
Modified:
hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh
hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/mapred-site.xml
Modified: hadoop/common/branches/branch-0.20-security-205/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/CHANGES.txt?rev=1190020&r1=1190019&r2=1190020&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.20-security-205/CHANGES.txt Thu Oct 27 20:53:37 2011
@@ -9,6 +9,9 @@ Release 0.20.205.1 - unreleased
HDFS-2501. Add version prefix and root methods to webhdfs. (szetszwo)
+ HADOOP-7728. Enable task memory management to be configurable in hadoop
+ config setup script. (ramya)
+
BUG FIXES
HDFS-2441. Remove the Content-Type set by HttpServer.QuotingInputFilter in
Modified: hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh?rev=1190020&r1=1190019&r2=1190020&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh (original)
+++ hadoop/common/branches/branch-0.20-security-205/src/packages/hadoop-setup-conf.sh Thu Oct 27 20:53:37 2011
@@ -55,6 +55,16 @@ usage: $0 <parameters>
--dfs-support-append=false|true Enable append
--hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts' Setup proxy users for hadoop
--hbase-user=hbase User which hbase is running as. Defaults to hbase
+ --mapred-cluster-map-memory-mb=memory Virtual memory of a map slot for the MR framework. Defaults to -1
+ --mapred-cluster-reduce-memory-mb=memory Virtual memory, of a reduce slot for the MR framework. Defaults to -1
+ --mapred-cluster-max-map-memory-mb=memory Maximum virtual memory of a single map task. Defaults to -1
+ This value should be set to (mapred.tasktracker.map.tasks.maximum * mapred.cluster.map.memory.mb)
+ --mapred-cluster-max-reduce-memory-mb=memory maximum virtual memory of a single reduce task. Defaults to -1
+ This value should be set to (mapred.tasktracker.reduce.tasks.maximum * mapred.cluster.reduce.memory.mb)
+ --mapred-job-map-memory-mb=memory Virtual memory of a single map slot for a job. Defaults to -1
+ This value should be <= mapred.cluster.max.map.memory.mb
+ --mapred-job-reduce-memory-mb=memory Virtual memory, of a single reduce slot for a job. Defaults to -1
+ This value should be <= mapred.cluster.max.reduce.memory.mb
"
exit 1
}
@@ -199,6 +209,12 @@ OPTS=$(getopt \
-l 'hadoop-proxy-users:' \
-l 'dfs-support-append:' \
-l 'hbase-user:' \
+ -l 'mapred-cluster-map-memory-mb:' \
+ -l 'mapred-cluster-reduce-memory-mb:' \
+ -l 'mapred-cluster-max-map-memory-mb:' \
+ -l 'mapred-cluster-max-reduce-memory-mb:' \
+ -l 'mapred-job-map-memory-mb:' \
+ -l 'mapred-job-reduce-memory-mb:' \
-o 'h' \
-- "$@")
@@ -334,6 +350,30 @@ while true ; do
HBASE_USER=$2; shift 2
AUTOMATED=1
;;
+ --mapred-cluster-map-memory-mb)
+ MAPRED_CLUSTER_MAP_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapred-cluster-reduce-memory-mb)
+ MAPRED_CLUSTER_REDUCE_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapred-cluster-max-map-memory-mb)
+ MAPRED_CLUSTER_MAX_MAP_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapred-cluster-max-reduce-memory-mb)
+ MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapred-job-map-memory-mb)
+ MAPRED_JOB_MAP_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapred-job-reduce-memory-mb)
+ MAPRED_JOB_REDUCE_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
--)
shift ; break
;;
@@ -365,6 +405,12 @@ HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
HBASE_USER=${HBASE_USER:-hbase}
+MAPRED_CLUSTER_MAP_MEMORY_MB=${MAPRED_CLUSTER_MAP_MEMORY_MB:--1}
+MAPRED_CLUSTER_REDUCE_MEMORY_MB=${MAPRED_CLUSTER_REDUCE_MEMORY_MB:--1}
+MAPRED_CLUSTER_MAX_MAP_MEMORY_MB=${MAPRED_CLUSTER_MAX_MAP_MEMORY_MB:--1}
+MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB=${MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB:--1}
+MAPRED_JOB_MAP_MEMORY_MB=${MAPRED_JOB_MAP_MEMORY_MB=:--1}
+MAPRED_JOB_REDUCE_MEMORY_MB=${MAPRED_JOB_REDUCE_MEMORY_MB:--1}
KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
Modified: hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/mapred-site.xml?rev=1190020&r1=1190019&r2=1190020&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/mapred-site.xml (original)
+++ hadoop/common/branches/branch-0.20-security-205/src/packages/templates/conf/mapred-site.xml Thu Oct 27 20:53:37 2011
@@ -259,4 +259,80 @@
<name>mapred.jobtracker.retirejob.interval</name>
<value>0</value>
</property>
+
+ <property>
+ <name>mapred.cluster.map.memory.mb</name>
+ <value>${MAPRED_CLUSTER_MAP_MEMORY_MB}</value>
+ <description>The size, in terms of virtual memory, of a single map slot
+ in the Map-Reduce framework, used by the scheduler.
+ A job can ask for multiple slots for a single map task via
+ mapred.job.map.memory.mb, upto the limit specified by
+ mapred.cluster.max.map.memory.mb, if the scheduler supports the feature.
+ The value of -1 indicates that this feature is turned off.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.cluster.reduce.memory.mb</name>
+ <value>${MAPRED_CLUSTER_REDUCE_MEMORY_MB}</value>
+ <description>The size, in terms of virtual memory, of a single reduce slot
+ in the Map-Reduce framework, used by the scheduler.
+ A job can ask for multiple slots for a single reduce task via
+ mapred.job.reduce.memory.mb, upto the limit specified by
+ mapred.cluster.max.reduce.memory.mb, if the scheduler supports the feature.
+ The value of -1 indicates that this feature is turned off.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.cluster.max.map.memory.mb</name>
+ <value>${MAPRED_CLUSTER_MAX_MAP_MEMORY_MB}</value>
+ <description>The maximum size, in terms of virtual memory, of a single map
+ task launched by the Map-Reduce framework, used by the scheduler.
+ A job can ask for multiple slots for a single map task via
+ mapred.job.map.memory.mb, upto the limit specified by
+ mapred.cluster.max.map.memory.mb, if the scheduler supports the feature.
+ The value of -1 indicates that this feature is turned off.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.cluster.max.reduce.memory.mb</name>
+ <value>${MAPRED_CLUSTER_MAX_REDUCE_MEMORY_MB}</value>
+ <description>The maximum size, in terms of virtual memory, of a single reduce
+ task launched by the Map-Reduce framework, used by the scheduler.
+ A job can ask for multiple slots for a single reduce task via
+ mapred.job.reduce.memory.mb, upto the limit specified by
+ mapred.cluster.max.reduce.memory.mb, if the scheduler supports the feature.
+ The value of -1 indicates that this feature is turned off.
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.job.map.memory.mb</name>
+ <value>${MAPRED_JOB_MAP_MEMORY_MB}</value>
+ <description>The size, in terms of virtual memory, of a single map task
+ for the job.
+ A job can ask for multiple slots for a single map task, rounded up to the
+ next multiple of mapred.cluster.map.memory.mb and upto the limit
+ specified by mapred.cluster.max.map.memory.mb, if the scheduler supports
+ the feature.
+ The value of -1 indicates that this feature is turned off iff
+ mapred.cluster.map.memory.mb is also turned off (-1).
+ </description>
+ </property>
+
+ <property>
+ <name>mapred.job.reduce.memory.mb</name>
+ <value>${MAPRED_JOB_REDUCE_MEMORY_MB}</value>
+ <description>The size, in terms of virtual memory, of a single reduce task
+ for the job.
+ A job can ask for multiple slots for a single map task, rounded up to the
+ next multiple of mapred.cluster.reduce.memory.mb and upto the limit
+ specified by mapred.cluster.max.reduce.memory.mb, if the scheduler supports
+ the feature.
+ The value of -1 indicates that this feature is turned off iff
+ mapred.cluster.reduce.memory.mb is also turned off (-1).
+ </description>
+ </property>
</configuration>