You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ra...@apache.org on 2011/10/27 23:31:37 UTC
svn commit: r1190044 - in
/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common:
CHANGES.txt src/main/packages/hadoop-setup-conf.sh
src/main/packages/templates/conf/mapred-site.xml
Author: ramya
Date: Thu Oct 27 21:31:37 2011
New Revision: 1190044
URL: http://svn.apache.org/viewvc?rev=1190044&view=rev
Log:
HADOOP-7728. Enable task memory management to be configurable in hadoop config setup script. (ramya)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1190044&r1=1190043&r2=1190044&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Thu Oct 27 21:31:37 2011
@@ -68,6 +68,9 @@ Release 0.23.0 - Unreleased
IMPROVEMENTS
+ HADOOP-7728. Enable task memory management to be configurable in hadoop
+ config setup script. (ramya)
+
HADOOP-7655. Provide a small validation script that smoke tests the installed
cluster. (Arpit Gupta via mattf)
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh?rev=1190044&r1=1190043&r2=1190044&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/hadoop-setup-conf.sh Thu Oct 27 21:31:37 2011
@@ -55,6 +55,16 @@ usage: $0 <parameters>
--dfs-support-append=false|true Enable append
--hadoop-proxy-users='user1:groups:hosts;user2:groups:hosts' Setup proxy users for hadoop
--hbase-user=hbase User which hbase is running as. Defaults to hbase
+ --mapreduce-cluster-mapmemory-mb=memory Virtual memory of a map slot for the MR framework. Defaults to -1
+ --mapreduce-cluster-reducememory-mb=memory Virtual memory, of a reduce slot for the MR framework. Defaults to -1
+ --mapreduce-jobtracker-maxmapmemory-mb=memory Maximum virtual memory of a single map task. Defaults to -1
+ This value should be set to (mapreduce.cluster.mapmemory.mb * mapreduce.tasktracker.map.tasks.maximum)
+ --mapreduce-jobtracker-maxreducememory-mb=memory Maximum virtual memory of a single reduce task. Defaults to -1
+ This value should be set to (mapreduce.cluster.reducememory.mb * mapreduce.tasktracker.reduce.tasks.maximum)
+ --mapreduce-map-memory-mb=memory Virtual memory of a single map slot for a job. Defaults to -1
+ This value should be <= mapred.cluster.max.map.memory.mb
+ --mapreduce-reduce-memory-mb=memory Virtual memory, of a single reduce slot for a job. Defaults to -1
+ This value should be <= mapred.cluster.max.reduce.memory.mb
"
exit 1
}
@@ -199,6 +209,12 @@ OPTS=$(getopt \
-l 'hadoop-proxy-users:' \
-l 'dfs-support-append:' \
-l 'hbase-user:' \
+ -l 'mapreduce-cluster-mapmemory-mb:' \
+ -l 'mapreduce-cluster-reducememory-mb:' \
+ -l 'mapreduce-jobtracker-maxmapmemory-mb:' \
+ -l 'mapreduce-jobtracker-maxreducememory-mb:' \
+ -l 'mapreduce-map-memory-mb:' \
+ -l 'mapreduce-reduce-memory-mb:' \
-o 'h' \
-- "$@")
@@ -334,6 +350,30 @@ while true ; do
HBASE_USER=$2; shift 2
AUTOMATED=1
;;
+ --mapreduce-cluster-mapmemory-mb)
+ MAPREDUCE_CLUSTER_MAPMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-cluster-reducememory-mb)
+ MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-jobtracker-maxmapmemory-mb)
+ MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-jobtracker-maxreducememory-mb)
+ MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-map-memory-mb)
+ MAPREDUCE_MAP_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
+ --mapreduce-reduce-memory-mb)
+ MAPREDUCE_REDUCE_MEMORY_MB=$2; shift 2
+ AUTOMATED=1
+ ;;
--)
shift ; break
;;
@@ -365,6 +405,12 @@ HADOOP_MR_USER=${HADOOP_MR_USER:-mr}
DFS_WEBHDFS_ENABLED=${DFS_WEBHDFS_ENABLED:-false}
DFS_SUPPORT_APPEND=${DFS_SUPPORT_APPEND:-false}
HBASE_USER=${HBASE_USER:-hbase}
+MAPREDUCE_CLUSTER_MAPMEMORY_MB=${MAPREDUCE_CLUSTER_MAPMEMORY_MB:--1}
+MAPREDUCE_CLUSTER_REDUCEMEMORY_MB=${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB:--1}
+MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB=${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB:--1}
+MAPREDUCE_MAP_MEMORY_MB=${MAPREDUCE_MAP_MEMORY_MB:--1}
+MAPREDUCE_REDUCE_MEMORY_MB=${MAPREDUCE_REDUCE_MEMORY_MB:--1}
KEYTAB_DIR=${KEYTAB_DIR:-/etc/security/keytabs}
HDFS_KEYTAB=${HDFS_KEYTAB:-/home/hdfs/hdfs.keytab}
MR_KEYTAB=${MR_KEYTAB:-/home/mr/mr.keytab}
Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml?rev=1190044&r1=1190043&r2=1190044&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/packages/templates/conf/mapred-site.xml Thu Oct 27 21:31:37 2011
@@ -272,4 +272,34 @@
<name>mapred.jobtracker.retirejob.interval</name>
<value>0</value>
</property>
+
+ <property>
+ <name>mapreduce.cluster.mapmemory.mb</name>
+ <value>${MAPREDUCE_CLUSTER_MAPMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.cluster.reducememory.mb</name>
+ <value>${MAPREDUCE_CLUSTER_REDUCEMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.maxmapmemory.mb</name>
+ <value>${MAPREDUCE_JOBTRACKER_MAXMAPMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.maxreducememory.mb</name>
+ <value>${MAPREDUCE_JOBTRACKER_MAXREDUCEMEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.map.memory.mb</name>
+ <value>${MAPREDUCE_MAP_MEMORY_MB}</value>
+ </property>
+
+ <property>
+ <name>mapreduce.reduce.memory.mb</name>
+ <value>${MAPREDUCE_REDUCE_MEMORY_MB}</value>
+ </property>
</configuration>