You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 10:41:08 UTC
svn commit: r1132099 - in /incubator/mesos/trunk/ec2/deploy.lucid64/root:
hadoop-0.20.2/conf/hadoop-env.sh mesos-ec2/copy-dir
mesos-ec2/hadoop-framework-conf/hadoop-env.sh
mesos-ec2/hadoop-framework-conf/mapred-site.xml
Author: benh
Date: Sun Jun 5 08:41:08 2011
New Revision: 1132099
URL: http://svn.apache.org/viewvc?rev=1132099&view=rev
Log:
Misc fixes (SSH timeouts and Hadoop memory)
Modified:
incubator/mesos/trunk/ec2/deploy.lucid64/root/hadoop-0.20.2/conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/hadoop-0.20.2/conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/hadoop-0.20.2/conf/hadoop-env.sh?rev=1132099&r1=1132098&r2=1132099&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/hadoop-0.20.2/conf/hadoop-env.sh (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/hadoop-0.20.2/conf/hadoop-env.sh Sun Jun 5 08:41:08 2011
@@ -12,7 +12,7 @@ export JAVA_HOME=/usr/lib/jvm/java-6-ope
# export HADOOP_CLASSPATH=
# The maximum amount of heap to use, in MB. Default is 1000.
-# export HADOOP_HEAPSIZE=2000
+export HADOOP_HEAPSIZE=1500
# Extra Java runtime options. Empty by default.
# export HADOOP_OPTS=-server
@@ -30,7 +30,7 @@ export HADOOP_JOBTRACKER_OPTS="-Dcom.sun
# Extra ssh options. Empty by default.
# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR"
-export HADOOP_SSH_OPTS="-o ConnectTimeout=2"
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5"
# Where log files are stored. $HADOOP_HOME/logs by default.
# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir?rev=1132099&r1=1132098&r2=1132099&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir Sun Jun 5 08:41:08 2011
@@ -11,11 +11,11 @@ DEST=`dirname "$DIR"`
SLAVES=`cat /root/mesos-ec2/slaves`
-SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=2"
+SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=5"
echo "RSYNC'ing $DIR to slaves..."
for slave in $SLAVES; do
echo $slave
- rsync -e "ssh $SSH_OPTS" -az "$DIR" "$slave:$DEST" & sleep 0.3
+ rsync -e "ssh $SSH_OPTS" -az "$DIR" "$slave:$DEST" & sleep 0.5
done
wait
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh?rev=1132099&r1=1132098&r2=1132099&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh Sun Jun 5 08:41:08 2011
@@ -12,7 +12,7 @@ export JAVA_HOME=/usr/lib/jvm/java-6-ope
# export HADOOP_CLASSPATH=
# The maximum amount of heap to use, in MB. Default is 1000.
-# export HADOOP_HEAPSIZE=2000
+export HADOOP_HEAPSIZE=1500
# Extra Java runtime options. Empty by default.
# export HADOOP_OPTS=-server
@@ -30,7 +30,7 @@ export HADOOP_JOBTRACKER_OPTS="-Dcom.sun
# Extra ssh options. Empty by default.
# export HADOOP_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HADOOP_CONF_DIR"
-export HADOOP_SSH_OPTS="-o ConnectTimeout=2"
+export HADOOP_SSH_OPTS="-o ConnectTimeout=5"
# Where log files are stored. $HADOOP_HOME/logs by default.
# export HADOOP_LOG_DIR=${HADOOP_HOME}/logs
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml?rev=1132099&r1=1132098&r2=1132099&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml Sun Jun 5 08:41:08 2011
@@ -36,8 +36,13 @@
</property>
<property>
+ <name>tasktracker.http.threads</name>
+ <value>50</value>
+ </property>
+
+ <property>
<name>mapred.child.java.opts</name>
- <value>-Xmx512m</value>
+ <value>-Xmx1512m</value>
</property>
<property>
@@ -70,4 +75,12 @@
<value>6</value>
</property>
+ <property>
+ <name>io.sort.mb</name>
+ <value>200</value>
+ <description>The total amount of buffer memory to use while sorting
+ files, in megabytes. By default, gives each merge stream 1MB, which
+ should minimize seeks.</description>
+ </property>
+
</configuration>