You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 11:03:25 UTC
svn commit: r1132222 - in /incubator/mesos/trunk/ec2:
deploy.centos64/root/ephemeral-hdfs/conf/ deploy.centos64/root/mesos-ec2/
deploy.centos64/root/mesos-ec2/hadoop-framework-conf/
deploy.lucid64/root/mesos-ec2/
Author: benh
Date: Sun Jun 5 09:03:25 2011
New Revision: 1132222
URL: http://svn.apache.org/viewvc?rev=1132222&view=rev
Log:
A few EC2 fixes:
- Decreased some of the default settings for Hadoop on EC2 to work better
on smaller nodes.
- Increased timeout in start-mesos to make sure master is up before
slaves try to connect to it.
- Fixed a potential infinite retry cycle in Doug's code that would
happen if a node was unavailable (e.g. was terminated in ElasticFox)
Modified:
incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup
incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos
Modified: incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh?rev=1132222&r1=1132221&r2=1132222&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh (original)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/ephemeral-hdfs/conf/hadoop-env.sh Sun Jun 5 09:03:25 2011
@@ -12,7 +12,7 @@ export JAVA_HOME=/usr/java/default
# export HADOOP_CLASSPATH=
# The maximum amount of heap to use, in MB. Default is 1000.
-export HADOOP_HEAPSIZE=1500
+export HADOOP_HEAPSIZE=1000
# Extra Java runtime options. Empty by default.
# export HADOOP_OPTS=-server
Modified: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh?rev=1132222&r1=1132221&r2=1132222&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh (original)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh Sun Jun 5 09:03:25 2011
@@ -12,7 +12,7 @@ export JAVA_HOME=/usr/java/default
# export HADOOP_CLASSPATH=
# The maximum amount of heap to use, in MB. Default is 1000.
-export HADOOP_HEAPSIZE=1500
+export HADOOP_HEAPSIZE=1000
# Extra Java runtime options. Empty by default.
# export HADOOP_OPTS=-server
Modified: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml?rev=1132222&r1=1132221&r2=1132222&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml (original)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml Sun Jun 5 09:03:25 2011
@@ -37,12 +37,17 @@
<property>
<name>tasktracker.http.threads</name>
- <value>50</value>
+ <value>40</value>
</property>
<property>
<name>mapred.child.java.opts</name>
- <value>-Xmx1512m</value>
+ <value>-Xmx500m</value>
+ </property>
+
+ <property>
+ <name>mapred.mesos.task.mem</name>
+ <value>500</value>
</property>
<property>
@@ -57,7 +62,7 @@
<property>
<name>io.sort.mb</name>
- <value>100</value>
+ <value>150</value>
</property>
<property>
@@ -75,12 +80,4 @@
<value>8</value>
</property>
- <property>
- <name>io.sort.mb</name>
- <value>200</value>
- <description>The total amount of buffer memory to use while sorting
- files, in megabytes. By default, gives each merge stream 1MB, which
- should minimize seeks.</description>
- </property>
-
</configuration>
Modified: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup?rev=1132222&r1=1132221&r2=1132222&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup (original)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/setup Sun Jun 5 09:03:25 2011
@@ -82,9 +82,12 @@ if [[ $NUM_ZOOS != 0 ]] ; then
done
fi
-TODO="$SLAVES"
+# Try to SSH to each slave to approve their key. Since some slaves may be
+# slow in starting, we retry failed slaves up to 3 times.
+TODO="$SLAVES" # List of slaves to try (initially all of them)
+TRIES="0" # Number of times we've tried so far
echo "SSH'ing to slaves to approve keys..."
-while [ "e$TODO" != "e" ] ; do
+while [ "e$TODO" != "e" ] && [ $TRIES -lt 4 ] ; do
NEW_TODO=
for slave in $TODO; do
echo $slave
@@ -93,8 +96,9 @@ while [ "e$TODO" != "e" ] ; do
NEW_TODO="$NEW_TODO $slave"
fi
done
- if [ "e$NEW_TODO" != "e" ] ; then
- sleep 10
+ TRIES=$[$TRIES + 1]
+ if [ "e$NEW_TODO" != "e" ] && [ $TRIES -lt 4 ] ; then
+ sleep 15
TODO="$NEW_TODO"
echo "Re-attempting SSH to slaves to approve keys..."
else
Modified: incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos?rev=1132222&r1=1132221&r2=1132222&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos (original)
+++ incubator/mesos/trunk/ec2/deploy.centos64/root/mesos-ec2/start-mesos Sun Jun 5 09:03:25 2011
@@ -32,7 +32,7 @@ else
ssh $SSH_OPTS $ACTIVE_MASTER "/root/mesos-ec2/mesos-daemon mesos-master -p 5050 $@ </dev/null >/dev/null"
fi
-sleep 2
+sleep 5
for slave in $SLAVES; do
echo "Starting slave on $slave"
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos?rev=1132222&r1=1132221&r2=1132222&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos Sun Jun 5 09:03:25 2011
@@ -32,7 +32,7 @@ else
ssh $SSH_OPTS $ACTIVE_MASTER "/root/mesos-ec2/mesos-daemon mesos-master -p 5050 $@ </dev/null >/dev/null"
fi
-sleep 2
+sleep 5
for slave in $SLAVES; do
echo "Starting slave on $slave"