You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mesos.apache.org by be...@apache.org on 2011/06/05 10:26:09 UTC
svn commit: r1132025 - in /incubator/mesos/trunk/ec2: ./
deploy.karmic64/root/hadoop-0.20.2/conf/ deploy.karmic64/root/mesos-ec2/
deploy.karmic64/root/mesos-ec2/hadoop-framework-conf/
deploy.karmic64/root/mesos-ec2/haproxy+apache/ deploy.lucid64/root/m...
Author: benh
Date: Sun Jun 5 08:26:07 2011
New Revision: 1132025
URL: http://svn.apache.org/viewvc?rev=1132025&view=rev
Log:
Updated EC2 scripts to work with new Mesos AMI.
Removed:
incubator/mesos/trunk/ec2/deploy.karmic64/root/hadoop-0.20.2/conf/core-site.xml
incubator/mesos/trunk/ec2/deploy.karmic64/root/hadoop-0.20.2/conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.karmic64/root/hadoop-0.20.2/conf/hdfs-site.xml
incubator/mesos/trunk/ec2/deploy.karmic64/root/hadoop-0.20.2/conf/masters
incubator/mesos/trunk/ec2/deploy.karmic64/root/hadoop-0.20.2/conf/slaves
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/copy-dir
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/epilogue.kill-mpi-ring
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/hadoop-framework-conf/core-site.xml
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/haproxy+apache/haproxy.config.template
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/master
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/mesos-daemon
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/prologue.setup-mpi-master
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/redeploy-mesos
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/runaway-kill
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/runaway-killall
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/setup
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/setup-apache
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/setup-mpi
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/setup-slave
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/setup-torque
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/setup-torque-server
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/slaves
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/ssh-no-keychecking
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/start-mesos
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/start-mos
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/start-torque-slaves
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/stop-mesos
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/stop-mos
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/stop-tasktracker-child
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/stop-torque-slaves
incubator/mesos/trunk/ec2/deploy.karmic64/root/mesos-ec2/zoo
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/runaway-kill
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/runaway-killall
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-apache
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mos
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/stop-mos
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/stop-tasktracker-child
incubator/mesos/trunk/ec2/deploy.solaris/root/hadoop-0.20.0/conf/core-site.xml
incubator/mesos/trunk/ec2/deploy.solaris/root/hadoop-0.20.0/conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.solaris/root/hadoop-0.20.0/conf/hdfs-site.xml
incubator/mesos/trunk/ec2/deploy.solaris/root/hadoop-0.20.0/conf/masters
incubator/mesos/trunk/ec2/deploy.solaris/root/hadoop-0.20.0/conf/slaves
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/hadoop-framework-conf/core-site.xml
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/hadoop-framework-conf/hadoop-env.sh
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/haproxy+apache/haproxy.config.template
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/master
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/mesos-daemon
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/redeploy-mesos
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/runaway-kill
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/runaway-killall
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/setup
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/setup-apache
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/setup-slave
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/slaves
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/start-mesos
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/start-mos
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/stop-mesos
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/stop-mos
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/stop-tasktracker-child
incubator/mesos/trunk/ec2/deploy.solaris/root/mesos-ec2/zoo
incubator/mesos/trunk/ec2/launch-torque-fw.sh
Modified:
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir (contents, props changed)
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/redeploy-mesos
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup (contents, props changed)
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/ssh-no-keychecking (contents, props changed)
incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos
incubator/mesos/trunk/ec2/mesos_ec2.py
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
(empty)
Propchange: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/copy-dir
------------------------------------------------------------------------------
svn:executable = *
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/hadoop-framework-conf/mapred-site.xml Sun Jun 5 08:26:07 2011
@@ -6,8 +6,13 @@
<configuration>
<property>
- <name>mapred.use.mesos</name>
- <value>true</value>
+ <name>mapred.job.tracker</name>
+ <value>{{master}}:9001</value>
+ </property>
+
+ <property>
+ <name>mapred.jobtracker.taskScheduler</name>
+ <value>org.apache.hadoop.mapred.MesosScheduler</value>
</property>
<property>
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon Sun Jun 5 08:26:07 2011
@@ -7,27 +7,24 @@ export MESOS_HOME=/root/mesos
export MESOS_PUBLIC_DNS=`wget -q -O - http://instance-data.ec2.internal/latest/meta-data/public-hostname`
# Set PATH to include Scala
-export PATH=$PATH:/root/scala-2.8.0.RC3/bin
+export PATH=$PATH:/root/scala-2.8.0.final/bin
# Set HADOOP variable to allow slaves to get executors from HDFS
export HADOOP=/root/hadoop-0.20.2/bin/hadoop
ulimit -n 8192
-export LD_PRELOAD_32=/usr/lib/extendedFILE.so.1
-
-export GOOGLE_LOG_DIR=/mnt
PROGRAM=$1
shift
EXTRA_OPTS=""
if [ "$PROGRAM" == "mesos-slave" ]; then
- # Compute CPU and memory resources on this machine (TODO: Solaris-specific)
+ # Compute CPU and memory resources on this machine (TODO: Linux-specific)
CPUS=`grep processor /proc/cpuinfo | wc -l`
MEM_KB=`cat /proc/meminfo | grep MemTotal | awk '{print $2}'`
- MEM=$[(MEM_KB - 1024 * 1024) * 1024]
- EXTRA_OPTS="--cpus $CPUS --mem $MEM --isolation process"
+ MEM=$[(MEM_KB - 1024 * 1024) / 1024]
+ EXTRA_OPTS="--cpus=$CPUS --mem=$MEM --isolation=process"
fi
-cd $MESOS_HOME/src
-nohup ./$PROGRAM $EXTRA_OPTS $@ </dev/null >/mnt/$PROGRAM.out 2>&1 &
+cd $MESOS_HOME/bin
+nohup ./$PROGRAM --log_dir=/mnt/mesos-logs --work_dir=/mnt/mesos-work $EXTRA_OPTS $@ </dev/null >/mnt/$PROGRAM.out 2>&1 &
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/redeploy-mesos
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/redeploy-mesos?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/redeploy-mesos (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/redeploy-mesos Sun Jun 5 08:26:07 2011
@@ -8,5 +8,5 @@ SSH_OPTS="-o StrictHostKeyChecking=no -o
echo "RSYNC'ing /root/mesos to slaves..."
for slave in `cat $SLAVES`; do
echo $slave
- rsync -e "ssh $SSH_OPTS" -az --exclude '*.d' --exclude '*.o' --exclude '*.cpp' --exclude '*.hpp' --exclude '*.pyc' --exclude 'mesos/frameworks/hadoop-0.20.0/logs/*' /root/mesos $slave:/root
+ rsync -e "ssh $SSH_OPTS" -az --exclude '*.d' --exclude '*.o' --exclude '*.cpp' --exclude '*.hpp' --exclude '*.pyc' --exclude 'mesos/frameworks/hadoop-0.20.0/logs/*' --exclude 'mesos/work' --exclude 'mesos/logs' --exclude 'mesos/test_output' /root/mesos $slave:/root
done
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup Sun Jun 5 08:26:07 2011
@@ -19,14 +19,10 @@ SCRIPTS="copy-dir
setup-torque-server
start-torque-slaves
stop-torque-slaves
- prologue.setup-mpi-master
setup-mpi
ssh-no-keychecking
- start-mos
start-mesos
- stop-mos
- stop-mesos
- stop-tasktracker-child"
+ stop-mesos"
HADOOP_HOME=/root/hadoop-0.20.2
@@ -87,7 +83,7 @@ done
DOWNLOADED=0
if [[ "$DOWNLOAD_METHOD" == "git" ]] ; then
- REPOSITORY=git@github.com:mesosproject/mesos.git
+ REPOSITORY=git@github.com:mesos/mesos.git
echo "Checking out Mesos from $REPOSITORY"
pushd /root > /dev/null 2>&1
rm -rf mesos mesos.tgz
@@ -98,14 +94,6 @@ if [[ "$DOWNLOAD_METHOD" == "git" ]] ; t
popd > /dev/null 2>&1
popd > /dev/null 2>&1
DOWNLOADED=1
-elif [[ "$DOWNLOAD_METHOD" == "web" ]] ; then
- echo "Downloading latest Mesos snapshot from mesos.berkeley.edu"
- pushd /root > /dev/null 2>&1
- rm -rf mesos mesos.tgz
- wget http://mesos:mesos@mesos.berkeley.edu/releases/mesos.tgz
- tar xzf mesos.tgz
- popd > /dev/null 2>&1
- DOWNLOADED=1
fi
@@ -113,7 +101,7 @@ fi
if [[ "$DOWNLOADED" == "1" ]] ; then
echo "Building Mesos..."
pushd /root/mesos > /dev/null 2>&1
- ./configure.OURS.$OS_NAME
+ ./configure.template.ubuntu-lucid-64
make clean
make
popd > /dev/null 2>&1
@@ -124,14 +112,14 @@ if [[ "$DOWNLOADED" == "1" ]] ; then
popd > /dev/null 2>&1
fi
echo "Building Hadoop framework..."
- pushd /root/mesos/frameworks/hadoop-0.20.0 > /dev/null 2>&1
+ pushd /root/mesos/frameworks/hadoop-0.20.2 > /dev/null 2>&1
ant
ant examples
popd > /dev/null 2>&1
fi
echo "Setting up Hadoop framework config files..."
-cp hadoop-framework-conf/* /root/mesos/frameworks/hadoop-0.20.0/conf
+cp hadoop-framework-conf/* /root/mesos/frameworks/hadoop-0.20.2/conf
echo "Setting up haproxy+apache framework config files..."
cp haproxy+apache/* /root/mesos/frameworks/haproxy+apache
@@ -164,11 +152,8 @@ $HADOOP_HOME/bin/hadoop namenode -format
echo "Starting HDFS..."
$HADOOP_HOME/bin/start-dfs.sh
-echo "Starting MapOutputServer(s)..."
-./start-mos
-
-echo "Setting up torque"
-./setup-torque
+#echo "Setting up torque"
+#./setup-torque
echo "Starting Mesos cluster..."
./start-mesos
Propchange: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup
------------------------------------------------------------------------------
svn:executable = *
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave Sun Jun 5 08:26:07 2011
@@ -12,6 +12,8 @@ for i in /mnt /mnt2 /mnt3 /mnt4; do
done
mkdir -p /mnt/hdfs-logs
mkdir -p /mnt/hadoop-logs
+mkdir -p /mnt/mesos-logs
+mkdir -p /mnt/mesos-work
# Mount cgroup file system
mkdir -p /cgroup
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/ssh-no-keychecking
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/ssh-no-keychecking?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
(empty)
Propchange: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/ssh-no-keychecking
------------------------------------------------------------------------------
svn:executable = *
Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/start-mesos Sun Jun 5 08:26:07 2011
@@ -7,11 +7,11 @@ SLAVES="`cat slaves`"
SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=2"
echo "Starting master on $MASTER"
-ssh $SSH_OPTS $MASTER "/root/mesos-ec2/mesos-daemon mesos-master -p 1111 </dev/null >/dev/null"
+ssh $SSH_OPTS $MASTER "/root/mesos-ec2/mesos-daemon mesos-master </dev/null >/dev/null"
for slave in $SLAVES; do
echo "Starting slave on $slave"
- ssh $SSH_OPTS $slave "/root/mesos-ec2/mesos-daemon mesos-slave 1@$MASTER:1111 </dev/null >/dev/null" &
+ ssh $SSH_OPTS $slave "/root/mesos-ec2/mesos-daemon mesos-slave -u 1@$MASTER:5050 </dev/null >/dev/null" &
sleep 0.1
done
wait
Modified: incubator/mesos/trunk/ec2/mesos_ec2.py
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/mesos_ec2.py?rev=1132025&r1=1132024&r2=1132025&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/mesos_ec2.py (original)
+++ incubator/mesos/trunk/ec2/mesos_ec2.py Sun Jun 5 08:26:07 2011
@@ -32,16 +32,15 @@ def parse_args():
help="Master instance type (leave empty for same as instance-type)")
parser.add_option("-z", "--zone", default="us-east-1b",
help="Availability zone to launch instances in")
- parser.add_option("-a", "--ami", default="ami-f2e20a9b", # mesos-lucid-0.4
+ parser.add_option("-a", "--ami", default="ami-f8806a91",
help="Amazon Machine Image ID to use")
parser.add_option("-o", "--os", default="lucid64",
- help="OS on the Amazon Machine Image (lucid6 | karmic64 | solaris)")
+ help="OS on the Amazon Machine Image (lucid64 for now)")
parser.add_option("-d", "--download", metavar="SOURCE", default="none",
help="Where to download latest code from: set to 'git' to check out " +
- "from git, 'web' for latest snapshot .tgz, or 'none' to use " +
- "the build of Mesos on the AMI (default)")
+ "from git, or 'none' to use the Mesos on the AMI (default)")
parser.add_option("-b", "--branch", default="master",
- help="if using git, which branch to check out. default is 'master'")
+ help="If using git, which branch to check out. Default is 'master'")
parser.add_option("-D", metavar="[ADDRESS:]PORT", dest="proxy_port",
help="Use SSH dynamic port forwarding to create a SOCKS proxy at " +
"the given local address (for use with login)")
@@ -50,7 +49,7 @@ def parse_args():
"(for debugging)")
parser.add_option("-f", "--ft", default="1",
help="Number of masters to run. Default is 1. " +
- "Greater values cause Mesos to run in FT mode with ZooKeeper")
+ "Greater values cause Mesos to run in FT mode with ZooKeeper.")
(opts, args) = parser.parse_args()
opts.ft = int(opts.ft)
if len(args) != 2: