svn commit: r1132061 - in /incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2: mesos-daemon setup setup-slave

[be...@apache.org]

Author: benh
Date: Sun Jun  5 08:30:45 2011
New Revision: 1132061

URL: http://svn.apache.org/viewvc?rev=1132061&view=rev
Log:
A couple of fixes to the EC2 scripts
- Setup was not working because it used relative paths without cd'ing into mesos-ec2
- Slave setup was creating /mnt2, /mnt3 and /mnt4 directories even on nodes that don't have all the ephemeral devices, which would result in Hadoop writing too much data to the root partition (since these were never mounted to some other device)

Modified:
    incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon
    incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup
    incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave

Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon?rev=1132061&r1=1132060&r2=1132061&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/mesos-daemon Sun Jun  5 08:30:45 2011
@@ -9,8 +9,8 @@ export MESOS_PUBLIC_DNS=`wget -q -O - ht
 # Set PATH to include Scala
 export PATH=$PATH:/root/scala-2.8.0.final/bin
 
-# Set HADOOP variable to allow slaves to get executors from HDFS
-export HADOOP=/root/hadoop-0.20.2/bin/hadoop
+# Set HADOOP_HOME variable to allow slaves to get executors from HDFS
+export HADOOP_HOME=/root/hadoop-0.20.2
 
 ulimit -n 8192
 

Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup?rev=1132061&r1=1132060&r2=1132061&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup Sun Jun  5 08:30:45 2011
@@ -1,4 +1,8 @@
 #!/bin/bash
+
+# Make sure we are in the mesos-ec2 directory
+cd /root/mesos-ec2
+
 OS_NAME=$1
 DOWNLOAD_METHOD=$2
 BRANCH=$3
@@ -22,13 +26,7 @@ fi
 SCRIPTS="copy-dir
          mesos-daemon
          redeploy-mesos
-         setup-apache
          setup-slave              
-         setup-torque
-         setup-torque-server
-         start-torque-slaves
-         stop-torque-slaves
-         setup-mpi
          ssh-no-keychecking
          start-mesos
          stop-mesos"

Modified: incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave
URL: http://svn.apache.org/viewvc/incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave?rev=1132061&r1=1132060&r2=1132061&view=diff
==============================================================================
--- incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave (original)
+++ incubator/mesos/trunk/ec2/deploy.lucid64/root/mesos-ec2/setup-slave Sun Jun  5 08:30:45 2011
@@ -2,19 +2,34 @@
 
 echo "Setting up Mesos slave on `hostname`..."
 
-# Mount any ephemeral volumes we might have beyond /mnt
-mkdir -p /mnt2 /mnt3 /mnt4
-mount /dev/sdc /mnt2
-mount /dev/sdd /mnt3
-mount /dev/sde /mnt4
-for i in /mnt /mnt2 /mnt3 /mnt4; do
-  mkdir -p $i/hdfs/dfs $i/hadoop/tmp $i/hadoop/mrlocal
-done
+# Create Hadoop and HDFS directories in a given parent directory
+# (for example /mnt, /mnt2, and so on)
+function create_hadoop_dirs {
+  location=$1
+  mkdir -p $location/hdfs/dfs $location/hadoop/tmp $location/hadoop/mrlocal
+}
+
+# Set up Hadoop and Mesos directories in /mnt
+create_hadoop_dirs /mnt
 mkdir -p /mnt/hdfs-logs
 mkdir -p /mnt/hadoop-logs
 mkdir -p /mnt/mesos-logs
 mkdir -p /mnt/mesos-work
 
+# Mount any ephemeral volumes we might have beyond /mnt
+function setup_extra_volume {
+  device=$1
+  mountpoint=$2
+  if [ -e $device ]; then
+    mkdir -p $mountpoint
+    mount $device $mountpoint
+    create_hadoop_dirs $mountpoint
+  fi
+}
+setup_extra_volume /dev/sdc /mnt2
+setup_extra_volume /dev/sdd /mnt3
+setup_extra_volume /dev/sde /mnt4
+
 # Mount cgroup file system
 mkdir -p /cgroup
 mount -t cgroup cgroup /cgroup