You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by to...@apache.org on 2008/09/18 12:55:33 UTC
svn commit: r696626 - in /hadoop/core/trunk: ./ src/contrib/ec2/bin/
Author: tomwhite
Date: Thu Sep 18 03:55:32 2008
New Revision: 696626
URL: http://svn.apache.org/viewvc?rev=696626&view=rev
Log:
HADOOP-2411. Add support for larger CPU EC2 instance types. Contributed by Chris K Wensel.
Removed:
hadoop/core/trunk/src/contrib/ec2/bin/destroy-hadoop-cluster
hadoop/core/trunk/src/contrib/ec2/bin/login-hadoop-cluster
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2
hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster
hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves
hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters
Modified: hadoop/core/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Thu Sep 18 03:55:32 2008
@@ -344,6 +344,9 @@
HADOOP-4117. Improve configurability of Hadoop EC2 instances.
(tomwhite)
+ HADOOP-2411. Add support for larger CPU EC2 instance types.
+ (Chris K Wensel via tomwhite)
+
OPTIMIZATIONS
HADOOP-3556. Removed lock contention in MD5Hash by changing the
Modified: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2 (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2 Thu Sep 18 03:55:32 2008
@@ -26,7 +26,7 @@
echo " launch-cluster <group> <num slaves> launch a cluster of Hadoop EC2 instances - launch-master then launch-slaves"
echo " launch-master <group> launch or find a cluster master"
echo " launch-slaves <group> <num slaves> launch the cluster slaves"
- echo " terminate-cluster terminate all Hadoop EC2 instances"
+ echo " terminate-cluster <group> terminate all Hadoop EC2 instances"
echo " login <group|instance id> login to the master node of the Hadoop EC2 cluster"
echo " screen <group|instance id> start or attach 'screen' on the master node of the Hadoop EC2 cluster"
echo " proxy <group|instance id> start a socks proxy on localhost:6666 (use w/foxyproxy)"
Modified: hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/hadoop-ec2-env.sh.template Thu Sep 18 03:55:32 2008
@@ -40,13 +40,13 @@
SSH_OPTS=`echo -i "$PRIVATE_KEY_PATH" -o StrictHostKeyChecking=no -o ServerAliveInterval=30`
# The version of Hadoop to use.
-HADOOP_VERSION=0.17.0
+HADOOP_VERSION=0.19.0
# The Amazon S3 bucket where the Hadoop AMI is stored.
# The default value is for public images, so can be left if you are using running a public image.
# Change this value only if you are creating your own (private) AMI
# so you can store it in a bucket you own.
-S3_BUCKET=hadoop-ec2-images
+S3_BUCKET=hadoop-images
# Enable public access to JobTracker and TaskTracker web interfaces
ENABLE_WEB_PORTS=true
@@ -58,6 +58,8 @@
INSTANCE_TYPE="m1.small"
#INSTANCE_TYPE="m1.large"
#INSTANCE_TYPE="m1.xlarge"
+#INSTANCE_TYPE="c1.medium"
+#INSTANCE_TYPE="c1.xlarge"
# The EC2 group master name. CLUSTER is set by calling scripts
CLUSTER_MASTER=$CLUSTER-master
@@ -72,16 +74,28 @@
#
# The version number of the installed JDK.
-JAVA_VERSION=1.6.0_05
+JAVA_VERSION=1.6.0_07
# SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
# The download URL for the Sun JDK. Visit http://java.sun.com/javase/downloads/index.jsp and get the URL for the "Linux self-extracting file".
-if [ "$INSTANCE_TYPE" == "m1.small" ]; then
+if [ "$INSTANCE_TYPE" == "m1.small" -o "$INSTANCE_TYPE" == "c1.medium" ]; then
ARCH='i386'
- BASE_AMI_IMAGE="ami-f51aff9c" # ec2-public-images/fedora-8-i386-base-v1.06.manifest.xml
- JAVA_BINARY_URL=
+ BASE_AMI_IMAGE="ami-2b5fba42" # ec2-public-images/fedora-8-i386-base-v1.07.manifest.xml
+ JAVA_BINARY_URL=''
else
ARCH='x86_64'
- BASE_AMI_IMAGE="ami-f21aff9b" # ec2-public-images/fedora-8-x86_64-base-v1.06.manifest.xml
- JAVA_BINARY_URL=
+ BASE_AMI_IMAGE="ami-2a5fba43" # ec2-public-images/fedora-8-x86_64-base-v1.07.manifest.xml
+ JAVA_BINARY_URL=''
+fi
+
+if [ "$INSTANCE_TYPE" == "c1.medium" ]; then
+ AMI_KERNEL=aki-9b00e5f2 # ec2-public-images/vmlinuz-2.6.18-xenU-ec2-v1.0.i386.aki.manifest.xml
+fi
+
+if [ "$INSTANCE_TYPE" == "c1.xlarge" ]; then
+ AMI_KERNEL=aki-9800e5f1 # ec2-public-images/vmlinuz-2.6.18-xenU-ec2-v1.0.x86_64.aki.manifest.xml
+fi
+
+if [ "$AMI_KERNEL" != "" ]; then
+ KERNEL_ARG="--kernel ${AMI_KERNEL}"
fi
Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-cluster Thu Sep 18 03:55:32 2008
@@ -28,6 +28,7 @@
if [ -z $2 ]; then
echo "Must specify the number of slaves to start."
+ exit -1
fi
if ! "$bin"/launch-hadoop-master $1 ; then
Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-master Thu Sep 18 03:55:32 2008
@@ -81,7 +81,7 @@
# Start a master
echo "Starting master with AMI $AMI_IMAGE"
USER_DATA="MASTER_HOST=master,MAX_MAP_TASKS=$MAX_MAP_TASKS,MAX_REDUCE_TASKS=$MAX_REDUCE_TASKS,COMPRESS=$COMPRESS"
-INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f "$bin"/$USER_DATA_FILE -t $INSTANCE_TYPE | grep INSTANCE | awk '{print $2}'`
+INSTANCE=`ec2-run-instances $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k $KEY_NAME -f "$bin"/$USER_DATA_FILE -t $INSTANCE_TYPE $KERNEL_ARG | grep INSTANCE | awk '{print $2}'`
echo "Waiting for instance $INSTANCE to start"
while true; do
printf "."
Modified: hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/launch-hadoop-slaves Thu Sep 18 03:55:32 2008
@@ -24,6 +24,7 @@
if [ -z $2 ]; then
echo "Must specify the number of slaves to start."
+ exit -1
fi
CLUSTER=$1
@@ -49,6 +50,6 @@
# Start slaves
echo "Adding $1 node(s) to cluster group $CLUSTER with AMI $AMI_IMAGE"
-ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -f "$bin"/$USER_DATA_FILE.slave -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
+ec2-run-instances $AMI_IMAGE -n "$NO_INSTANCES" -g "$CLUSTER" -k "$KEY_NAME" -f "$bin"/$USER_DATA_FILE.slave -t "$INSTANCE_TYPE" -z "$MASTER_ZONE" $KERNEL_ARG | grep INSTANCE | awk '{print $2}'
rm "$bin"/$USER_DATA_FILE.slave
Modified: hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters
URL: http://svn.apache.org/viewvc/hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters?rev=696626&r1=696625&r2=696626&view=diff
==============================================================================
--- hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters (original)
+++ hadoop/core/trunk/src/contrib/ec2/bin/list-hadoop-clusters Thu Sep 18 03:55:32 2008
@@ -23,7 +23,7 @@
. "$bin"/hadoop-ec2-env.sh
# Finding Hadoop clusters
-CLUSTERS=`ec2-describe-instances | awk '"RESERVATION" == $1 {print $4}' | grep -v -e "-master$" | sort | uniq`
+CLUSTERS=`ec2-describe-instances | awk '"RESERVATION" == $1 && $4 ~ /-master$/, "INSTANCE" == $1' | tr '\n' '\t' | grep running | cut -f4 | rev | cut -d'-' -f2- | rev`
[ -z "$CLUSTERS" ] && echo "No running clusters." && exit 0