You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2010/03/27 21:13:21 UTC

svn commit: r928258 - in /hadoop/hbase/branches/0.20/src/contrib/ec2/bin: ./ image/

Author: apurtell
Date: Sat Mar 27 20:13:20 2010
New Revision: 928258

URL: http://svn.apache.org/viewvc?rev=928258&view=rev
Log:
[EC2] consider multiple regions in config; added 'reboot-hbase-cluster

Added:
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster   (with props)
Modified:
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
    hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster Sat Mar 27 20:13:20 2010
@@ -26,6 +26,7 @@ fi
 # get arguments
 COMMAND="$1"
 shift
+
 # get group
 CLUSTER="$1"
 shift
@@ -44,8 +45,8 @@ if [[ "$CLUSTER" = "i-*" ]]; then
   HOST=`ec2-describe-instances $TOOL_OPTS $CLUSTER | grep running | awk '{print $4}'`
   [ -z $HOST ] && echo "Instance still pending or no longer running: $CLUSTER" && exit 1
 else
-  [ ! -f $MASTER_IP_PATH ] && echo "Wrong group name, or cluster not launched! $CLUSTER" && exit 1
-  HOST=`cat $MASTER_IP_PATH`
+  [ ! -f $MASTER_ADDR_PATH ] && echo "Wrong group name, or cluster not launched! $CLUSTER" && exit 1
+  HOST=`cat $MASTER_ADDR_PATH`
 fi
 
 if [ "$COMMAND" = "login" ] ; then
@@ -64,6 +65,5 @@ elif [ "$COMMAND" = "screen" ] ; then
   echo "Logging in and attaching screen on host $HOST."
   ssh $SSH_OPTS -t "root@$HOST" 'screen -D -R'
 else
-  echo "Executing command on host $HOST."
-  ssh $SSH_OPTS -t "root@$HOST" "$COMMAND"
-fi
\ No newline at end of file
+  ssh $SSH_OPTS -t "root@$HOST" "$COMMAND $@"
+fi

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster Sat Mar 27 20:13:20 2010
@@ -41,7 +41,5 @@ fi
 
 "$bin"/revoke-hbase-cluster-secgroups $CLUSTER
 
-rm -f $MASTER_IP_PATH
-rm -f $MASTER_PRIVATE_IP_PATH
-rm -f $MASTER_ZONE_PATH
-rm -f $ZOOKEEPER_QUORUM_PATH
+rm -f $ZOOKEEPER_ADDR_PATH $ZOOKEEPER_QUORUM_PATH
+rm -f $MASTER_IP_PATH $MASTER_ADDR_PATH $MASTER_ZONE_PATH

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh Sat Mar 27 20:13:20 2010
@@ -1,13 +1,9 @@
-#
-# Copyright 2010 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
 #
 #     http://www.apache.org/licenses/LICENSE-2.0
 #
@@ -16,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
 
 function getCredentialSetting {
   name=$1

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2 (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2 Sat Mar 27 20:13:20 2010
@@ -28,6 +28,7 @@ if [ $# = 0 ]; then
   echo "  launch-master  <name>                 launch or find a cluster master"
   echo "  launch-slaves  <name> <slaves>        launch the cluster slaves"
   echo "  terminate-cluster  <name>             terminate all HBase EC2 instances"
+  echo "  reboot-cluster  <name>                reboot all HBase EC2 instances"
   echo "  delete-cluster <name>                 clean up after a terminated cluster"
   echo "  login  <name|instance id>             login to the master node"
   echo "  screen <name|instance id>             start or attach 'screen' on the master"
@@ -56,6 +57,8 @@ elif [ "$COMMAND" = "delete-cluster" ] ;
   . "$bin"/delete-hbase-cluster $*
 elif [ "$COMMAND" = "terminate-cluster" ] ; then
   . "$bin"/terminate-hbase-cluster $*
+elif [ "$COMMAND" = "reboot-cluster" ] ; then
+  . "$bin"/reboot-hbase-cluster $*
 elif [ "$COMMAND" = "list" ] ; then
   . "$bin"/list-hbase-clusters
 else

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh Sat Mar 27 20:13:20 2010
@@ -46,14 +46,18 @@ HADOOP_URL=http://hbase.s3.amazonaws.com
 LZO_URL=http://hbase.s3.amazonaws.com/hadoop/lzo-linux-$HADOOP_VERSION.tar.gz
 
 # The Amazon S3 bucket where the HBase AMI is stored.
-# Change this value only if you are creating your own (private) AMI
+# Change this value if you are launching instances in regions other than 
+# us-east-1 (the default).
+# Change this value otherwise only if you are creating your own (private) AMI
 # so you can store it in a bucket you own.
-S3_BUCKET=apache-hbase-images
+S3_BUCKET=apache-hbase-images            # us-east-1
+#S3_BUCKET=apache-hbase-images-eu         # eu-west-1
+#S3_BUCKET=apache-hbase-images-us-west-1  # us-west-1
 
 # Enable public access web interfaces
 ENABLE_WEB_PORTS=false
 
-# Enable mapping of elastic IPs to ZK and master instances
+# Enable use of elastic IPs for ZK and master instances
 ENABLE_ELASTIC_IPS=false
 
 # Extra packages
@@ -62,11 +66,11 @@ ENABLE_ELASTIC_IPS=false
 # The repository descriptor will be fetched into /etc/yum/repos.d.
 EXTRA_PACKAGES=
 
-# Use only m1.large or c1.xlarge unless you know what you are doing
-MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-m1.large}
+# Use only c1.xlarge unless you know what you are doing
+MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
 
 # Use only m1.large or c1.xlarge unless you know what you are doing
-SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-m1.large}
+SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-c1.xlarge}
 
 # Use only m1.small or c1.medium unless you know what you are doing
 ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-m1.small}
@@ -107,6 +111,9 @@ CLUSTER_ZOOKEEPER=$CLUSTER-zk
 ZOOKEEPER_QUORUM_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-quorum
 ZOOKEEPER_ADDR_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-addrs
 
+# Instances path
+INSTANCES_PATH=$HOME/.hbase-${CLUSTER}-instances
+
 # The script to run on instance boot.
 USER_DATA_FILE=hbase-ec2-init-remote.sh
 
@@ -116,12 +123,18 @@ JAVA_VERSION=1.6.0_17
 JAVA_URL=http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-@arch@.bin
 
 # SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
+# Change the BASE_AMI_IMAGE setting if you are creating custom AMI in a region
+# other than us-east-1 (the default).
 if [ "$SLAVE_INSTANCE_TYPE" = "m1.small" -o "$SLAVE_INSTANCE_TYPE" = "c1.medium" ]; then
   SLAVE_ARCH='i386'
-  BASE_AMI_IMAGE="ami-48aa4921"  # ec2-public-images/fedora-8-i386-base-v1.10.manifest.xml
+  BASE_AMI_IMAGE="ami-48aa4921"  # us-east-1 ec2-public-images/fedora-8-i386-base-v1.10.manifest.xml
+  #BASE_AMI_IMAGE="ami-810657c4"  # us-west-1
+  #BASE_AMI_IMAGE="ami-0a48637e"  # eu-west-1
 else
   SLAVE_ARCH='x86_64'
-  BASE_AMI_IMAGE="ami-f61dfd9f"  # ec2-public-images/fedora-8-x86_64-base-v1.10.manifest.xml
+  BASE_AMI_IMAGE="ami-f61dfd9f"  # us-east-1 ec2-public-images/fedora-8-x86_64-base-v1.10.manifest.xml
+  #BASE_AMI_IMAGE="ami-870657c2"  # us-west-1
+  #BASE_AMI_IMAGE="ami-927a51e6"  # eu-west-1
 fi
 
 if [ "$MASTER_INSTANCE_TYPE" = "m1.small" -o "$MASTER_INSTANCE_TYPE" = "c1.medium" ]; then

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh Sat Mar 27 20:13:20 2010
@@ -1,28 +1,12 @@
 #!/usr/bin/env bash
 
-#
-# Copyright 2010 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
+# ZOOKEEPER_QUORUM set in the environment by the caller
 HBASE_HOME=`ls -d /usr/local/hbase-*`
 
+###############################################################################
 # HBase configuration (Zookeeper)
+###############################################################################
+
 cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
 <?xml version="1.0"?>
 <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
@@ -46,7 +30,9 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<
 </configuration>
 EOF
 
+###############################################################################
 # Start services
+###############################################################################
 
 # up open file descriptor limits
 echo "root soft nofile 32768" >> /etc/security/limits.conf

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote Sat Mar 27 20:13:20 2010
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master Sat Mar 27 20:13:20 2010
@@ -67,7 +67,7 @@ sed -e "s|%ZOOKEEPER_QUORUM%|$quorum|" \
     -e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
     "$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
 
-inst=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
+inst=`ec2-run-instances $TOOL_OPTS $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
 if [ "$ENABLE_ELASTIC_IPS" = "true" ] ; then
   addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'`
   ec2-associate-address $TOOL_OPTS $addr -i $inst

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves Sat Mar 27 20:13:20 2010
@@ -56,6 +56,6 @@ sed -e "s|%MASTER_HOST%|$master|" \
 
 # Start slaves
 echo "Starting $NUM_SLAVES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $zone"
-ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$zone" | grep INSTANCE | awk '{print $2}'
+ec2-run-instances $TOOL_OPTS $AMI_IMAGE -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$zone" | grep INSTANCE | awk '{print $2}' > $INSTANCES_PATH
 
 rm "$bin"/$USER_DATA_FILE.slave

Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper Sat Mar 27 20:13:20 2010
@@ -52,7 +52,7 @@ peer_addrs=""
 i=0
 while [ $i -lt $NO_INSTANCES ] ; do
   echo "Starting an AMI with ID $ZOO_AMI_IMAGE (arch $arch) in group $CLUSTER_ZOOKEEPER"
-  inst=`ec2-run-instances $ZOO_AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'`
+  inst=`ec2-run-instances $TOOL_OPTS $ZOO_AMI_IMAGE -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'`
   if [ "$ENABLE_ELASTIC_IPS" = "true" ] ; then
     addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'`
     ec2-associate-address $TOOL_OPTS $addr -i $inst

Added: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster?rev=928258&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster (added)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster Sat Mar 27 20:13:20 2010
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Reboot a cluster.
+
+if [ -z $1 ]; then
+  echo "Cluster name required!"
+  exit 1
+fi
+
+CLUSTER=$1
+
+# Import variables
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/hbase-ec2-env.sh
+
+# Finding HBase image
+HBASE_INSTANCES=`ec2-describe-instances $TOOL_OPTS | awk '"RESERVATION" == $1 && ("'$CLUSTER'" == $4 || "'$CLUSTER_MASTER'" == $4 || "'$CLUSTER_ZOOKEEPER'" == $4), "RESERVATION" == $1 && ("'$CLUSTER'" != $4 && "'$CLUSTER_MASTER'" != $4 && "'$CLUSTER_ZOOKEEPER'" != $4)'`
+HBASE_INSTANCES=`echo "$HBASE_INSTANCES" | grep INSTANCE | grep running`
+
+[ -z "$HBASE_INSTANCES" ] && echo "No running instances in cluster $CLUSTER." && exit 0
+
+echo "Running HBase instances:"
+echo "$HBASE_INSTANCES"
+read -p "Reboot all instances? [yes or no]: " answer
+
+if [ "$answer" != "yes" ]; then
+  exit 1
+fi
+
+ec2-reboot-instances $TOOL_OPTS `echo "$HBASE_INSTANCES" | awk '{print $2}'`

Propchange: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster
------------------------------------------------------------------------------
    svn:executable = *