You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2010/03/27 21:13:21 UTC
svn commit: r928258 - in /hadoop/hbase/branches/0.20/src/contrib/ec2/bin: ./
image/
Author: apurtell
Date: Sat Mar 27 20:13:20 2010
New Revision: 928258
URL: http://svn.apache.org/viewvc?rev=928258&view=rev
Log:
[EC2] consider multiple regions in config; added 'reboot-hbase-cluster
Added:
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster (with props)
Modified:
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/cmd-hbase-cluster Sat Mar 27 20:13:20 2010
@@ -26,6 +26,7 @@ fi
# get arguments
COMMAND="$1"
shift
+
# get group
CLUSTER="$1"
shift
@@ -44,8 +45,8 @@ if [[ "$CLUSTER" = "i-*" ]]; then
HOST=`ec2-describe-instances $TOOL_OPTS $CLUSTER | grep running | awk '{print $4}'`
[ -z $HOST ] && echo "Instance still pending or no longer running: $CLUSTER" && exit 1
else
- [ ! -f $MASTER_IP_PATH ] && echo "Wrong group name, or cluster not launched! $CLUSTER" && exit 1
- HOST=`cat $MASTER_IP_PATH`
+ [ ! -f $MASTER_ADDR_PATH ] && echo "Wrong group name, or cluster not launched! $CLUSTER" && exit 1
+ HOST=`cat $MASTER_ADDR_PATH`
fi
if [ "$COMMAND" = "login" ] ; then
@@ -64,6 +65,5 @@ elif [ "$COMMAND" = "screen" ] ; then
echo "Logging in and attaching screen on host $HOST."
ssh $SSH_OPTS -t "root@$HOST" 'screen -D -R'
else
- echo "Executing command on host $HOST."
- ssh $SSH_OPTS -t "root@$HOST" "$COMMAND"
-fi
\ No newline at end of file
+ ssh $SSH_OPTS -t "root@$HOST" "$COMMAND $@"
+fi
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/delete-hbase-cluster Sat Mar 27 20:13:20 2010
@@ -41,7 +41,5 @@ fi
"$bin"/revoke-hbase-cluster-secgroups $CLUSTER
-rm -f $MASTER_IP_PATH
-rm -f $MASTER_PRIVATE_IP_PATH
-rm -f $MASTER_ZONE_PATH
-rm -f $ZOOKEEPER_QUORUM_PATH
+rm -f $ZOOKEEPER_ADDR_PATH $ZOOKEEPER_QUORUM_PATH
+rm -f $MASTER_IP_PATH $MASTER_ADDR_PATH $MASTER_ZONE_PATH
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/functions.sh Sat Mar 27 20:13:20 2010
@@ -1,13 +1,9 @@
-#
-# Copyright 2010 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
@@ -16,7 +12,6 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
function getCredentialSetting {
name=$1
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2 (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2 Sat Mar 27 20:13:20 2010
@@ -28,6 +28,7 @@ if [ $# = 0 ]; then
echo " launch-master <name> launch or find a cluster master"
echo " launch-slaves <name> <slaves> launch the cluster slaves"
echo " terminate-cluster <name> terminate all HBase EC2 instances"
+ echo " reboot-cluster <name> reboot all HBase EC2 instances"
echo " delete-cluster <name> clean up after a terminated cluster"
echo " login <name|instance id> login to the master node"
echo " screen <name|instance id> start or attach 'screen' on the master"
@@ -56,6 +57,8 @@ elif [ "$COMMAND" = "delete-cluster" ] ;
. "$bin"/delete-hbase-cluster $*
elif [ "$COMMAND" = "terminate-cluster" ] ; then
. "$bin"/terminate-hbase-cluster $*
+elif [ "$COMMAND" = "reboot-cluster" ] ; then
+ . "$bin"/reboot-hbase-cluster $*
elif [ "$COMMAND" = "list" ] ; then
. "$bin"/list-hbase-clusters
else
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-env.sh Sat Mar 27 20:13:20 2010
@@ -46,14 +46,18 @@ HADOOP_URL=http://hbase.s3.amazonaws.com
LZO_URL=http://hbase.s3.amazonaws.com/hadoop/lzo-linux-$HADOOP_VERSION.tar.gz
# The Amazon S3 bucket where the HBase AMI is stored.
-# Change this value only if you are creating your own (private) AMI
+# Change this value if you are launching instances in regions other than
+# us-east-1 (the default).
+# Change this value otherwise only if you are creating your own (private) AMI
# so you can store it in a bucket you own.
-S3_BUCKET=apache-hbase-images
+S3_BUCKET=apache-hbase-images # us-east-1
+#S3_BUCKET=apache-hbase-images-eu # eu-west-1
+#S3_BUCKET=apache-hbase-images-us-west-1 # us-west-1
# Enable public access web interfaces
ENABLE_WEB_PORTS=false
-# Enable mapping of elastic IPs to ZK and master instances
+# Enable use of elastic IPs for ZK and master instances
ENABLE_ELASTIC_IPS=false
# Extra packages
@@ -62,11 +66,11 @@ ENABLE_ELASTIC_IPS=false
# The repository descriptor will be fetched into /etc/yum/repos.d.
EXTRA_PACKAGES=
-# Use only m1.large or c1.xlarge unless you know what you are doing
-MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-m1.large}
+# Use only c1.xlarge unless you know what you are doing
+MASTER_INSTANCE_TYPE=${MASTER_INSTANCE_TYPE:-c1.xlarge}
# Use only m1.large or c1.xlarge unless you know what you are doing
-SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-m1.large}
+SLAVE_INSTANCE_TYPE=${SLAVE_INSTANCE_TYPE:-c1.xlarge}
# Use only m1.small or c1.medium unless you know what you are doing
ZOO_INSTANCE_TYPE=${ZOO_INSTANCE_TYPE:-m1.small}
@@ -107,6 +111,9 @@ CLUSTER_ZOOKEEPER=$CLUSTER-zk
ZOOKEEPER_QUORUM_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-quorum
ZOOKEEPER_ADDR_PATH=$HOME/.hbase-${CLUSTER_ZOOKEEPER}-addrs
+# Instances path
+INSTANCES_PATH=$HOME/.hbase-${CLUSTER}-instances
+
# The script to run on instance boot.
USER_DATA_FILE=hbase-ec2-init-remote.sh
@@ -116,12 +123,18 @@ JAVA_VERSION=1.6.0_17
JAVA_URL=http://hbase.s3.amazonaws.com/jdk/jdk-${JAVA_VERSION}-linux-@arch@.bin
# SUPPORTED_ARCHITECTURES = ['i386', 'x86_64']
+# Change the BASE_AMI_IMAGE setting if you are creating custom AMI in a region
+# other than us-east-1 (the default).
if [ "$SLAVE_INSTANCE_TYPE" = "m1.small" -o "$SLAVE_INSTANCE_TYPE" = "c1.medium" ]; then
SLAVE_ARCH='i386'
- BASE_AMI_IMAGE="ami-48aa4921" # ec2-public-images/fedora-8-i386-base-v1.10.manifest.xml
+ BASE_AMI_IMAGE="ami-48aa4921" # us-east-1 ec2-public-images/fedora-8-i386-base-v1.10.manifest.xml
+ #BASE_AMI_IMAGE="ami-810657c4" # us-west-1
+ #BASE_AMI_IMAGE="ami-0a48637e" # eu-west-1
else
SLAVE_ARCH='x86_64'
- BASE_AMI_IMAGE="ami-f61dfd9f" # ec2-public-images/fedora-8-x86_64-base-v1.10.manifest.xml
+ BASE_AMI_IMAGE="ami-f61dfd9f" # us-east-1 ec2-public-images/fedora-8-x86_64-base-v1.10.manifest.xml
+ #BASE_AMI_IMAGE="ami-870657c2" # us-west-1
+ #BASE_AMI_IMAGE="ami-927a51e6" # eu-west-1
fi
if [ "$MASTER_INSTANCE_TYPE" = "m1.small" -o "$MASTER_INSTANCE_TYPE" = "c1.medium" ]; then
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-zookeeper-remote.sh Sat Mar 27 20:13:20 2010
@@ -1,28 +1,12 @@
#!/usr/bin/env bash
-#
-# Copyright 2010 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
+# ZOOKEEPER_QUORUM set in the environment by the caller
HBASE_HOME=`ls -d /usr/local/hbase-*`
+###############################################################################
# HBase configuration (Zookeeper)
+###############################################################################
+
cat > $HBASE_HOME/conf/hbase-site.xml <<EOF
<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
@@ -46,7 +30,9 @@ cat > $HBASE_HOME/conf/hbase-site.xml <<
</configuration>
EOF
+###############################################################################
# Start services
+###############################################################################
# up open file descriptor limits
echo "root soft nofile 32768" >> /etc/security/limits.conf
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/image/create-hbase-image-remote Sat Mar 27 20:13:20 2010
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master Sat Mar 27 20:13:20 2010
@@ -67,7 +67,7 @@ sed -e "s|%ZOOKEEPER_QUORUM%|$quorum|" \
-e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
"$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
-inst=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
+inst=`ec2-run-instances $TOOL_OPTS $AMI_IMAGE -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
if [ "$ENABLE_ELASTIC_IPS" = "true" ] ; then
addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'`
ec2-associate-address $TOOL_OPTS $addr -i $inst
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves Sat Mar 27 20:13:20 2010
@@ -56,6 +56,6 @@ sed -e "s|%MASTER_HOST%|$master|" \
# Start slaves
echo "Starting $NUM_SLAVES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $zone"
-ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$zone" | grep INSTANCE | awk '{print $2}'
+ec2-run-instances $TOOL_OPTS $AMI_IMAGE -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$zone" | grep INSTANCE | awk '{print $2}' > $INSTANCES_PATH
rm "$bin"/$USER_DATA_FILE.slave
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper?rev=928258&r1=928257&r2=928258&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-zookeeper Sat Mar 27 20:13:20 2010
@@ -52,7 +52,7 @@ peer_addrs=""
i=0
while [ $i -lt $NO_INSTANCES ] ; do
echo "Starting an AMI with ID $ZOO_AMI_IMAGE (arch $arch) in group $CLUSTER_ZOOKEEPER"
- inst=`ec2-run-instances $ZOO_AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'`
+ inst=`ec2-run-instances $TOOL_OPTS $ZOO_AMI_IMAGE -n 1 -g $CLUSTER_ZOOKEEPER -k root -t $type | grep INSTANCE | awk '{print $2}'`
if [ "$ENABLE_ELASTIC_IPS" = "true" ] ; then
addr=`ec2-allocate-address $TOOL_OPTS | awk '{print $2}'`
ec2-associate-address $TOOL_OPTS $addr -i $inst
Added: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster?rev=928258&view=auto
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster (added)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster Sat Mar 27 20:13:20 2010
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Reboot a cluster.
+
+if [ -z $1 ]; then
+ echo "Cluster name required!"
+ exit 1
+fi
+
+CLUSTER=$1
+
+# Import variables
+bin=`dirname "$0"`
+bin=`cd "$bin"; pwd`
+. "$bin"/hbase-ec2-env.sh
+
+# Finding HBase image
+HBASE_INSTANCES=`ec2-describe-instances $TOOL_OPTS | awk '"RESERVATION" == $1 && ("'$CLUSTER'" == $4 || "'$CLUSTER_MASTER'" == $4 || "'$CLUSTER_ZOOKEEPER'" == $4), "RESERVATION" == $1 && ("'$CLUSTER'" != $4 && "'$CLUSTER_MASTER'" != $4 && "'$CLUSTER_ZOOKEEPER'" != $4)'`
+HBASE_INSTANCES=`echo "$HBASE_INSTANCES" | grep INSTANCE | grep running`
+
+[ -z "$HBASE_INSTANCES" ] && echo "No running instances in cluster $CLUSTER." && exit 0
+
+echo "Running HBase instances:"
+echo "$HBASE_INSTANCES"
+read -p "Reboot all instances? [yes or no]: " answer
+
+if [ "$answer" != "yes" ]; then
+ exit 1
+fi
+
+ec2-reboot-instances $TOOL_OPTS `echo "$HBASE_INSTANCES" | awk '{print $2}'`
Propchange: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/reboot-hbase-cluster
------------------------------------------------------------------------------
svn:executable = *