You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2010/03/08 10:53:54 UTC
svn commit: r920253 - in /hadoop/hbase/branches/0.20: CHANGES.txt
src/contrib/ec2/bin/hbase-ec2-init-remote.sh
src/contrib/ec2/bin/launch-hbase-cluster
src/contrib/ec2/bin/launch-hbase-master
src/contrib/ec2/bin/launch-hbase-slaves
Author: apurtell
Date: Mon Mar 8 09:53:53 2010
New Revision: 920253
URL: http://svn.apache.org/viewvc?rev=920253&view=rev
Log:
HBASE-2299 [EC2] mapreduce fixups for PE
Modified:
hadoop/hbase/branches/0.20/CHANGES.txt
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-cluster
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
Modified: hadoop/hbase/branches/0.20/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/CHANGES.txt?rev=920253&r1=920252&r2=920253&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/CHANGES.txt (original)
+++ hadoop/hbase/branches/0.20/CHANGES.txt Mon Mar 8 09:53:53 2010
@@ -30,6 +30,7 @@
HBASE-2293 CME in RegionManager#isMetaServer
HBASE-2261 The javadoc in WhileMatchFilter and it's tests in TestFilter are
not accurate/wrong
+ HBASE-2299 [EC2] mapreduce fixups for PE
IMPROVEMENTS
HBASE-2180 Bad read performance from synchronizing hfile.fddatainputstream
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh?rev=920253&r1=920252&r2=920253&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/hbase-ec2-init-remote.sh Mon Mar 8 09:53:53 2010
@@ -5,6 +5,7 @@
MASTER_HOST="%MASTER_HOST%"
ZOOKEEPER_QUORUM="%ZOOKEEPER_QUORUM%"
+NUM_SLAVES="%NUM_SLAVES%"
EXTRA_PACKAGES="%EXTRA_PACKAGES%"
SECURITY_GROUPS=`wget -q -O - http://169.254.169.254/latest/meta-data/security-groups`
IS_MASTER=`echo $SECURITY_GROUPS | awk '{ a = match ($0, "-master$"); if (a) print "true"; else print "false"; }'`
@@ -120,6 +121,10 @@
<value>$DFS_DATA_DIR</value>
</property>
<property>
+ <name>dfs.replication</name>
+ <value>3</value>
+</property>
+<property>
<name>dfs.datanode.handler.count</name>
<value>10</value>
</property>
@@ -138,14 +143,6 @@
<value>$MASTER_HOST:8021</value>
</property>
<property>
- <name>mapred.output.compress</name>
- <value>true</value>
-</property>
-<property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
-</property>
-<property>
<name>io.compression.codecs</name>
<value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.BZip2Codec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec</value>
</property>
@@ -154,14 +151,22 @@
<value>com.hadoop.compression.lzo.LzoCodec</value>
</property>
<property>
- <name>mapred.map.output.compression.codec</name>
- <value>com.hadoop.compression.lzo.LzoCodec</value>
+ <name>mapred.map.tasks.speculative.execution</name>
+ <value>false</value>
+</property>
+<property>
+ <name>mapred.child.java.opts</name>
+ <value>-Xmx512m -XX:+UseCompressedOops</value>
</property>
</configuration>
EOF
+# Add JVM options
+cat >> $HADOOP_HOME/conf/hadoop-env.sh <<EOF
+export HADOOP_OPTS="$HADOOP_OPTS -XX:+UseCompressedOops"
+EOF
# Update classpath to include HBase jars and config
cat >> $HADOOP_HOME/conf/hadoop-env.sh <<EOF
-HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/AgileJSON-2009-03-30.jar:$HBASE_HOME/lib/json.jar:$HBASE_HOME/lib/zookeeper-3.2.1.jar:$HBASE_HOME/conf"
+export HADOOP_CLASSPATH="$HBASE_HOME/hbase-${HBASE_VERSION}.jar:$HBASE_HOME/lib/AgileJSON-2009-03-30.jar:$HBASE_HOME/lib/json.jar:$HBASE_HOME/lib/zookeeper-3.2.2.jar:$HBASE_HOME/conf"
EOF
# Configure Hadoop for Ganglia
cat > $HADOOP_HOME/conf/hadoop-metrics.properties <<EOF
@@ -191,6 +196,10 @@
<value>true</value>
</property>
<property>
+ <name>hbase.regions.server.count.min</name>
+ <value>$NUM_SLAVES</value>
+</property>
+<property>
<name>hbase.zookeeper.quorum</name>
<value>$ZOOKEEPER_QUORUM</value>
</property>
@@ -224,10 +233,12 @@
</property>
</configuration>
EOF
+# Copy over mapred configuration for jobs started with 'hbase ...'
+cp $HADOOP_HOME/conf/mapred-site.xml $HBASE_HOME/conf/mapred-site.xml
# Override JVM options
cat >> $HBASE_HOME/conf/hbase-env.sh <<EOF
-export HBASE_MASTER_OPTS="-Xmx1000m -XX:+UseConcMarkSweepGC -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
-export HBASE_REGIONSERVER_OPTS="-Xmx2000m -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:NewSize=64m -XX:MaxNewSize=64m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
+export HBASE_MASTER_OPTS="-Xmx1000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-master-gc.log"
+export HBASE_REGIONSERVER_OPTS="-Xmx2000m -XX:+UseCompressedOops -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=88 -XX:NewSize=64m -XX:MaxNewSize=64m -XX:+DoEscapeAnalysis -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:/mnt/hbase/logs/hbase-regionserver-gc.log"
EOF
# Configure HBase for Ganglia
cat > $HBASE_HOME/conf/hadoop-metrics.properties <<EOF
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-cluster
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-cluster?rev=920253&r1=920252&r2=920253&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-cluster (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-cluster Mon Mar 8 09:53:53 2010
@@ -52,7 +52,7 @@
# Launch the HBase master
-if ! "$bin"/launch-hbase-master $CLUSTER ; then
+if ! "$bin"/launch-hbase-master $CLUSTER $SLAVES ; then
exit $?
fi
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master?rev=920253&r1=920252&r2=920253&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-master Mon Mar 8 09:53:53 2010
@@ -24,6 +24,13 @@
CLUSTER=$1
+if [ -z $2 ]; then
+ echo "Must specify the number of slaves to start."
+ exit 1
+fi
+
+NUM_SLAVES=$2
+
# Import variables
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
@@ -52,13 +59,14 @@
fi
# Finding HBase image
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$arch | grep available | awk '{print $2}'`
# Start a master
echo "Starting master with AMI $AMI_IMAGE (arch $arch)"
# Substituting zookeeper quorum
ZOOKEEPER_QUORUM=`cat $ZOOKEEPER_QUORUM_PATH`
sed -e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
+ -e "s|%NUM_SLAVES%|$NUM_SLAVES|" \
-e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
"$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.master
INSTANCE=`ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n 1 -g $CLUSTER_MASTER -k root -f "$bin"/$USER_DATA_FILE.master -t $type | grep INSTANCE | awk '{print $2}'`
Modified: hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves
URL: http://svn.apache.org/viewvc/hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves?rev=920253&r1=920252&r2=920253&view=diff
==============================================================================
--- hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves (original)
+++ hadoop/hbase/branches/0.20/src/contrib/ec2/bin/launch-hbase-slaves Mon Mar 8 09:53:53 2010
@@ -29,7 +29,7 @@
exit 1
fi
-NO_INSTANCES=$2
+NUM_SLAVES=$2
# Import variables
bin=`dirname "$0"`
@@ -41,7 +41,7 @@
exit 1
fi
-[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep $HBASE_VERSION-$SLAVE_ARCH |grep available | awk '{print $2}'`
+[ -z "$AMI_IMAGE" ] && AMI_IMAGE=`ec2-describe-images $TOOL_OPTS -a | grep $S3_BUCKET | grep hbase | grep $HBASE_VERSION-$SLAVE_ARCH |grep available | awk '{print $2}'`
MASTER_HOST=`cat $MASTER_PRIVATE_IP_PATH`
MASTER_ZONE=`cat $MASTER_ZONE_PATH`
@@ -49,12 +49,13 @@
# Substituting master hostname and zookeeper quorum
sed -e "s|%MASTER_HOST%|$MASTER_HOST|" \
+ -e "s|%NUM_SLAVES%|$NUM_SLAVES|" \
-e "s|%ZOOKEEPER_QUORUM%|$ZOOKEEPER_QUORUM|" \
-e "s|%EXTRA_PACKAGES%|$EXTRA_PACKAGES|" \
"$bin"/$USER_DATA_FILE > "$bin"/$USER_DATA_FILE.slave
# Start slaves
-echo "Starting $NO_INSTANCES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $MASTER_ZONE"
-ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NO_INSTANCES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
+echo "Starting $NUM_SLAVES AMI(s) with ID $AMI_IMAGE (arch $SLAVE_ARCH) in group $CLUSTER in zone $MASTER_ZONE"
+ec2-run-instances $AMI_IMAGE $TOOL_OPTS -n "$NUM_SLAVES" -g "$CLUSTER" -k root -f "$bin"/$USER_DATA_FILE.slave -t "$SLAVE_INSTANCE_TYPE" -z "$MASTER_ZONE" | grep INSTANCE | awk '{print $2}'
rm "$bin"/$USER_DATA_FILE.slave