You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by as...@apache.org on 2011/08/04 07:36:40 UTC
svn commit: r1153757 - in /incubator/whirr/trunk: ./ recipes/
services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/
services/hbase/src/main/resources/functions/
services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/
ser...
Author: asavu
Date: Thu Aug 4 05:36:38 2011
New Revision: 1153757
URL: http://svn.apache.org/viewvc?rev=1153757&view=rev
Log:
WHIRR-240. [HBase] Enable support for HBase 0.90.x (Bruno Dumon via asavu)
Added:
incubator/whirr/trunk/recipes/hbase-ec2-0.90.properties
incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase089ServiceTest.java
incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase090ServiceTest.java
incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-0.90-test.properties
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java
incubator/whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceController.java
incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceTest.java
incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-test.properties
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1153757&r1=1153756&r2=1153757&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Thu Aug 4 05:36:38 2011
@@ -13,6 +13,8 @@ Trunk (unreleased changes)
WHIRR-260. Support Spot Instances (asavu)
+ WHIRR-240. [HBase] Enable support for HBase 0.90.x (Bruno Dumon via asavu)
+
IMPROVEMENTS
WHIRR-28. Add examples module (asavu)
Added: incubator/whirr/trunk/recipes/hbase-ec2-0.90.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/recipes/hbase-ec2-0.90.properties?rev=1153757&view=auto
==============================================================================
--- incubator/whirr/trunk/recipes/hbase-ec2-0.90.properties (added)
+++ incubator/whirr/trunk/recipes/hbase-ec2-0.90.properties Thu Aug 4 05:36:38 2011
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# HBase 0.90.x on Cloudera Hadoop Cluster on AWS EC2
+#
+
+# Read the Configuration Guide for more info:
+# http://incubator.apache.org/whirr/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=hbase-0.90
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=1 zookeeper+hadoop-namenode+hadoop-jobtracker+hbase-master,3 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/
+whirr.hardware-id=c1.xlarge
+# Ubuntu 10.04 LTS Lucid. See http://alestic.com/
+whirr.image-id=us-east-1/ami-da0cf8b3
+# If you choose a different location, make sure whirr.image-id is updated too
+whirr.location-id=us-east-1
+
+# By default use the user system SSH keys. Override them here.
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+# The HBase version to use.
+whirr.hbase.tarball.url=http://apache.cu.be/hbase/hbase-0.90.3/hbase-0.90.3.tar.gz
+
+# The Hadoop version to use. See http://hbase.apache.org/book/hadoop.html
+# The default Hadoop version used by Whirr does not fulfill the HBase 0.90.x requirements.
+# Whirr will replace the hadoop-core jar in HBase with the one from the actually installed Hadoop.
+# This example uses Cloudera's CDH3.
+whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u1.tar.gz
+
Modified: incubator/whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java?rev=1153757&r1=1153756&r2=1153757&view=diff
==============================================================================
--- incubator/whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java (original)
+++ incubator/whirr/trunk/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java Thu Aug 4 05:36:38 2011
@@ -48,7 +48,7 @@ public class CdhHBaseServiceTest {
private static final byte[] VALUE = Bytes.toBytes("testValue");
private static HBaseServiceController controller =
- HBaseServiceController.getInstance();
+ HBaseServiceController.getInstance("whirr-hbase-test.properties");
@BeforeClass
public static void setUp() throws Exception {
Modified: incubator/whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh?rev=1153757&r1=1153756&r2=1153757&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh (original)
+++ incubator/whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh Thu Aug 4 05:36:38 2011
@@ -130,6 +130,10 @@ function configure_hbase() {
<name>hbase.client.retries.number</name>
<value>100</value>
</property>
+<property>
+ <name>hbase.zookeeper.recoverable.waittime</name>
+ <value>600000</value>
+</property>
</configuration>
EOF
@@ -169,6 +173,22 @@ EOF
#mapred.servers=$MASTER_HOST:8649
#EOF
+ # Replace Hadoop jar of HBase with the ones from the actually installed Hadoop version
+ # This assumes there will always be Hadoop installed on each HBase node
+
+ if [ -d /usr/local/hadoop ] ; then
+ # First, remove existing jar file
+ rm -f $HBASE_HOME/lib/hadoop*
+
+ # This makes the assumption there will be exactly one file matching
+ # The stars around core is because the file is named differently in CDH vs Apache
+ # Hadoop distributon (hadoop-core-version vs hadoop-version-core).
+ HADOOP_JAR=`ls /usr/local/hadoop-*/hadoop*core*.jar`
+ ln -s $HADOOP_JAR $HBASE_HOME/lib/hadoop-core.jar
+ else
+ echo Copy hadoop jar to HBase error: did not find your Hadoop installation
+ fi
+
# keep PID files in a non-temporary directory
sed -i -e "s|# export HBASE_PID_DIR=.*|export HBASE_PID_DIR=/var/run/hbase|" \
$HBASE_CONF_DIR/hbase-env.sh
Added: incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase089ServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase089ServiceTest.java?rev=1153757&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase089ServiceTest.java (added)
+++ incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase089ServiceTest.java Thu Aug 4 05:36:38 2011
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hbase.integration;
+
+import org.junit.BeforeClass;
+
+public class HBase089ServiceTest extends HBaseServiceTest {
+ @BeforeClass
+ public static void setUp() throws Exception {
+ controller = HBaseServiceController.getInstance("whirr-hbase-test.properties");
+ controller.ensureClusterRunning();
+ }
+}
Added: incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase090ServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase090ServiceTest.java?rev=1153757&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase090ServiceTest.java (added)
+++ incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBase090ServiceTest.java Thu Aug 4 05:36:38 2011
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hbase.integration;
+
+import org.junit.BeforeClass;
+
+public class HBase090ServiceTest extends HBaseServiceTest {
+ @BeforeClass
+ public static void setUp() throws Exception {
+ controller = HBaseServiceController.getInstance("whirr-hbase-0.90-test.properties");
+ controller.ensureClusterRunning();
+ }
+}
Modified: incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceController.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceController.java?rev=1153757&r1=1153756&r2=1153757&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceController.java (original)
+++ incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceController.java Thu Aug 4 05:36:38 2011
@@ -20,6 +20,8 @@ package org.apache.whirr.service.hbase.i
import java.io.IOException;
import java.net.InetAddress;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Map.Entry;
import org.apache.commons.configuration.CompositeConfiguration;
@@ -47,14 +49,20 @@ import org.slf4j.LoggerFactory;
public class HBaseServiceController {
+ private final String configResource;
+
private static final Logger LOG =
LoggerFactory.getLogger(HBaseServiceController.class);
- private static final HBaseServiceController INSTANCE =
- new HBaseServiceController();
+ private static final Map<String, HBaseServiceController> INSTANCES = new HashMap<String, HBaseServiceController>();
- public static HBaseServiceController getInstance() {
- return INSTANCE;
+ public static HBaseServiceController getInstance(String configResource) {
+ HBaseServiceController controller = INSTANCES.get(configResource);
+ if (controller == null) {
+ controller = new HBaseServiceController(configResource);
+ INSTANCES.put(configResource, controller);
+ }
+ return controller;
}
private boolean running;
@@ -64,7 +72,8 @@ public class HBaseServiceController {
private Cluster cluster;
private Hbase.Client thriftClient;
- private HBaseServiceController() {
+ private HBaseServiceController(String configResource) {
+ this.configResource = configResource;
}
public synchronized boolean ensureClusterRunning() throws Exception {
@@ -83,7 +92,7 @@ public class HBaseServiceController {
if (System.getProperty("config") != null) {
config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
}
- config.addConfiguration(new PropertiesConfiguration("whirr-hbase-test.properties"));
+ config.addConfiguration(new PropertiesConfiguration(this.configResource));
clusterSpec = ClusterSpec.withTemporaryKeys(config);
controller = new ClusterController();
Modified: incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceTest.java?rev=1153757&r1=1153756&r2=1153757&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceTest.java (original)
+++ incubator/whirr/trunk/services/hbase/src/test/java/org/apache/whirr/service/hbase/integration/HBaseServiceTest.java Thu Aug 4 05:36:38 2011
@@ -33,10 +33,9 @@ import org.apache.hadoop.hbase.thrift.ge
import org.apache.hadoop.hbase.thrift.generated.TRowResult;
import org.apache.hadoop.hbase.util.Bytes;
import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Test;
-public class HBaseServiceTest {
+public abstract class HBaseServiceTest {
private static final byte[] FIRST = Bytes.toBytes("");
private static final byte[] TABLE = Bytes.toBytes("testtable");
@@ -46,13 +45,7 @@ public class HBaseServiceTest {
private static final byte[] COLUMN = Bytes.toBytes("testFamily1:testColumn");
private static final byte[] VALUE = Bytes.toBytes("testValue");
- private static HBaseServiceController controller =
- HBaseServiceController.getInstance();
-
- @BeforeClass
- public static void setUp() throws Exception {
- controller.ensureClusterRunning();
- }
+ protected static HBaseServiceController controller;
@AfterClass
public static void tearDown() throws Exception {
Added: incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-0.90-test.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-0.90-test.properties?rev=1153757&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-0.90-test.properties (added)
+++ incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-0.90-test.properties Thu Aug 4 05:36:38 2011
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=hbaseclustertest
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+zookeeper+hbase-master+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.hbase.tarball.url=http://apache.cu.be/hbase/hbase-0.90.3/hbase-0.90.3.tar.gz
+whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u0.tar.gz
Modified: incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-test.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-test.properties?rev=1153757&r1=1153756&r2=1153757&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-test.properties (original)
+++ incubator/whirr/trunk/services/hbase/src/test/resources/whirr-hbase-test.properties Thu Aug 4 05:36:38 2011
@@ -16,7 +16,7 @@
# limitations under the License.
#
whirr.cluster-name=hbaseclustertest
-whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+hbase-master,1 zookeeper+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+zookeeper+hbase-master+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
whirr.provider=${sys:whirr.test.provider}
whirr.identity=${sys:whirr.test.identity}