You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by to...@apache.org on 2010/09/28 05:28:41 UTC
svn commit: r1001999 - in /incubator/whirr/trunk: ./ core/
core/src/main/java/org/apache/whirr/service/
core/src/main/java/org/apache/whirr/service/jclouds/
core/src/test/java/org/apache/whirr/service/ scripts/apache/hadoop/
scripts/cloudera/cdh/ scrip...
Author: tomwhite
Date: Tue Sep 28 03:28:40 2010
New Revision: 1001999
URL: http://svn.apache.org/viewvc?rev=1001999&view=rev
Log:
WHIRR-97. Lucid is not stable on EC2.
Added:
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java (with props)
incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java (with props)
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopTemplateBuilderStrategy.java (with props)
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/core/pom.xml
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java
incubator/whirr/trunk/scripts/apache/hadoop/install
incubator/whirr/trunk/scripts/cloudera/cdh/install
incubator/whirr/trunk/scripts/sun/java/install
incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraService.java
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopService.java
incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperService.java
incubator/whirr/trunk/src/site/confluence/configuration-guide.confluence
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Tue Sep 28 03:28:40 2010
@@ -14,6 +14,9 @@ Trunk (unreleased changes)
WHIRR-93. Fail on checkstyle violation. (tomwhite)
+ WHIRR-97. Lucid is not stable on EC2. Default is now the Amazon Linux AMI
+ on EC2. (tomwhite)
+
Release 0.1.0 - 2010-09-02
INCOMPATIBLE CHANGES
Modified: incubator/whirr/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/pom.xml?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/core/pom.xml (original)
+++ incubator/whirr/trunk/core/pom.xml Tue Sep 28 03:28:40 2010
@@ -62,6 +62,10 @@
<artifactId>junit</artifactId>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-all</artifactId>
</dependency>
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java Tue Sep 28 03:28:40 2010
@@ -54,6 +54,9 @@ public class ClusterSpec {
CLUSTER_NAME(String.class, false),
PUBLIC_KEY_FILE(String.class, false),
PRIVATE_KEY_FILE(String.class, false),
+ IMAGE_ID(String.class, false),
+ HARDWARE_ID(String.class, false),
+ LOCATION_ID(String.class, false),
CLIENT_CIDRS(String.class, true),
RUN_URL_BASE(String.class, false);
@@ -147,6 +150,9 @@ public class ClusterSpec {
private String clusterName;
private Payload privateKey;
private Payload publicKey;
+ private String imageId;
+ private String hardwareId;
+ private String locationId;
private List<String> clientCidrs;
private String runUrlBase;
@@ -186,6 +192,9 @@ public class ClusterSpec {
} catch (IOException e) {
throw new ConfigurationException("error reading key from file", e);
}
+ setImageId(config.getString(Property.IMAGE_ID.getConfigName()));
+ setHardwareId(config.getString(Property.HARDWARE_ID.getConfigName()));
+ setLocationId(config.getString(Property.LOCATION_ID.getConfigName()));
setClientCidrs(c.getList(Property.CLIENT_CIDRS.getConfigName()));
setRunUrlBase(c.getString(Property.RUN_URL_BASE.getConfigName()));
this.config = c;
@@ -243,6 +252,15 @@ public class ClusterSpec {
public String readPublicKey() throws IOException {
return toStringAndClose(getPublicKey().getInput());
}
+ public String getImageId() {
+ return imageId;
+ }
+ public String getHardwareId() {
+ return hardwareId;
+ }
+ public String getLocationId() {
+ return locationId;
+ }
public List<String> getClientCidrs() {
return clientCidrs;
}
@@ -321,6 +339,18 @@ public class ClusterSpec {
checkNotNull(privateKey, "privateKey"));
}
+ public void setImageId(String imageId) {
+ this.imageId = imageId;
+ }
+
+ public void setHardwareId(String hardwareId) {
+ this.hardwareId = hardwareId;
+ }
+
+ public void setLocationId(String locationId) {
+ this.locationId = locationId;
+ }
+
public void setClientCidrs(List<String> clientCidrs) {
this.clientCidrs = clientCidrs;
}
@@ -344,6 +374,9 @@ public class ClusterSpec {
&& Objects.equal(identity, that.identity)
&& Objects.equal(credential, that.credential)
&& Objects.equal(clusterName, that.clusterName)
+ && Objects.equal(imageId, that.imageId)
+ && Objects.equal(hardwareId, that.hardwareId)
+ && Objects.equal(locationId, that.locationId)
&& Objects.equal(clientCidrs, that.clientCidrs)
&& Objects.equal(runUrlBase, that.runUrlBase)
;
@@ -354,7 +387,8 @@ public class ClusterSpec {
public int hashCode() {
return Objects.hashCode(instanceTemplates, serviceName,
provider, identity, credential, clusterName, publicKey,
- privateKey, clientCidrs, runUrlBase);
+ privateKey, imageId, hardwareId, locationId, clientCidrs,
+ runUrlBase);
}
public String toString() {
@@ -367,6 +401,9 @@ public class ClusterSpec {
.add("clusterName", clusterName)
.add("publicKey", publicKey)
.add("privateKey", privateKey)
+ .add("imageId", imageId)
+ .add("instanceSizeId", hardwareId)
+ .add("locationId", locationId)
.add("clientCidrs", clientCidrs)
.add("runUrlBase", runUrlBase)
.toString();
Added: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java?rev=1001999&view=auto
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java (added)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java Tue Sep 28 03:28:40 2010
@@ -0,0 +1,26 @@
+package org.apache.whirr.service.jclouds;
+
+import org.apache.whirr.service.ClusterSpec;
+import org.jclouds.compute.domain.TemplateBuilder;
+
+/**
+ * A class to configure a {@link TemplateBuilder}.
+ */
+public class TemplateBuilderStrategy {
+
+ public void configureTemplateBuilder(ClusterSpec clusterSpec,
+ TemplateBuilder templateBuilder) {
+
+ if (clusterSpec.getImageId() != null) {
+ templateBuilder.imageId(clusterSpec.getImageId());
+ }
+
+ if (clusterSpec.getHardwareId() != null) {
+ templateBuilder.hardwareId(clusterSpec.getHardwareId());
+ }
+
+ if (clusterSpec.getLocationId() != null) {
+ templateBuilder.locationId(clusterSpec.getLocationId());
+ }
+ }
+}
Propchange: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java?rev=1001999&view=auto
==============================================================================
--- incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java (added)
+++ incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java Tue Sep 28 03:28:40 2010
@@ -0,0 +1,46 @@
+package org.apache.whirr.service;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
+import org.jclouds.compute.domain.TemplateBuilder;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TemplateBuilderStrategyTest {
+
+ private TemplateBuilderStrategy strategy = new TemplateBuilderStrategy();
+ private ClusterSpec spec;
+
+ @Before
+ public void setUp() throws ConfigurationException {
+ spec = new ClusterSpec();
+ }
+
+ @Test
+ public void testImageIdIsPassedThrough() {
+ spec.setImageId("my-image-id");
+ TemplateBuilder builder = mock(TemplateBuilder.class);
+ strategy.configureTemplateBuilder(spec, builder);
+ verify(builder).imageId("my-image-id");
+ }
+
+ @Test
+ public void testHardwareIdIsPassedThrough() {
+ spec.setHardwareId("my-hardware-id");
+ TemplateBuilder builder = mock(TemplateBuilder.class);
+ strategy.configureTemplateBuilder(spec, builder);
+ verify(builder).hardwareId("my-hardware-id");
+ }
+
+ @Test
+ public void testLocationIdIsPassedThrough() {
+ spec.setLocationId("my-location-id");
+ TemplateBuilder builder = mock(TemplateBuilder.class);
+ strategy.configureTemplateBuilder(spec, builder);
+ verify(builder).locationId("my-location-id");
+ }
+
+}
Propchange: incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/TemplateBuilderStrategyTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/whirr/trunk/scripts/apache/hadoop/install
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/scripts/apache/hadoop/install?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/scripts/apache/hadoop/install (original)
+++ incubator/whirr/trunk/scripts/apache/hadoop/install Tue Sep 28 03:28:40 2010
@@ -371,7 +371,10 @@ function setup_web() {
apt-get -y install thttpd
WWW_BASE=/var/www
elif which rpm &> /dev/null; then
- yum install -y thttpd
+ # enable EPEL for thttpd package
+ rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-4.noarch.rpm
+ sed -i -e 's/enabled=1/enabled=0/' /etc/yum.repos.d/epel.repo
+ yum install -y --enablerepo=epel thttpd
chkconfig --add thttpd
WWW_BASE=/var/www/thttpd/html
fi
Modified: incubator/whirr/trunk/scripts/cloudera/cdh/install
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/scripts/cloudera/cdh/install?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/scripts/cloudera/cdh/install (original)
+++ incubator/whirr/trunk/scripts/cloudera/cdh/install Tue Sep 28 03:28:40 2010
@@ -399,18 +399,20 @@ function install_hue() {
apt-get -y install hue-useradmin hue-jobsub hue-beeswax
elif which rpm &> /dev/null; then
yum install -y hue-common
- yum install hue-useradmin hue-jobsub hue-beeswax
+ yum install -y hue-useradmin hue-jobsub hue-beeswax
fi
# Configure hue
sed -i -e "s|http_port=8088|http_port=80|" /etc/hue/hue.ini
# Hue logs should be on the /mnt partition
- rm -rf /var/log/hue
+ mv /var/log/hue /var/log/hue.tmp
mkdir -p $MOUNT/hue/logs
chown hue:hue $MOUNT/hue/logs
ln -s $MOUNT/hue/logs /var/log/hue
chown -R hue:hue /var/log/hue
+ mv /var/log/hue.tmp/* /var/log/hue
+ rm -rf /var/log/hue.tmp
}
function install_hue_plugins() {
@@ -431,17 +433,14 @@ function start_namenode() {
# Format HDFS
[ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HADOOP "$HADOOP namenode -format"
update-rc.d $HADOOP-namenode defaults
- update-rc.d $HADOOP-secondarynamenode defaults
elif which rpm &> /dev/null; then
AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
# Format HDFS
[ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HADOOP "$HADOOP namenode -format"
chkconfig --add $HADOOP-namenode
- chkconfig --add $HADOOP-secondarynamenode
fi
service $HADOOP-namenode start
- service $HADOOP-secondarynamenode start
$AS_HADOOP "$HADOOP dfsadmin -safemode wait"
$AS_HADOOP "/usr/bin/$HADOOP fs -mkdir /user"
Modified: incubator/whirr/trunk/scripts/sun/java/install
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/scripts/sun/java/install?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/scripts/sun/java/install (original)
+++ incubator/whirr/trunk/scripts/sun/java/install Tue Sep 28 03:28:40 2010
@@ -17,23 +17,24 @@
#
# Install Sun Java 6.
#
-# TODO: Update for RPM-based systems too.
set -x
set -e
-# Enable multiverse
-# TODO: check that it is not already enabled
-sed -i -e 's/universe$/universe multiverse/' /etc/apt/sources.list
-
-cat > /etc/apt/sources.list.d/canonical.com.list <<EOF
-deb http://archive.canonical.com/ubuntu lucid partner
-deb-src http://archive.canonical.com/ubuntu lucid partner
+function install_java_deb() {
+ # Enable multiverse
+ # TODO: check that it is not already enabled
+ sed -i -e 's/universe$/universe multiverse/' /etc/apt/sources.list
+
+ DISTRO=`lsb_release -s -c`
+ cat > /etc/apt/sources.list.d/canonical.com.list <<EOF
+deb http://archive.canonical.com/ubuntu $DISTRO partner
+deb-src http://archive.canonical.com/ubuntu $DISTRO partner
EOF
-
-apt-get update
-
-echo 'sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true
+
+ apt-get update
+
+ echo 'sun-java6-bin shared/accepted-sun-dlj-v1-1 boolean true
sun-java6-jdk shared/accepted-sun-dlj-v1-1 boolean true
sun-java6-jre shared/accepted-sun-dlj-v1-1 boolean true
sun-java6-jre sun-java6-jre/stopthread boolean true
@@ -42,9 +43,41 @@ sun-java6-bin shared/present-sun-dlj-v
sun-java6-jdk shared/present-sun-dlj-v1-1 note
sun-java6-jre shared/present-sun-dlj-v1-1 note
' | debconf-set-selections
-
-apt-get -y install sun-java6-jdk
-
-echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> /etc/profile
-export JAVA_HOME=/usr/lib/jvm/java-6-sun
-java -version
\ No newline at end of file
+
+ apt-get -y install sun-java6-jdk
+
+ echo "export JAVA_HOME=/usr/lib/jvm/java-6-sun" >> /etc/profile
+ export JAVA_HOME=/usr/lib/jvm/java-6-sun
+ java -version
+
+}
+
+function install_java_rpm() {
+ MACHINE_TYPE=`uname -m`
+ if [ ${MACHINE_TYPE} == 'x86_64' ]; then
+ JDK_PACKAGE=jdk-6u21-linux-x64-rpm.bin
+ else
+ JDK_PACKAGE=jdk-6u21-linux-i586-rpm.bin
+ fi
+ JDK_INSTALL_PATH=/usr/java
+ mkdir -p $JDK_INSTALL_PATH
+ cd $JDK_INSTALL_PATH
+ wget http://whirr-third-party.s3.amazonaws.com/$JDK_PACKAGE
+ chmod +x $JDK_PACKAGE
+ mv /bin/more /bin/more.no
+ yes | ./$JDK_PACKAGE -noregister
+ mv /bin/more.no /bin/more
+ rm -f *.rpm $JDK_PACKAGE
+
+ export JAVA_HOME=$(ls -d $JDK_INSTALL_PATH/jdk*)
+ echo "export JAVA_HOME=$JAVA_HOME" >> /etc/profile
+ alternatives --install /usr/bin/java java $JAVA_HOME/bin/java 17000
+ alternatives --set java $JAVA_HOME/bin/java
+ java -version
+}
+
+if which dpkg &> /dev/null; then
+ install_java_deb
+elif which rpm &> /dev/null; then
+ install_java_rpm
+fi
Modified: incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraService.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraService.java?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraService.java (original)
+++ incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraService.java Tue Sep 28 03:28:40 2010
@@ -20,11 +20,17 @@ package org.apache.whirr.service.cassand
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.whirr.service.RunUrlBuilder.runUrls;
-import static org.jclouds.compute.domain.OsFamily.UBUNTU;
import static org.jclouds.compute.options.TemplateOptions.Builder.runScript;
import static org.jclouds.compute.predicates.NodePredicates.runningWithTag;
import static org.jclouds.io.Payloads.newStringPayload;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
@@ -35,13 +41,13 @@ import java.util.Map;
import java.util.Set;
import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.Cluster.Instance;
import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
import org.apache.whirr.service.ComputeServiceContextBuilder;
import org.apache.whirr.service.Service;
-import org.apache.whirr.service.Cluster.Instance;
-import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
import org.apache.whirr.service.jclouds.FirewallSettings;
-import org.jclouds.aws.ec2.domain.InstanceType;
+import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.RunNodesException;
@@ -52,13 +58,6 @@ import org.jclouds.compute.domain.Templa
import org.jclouds.io.Payload;
import org.jclouds.ssh.ExecResponse;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
public class CassandraService extends Service {
public static final String CASSANDRA_ROLE = "cassandra";
@@ -80,16 +79,13 @@ public class CassandraService extends Se
"apache/cassandra/install"));
TemplateBuilder templateBuilder = computeService.templateBuilder()
- .osFamily(UBUNTU).options(
+ .options(
runScript(bootScript).installPrivateKey(
clusterSpec.getPrivateKey()).authorizePublicKey(
clusterSpec.getPublicKey()));
- // TODO extract this logic elsewhere
- if (clusterSpec.getProvider().equals("ec2"))
- templateBuilder.osVersionMatches("10.04")
- .imageDescriptionMatches(".*ubuntu-images.*")
- .hardwareId(InstanceType.M1_SMALL);
+ new TemplateBuilderStrategy().configureTemplateBuilder(clusterSpec,
+ templateBuilder);
Template template = templateBuilder.build();
Modified: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopService.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopService.java?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopService.java (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopService.java Tue Sep 28 03:28:40 2010
@@ -18,10 +18,9 @@
package org.apache.whirr.service.hadoop;
-import static org.apache.whirr.service.RunUrlBuilder.runUrls;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
-import static org.jclouds.compute.domain.OsFamily.UBUNTU;
+import static org.apache.whirr.service.RunUrlBuilder.runUrls;
import static org.jclouds.compute.options.TemplateOptions.Builder.runScript;
import static org.jclouds.io.Payloads.newStringPayload;
@@ -41,13 +40,13 @@ import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
+import org.apache.whirr.service.Cluster.Instance;
import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
import org.apache.whirr.service.ComputeServiceContextBuilder;
import org.apache.whirr.service.Service;
-import org.apache.whirr.service.Cluster.Instance;
-import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
import org.apache.whirr.service.jclouds.FirewallSettings;
-import org.jclouds.aws.ec2.domain.InstanceType;
+import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.RunNodesException;
@@ -88,16 +87,12 @@ public class HadoopService extends Servi
clusterSpec.getProvider())));
TemplateBuilder masterTemplateBuilder = computeService.templateBuilder()
- .osFamily(UBUNTU)
.options(runScript(nnjtBootScript)
.installPrivateKey(clusterSpec.getPrivateKey())
.authorizePublicKey(clusterSpec.getPublicKey()));
- // TODO extract this logic elsewhere
- if (clusterSpec.getProvider().equals("ec2"))
- masterTemplateBuilder.osVersionMatches("10.04")
- .imageDescriptionMatches(".*ubuntu-images.*")
- .hardwareId(InstanceType.M1_SMALL);
+ TemplateBuilderStrategy strategy = new HadoopTemplateBuilderStrategy();
+ strategy.configureTemplateBuilder(clusterSpec, masterTemplateBuilder);
Template masterTemplate = masterTemplateBuilder.build();
@@ -142,16 +137,11 @@ public class HadoopService extends Servi
jobtrackerPublicAddress.getHostName())));
TemplateBuilder slaveTemplateBuilder = computeService.templateBuilder()
- .osFamily(UBUNTU)
.options(runScript(slaveBootScript)
.installPrivateKey(clusterSpec.getPrivateKey())
.authorizePublicKey(clusterSpec.getPublicKey()));
- // TODO extract this logic elsewhere
- if (clusterSpec.getProvider().equals("ec2"))
- slaveTemplateBuilder.osVersionMatches("10.04")
- .imageDescriptionMatches(".*ubuntu-images.*")
- .hardwareId(InstanceType.M1_SMALL);
+ strategy.configureTemplateBuilder(clusterSpec, slaveTemplateBuilder);
Template slaveTemplate = slaveTemplateBuilder.build();
Added: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopTemplateBuilderStrategy.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopTemplateBuilderStrategy.java?rev=1001999&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopTemplateBuilderStrategy.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopTemplateBuilderStrategy.java Tue Sep 28 03:28:40 2010
@@ -0,0 +1,20 @@
+package org.apache.whirr.service.hadoop;
+
+import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
+import org.jclouds.aws.ec2.compute.domain.EC2Hardware;
+import org.jclouds.compute.domain.TemplateBuilder;
+
+public class HadoopTemplateBuilderStrategy extends TemplateBuilderStrategy {
+
+ public void configureTemplateBuilder(ClusterSpec clusterSpec,
+ TemplateBuilder templateBuilder) {
+ super.configureTemplateBuilder(clusterSpec, templateBuilder);
+
+ if ("ec2".equals(clusterSpec.getProvider())
+ && clusterSpec.getHardwareId() == null) {
+ // micro is too small for Hadoop (even for testing)
+ templateBuilder.fromHardware(EC2Hardware.M1_SMALL);
+ }
+ }
+}
Propchange: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopTemplateBuilderStrategy.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperService.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperService.java?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperService.java (original)
+++ incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperService.java Tue Sep 28 03:28:40 2010
@@ -20,11 +20,18 @@ package org.apache.whirr.service.zookeep
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.whirr.service.RunUrlBuilder.runUrls;
-import static org.jclouds.compute.domain.OsFamily.UBUNTU;
import static org.jclouds.compute.options.TemplateOptions.Builder.runScript;
import static org.jclouds.compute.predicates.NodePredicates.runningWithTag;
import static org.jclouds.io.Payloads.newStringPayload;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
import java.io.IOException;
import java.net.InetAddress;
import java.net.UnknownHostException;
@@ -32,13 +39,13 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
+import org.apache.whirr.service.Cluster.Instance;
import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
import org.apache.whirr.service.ComputeServiceContextBuilder;
import org.apache.whirr.service.Service;
-import org.apache.whirr.service.Cluster.Instance;
-import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
import org.apache.whirr.service.jclouds.FirewallSettings;
-import org.jclouds.aws.ec2.domain.InstanceType;
+import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.RunNodesException;
@@ -48,14 +55,6 @@ import org.jclouds.compute.domain.Templa
import org.jclouds.compute.domain.TemplateBuilder;
import org.jclouds.io.Payload;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Throwables;
-import com.google.common.collect.Collections2;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
public class ZooKeeperService extends Service {
public static final String ZOOKEEPER_ROLE = "zk";
@@ -77,16 +76,12 @@ public class ZooKeeperService extends Se
"apache/zookeeper/install"));
TemplateBuilder templateBuilder = computeService.templateBuilder()
- .osFamily(UBUNTU)
.options(runScript(bootScript)
.installPrivateKey(clusterSpec.getPrivateKey())
.authorizePublicKey(clusterSpec.getPublicKey()));
- // TODO extract this logic elsewhere
- if (clusterSpec.getProvider().equals("ec2"))
- templateBuilder.osVersionMatches("10.04")
- .imageDescriptionMatches(".*ubuntu-images.*")
- .hardwareId(InstanceType.M1_SMALL);
+ new TemplateBuilderStrategy().configureTemplateBuilder(clusterSpec,
+ templateBuilder);
Template template = templateBuilder.build();
Modified: incubator/whirr/trunk/src/site/confluence/configuration-guide.confluence
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/src/site/confluence/configuration-guide.confluence?rev=1001999&r1=1001998&r2=1001999&view=diff
==============================================================================
--- incubator/whirr/trunk/src/site/confluence/configuration-guide.confluence (original)
+++ incubator/whirr/trunk/src/site/confluence/configuration-guide.confluence Tue Sep 28 03:28:40 2010
@@ -11,6 +11,9 @@ Whirr is configured using a properties f
| {{identity}} | {{\--identity}} | none | The cloud identity. See the table below for how this maps to the credentials for your provider. |
| {{credential}} | {{\--credential}} | none | The cloud credential. See the table below for how this maps to the credentials for your provider. |
| {{secret-key-file}} | {{\--secret-key-file}} | _\~/.ssh/id\_rsa_ | The filename of the private key used to connect to instances. |
+| {{image-id}} | {{\--image-id}} | none | The ID of the image to use for instances. If not specified then a vanilla Ubuntu image is chosen. |
+| {{hardware-id}} | {{\--hardware-id}} | none | The type of hardware to use for the instance. This must be compatible with the image ID. |
+| {{location-id}} | {{\--location-id}} | none | The location to launch instances in. If not specified then an arbitrary location will be chosen. |
| {{client-cidrs}} | {{\--client-cidrs}} | none | A comma-separated list of [CIDR |http://en.wikipedia.org/wiki/Classless\_Inter-Domain\_Routing] blocks. E.g. {{208.128.0.0/11,108.128.0.0/11}} |
| {{run-url-base}} | {{\--run-url-base}} | {{http://whirr.s3.amazonaws.com/VERSION/}} | The base URL for forming run urls from. Change this to host your own set of launch scripts. |
@@ -38,9 +41,9 @@ h2. Comparison
| {{client-cidr}} | {{client-cidrs}} | Python {{client-cidr}} option may be repeated multiple times, whereas Java {{client-cidrs}} contains comma-separated CIDRs. |
| none | {{run-url-base}} | Specified using {{user-data-file}} in Python. |
| {{public-key}} | none | Based on secret key in Java (add {{.pub}}). |
-| {{image-id}} | none | Need to add to Java. Along with more flexible image matching (pattern, OS, arch). See jclouds TemplateBuilder. |
-| {{instance-type}} | none | Ditto. |
-| {{availability-zone}} | none | Ditto. |
+| {{image-id}} | {{image-id}} | |
+| {{instance-type}} | {{hardware-id}} | |
+| {{availability-zone}} | {{location-id}} | Location is more general than availability zone. |
| {{security-group}} | none | Amazon-specific. However, Amazon users may wish to start a cluster in additional security groups. |
| {{env}} | none | May not be needed in Java with runurls. |
| {{user-data-file}} | none | Amazon-specific. Use runurls. |