You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by as...@apache.org on 2011/07/15 07:55:36 UTC
svn commit: r1146966 - in /incubator/whirr/trunk: ./
core/src/main/java/org/apache/whirr/
core/src/main/java/org/apache/whirr/actions/
core/src/main/java/org/apache/whirr/service/jclouds/ recipes/ src/site/xdoc/
Author: asavu
Date: Fri Jul 15 05:55:35 2011
New Revision: 1146966
URL: http://svn.apache.org/viewvc?rev=1146966&view=rev
Log:
WHIRR-260. Support Spot Instances (asavu)
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
incubator/whirr/trunk/recipes/hadoop-ec2.properties
incubator/whirr/trunk/src/site/xdoc/configuration-guide.xml
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1146966&r1=1146965&r2=1146966&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Fri Jul 15 05:55:35 2011
@@ -11,6 +11,8 @@ Trunk (unreleased changes)
WHIRR-76. Support spot instances in python scripts (Soren Macbeth via asavu)
+ WHIRR-260. Support Spot Instances (asavu)
+
IMPROVEMENTS
WHIRR-28. Add examples module (asavu)
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java?rev=1146966&r1=1146965&r2=1146966&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ClusterSpec.java Fri Jul 15 05:55:35 2011
@@ -139,6 +139,8 @@ public class ClusterSpec {
STATE_STORE_BLOB(String.class, false, "Blob name for state storage. " +
"Valid only for the blob state store. Defaults to whirr-<cluster-name>"),
+
+ AWS_EC2_SPOT_PRICE(Float.class, false, "Spot instance price (aws-ec2 specific option)"),
IMAGE_ID(String.class, false, "The ID of the image to use for " +
"instances. If not specified then a vanilla Linux image is " +
@@ -256,6 +258,8 @@ public class ClusterSpec {
private String stateStoreContainer;
private String stateStoreBlob;
+ private float awsEc2SpotPrice;
+
private String privateKey;
private File privateKeyFile;
private String publicKey;
@@ -316,6 +320,8 @@ public class ClusterSpec {
setStateStoreContainer(getString(Property.STATE_STORE_CONTAINER));
setStateStoreBlob(getString(Property.STATE_STORE_BLOB));
+ setAwsEc2SpotPrice(getFloat(Property.AWS_EC2_SPOT_PRICE, -1));
+
checkAndSetKeyPair();
setImageId(getString(Property.IMAGE_ID));
@@ -356,6 +362,8 @@ public class ClusterSpec {
r.setBlobStoreCredential(getBlobStoreCredential());
r.setBlobStoreCacheContainer(getBlobStoreCacheContainer());
+ r.setAwsEc2SpotPrice(getAwsEc2SpotPrice());
+
r.setStateStore(getStateStore());
r.setStateStoreContainer(getStateStoreContainer());
r.setStateStoreBlob(getStateStoreBlob());
@@ -385,6 +393,10 @@ public class ClusterSpec {
return config.getInt(key.getConfigName(), defaultValue);
}
+ private float getFloat(Property key, float defaultValue) {
+ return config.getFloat(key.getConfigName(), defaultValue);
+ }
+
private List<String> getList(Property key) {
return config.getList(key.getConfigName());
}
@@ -540,6 +552,10 @@ public class ClusterSpec {
return stateStoreBlob;
}
+ public float getAwsEc2SpotPrice() {
+ return awsEc2SpotPrice;
+ }
+
public String getServiceName() {
return serviceName;
}
@@ -663,6 +679,10 @@ public class ClusterSpec {
this.stateStoreBlob = blob;
}
+ public void setAwsEc2SpotPrice(float value) {
+ this.awsEc2SpotPrice = value;
+ }
+
public void setClusterName(String clusterName) {
this.clusterName = clusterName;
}
@@ -826,6 +846,7 @@ public class ClusterSpec {
&& Objects.equal(getStateStore(), that.getStateStore())
&& Objects.equal(getStateStoreContainer(), that.getStateStoreContainer())
&& Objects.equal(getStateStoreBlob(), that.getStateStoreBlob())
+ && Objects.equal(getAwsEc2SpotPrice(), that.getAwsEc2SpotPrice())
;
}
return false;
@@ -858,7 +879,8 @@ public class ClusterSpec {
getRunUrlBase(),
getStateStore(),
getStateStoreBlob(),
- getStateStoreContainer()
+ getStateStoreContainer(),
+ getAwsEc2SpotPrice()
);
}
@@ -890,6 +912,7 @@ public class ClusterSpec {
.add("stateStore", getStateStore())
.add("stateStoreContainer", getStateStoreContainer())
.add("stateStoreBlob", getStateStoreBlob())
+ .add("awsEc2SpotPrice", getAwsEc2SpotPrice())
.toString();
}
}
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java?rev=1146966&r1=1146965&r2=1146966&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/BootstrapClusterAction.java Fri Jul 15 05:55:35 2011
@@ -53,12 +53,14 @@ import org.apache.whirr.service.ClusterA
import org.apache.whirr.service.ClusterActionHandler;
import org.apache.whirr.service.jclouds.StatementBuilder;
import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
+import org.jclouds.aws.ec2.compute.AWSEC2TemplateOptions;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.RunNodesException;
import org.jclouds.compute.domain.NodeMetadata;
import org.jclouds.compute.domain.Template;
import org.jclouds.compute.domain.TemplateBuilder;
+import org.jclouds.aws.ec2.AWSEC2Client;
import org.jclouds.scriptbuilder.InitBuilder;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.jclouds.scriptbuilder.domain.Statement;
@@ -148,19 +150,39 @@ public class BootstrapClusterAction exte
ComputeService computeService, StatementBuilder statementBuilder,
TemplateBuilderStrategy strategy)
throws MalformedURLException {
+
LOG.info("Configuring template");
if (LOG.isDebugEnabled())
LOG.debug("Running script:\n{}", statementBuilder.render(OsFamily.UNIX));
+
Statement runScript = addUserAndAuthorizeSudo(
clusterSpec.getClusterUser(),
clusterSpec.getPublicKey(),
clusterSpec.getPrivateKey(),
statementBuilder);
+
TemplateBuilder templateBuilder = computeService.templateBuilder()
.options(runScript(runScript));
strategy.configureTemplateBuilder(clusterSpec, templateBuilder);
- return templateBuilder.build();
-
+
+ return setSpotInstancePriceIfSpecified(
+ computeService.getContext(), clusterSpec, templateBuilder.build());
+ }
+
+ /**
+ * Set maximum spot instance price based on the configuration
+ */
+ private Template setSpotInstancePriceIfSpecified(
+ ComputeServiceContext context, ClusterSpec spec, Template template) {
+
+ if (context != null && context.getProviderSpecificContext().getApi() instanceof AWSEC2Client) {
+ if (spec.getAwsEc2SpotPrice() > 0) {
+ template.getOptions().as(AWSEC2TemplateOptions.class)
+ .spotPrice(spec.getAwsEc2SpotPrice());
+ }
+ }
+
+ return template;
}
private static Statement addUserAndAuthorizeSudo(String user,
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java?rev=1146966&r1=1146965&r2=1146966&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/TemplateBuilderStrategy.java Fri Jul 15 05:55:35 2011
@@ -45,7 +45,7 @@ public class TemplateBuilderStrategy {
} else {
templateBuilder.minRam(1024);
}
-
+
if (clusterSpec.getLocationId() != null) {
templateBuilder.locationId(clusterSpec.getLocationId());
}
Modified: incubator/whirr/trunk/recipes/hadoop-ec2.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/recipes/hadoop-ec2.properties?rev=1146966&r1=1146965&r2=1146966&view=diff
==============================================================================
--- incubator/whirr/trunk/recipes/hadoop-ec2.properties (original)
+++ incubator/whirr/trunk/recipes/hadoop-ec2.properties Fri Jul 15 05:55:35 2011
@@ -44,6 +44,10 @@ whirr.image-id=us-east-1/ami-da0cf8b3
# If you choose a different location, make sure whirr.image-id is updated too
whirr.location-id=us-east-1
+# You can also specify the spot instance price
+# http://aws.amazon.com/ec2/spot-instances/
+# whirr.aws-ec2-spot-price=0.15
+
# By default use the user system SSH keys. Override them here.
# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
# whirr.public-key-file=${whirr.private-key-file}.pub
@@ -57,4 +61,4 @@ whirr.location-id=us-east-1
# Expert: specify the version of Hadoop to install.
#whirr.hadoop.version=0.20.2
-#whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
\ No newline at end of file
+#whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
Modified: incubator/whirr/trunk/src/site/xdoc/configuration-guide.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/src/site/xdoc/configuration-guide.xml?rev=1146966&r1=1146965&r2=1146966&view=diff
==============================================================================
--- incubator/whirr/trunk/src/site/xdoc/configuration-guide.xml (original)
+++ incubator/whirr/trunk/src/site/xdoc/configuration-guide.xml Fri Jul 15 05:55:35 2011
@@ -182,6 +182,25 @@ xsi:schemaLocation="http://maven.apache.
<td>The number of retries in case of insufficient successfully started
instances.</td>
</tr>
+ <tr valign="top">
+ <td>
+ <tt>whirr.aws-ec2-spot-price</tt>
+ </td>
+ <td>
+ <tt>--aws-ec2-spot-price</tt>
+ </td>
+ <td>
+ <tt>none</tt>
+ </td>
+ <td>
+ Spot instance price. If the price isn't fulfilled it times out after 20 minutes
+ (<tt>jclouds.compute.timeout.node-running</tt>). We recommended that you set the spot price
+ to current real price so that you will always save money while keeping the instances
+ running because you will still pay the spot price and not the max value you put in. Keep
+ in mind that price spikes can still take out some instances, or prevent requests being
+ filled, but it works well most of the time. Note: this is an EC2 specific option.
+ </td>
+ </tr>
</table>
<subsection name="Cloud Provider Options"></subsection>
<table border="0">