You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by to...@apache.org on 2010/12/16 22:19:31 UTC
svn commit: r1050164 - in /incubator/whirr/trunk: ./
core/src/main/java/org/apache/whirr/service/
core/src/main/java/org/apache/whirr/service/jclouds/
core/src/test/java/org/apache/whirr/service/jclouds/
services/cassandra/src/main/java/org/apache/whir...
Author: tomwhite
Date: Thu Dec 16 21:19:30 2010
New Revision: 1050164
URL: http://svn.apache.org/viewvc?rev=1050164&view=rev
Log:
WHIRR-123. Cassandra integration tests hang if whirr's scripts bucket is missing.
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/RunUrlStatement.java
incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/RunUrlStatementTest.java
incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java
incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Thu Dec 16 21:19:30 2010
@@ -38,6 +38,9 @@ Trunk (unreleased changes)
WHIRR-159. Cassandra and ZooKeeper fail on Ubuntu on Rackspace. (tomwhite)
+ WHIRR-123. Cassandra integration tests hang if whirr's scripts bucket is
+ missing. (tomwhite)
+
Release 0.2.0 - 2010-11-04
NEW FEATURES
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionHandlerSupport.java Thu Dec 16 21:19:30 2010
@@ -19,7 +19,6 @@
package org.apache.whirr.service;
import java.io.IOException;
-import java.net.MalformedURLException;
import org.apache.whirr.service.jclouds.RunUrlStatement;
import org.jclouds.scriptbuilder.domain.Statement;
@@ -82,10 +81,11 @@ public abstract class ClusterActionHandl
* A convenience method for adding a {@link RunUrlStatement} to a
* {@link ClusterActionEvent}.
*/
- public static void addRunUrl(ClusterActionEvent event, String runUrl)
- throws MalformedURLException {
- Statement statement = new RunUrlStatement(event.getClusterSpec()
- .getRunUrlBase(), runUrl);
+ public static void addRunUrl(ClusterActionEvent event, String runUrl,
+ String... args)
+ throws IOException {
+ Statement statement = new RunUrlStatement(
+ event.getClusterSpec().getRunUrlBase(), runUrl, args);
event.getStatementBuilder().addStatement(statement);
}
}
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/RunUrlStatement.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/RunUrlStatement.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/RunUrlStatement.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/RunUrlStatement.java Thu Dec 16 21:19:30 2010
@@ -18,29 +18,46 @@
package org.apache.whirr.service.jclouds;
+import com.google.common.base.Joiner;
import com.google.common.base.Objects;
import com.google.common.collect.ImmutableSet;
-import java.net.MalformedURLException;
+import java.io.IOException;
+import java.net.HttpURLConnection;
import java.net.URL;
+import java.util.Arrays;
+import java.util.List;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.jclouds.scriptbuilder.domain.Statement;
import org.jclouds.scriptbuilder.domain.Statements;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class RunUrlStatement implements Statement {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RunUrlStatement.class);
+
private String runUrl;
+ private List<String> args;
- public RunUrlStatement(String runUrl) {
- this.runUrl = runUrl;
+ public RunUrlStatement(String runUrlBase, String url, String... args)
+ throws IOException {
+ this(true, runUrlBase, url, args);
}
- public RunUrlStatement(String runUrlBase, String url)
- throws MalformedURLException {
- this(new URL(new URL(runUrlBase), url).toExternalForm());
+ public RunUrlStatement(boolean checkUrlExists, String runUrlBase, String url,
+ String... args)
+ throws IOException {
+ URL runUrl = new URL(new URL(runUrlBase), url);
+ if (checkUrlExists) {
+ checkUrlExists(runUrl, "Runurl %s not found.", runUrl);
+ }
+ this.runUrl = runUrl.toExternalForm();
+ this.args = Arrays.asList(args);
}
-
+
@Override
public Iterable<String> functionDependecies(OsFamily family) {
return ImmutableSet.<String>of("installRunUrl");
@@ -48,21 +65,49 @@ public class RunUrlStatement implements
@Override
public String render(OsFamily family) {
- return Statements.exec("runurl " + runUrl).render(family);
+ StringBuilder command = new StringBuilder("runurl ");
+ command.append(runUrl);
+ if (!args.isEmpty()) {
+ command.append(' ');
+ command.append(Joiner.on(' ').join(args));
+ }
+ return Statements.exec(command.toString()).render(family);
}
@Override
public boolean equals(Object o) {
if (o instanceof RunUrlStatement) {
RunUrlStatement that = (RunUrlStatement) o;
- return Objects.equal(runUrl, that.runUrl);
+ return Objects.equal(runUrl, that.runUrl)
+ && Objects.equal(args, that.args);
}
return false;
}
@Override
public int hashCode() {
- return Objects.hashCode(runUrl);
+ return Objects.hashCode(runUrl, args);
}
+ public static void checkUrlExists(URL url, String errorMessageTemplate,
+ Object... errorMessageArgs)
+ throws IOException {
+ if (!urlExists(url)) {
+ throw new IllegalArgumentException(
+ String.format(errorMessageTemplate, errorMessageArgs));
+ }
+ }
+
+ private static boolean urlExists(URL url) throws IOException {
+ HttpURLConnection connection = (HttpURLConnection) url.openConnection();
+ try {
+ connection.setRequestMethod("HEAD");
+ connection.connect();
+ int responseCode = connection.getResponseCode();
+ LOG.debug("Response code {} from {}", responseCode, url);
+ return responseCode == HttpURLConnection.HTTP_OK;
+ } finally {
+ connection.disconnect();
+ }
+ }
}
Modified: incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/RunUrlStatementTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/RunUrlStatementTest.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/RunUrlStatementTest.java (original)
+++ incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/RunUrlStatementTest.java Thu Dec 16 21:19:30 2010
@@ -21,7 +21,8 @@ package org.apache.whirr.service.jclouds
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
-import java.net.MalformedURLException;
+import java.io.IOException;
+import java.net.URL;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.junit.Test;
@@ -31,25 +32,44 @@ public class RunUrlStatementTest {
private static final String NL = System.getProperty("line.separator");
@Test
- public void testBaseWithTrailingSlash() throws MalformedURLException {
+ public void testBaseWithTrailingSlash() throws IOException {
assertThat(
- new RunUrlStatement("http://example.org/", "a/b").render(OsFamily.UNIX),
+ new RunUrlStatement(false, "http://example.org/", "a/b").render(OsFamily.UNIX),
is("runurl http://example.org/a/b" + NL));
}
+
+ @Test
+ public void testWithArgs() throws IOException {
+ assertThat(
+ new RunUrlStatement(false, "http://example.org/", "a/b", "x", "y").render(OsFamily.UNIX),
+ is("runurl http://example.org/a/b x y" + NL));
+ }
@Test
- public void testBaseWithoutTrailingSlash() throws MalformedURLException {
+ public void testBaseWithoutTrailingSlash() throws IOException {
assertThat(
- new RunUrlStatement("http://example.org", "a/b").render(OsFamily.UNIX),
+ new RunUrlStatement(false, "http://example.org", "a/b").render(OsFamily.UNIX),
is("runurl http://example.org/a/b" + NL));
}
@Test
- public void testAbsolutePath() throws MalformedURLException {
+ public void testAbsolutePath() throws IOException {
assertThat(
- new RunUrlStatement("http://example.org/", "http://example2.org/a/b")
+ new RunUrlStatement(false, "http://example.org/", "http://example2.org/a/b")
.render(OsFamily.UNIX),
is("runurl http://example2.org/a/b" + NL));
}
+
+ @Test
+ public void testCheckUrlExists() throws IOException {
+ RunUrlStatement.checkUrlExists(new URL("http://whirr.s3.amazonaws.com/"),
+ "Exists");
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testCheckUrlDoesNotExist() throws IOException {
+ RunUrlStatement.checkUrlExists(
+ new URL("http://whirr.s3.amazonaws.com/non-existent"), "Doesn't exist");
+ }
}
Modified: incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java (original)
+++ incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/jclouds/StatementBuilderTest.java Thu Dec 16 21:19:30 2010
@@ -23,7 +23,7 @@ import static org.hamcrest.Matchers.grea
import static org.hamcrest.Matchers.is;
import static org.junit.Assert.assertThat;
-import java.net.MalformedURLException;
+import java.io.IOException;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.junit.Test;
@@ -31,11 +31,14 @@ import org.junit.Test;
public class StatementBuilderTest {
@Test
- public void testDeduplication() throws MalformedURLException {
+ public void testDeduplication() throws IOException {
StatementBuilder builder = new StatementBuilder();
- builder.addStatement(new RunUrlStatement("http://example.org/a/b c"));
- builder.addStatement(new RunUrlStatement("http://example.org/d/e f"));
- builder.addStatement(new RunUrlStatement("http://example.org/a/b c"));
+ builder.addStatement(
+ new RunUrlStatement(false, "http://example.org/", "a/b", "c"));
+ builder.addStatement(
+ new RunUrlStatement(false, "http://example.org/", "d/e", "f"));
+ builder.addStatement(
+ new RunUrlStatement(false, "http://example.org/", "a/b", "c"));
String script = builder.render(OsFamily.UNIX);
int first = script.indexOf("runurl http://example.org/a/b c");
assertThat(first, greaterThan(-1));
Modified: incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/cassandra/src/main/java/org/apache/whirr/service/cassandra/CassandraClusterActionHandler.java Thu Dec 16 21:19:30 2010
@@ -72,10 +72,8 @@ public class CassandraClusterActionHandl
List<Instance> seeds = getSeeds(cluster.getInstances());
String servers = Joiner.on(' ').join(getPrivateIps(seeds));
- addRunUrl(event,
- String.format("apache/cassandra/post-configure -c %s %s",
- clusterSpec.getProvider(),
- servers));
+ addRunUrl(event, "apache/cassandra/post-configure",
+ "-c", clusterSpec.getProvider(), servers);
}
private List<String> getPrivateIps(List<Instance> instances) {
Modified: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java Thu Dec 16 21:19:30 2010
@@ -41,13 +41,11 @@ public class HadoopDataNodeClusterAction
@Override
protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
ClusterSpec clusterSpec = event.getClusterSpec();
- addRunUrl(event, String.format("util/configure-hostnames -c %s",
- clusterSpec.getProvider()));
+ addRunUrl(event, "util/configure-hostnames", "-c", clusterSpec.getProvider());
String hadoopInstallRunUrl = clusterSpec.getConfiguration().getString(
"whirr.hadoop-install-runurl", "apache/hadoop/install");
addRunUrl(event, "sun/java/install");
- addRunUrl(event, String.format("%s -c %s", hadoopInstallRunUrl,
- clusterSpec.getProvider()));
+ addRunUrl(event, hadoopInstallRunUrl, "-c", clusterSpec.getProvider());
event.setTemplateBuilderStrategy(new HadoopTemplateBuilderStrategy());
}
@@ -64,13 +62,11 @@ public class HadoopDataNodeClusterAction
String hadoopConfigureRunUrl = clusterSpec.getConfiguration().getString(
"whirr.hadoop-configure-runurl", "apache/hadoop/post-configure");
- addRunUrl(event,
- String.format(
- "%s dn,tt -n %s -j %s -c %s",
- hadoopConfigureRunUrl,
- DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
- DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
- clusterSpec.getProvider()));
+ addRunUrl(event, hadoopConfigureRunUrl,
+ "dn,tt",
+ "-n", DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
+ "-j", DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
+ clusterSpec.getProvider());
}
}
Modified: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java Thu Dec 16 21:19:30 2010
@@ -62,13 +62,11 @@ public class HadoopNameNodeClusterAction
@Override
protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
ClusterSpec clusterSpec = event.getClusterSpec();
- addRunUrl(event, String.format("util/configure-hostnames -c %s",
- clusterSpec.getProvider()));
+ addRunUrl(event, "util/configure-hostnames", "-c", clusterSpec.getProvider());
String hadoopInstallRunUrl = clusterSpec.getConfiguration().getString(
"whirr.hadoop-install-runurl", "apache/hadoop/install");
addRunUrl(event, "sun/java/install");
- addRunUrl(event, String.format("%s -c %s", hadoopInstallRunUrl,
- clusterSpec.getProvider()));
+ addRunUrl(event, hadoopInstallRunUrl, "-c", clusterSpec.getProvider());
event.setTemplateBuilderStrategy(new HadoopTemplateBuilderStrategy());
}
@@ -103,12 +101,11 @@ public class HadoopNameNodeClusterAction
String hadoopConfigureRunUrl = clusterSpec.getConfiguration().getString(
"whirr.hadoop-configure-runurl", "apache/hadoop/post-configure");
- addRunUrl(event,
- String.format("%s nn,jt -n %s -j %s -c %s",
- hadoopConfigureRunUrl,
- DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
- DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
- clusterSpec.getProvider()));
+ addRunUrl(event, hadoopConfigureRunUrl,
+ "nn,jt",
+ "-n", DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
+ "-j", DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
+ clusterSpec.getProvider());
}
@Override
Modified: incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java?rev=1050164&r1=1050163&r2=1050164&view=diff
==============================================================================
--- incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/zookeeper/src/main/java/org/apache/whirr/service/zookeeper/ZooKeeperClusterActionHandler.java Thu Dec 16 21:19:30 2010
@@ -68,10 +68,9 @@ public class ZooKeeperClusterActionHandl
// Pass list of all servers in ensemble to configure script.
// Position is significant: i-th server has id i.
String servers = Joiner.on(' ').join(getPrivateIps(cluster.getInstances()));
- addRunUrl(event,
- String.format("apache/zookeeper/post-configure -c %s %s",
+ addRunUrl(event, "apache/zookeeper/post-configure", "-c",
clusterSpec.getProvider(),
- servers));
+ servers);
}
@Override