You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by to...@apache.org on 2011/04/27 06:07:02 UTC
svn commit: r1096985 - in /incubator/whirr/trunk: ./ core/
core/src/main/java/org/apache/whirr/
core/src/main/java/org/apache/whirr/actions/
core/src/main/java/org/apache/whirr/service/
core/src/main/resources/META-INF/ core/src/main/resources/META-INF...
Author: tomwhite
Date: Wed Apr 27 04:07:02 2011
New Revision: 1096985
URL: http://svn.apache.org/viewvc?rev=1096985&view=rev
Log:
WHIRR-285. Add support for BYON.
Added:
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ByonClusterController.java (with props)
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ByonClusterAction.java (with props)
incubator/whirr/trunk/core/src/main/resources/META-INF/
incubator/whirr/trunk/core/src/main/resources/META-INF/services/
incubator/whirr/trunk/core/src/main/resources/META-INF/services/org.apache.whirr.ClusterController
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/core/pom.xml
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeServiceContextBuilder.java
incubator/whirr/trunk/pom.xml
incubator/whirr/trunk/services/zookeeper/src/main/resources/functions/configure_zookeeper.sh
incubator/whirr/trunk/services/zookeeper/src/test/java/org/apache/whirr/service/zookeeper/integration/ZooKeeperServiceTest.java
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1096985&r1=1096984&r2=1096985&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Wed Apr 27 04:07:02 2011
@@ -12,6 +12,8 @@ Trunk (unreleased changes)
WHIRR-220. Support local tarball upload (asavu)
+ WHIRR-285. Add support for BYON. (tomwhite)
+
IMPROVEMENTS
WHIRR-262. Services should not have to do reverse DNS lookups. (tomwhite)
Modified: incubator/whirr/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/pom.xml?rev=1096985&r1=1096984&r2=1096985&view=diff
==============================================================================
--- incubator/whirr/trunk/core/pom.xml (original)
+++ incubator/whirr/trunk/core/pom.xml Wed Apr 27 04:07:02 2011
@@ -42,6 +42,10 @@
<artifactId>jclouds-core</artifactId>
</dependency>
<dependency>
+ <groupId>org.jclouds.api</groupId>
+ <artifactId>byon</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-jsch</artifactId>
</dependency>
Added: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ByonClusterController.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ByonClusterController.java?rev=1096985&view=auto
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ByonClusterController.java (added)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ByonClusterController.java Wed Apr 27 04:07:02 2011
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr;
+
+import static org.apache.whirr.service.ClusterActionHandler.BOOTSTRAP_ACTION;
+import static org.apache.whirr.service.ClusterActionHandler.CONFIGURE_ACTION;
+import static org.apache.whirr.service.ClusterActionHandler.DESTROY_ACTION;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.whirr.actions.ByonClusterAction;
+import org.apache.whirr.service.ClusterActionHandler;
+import org.jclouds.compute.ComputeServiceContextFactory;
+import org.jclouds.compute.domain.NodeMetadata;
+
+/**
+ * Equivalent of {@link ClusterController}, but for execution in BYON mode
+ * ("bring your own nodes").
+ */
+public class ByonClusterController extends ClusterController {
+
+ @Override
+ public String getName() {
+ return "byon";
+ }
+
+ public Cluster launchCluster(ClusterSpec clusterSpec) throws IOException,
+ InterruptedException {
+
+ ComputeServiceContextFactory computeServiceFactory = new ComputeServiceContextFactory();
+ Map<String, ClusterActionHandler> handlerMap = new HandlerMapFactory()
+ .create();
+
+ ClusterAction bootstrapper = new ByonClusterAction(BOOTSTRAP_ACTION,
+ computeServiceFactory, handlerMap);
+ Cluster cluster = bootstrapper.execute(clusterSpec, null);
+
+ ClusterAction configurer = new ByonClusterAction(CONFIGURE_ACTION,
+ computeServiceFactory, handlerMap);
+ cluster = configurer.execute(clusterSpec, cluster);
+
+ return cluster;
+ }
+
+ public void destroyCluster(ClusterSpec clusterSpec) throws IOException,
+ InterruptedException {
+ ComputeServiceContextFactory computeServiceFactory = new ComputeServiceContextFactory();
+ Map<String, ClusterActionHandler> handlerMap = new HandlerMapFactory()
+ .create();
+
+ ClusterAction destroyer = new ByonClusterAction(DESTROY_ACTION,
+ computeServiceFactory, handlerMap);
+ destroyer.execute(clusterSpec, null);
+ }
+
+ @Override
+ public void destroyInstance(ClusterSpec clusterSpec, String instanceId)
+ throws IOException {
+ // TODO
+ }
+
+ @Override
+ public Set<? extends NodeMetadata> getNodes(ClusterSpec clusterSpec)
+ throws IOException, InterruptedException {
+ // TODO return singleton with trivial NodeMetadata for localhost?
+ return null;
+ }
+}
Propchange: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/ByonClusterController.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ByonClusterAction.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ByonClusterAction.java?rev=1096985&view=auto
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ByonClusterAction.java (added)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ByonClusterAction.java Wed Apr 27 04:07:02 2011
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.actions;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicates;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.whirr.Cluster;
+import org.apache.whirr.Cluster.Instance;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.InstanceTemplate;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandler;
+import org.apache.whirr.service.ComputeServiceContextBuilder;
+import org.apache.whirr.service.jclouds.StatementBuilder;
+import org.jclouds.compute.ComputeService;
+import org.jclouds.compute.ComputeServiceContext;
+import org.jclouds.compute.ComputeServiceContextFactory;
+import org.jclouds.compute.domain.ComputeMetadata;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.domain.Credentials;
+import org.jclouds.scriptbuilder.domain.OsFamily;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ByonClusterAction extends ScriptBasedClusterAction {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ByonClusterAction.class);
+
+ private final String action;
+
+ public ByonClusterAction(String action, final ComputeServiceContextFactory computeServiceContextFactory,
+ final Map<String, ClusterActionHandler> handlerMap) {
+ super(computeServiceContextFactory, handlerMap);
+ this.action = action;
+ }
+
+ @Override
+ protected String getAction() {
+ return action;
+ }
+
+ @Override
+ protected void doAction(Map<InstanceTemplate, ClusterActionEvent> eventMap)
+ throws IOException, InterruptedException {
+
+ ExecutorService executorService = Executors.newCachedThreadPool();
+
+ Set<Future<Void>> futures = Sets.newHashSet();
+
+ List<NodeMetadata> nodes = Lists.newArrayList();
+ int numberAllocated = 0;
+ Set<Instance> instances = Sets.newLinkedHashSet();
+ for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
+
+ ClusterSpec clusterSpec = entry.getValue().getClusterSpec();
+ final StatementBuilder statementBuilder = entry.getValue().getStatementBuilder();
+ ComputeServiceContext computeServiceContext =
+ ComputeServiceContextBuilder.build(getComputeServiceContextFactory(), clusterSpec);
+ final ComputeService computeService = computeServiceContext.getComputeService();
+ Credentials credentials = new Credentials(clusterSpec.getIdentity(), clusterSpec.getCredential());
+
+ if (numberAllocated == 0) {
+ for (ComputeMetadata compute : computeService.listNodes()) {
+ if (!(compute instanceof NodeMetadata)) {
+ throw new IllegalArgumentException("Not an instance of NodeMetadata: " + compute);
+ }
+ nodes.add((NodeMetadata) compute);
+ }
+ }
+ int num = entry.getKey().getNumberOfInstances();
+ final List<NodeMetadata> templateNodes =
+ nodes.subList(numberAllocated, numberAllocated + num);
+ numberAllocated += num;
+
+ instances.addAll(getInstances(credentials,
+ entry.getKey().getRoles(), templateNodes));
+
+ futures.add(executorService.submit(new Callable<Void>() {
+ @Override
+ public Void call() throws Exception {
+ LOG.info("Running script");
+ if (LOG.isDebugEnabled())
+ LOG.debug("Running script:\n{}", statementBuilder.render(OsFamily.UNIX));
+ computeService.runScriptOnNodesMatching(
+ Predicates.in(templateNodes),
+ statementBuilder);
+ LOG.info("Script run completed");
+ return null;
+ }
+ }));
+ }
+
+ for (Future<Void> future : futures) {
+ try {
+ future.get();
+ } catch (ExecutionException e) {
+ throw new IOException(e.getCause());
+ }
+ }
+
+ if (action.equals(ClusterActionHandler.BOOTSTRAP_ACTION)) {
+ Cluster cluster = new Cluster(instances);
+ for (ClusterActionEvent event : eventMap.values()) {
+ event.setCluster(cluster);
+ }
+ }
+ }
+
+ private Set<Instance> getInstances(final Credentials credentials, final Set<String> roles,
+ Collection<NodeMetadata> nodes) {
+ return Sets.newLinkedHashSet(Collections2.transform(Sets.newLinkedHashSet(nodes),
+ new Function<NodeMetadata, Instance>() {
+ @Override
+ public Instance apply(NodeMetadata node) {
+ String publicIp = Iterables.get(node.getPublicAddresses(), 0);
+ return new Instance(credentials, roles, publicIp, publicIp, node.getId(), null);
+ }
+ }));
+ }
+
+}
Propchange: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ByonClusterAction.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeServiceContextBuilder.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeServiceContextBuilder.java?rev=1096985&r1=1096984&r2=1096985&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeServiceContextBuilder.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ComputeServiceContextBuilder.java Wed Apr 27 04:07:02 2011
@@ -50,6 +50,12 @@ public class ComputeServiceContextBuilde
public static ComputeServiceContext build(final ComputeServiceContextFactory factory, final ClusterSpec spec) throws IOException {
Configuration jcloudsConfig =
spec.getConfigurationForKeysWithPrefix("jclouds");
+
+ // jclouds byon.endpoint property does not follow convention of starting
+ // with "jclouds." prefix, so we special case it here
+ if (jcloudsConfig.containsKey("jclouds.byon.endpoint")) {
+ jcloudsConfig.setProperty("byon.endpoint", jcloudsConfig.getProperty("jclouds.byon.endpoint"));
+ }
Set<AbstractModule> wiring = ImmutableSet.of(new JschSshClientModule(),
new Log4JLoggingModule(), new BindLoginCredentialsPatchForEC2());
if (spec.getProvider().equals("ec2")){
Added: incubator/whirr/trunk/core/src/main/resources/META-INF/services/org.apache.whirr.ClusterController
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/resources/META-INF/services/org.apache.whirr.ClusterController?rev=1096985&view=auto
==============================================================================
--- incubator/whirr/trunk/core/src/main/resources/META-INF/services/org.apache.whirr.ClusterController (added)
+++ incubator/whirr/trunk/core/src/main/resources/META-INF/services/org.apache.whirr.ClusterController Wed Apr 27 04:07:02 2011
@@ -0,0 +1 @@
+org.apache.whirr.ByonClusterController
\ No newline at end of file
Modified: incubator/whirr/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/pom.xml?rev=1096985&r1=1096984&r2=1096985&view=diff
==============================================================================
--- incubator/whirr/trunk/pom.xml (original)
+++ incubator/whirr/trunk/pom.xml Wed Apr 27 04:07:02 2011
@@ -70,6 +70,11 @@
<version>${jclouds.version}</version>
</dependency>
<dependency>
+ <groupId>org.jclouds.api</groupId>
+ <artifactId>byon</artifactId>
+ <version>${jclouds.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.jclouds.driver</groupId>
<artifactId>jclouds-jsch</artifactId>
<version>${jclouds.version}</version>
Modified: incubator/whirr/trunk/services/zookeeper/src/main/resources/functions/configure_zookeeper.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/zookeeper/src/main/resources/functions/configure_zookeeper.sh?rev=1096985&r1=1096984&r2=1096985&view=diff
==============================================================================
--- incubator/whirr/trunk/services/zookeeper/src/main/resources/functions/configure_zookeeper.sh (original)
+++ incubator/whirr/trunk/services/zookeeper/src/main/resources/functions/configure_zookeeper.sh Wed Apr 27 04:07:02 2011
@@ -34,9 +34,12 @@ function configure_zookeeper() {
ec2 | aws-ec2 )
SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/local-ipv4`
;;
- *)
+ cloudservers-uk | cloudservers-us)
SELF_HOST=`/sbin/ifconfig eth1 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
;;
+ *)
+ SELF_HOST=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
+ ;;
esac
myid_file=/var/log/zookeeper/txlog/myid
Modified: incubator/whirr/trunk/services/zookeeper/src/test/java/org/apache/whirr/service/zookeeper/integration/ZooKeeperServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/zookeeper/src/test/java/org/apache/whirr/service/zookeeper/integration/ZooKeeperServiceTest.java?rev=1096985&r1=1096984&r2=1096985&view=diff
==============================================================================
--- incubator/whirr/trunk/services/zookeeper/src/test/java/org/apache/whirr/service/zookeeper/integration/ZooKeeperServiceTest.java (original)
+++ incubator/whirr/trunk/services/zookeeper/src/test/java/org/apache/whirr/service/zookeeper/integration/ZooKeeperServiceTest.java Wed Apr 27 04:07:02 2011
@@ -27,6 +27,7 @@ import org.apache.commons.configuration.
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.whirr.Cluster;
import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterControllerFactory;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.service.zookeeper.ZooKeeperCluster;
import org.apache.zookeeper.CreateMode;
@@ -54,7 +55,7 @@ public class ZooKeeperServiceTest {
}
config.addConfiguration(new PropertiesConfiguration("whirr-zookeeper-test.properties"));
clusterSpec = ClusterSpec.withTemporaryKeys(config);
- controller = new ClusterController();
+ controller = new ClusterControllerFactory().create(clusterSpec.getServiceName());
cluster = controller.launchCluster(clusterSpec);
hosts = ZooKeeperCluster.getHosts(cluster);