You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by as...@apache.org on 2011/04/12 12:37:25 UTC
svn commit: r1091362 - in /incubator/whirr/trunk: ./ cli/ recipes/
services/voldemort/ services/voldemort/lib/ services/voldemort/src/
services/voldemort/src/main/ services/voldemort/src/main/java/
services/voldemort/src/main/java/org/ services/voldemo...
Author: asavu
Date: Tue Apr 12 10:37:24 2011
New Revision: 1091362
URL: http://svn.apache.org/viewvc?rev=1091362&view=rev
Log:
WHIRR-237. Add Voldemort as a service. (Kirk True via asavu)
Added:
incubator/whirr/trunk/recipes/voldemort-ec2.properties
incubator/whirr/trunk/services/voldemort/ (with props)
incubator/whirr/trunk/services/voldemort/lib/
incubator/whirr/trunk/services/voldemort/lib/linkedin-voldemort-0.90.RC3.jar (with props)
incubator/whirr/trunk/services/voldemort/pom.xml
incubator/whirr/trunk/services/voldemort/src/
incubator/whirr/trunk/services/voldemort/src/main/
incubator/whirr/trunk/services/voldemort/src/main/java/
incubator/whirr/trunk/services/voldemort/src/main/java/org/
incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/
incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/
incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/
incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/
incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java
incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortConstants.java
incubator/whirr/trunk/services/voldemort/src/main/resources/
incubator/whirr/trunk/services/voldemort/src/main/resources/META-INF/
incubator/whirr/trunk/services/voldemort/src/main/resources/META-INF/services/
incubator/whirr/trunk/services/voldemort/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
incubator/whirr/trunk/services/voldemort/src/main/resources/functions/
incubator/whirr/trunk/services/voldemort/src/main/resources/functions/configure_voldemort.sh
incubator/whirr/trunk/services/voldemort/src/main/resources/functions/install_voldemort.sh
incubator/whirr/trunk/services/voldemort/src/test/
incubator/whirr/trunk/services/voldemort/src/test/java/
incubator/whirr/trunk/services/voldemort/src/test/java/org/
incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/
incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/
incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/
incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/voldemort/
incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/voldemort/integration/
incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/voldemort/integration/VoldemortServiceTest.java
incubator/whirr/trunk/services/voldemort/src/test/resources/
incubator/whirr/trunk/services/voldemort/src/test/resources/log4j.xml
incubator/whirr/trunk/services/voldemort/src/test/resources/whirr-voldemort-test.properties
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/cli/pom.xml
incubator/whirr/trunk/pom.xml
incubator/whirr/trunk/src/site/confluence/index.confluence
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1091362&r1=1091361&r2=1091362&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Tue Apr 12 10:37:24 2011
@@ -8,6 +8,8 @@ Trunk (unreleased changes)
WHIRR-222. Support multiple versions of Hadoop. (tomwhite)
+ WHIRR-237. Add Voldemort as a service. (Kirk True via asavu)
+
IMPROVEMENTS
WHIRR-262. Services should not have to do reverse DNS lookups. (tomwhite)
Modified: incubator/whirr/trunk/cli/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/pom.xml?rev=1091362&r1=1091361&r2=1091362&view=diff
==============================================================================
--- incubator/whirr/trunk/cli/pom.xml (original)
+++ incubator/whirr/trunk/cli/pom.xml Tue Apr 12 10:37:24 2011
@@ -31,6 +31,11 @@
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>whirr-voldemort</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>whirr-core</artifactId>
<version>${project.version}</version>
</dependency>
Modified: incubator/whirr/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/pom.xml?rev=1091362&r1=1091361&r2=1091362&view=diff
==============================================================================
--- incubator/whirr/trunk/pom.xml (original)
+++ incubator/whirr/trunk/pom.xml Tue Apr 12 10:37:24 2011
@@ -44,6 +44,7 @@
<module>services/hadoop</module>
<module>services/zookeeper</module>
<module>services/hbase</module>
+ <module>services/voldemort</module>
<module>services/elasticsearch</module>
</modules>
Added: incubator/whirr/trunk/recipes/voldemort-ec2.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/recipes/voldemort-ec2.properties?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/recipes/voldemort-ec2.properties (added)
+++ incubator/whirr/trunk/recipes/voldemort-ec2.properties Tue Apr 12 10:37:24 2011
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Voldemort Cluster on AWS EC2
+#
+
+# Read the Configuration Guide for more info:
+# http://incubator.apache.org/whirr/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=voldemort
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=2 voldemort
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+# By default use the user system SSH keys. Override them here.
+whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+whirr.public-key-file=${whirr.private-key-file}.pub
+
+# By default we use a stock build and configuration, but these can be overridden
+# by providing URLs for the build and configuration. Note: the configuration is
+# a flat hierarchy, there are no expected sub-directories in whirr.voldemort.conf.url,
+# just the two files server.properties and stores.xml.
+#whirr.voldemort.tarball.url=http://www.example.com/my/custom/build/of/voldemort.tar.gz
+#whirr.voldemort.conf.url=http://www.example.com/my/custom/voldemort/config.tar.gz
+
Propchange: incubator/whirr/trunk/services/voldemort/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Tue Apr 12 10:37:24 2011
@@ -0,0 +1,2 @@
+target
+
Added: incubator/whirr/trunk/services/voldemort/lib/linkedin-voldemort-0.90.RC3.jar
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/lib/linkedin-voldemort-0.90.RC3.jar?rev=1091362&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/whirr/trunk/services/voldemort/lib/linkedin-voldemort-0.90.RC3.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/whirr/trunk/services/voldemort/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/pom.xml?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/pom.xml (added)
+++ incubator/whirr/trunk/services/voldemort/pom.xml Tue Apr 12 10:37:24 2011
@@ -0,0 +1,109 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-voldemort</artifactId>
+ <packaging>jar</packaging>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ <name>Apache Whirr Voldemort</name>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds</groupId>
+ <artifactId>jclouds-compute</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds</groupId>
+ <artifactId>jclouds-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds.driver</groupId>
+ <artifactId>jclouds-jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds.driver</groupId>
+ <artifactId>jclouds-log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <!-- replace these with just a Voldemort entry once it is in a maven repo -->
+ <dependency>
+ <groupId>linkedin</groupId>
+ <artifactId>voldemort</artifactId>
+ <version>0.90</version>
+ <scope>system</scope>
+ <systemPath>${basedir}/lib/linkedin-voldemort-0.90.RC3.jar</systemPath>
+ </dependency>
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>2.3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.jdom</groupId>
+ <artifactId>jdom</artifactId>
+ <version>1.1</version>
+ </dependency>
+ </dependencies>
+</project>
Added: incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java (added)
+++ incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortClusterActionHandler.java Tue Apr 12 10:37:24 2011
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.voldemort;
+
+import static org.apache.whirr.service.FirewallManager.Rule;
+import static org.apache.whirr.service.RolePredicates.role;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.ADMIN_PORT;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.CLIENT_PORT;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.FUNCTION_CONFIGURE;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.FUNCTION_INSTALL;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.HTTP_PORT;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.KEY_CONF_URL;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.KEY_TARBALL_URL;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.PARAM_CLUSTER_NAME;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.PARAM_CONF_URL;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.PARAM_PARTITIONS_PER_NODE;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.PARAM_PROVIDER;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.PARAM_TARBALL_URL;
+import static org.apache.whirr.service.voldemort.VoldemortConstants.ROLE;
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.ComputeServiceContextBuilder;
+import org.apache.whirr.service.Cluster.Instance;
+import org.apache.whirr.service.FirewallManager;
+import org.apache.whirr.service.jclouds.FirewallSettings;
+import org.jclouds.compute.ComputeServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Lists;
+
+public class VoldemortClusterActionHandler extends ClusterActionHandlerSupport {
+
+ private static final Logger LOG = LoggerFactory.getLogger(VoldemortClusterActionHandler.class);
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
+ Configuration config = event.getClusterSpec().getConfiguration();
+ List<String> optArgs = new ArrayList<String>();
+
+ String tarUrl = config.getString(KEY_TARBALL_URL);
+
+ if (tarUrl != null && !tarUrl.trim().isEmpty()) {
+ optArgs.add(PARAM_TARBALL_URL);
+ optArgs.add(tarUrl);
+ }
+
+ String confUrl = config.getString(KEY_CONF_URL);
+
+ if (confUrl != null && !confUrl.trim().isEmpty()) {
+ optArgs.add(PARAM_CONF_URL);
+ optArgs.add(confUrl);
+ }
+
+ addStatement(event, call("install_java"));
+ addStatement(event, call(FUNCTION_INSTALL, optArgs.toArray(new String[optArgs.size()])));
+ }
+
+ @Override
+ protected void beforeConfigure(ClusterActionEvent event) throws IOException, InterruptedException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Cluster cluster = event.getCluster();
+
+ LOG.info("Authorizing firewall");
+ event.getFirewallManager().addRule(
+ Rule.create()
+ .destination(cluster.getInstancesMatching(role(ROLE)))
+ .ports(CLIENT_PORT, ADMIN_PORT, HTTP_PORT)
+ );
+
+ String servers = Joiner.on(' ').join(getPrivateIps(cluster.getInstances()));
+
+ Configuration config = event.getClusterSpec().getConfiguration();
+ int partitionsPerNode = config.getInt(PARAM_PARTITIONS_PER_NODE, 10);
+ addStatement(event, call(FUNCTION_CONFIGURE,
+ PARAM_PROVIDER,
+ clusterSpec.getProvider(),
+ PARAM_PARTITIONS_PER_NODE,
+ Integer.toString(partitionsPerNode),
+ PARAM_CLUSTER_NAME,
+ clusterSpec.getClusterName(),
+ servers));
+ }
+
+ @Override
+ protected void afterConfigure(ClusterActionEvent event) {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Cluster cluster = event.getCluster();
+
+ String servers = Joiner.on(' ').join(getPrivateIps(cluster.getInstances()));
+
+ LOG.info("Completed setup of Voldemort {} with servers {}", clusterSpec.getClusterName(), servers);
+ }
+
+ /**
+ * Given a set of instances returns a list of their private IPs.
+ *
+ * @param instances Set of instances in the cluster
+ * @return List of all private IPs as strings
+ */
+ private List<String> getPrivateIps(Set<Instance> instances) {
+ return Lists.transform(Lists.newArrayList(instances), new Function<Instance, String>() {
+
+ @Override
+ public String apply(Instance instance) {
+ return instance.getPrivateIp();
+ }
+ });
+ }
+
+}
Added: incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortConstants.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortConstants.java?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortConstants.java (added)
+++ incubator/whirr/trunk/services/voldemort/src/main/java/org/apache/whirr/service/voldemort/VoldemortConstants.java Tue Apr 12 10:37:24 2011
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.voldemort;
+
+public final class VoldemortConstants {
+
+ public static final String KEY_TARBALL_URL = "whirr.voldemort.tarball.url";
+
+ public static final String KEY_CONF_URL = "whirr.voldemort.conf.url";
+
+ public static final String KEY_PARTITIONS_PER_NODE = "whirr.voldemort.partitions";
+
+ public static final String FUNCTION_INSTALL = "install_voldemort";
+
+ public static final String FUNCTION_CONFIGURE = "configure_voldemort";
+
+ public static final String PARAM_PROVIDER = "-c";
+
+ public static final String PARAM_TARBALL_URL = "-u";
+
+ public static final String PARAM_CONF_URL = "-f";
+
+ public static final String PARAM_PARTITIONS_PER_NODE = "-p";
+
+ public static final String PARAM_CLUSTER_NAME = "-n";
+
+ public static final String ROLE = "voldemort";
+
+ public static final int CLIENT_PORT = 6666;
+
+ public static final int ADMIN_PORT = 6667;
+
+ public static final int HTTP_PORT = 8081;
+
+ private VoldemortConstants() {
+ }
+
+}
Added: incubator/whirr/trunk/services/voldemort/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler (added)
+++ incubator/whirr/trunk/services/voldemort/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler Tue Apr 12 10:37:24 2011
@@ -0,0 +1 @@
+org.apache.whirr.service.voldemort.VoldemortClusterActionHandler
Added: incubator/whirr/trunk/services/voldemort/src/main/resources/functions/configure_voldemort.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/main/resources/functions/configure_voldemort.sh?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/main/resources/functions/configure_voldemort.sh (added)
+++ incubator/whirr/trunk/services/voldemort/src/main/resources/functions/configure_voldemort.sh Tue Apr 12 10:37:24 2011
@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# -c <cloud-provider> -p <number_of_partitions_per_node> -n <cluster_name> <server-ip>*
+function configure_voldemort() {
+ . /etc/profile
+
+ CLUSTER_NAME=voldemort
+
+ while getopts "c:p:n:" OPTION; do
+ case $OPTION in
+ c)
+ CLOUD_PROVIDER="$OPTARG"
+ ;;
+ p)
+ PARTITIONS_PER_NODE="$OPTARG"
+ ;;
+ n)
+ CLUSTER_NAME="$OPTARG"
+ ;;
+ esac
+ done
+ shift $((OPTIND-1));
+
+ # Use public IPs for all Voldemort nodes
+ case $CLOUD_PROVIDER in
+ ec2)
+ SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/local-ipv4`
+ ;;
+ *)
+ SELF_HOST=`/sbin/ifconfig eth1 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
+ ;;
+ esac
+
+ prop_file=$VOLDEMORT_HOME/config/server.properties
+ hostname_file=$VOLDEMORT_HOME/hostnames
+
+ if [ -e $hostname_file ]; then
+ rm -rf $hostname_file
+ fi
+ touch $hostname_file
+
+ # Remove the node id from the existing file
+ mv $prop_file $prop_file.bak
+ grep -v "^node.id=" $prop_file.bak > $prop_file
+
+ # Set the node id in the server properties file
+ # Also generate the hostname files
+ if [[ $# -gt 0 ]]; then
+ id=0
+ for server in "$@"; do
+ if [[ $server == *$SELF_HOST* ]]; then
+ myid=$id
+ echo -e "\nnode.id=$id" >> $prop_file
+ fi
+ echo -e "$server\n" >> $hostname_file
+ id=$((id+1))
+ done
+
+ if [ -z $myid ]; then
+ echo "Could not determine id for my host $SELF_HOST against servers $@."
+ exit 1
+ fi
+ else
+ echo "Missing server names"
+ exit 1
+ fi
+
+
+ # Set up the cluster metadata
+ chmod +x $VOLDEMORT_ROOT/contrib/ec2-testing/bin/run-class.sh
+ chmod +x $VOLDEMORT_ROOT/contrib/ec2-testing/bin/voldemort-clustergenerator.sh
+ $VOLDEMORT_ROOT/contrib/ec2-testing/bin/voldemort-clustergenerator.sh --useinternal true --clustername $CLUSTER_NAME --partitions $PARTITIONS_PER_NODE --hostnames $hostname_file > $VOLDEMORT_HOME/config/cluster.xml
+
+ # Start Voldemort
+ nohup /etc/rc.local &
+}
Added: incubator/whirr/trunk/services/voldemort/src/main/resources/functions/install_voldemort.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/main/resources/functions/install_voldemort.sh?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/main/resources/functions/install_voldemort.sh (added)
+++ incubator/whirr/trunk/services/voldemort/src/main/resources/functions/install_voldemort.sh Tue Apr 12 10:37:24 2011
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function download() {
+ tar_url=$1
+ tar_file=`basename $tar_url`
+ dest_dir=$2
+
+ for i in `seq 1 3`; do
+ curl --retry 3 --show-error --fail -O $tar_url
+ if tar zxf $tar_file -C $dest_dir ; then
+ break;
+ else
+ rm -f $tar_file
+ fi
+ done
+
+ if [ ! -e $tar_file ]; then
+ echo "Failed to download $tar_url. Aborting."
+ exit 1
+ fi
+}
+
+# -f <configuration URL> -u <voldemort_tar_url>
+function install_voldemort() {
+ VOLDEMORT_VERSION="0.90.RC3"
+ VOLDEMORT_TAR_URL=http://test.project-voldemort.com:8080/job/release-090/lastSuccessfulBuild/artifact/dist/voldemort-${VOLDEMORT_VERSION}.tar.gz
+ VOLDEMORT_CONF_URL=
+ VOLDEMORT_ROOT=/usr/local/voldemort
+ VOLDEMORT_HOME=/etc/voldemort
+
+ while getopts "f:u:" OPTION; do
+ case $OPTION in
+ f)
+ VOLDEMORT_CONF_URL="$OPTARG"
+ ;;
+ u)
+ VOLDEMORT_TAR_URL="$OPTARG"
+ ;;
+ esac
+ done
+
+ # Download the binary distribution and put it in the correct directory.
+ download $VOLDEMORT_TAR_URL `dirname $VOLDEMORT_ROOT`
+ mv ${VOLDEMORT_ROOT}* $VOLDEMORT_ROOT
+
+ # Create our config directory which Voldemort expects to live under $VOLDEMORT_HOME.
+ mkdir -p $VOLDEMORT_HOME/config
+
+ if [ "$VOLDEMORT_CONF_URL" = "" ] ; then
+ # Copy sample property files over to config dir if the user didn't provide a config URL.
+ cp $VOLDEMORT_ROOT/config/single_node_cluster/config/* $VOLDEMORT_HOME/config/.
+ else
+ # Otherwise, download the configuration file contents to the configuration directory.
+ download $VOLDEMORT_CONF_URL $VOLDEMORT_HOME/config
+ fi
+
+ # Set Voldemort vars
+ echo "export VOLDEMORT_ROOT=$VOLDEMORT_ROOT" >> /etc/profile
+ echo "export VOLDEMORT_HOME=$VOLDEMORT_HOME" >> /etc/profile
+
+ # Start
+ sed -i -e "s/exit 0//" /etc/rc.local
+ sed -i -e "s/voldemort//" /etc/rc.local
+
+ cat >> /etc/rc.local <<RC_LOCAL
+cd \$VOLDEMORT_ROOT
+./bin/voldemort-server.sh > /var/log/voldemort.log 2>&1 &
+cd -
+RC_LOCAL
+}
Added: incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/voldemort/integration/VoldemortServiceTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/voldemort/integration/VoldemortServiceTest.java?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/voldemort/integration/VoldemortServiceTest.java (added)
+++ incubator/whirr/trunk/services/voldemort/src/test/java/org/apache/whirr/service/voldemort/integration/VoldemortServiceTest.java Tue Apr 12 10:37:24 2011
@@ -0,0 +1,117 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.voldemort.integration;
+
+import static org.apache.whirr.service.voldemort.VoldemortConstants.ADMIN_PORT;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.Service;
+import org.apache.whirr.service.Cluster.Instance;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import voldemort.client.protocol.admin.AdminClient;
+import voldemort.client.protocol.admin.AdminClientConfig;
+import voldemort.cluster.Node;
+
+public class VoldemortServiceTest {
+
+ private ClusterSpec clusterSpec;
+
+ private Service service;
+
+ private Cluster cluster;
+
+ @Before
+ public void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(new PropertiesConfiguration("whirr-voldemort-test.properties"));
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+
+ service = new Service();
+ cluster = service.launchCluster(clusterSpec);
+
+ waitForBootstrap();
+ }
+
+ private void waitForBootstrap() {
+ for (Instance instance : cluster.getInstances()) {
+ while (true) {
+ try {
+ String url = "tcp://" + instance.getPublicAddress().getHostAddress() + ":" + ADMIN_PORT;
+ AdminClient client = new AdminClient(url, new AdminClientConfig());
+ client.getAdminClientCluster();
+ break;
+ } catch (Exception e) {
+ System.out.print(".");
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e1) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testInstances() throws Exception {
+ Set<Instance> instances = cluster.getInstances();
+ String url = "tcp://" + instances.iterator().next().getPublicAddress().getHostAddress() + ":" + ADMIN_PORT;
+ AdminClient client = new AdminClient(url, new AdminClientConfig());
+ voldemort.cluster.Cluster c = client.getAdminClientCluster();
+
+ List<String> voldemortBasedHosts = new ArrayList<String>();
+
+ for (Node node : c.getNodes())
+ voldemortBasedHosts.add(node.getHost());
+
+ List<String> whirrBasedHosts = new ArrayList<String>();
+
+ for (Instance instance : instances)
+ whirrBasedHosts.add(instance.getPrivateAddress().getHostAddress());
+
+ Collections.sort(voldemortBasedHosts);
+ Collections.sort(whirrBasedHosts);
+
+ Assert.assertEquals(whirrBasedHosts, voldemortBasedHosts);
+ }
+
+ @After
+ public void tearDown() throws IOException, InterruptedException {
+ if (service != null) {
+ service.destroyCluster(clusterSpec);
+ }
+ }
+
+}
Added: incubator/whirr/trunk/services/voldemort/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/test/resources/log4j.xml?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/test/resources/log4j.xml (added)
+++ incubator/whirr/trunk/services/voldemort/src/test/resources/log4j.xml Tue Apr 12 10:37:24 2011
@@ -0,0 +1,196 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+ <!--
+ For more configuration infromation and examples see the Apache
+ Log4j website: http://logging.apache.org/log4j/
+ -->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
+ debug="false">
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <param name="Threshold" value="INFO" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-wire.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-blobstore.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+
+ <!-- A time/date based rolling appender -->
+ <appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-compute.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WHIRRFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/whirr.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="COMPUTEFILE" />
+ </appender>
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="FILE" />
+ </appender>
+
+ <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="WIREFILE" />
+ </appender>
+
+ <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="BLOBSTOREFILE" />
+ </appender>
+ <!-- ================ -->
+ <!-- Limit categories -->
+ <!-- ================ -->
+ <category name="jclouds.blobstore">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCBLOBSTORE" />
+ </category>
+
+ <category name="org.jclouds">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNC" />
+ </category>
+
+ <category name="org.apache.whirr">
+ <priority value="DEBUG" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
+ <category name="jclouds.headers">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+ <category name="jclouds.compute">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCCOMPUTE" />
+ </category>
+
+ <category name="jclouds.wire">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category><!--
+
+ <category name="jclouds.signature">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+
+
+ --><!-- ======================= -->
+ <!-- Setup the Root category -->
+ <!-- ======================= -->
+
+ <root>
+ <priority value="WARN" />
+ <appender-ref ref="CONSOLE" />
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
Added: incubator/whirr/trunk/services/voldemort/src/test/resources/whirr-voldemort-test.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/voldemort/src/test/resources/whirr-voldemort-test.properties?rev=1091362&view=auto
==============================================================================
--- incubator/whirr/trunk/services/voldemort/src/test/resources/whirr-voldemort-test.properties (added)
+++ incubator/whirr/trunk/services/voldemort/src/test/resources/whirr-voldemort-test.properties Tue Apr 12 10:37:24 2011
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=voldemort
+whirr.instance-templates=2 voldemort
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
\ No newline at end of file
Modified: incubator/whirr/trunk/src/site/confluence/index.confluence
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/src/site/confluence/index.confluence?rev=1091362&r1=1091361&r2=1091362&view=diff
==============================================================================
--- incubator/whirr/trunk/src/site/confluence/index.confluence (original)
+++ incubator/whirr/trunk/src/site/confluence/index.confluence Tue Apr 12 10:37:24 2011
@@ -14,9 +14,9 @@ h2. Which services and cloud providers a
Whirr uses [jclouds|http://code.google.com/p/jclouds/] for provisioning, so in principle it should support all the cloud providers that jclouds supports. The following table shows
the cloud provider and service combinations that have been tested.
-||Cloud provider||Cassandra||Hadoop||ZooKeeper||HBase||elasticsearch||
-|Amazon EC2|Yes|Yes|Yes|Yes|Yes|
-|Rackspace Cloud Servers|Yes|Yes|Yes|Yes|Yes|
+||Cloud provider||Cassandra||Hadoop||ZooKeeper||HBase||elasticsearch||Voldemort||
+|Amazon EC2|Yes|Yes|Yes|Yes|Yes|Yes|
+|Rackspace Cloud Servers|Yes|Yes|Yes|Yes|Yes|Yes|
h2. Download