You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by as...@apache.org on 2011/03/29 14:13:28 UTC
svn commit: r1086564 - in /incubator/whirr/trunk: ./ cli/ recipes/
services/elasticsearch/ services/elasticsearch/src/
services/elasticsearch/src/main/ services/elasticsearch/src/main/java/
services/elasticsearch/src/main/java/org/ services/elasticsear...
Author: asavu
Date: Tue Mar 29 12:13:27 2011
New Revision: 1086564
URL: http://svn.apache.org/viewvc?rev=1086564&view=rev
Log:
WHIRR-261. Add ElasticSearch as a service
Added:
incubator/whirr/trunk/recipes/elasticsearch-ec2.properties
incubator/whirr/trunk/recipes/elasticsearch-rackspace.properties
incubator/whirr/trunk/services/elasticsearch/
incubator/whirr/trunk/services/elasticsearch/pom.xml
incubator/whirr/trunk/services/elasticsearch/src/
incubator/whirr/trunk/services/elasticsearch/src/main/
incubator/whirr/trunk/services/elasticsearch/src/main/java/
incubator/whirr/trunk/services/elasticsearch/src/main/java/org/
incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/
incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/
incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/
incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/
incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java
incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java
incubator/whirr/trunk/services/elasticsearch/src/main/resources/
incubator/whirr/trunk/services/elasticsearch/src/main/resources/META-INF/
incubator/whirr/trunk/services/elasticsearch/src/main/resources/META-INF/services/
incubator/whirr/trunk/services/elasticsearch/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/
incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/configure_elasticsearch.sh
incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/install_elasticsearch.sh
incubator/whirr/trunk/services/elasticsearch/src/main/resources/whirr-elasticsearch-default.properties
incubator/whirr/trunk/services/elasticsearch/src/test/
incubator/whirr/trunk/services/elasticsearch/src/test/java/
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilderTest.java
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/integration/
incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/integration/ElasticSearchTest.java
incubator/whirr/trunk/services/elasticsearch/src/test/resources/
incubator/whirr/trunk/services/elasticsearch/src/test/resources/whirr-elasticsearch-test.properties
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/cli/pom.xml
incubator/whirr/trunk/pom.xml
incubator/whirr/trunk/src/site/confluence/index.confluence
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1086564&r1=1086563&r2=1086564&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Tue Mar 29 12:13:27 2011
@@ -2,6 +2,10 @@ Apache Whirr Change Log
Trunk (unreleased changes)
+ NEW FEATURES
+
+ WHIRR-261. Add ElasticSearch as a service (asavu)
+
BUG FIXES
WHIRR-253. ZooKeeper service should only authorize ingress to ZooKeeper
Modified: incubator/whirr/trunk/cli/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/pom.xml?rev=1086564&r1=1086563&r2=1086564&view=diff
==============================================================================
--- incubator/whirr/trunk/cli/pom.xml (original)
+++ incubator/whirr/trunk/cli/pom.xml Tue Mar 29 12:13:27 2011
@@ -56,6 +56,11 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>whirr-elasticsearch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>whirr-hbase</artifactId>
<version>${project.version}</version>
</dependency>
Modified: incubator/whirr/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/pom.xml?rev=1086564&r1=1086563&r2=1086564&view=diff
==============================================================================
--- incubator/whirr/trunk/pom.xml (original)
+++ incubator/whirr/trunk/pom.xml Tue Mar 29 12:13:27 2011
@@ -44,6 +44,7 @@
<module>services/hadoop</module>
<module>services/zookeeper</module>
<module>services/hbase</module>
+ <module>services/elasticsearch</module>
</modules>
<properties>
Added: incubator/whirr/trunk/recipes/elasticsearch-ec2.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/recipes/elasticsearch-ec2.properties?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/recipes/elasticsearch-ec2.properties (added)
+++ incubator/whirr/trunk/recipes/elasticsearch-ec2.properties Tue Mar 29 12:13:27 2011
@@ -0,0 +1,88 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# ElasticSearch Cluster on AWS EC2
+#
+
+# Read the Configuration Guide for more info:
+# http://incubator.apache.org/whirr/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=elasticsearch
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=2 elasticsearch
+whirr.hardware-min-ram=2048
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+# By default use the user system SSH keys. Override them here.
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+# You can specify the version by setting the tarball url
+# whirr.elasticsearch.tarball.url=http://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.15.2.tar.gz
+
+#
+# elasticsearch specific settings (Expert)
+#
+
+# 1. Gateway Persistence settings
+# See: http://www.elasticsearch.org/guide/reference/modules/gateway/
+# Defaults: the index is only stored in memory and all data is lost on shutdown
+
+# 1.1 Enable persistence on S3
+# See: http://www.elasticsearch.org/guide/reference/modules/gateway/s3.html
+
+# es.gateway.type=s3
+# es.gateway.s3.bucket: elasticsearch
+
+# 1.2 Enable persistence on the local filesystem
+# See: http://www.elasticsearch.org/guide/reference/modules/gateway/local.html
+
+# es.gateway.type=fs
+# es.gateway.recovery_after_nodes=1
+# es.gateway.recovery_after_time=5m
+# es.expected_nodes=2
+
+# 1.3 Enable persistence on HDFS
+# See: http://www.elasticsearch.org/guide/reference/modules/gateway/hadoop.html
+
+# es.gateway.type=hdfs
+# es.gateway.hdfs.uri=hdfs://myhost:8022
+# es.gateway.hdfs.path=/some/path
+
+# 2. Scripting Support
+# See: http://www.elasticsearch.org/guide/reference/modules/scripting.html
+# The scripting module uses by default mvel
+
+# Just add them to the list of installed plugins
+# es.plugins=lang-javascript, lang-groovy, lang-python,
+
+# 3. Memcached protocol support
+# See: http://www.elasticsearch.org/guide/reference/modules/memcached.html
+
+# es.plugins=transport-memcached
+
+# 4. Thrift protocol support
+# See: http://www.elasticsearch.org/guide/reference/modules/thrift.html
+
+# es.plugins=transport-thrift
Added: incubator/whirr/trunk/recipes/elasticsearch-rackspace.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/recipes/elasticsearch-rackspace.properties?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/recipes/elasticsearch-rackspace.properties (added)
+++ incubator/whirr/trunk/recipes/elasticsearch-rackspace.properties Tue Mar 29 12:13:27 2011
@@ -0,0 +1,91 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# ElasticSearch Cluster on Rackspace Cloud
+#
+
+# Read the Configuration Guide for more info:
+# http://incubator.apache.org/whirr/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=elasticsearch
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=2 elasticsearch
+
+# For Rackspace set RACKSPACE_USERNAME and RACKSPACE_API_KEY environment variables.
+whirr.provider=cloudservers-us
+whirr.identity=${env:RACKSPACE_USERNAME}
+whirr.credential=${env:RACKSPACE_API_KEY}
+
+# The size of the instance to use. See http://www.rackspacecloud.com/cloud_hosting_products/servers/faq/
+# id 3: 1GB, 1 virtual core
+# id 4: 2GB, 2 virtual cores
+# id 5: 4GB, 2 virtual cores
+# id 6: 8GB, 4 virtual cores
+# id 7: 15.5GB, 4 virtual cores
+whirr.hardware-id=6
+# Ubuntu 10.04 LTS Lucid
+whirr.image-id=49
+
+# By default use the user system SSH keys. Override them here.
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+# You can specify the version by setting the tarball url
+# whirr.elasticsearch.tarball.url=http://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.15.2.tar.gz
+
+#
+# elasticsearch specific settings (Expert)
+#
+
+# 1. Gateway Persistence settings
+# See: http://www.elasticsearch.org/guide/reference/modules/gateway/
+# Defaults: the index is only stored in memory and all data is lost on shutdown
+
+# 1.1 Enable persistence on the local filesystem
+# See: http://www.elasticsearch.org/guide/reference/modules/gateway/local.html
+
+# es.gateway.type=fs
+# es.gateway.recovery_after_nodes=1
+# es.gateway.recovery_after_time=5m
+# es.expected_nodes=2
+
+# 1.2 Enable persistence on HDFS
+# See: http://www.elasticsearch.org/guide/reference/modules/gateway/hadoop.html
+
+# es.gateway.type=hdfs
+# es.gateway.hdfs.uri=hdfs://myhost:8022
+# es.gateway.hdfs.path=/some/path
+
+# 2. Scripting Support
+# See: http://www.elasticsearch.org/guide/reference/modules/scripting.html
+# The scripting module uses by default mvel
+
+# Just add them to the list of installed plugins
+# es.plugins=lang-javascript, lang-groovy, lang-python,
+
+# 3. Memcached protocol support
+# See: http://www.elasticsearch.org/guide/reference/modules/memcached.html
+
+# es.plugins=transport-memcached
+
+# 4. Thrift protocol support
+# See: http://www.elasticsearch.org/guide/reference/modules/thrift.html
+
+# es.plugins=transport-thrift
Added: incubator/whirr/trunk/services/elasticsearch/pom.xml
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/pom.xml?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/pom.xml (added)
+++ incubator/whirr/trunk/services/elasticsearch/pom.xml Tue Mar 29 12:13:27 2011
@@ -0,0 +1,95 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-elasticsearch</artifactId>
+ <packaging>jar</packaging>
+ <version>0.5.0-incubating-SNAPSHOT</version>
+ <name>Apache Whirr ElasticSearch</name>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds</groupId>
+ <artifactId>jclouds-compute</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds</groupId>
+ <artifactId>jclouds-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds.driver</groupId>
+ <artifactId>jclouds-jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.jclouds.driver</groupId>
+ <artifactId>jclouds-log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
Added: incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilder.java Tue Mar 29 12:13:27 2011
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.elasticsearch;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.ClusterSpec;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.whirr.service.RolePredicates.role;
+
+public class ElasticSearchConfigurationBuilder {
+
+ private static final org.slf4j.Logger LOG =
+ LoggerFactory.getLogger(ElasticSearchConfigurationBuilder.class);
+
+ public static final String ES_PREFIX = "es";
+
+ /**
+ * Generate an appendFile statement for jclouds
+ */
+ public static Statement build(String path, Configuration config) {
+ return asFileStatement(path, config);
+ }
+
+ public static Statement build(String path, ClusterSpec spec, Cluster cluster) {
+ return build(path, buildConfig(spec, cluster));
+ }
+
+ /**
+ * Build a configuration by adding the expected defaults
+ */
+ public static Configuration buildConfig(ClusterSpec spec, Cluster cluster) {
+ CompositeConfiguration config = new CompositeConfiguration();
+
+ config.addConfiguration(spec.getConfiguration());
+ try {
+ config.addConfiguration(
+ new PropertiesConfiguration("whirr-elasticsearch-default.properties"));
+ } catch (ConfigurationException e) {
+ LOG.error("Configuration error", e); // this should never happen
+ }
+
+ if ("aws-ec2".equals(spec.getProvider()) || "ec2".equals(spec.getProvider())) {
+ addDefaultsForEC2(spec, config);
+ } else {
+ addDefaultsForUnicast(cluster, config);
+ }
+ if (!config.containsKey("es.cluster.name")) {
+ config.addProperty("es.cluster.name", spec.getClusterName());
+ }
+
+ return config;
+ }
+
+ /**
+ * Use the native EC2 discovery module on AWS
+ */
+ private static void addDefaultsForEC2(ClusterSpec spec, CompositeConfiguration config) {
+ config.addProperty("es.discovery.type", "ec2");
+ if (!config.containsKey("es.cloud.aws.access_key")) {
+ config.addProperty("es.cloud.aws.access_key", spec.getIdentity());
+ }
+ if (!config.containsKey("es.cloud.aws.secret_key")) {
+ config.addProperty("es.cloud.aws.secret_key", spec.getCredential());
+ }
+ if (!config.getList("es.plugins", Lists.newLinkedList()).contains("cloud-aws")) {
+ config.addProperty("es.plugins", "cloud-aws");
+ }
+ }
+
+ /**
+ * Use unicast if not on AWS (most of the cloud providers deny multicast traffic).
+ */
+ private static void addDefaultsForUnicast(Cluster cluster, CompositeConfiguration config) {
+ List<String> hosts = Lists.newLinkedList();
+ for(Cluster.Instance instance : cluster.getInstancesMatching(role(ElasticSearchHandler.ROLE))) {
+ hosts.add(String.format("\"%s:9300\"", instance.getPrivateAddress().getHostAddress()));
+ }
+ config.addProperty("es.discovery.zen.ping.multicast.enabled", "false");
+ config.addProperty("es.discovery.zen.ping.unicast.hosts", StringUtils.join(hosts, ","));
+ }
+
+ private static Statement asFileStatement(String path, Configuration configuration) {
+ return Statements.appendFile(path, asYamlLines(configuration.subset(ES_PREFIX)));
+ }
+
+ /**
+ * Create the YAML configuration file lines from the configuration.
+ *
+ * This functions transforms cloud.aws.id=1 to
+ *
+ * cloud:
+ * aws:
+ * id: 1
+ */
+ @VisibleForTesting
+ public static List<String> asYamlLines(Configuration config) {
+ return asYamlLines(config, 0);
+ }
+
+ private static List<String> asYamlLines(Configuration config, int depth) {
+ List<String> lines = Lists.newArrayList();
+ Set<String> prefixes = Sets.newHashSet();
+
+ Iterator<String> keys = config.getKeys();
+ while(keys.hasNext()) {
+ String key = keys.next();
+
+ String[] parts = key.split("\\.");
+ String prefix = parts[0];
+
+ if (prefixes.contains(prefix)) {
+ continue; // skip parsed set of keys
+ }
+
+ if (parts.length == 1) {
+ lines.add(spaces(depth * 2) + key + ": " + config.getProperty(key));
+
+ } else if (parts.length > 1) {
+ lines.add(spaces(depth * 2) + prefix + ":");
+ lines.addAll(asYamlLines(config.subset(prefix), depth + 1));
+ }
+
+ prefixes.add(prefix);
+ }
+
+ return lines;
+ }
+
+ /**
+ * Generate a string with spaces having the requested length
+ */
+ private static String spaces(int length) {
+ StringBuilder builder = new StringBuilder();
+ for(int i=0; i<length; i++) {
+ builder.append(" ");
+ }
+ return builder.toString();
+ }
+}
Added: incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/main/java/org/apache/whirr/service/elasticsearch/ElasticSearchHandler.java Tue Mar 29 12:13:27 2011
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.elasticsearch;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.ComputeServiceContextBuilder;
+import org.apache.whirr.service.jclouds.FirewallSettings;
+import org.jclouds.compute.ComputeServiceContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static org.apache.whirr.service.RolePredicates.role;
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+public class ElasticSearchHandler extends ClusterActionHandlerSupport {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ElasticSearchHandler.class);
+
+ public static final String ROLE = "elasticsearch";
+
+ public static final int HTTP_CLIENT_PORT = 9200;
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) {
+ ClusterSpec spec = event.getClusterSpec();
+ Configuration config = spec.getConfiguration();
+
+ addStatement(event, call("install_java"));
+ addStatement(event, call("install_elasticsearch",
+ config.getString("whirr.elasticsearch.tarball.url", "")));
+ }
+
+ @Override
+ protected void beforeConfigure(ClusterActionEvent event) throws IOException {
+ ClusterSpec spec = event.getClusterSpec();
+ Cluster cluster = event.getCluster();
+
+ LOG.info("Authorizing firewall port {}", HTTP_CLIENT_PORT);
+ ComputeServiceContext computeServiceContext =
+ ComputeServiceContextBuilder.build(spec);
+
+ FirewallSettings.authorizeIngress(computeServiceContext,
+ cluster.getInstancesMatching(role(ROLE)), spec, HTTP_CLIENT_PORT);
+
+ Configuration config = ElasticSearchConfigurationBuilder.buildConfig(spec, cluster);
+ addStatement(event,
+ ElasticSearchConfigurationBuilder.build("/tmp/elasticsearch.yml", config));
+ addStatement(event, call("configure_elasticsearch",
+ config.getStringArray("es.plugins")));
+ }
+}
Added: incubator/whirr/trunk/services/elasticsearch/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler Tue Mar 29 12:13:27 2011
@@ -0,0 +1 @@
+org.apache.whirr.service.elasticsearch.ElasticSearchHandler
\ No newline at end of file
Added: incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/configure_elasticsearch.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/configure_elasticsearch.sh?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/configure_elasticsearch.sh (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/configure_elasticsearch.sh Tue Mar 29 12:13:27 2011
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function configure_elasticsearch() {
+ cd /usr/local/elasticsearch-0.15.2/
+
+ for plugin in $@
+ do
+ ./bin/plugin install $plugin
+ done
+
+ cp /tmp/elasticsearch.yml config/elasticsearch.yml
+
+ # use 80% of memory for jvm heap
+ local MAXMEM=$(($(free|awk '/^Mem:/{print $2}') * 8 / 10 / 1024))m
+
+ ./bin/elasticsearch -Xmx$MAXMEM -Xms$MAXMEM -p elasticsearch.pid
+}
+
Added: incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/install_elasticsearch.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/install_elasticsearch.sh?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/install_elasticsearch.sh (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/main/resources/functions/install_elasticsearch.sh Tue Mar 29 12:13:27 2011
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function install_elasticsearch() {
+ local CURL="curl -L --silent --show-error --fail --connect-timeout 10 --max-time 600 --retry 5"
+
+ local ES_URL=${1:-http://github.com/downloads/elasticsearch/elasticsearch/elasticsearch-0.15.2.tar.gz}
+ local TAR_FILE=`basename $ES_URL`
+
+ for retry_count in `seq 1 3`
+ do
+ $CURL -O $ES_URL || true
+ if [ -f $TAR_FILE ]; then
+ break;
+ fi
+ if [ ! $retry_count -eq "3" ]; then
+ sleep 10
+ fi
+ done
+
+ if [ -f $TAR_FILE ]; then
+ tar xzf $TAR_FILE -C /usr/local
+ rm -f $TAR_FILE
+ else
+ echo "Unable to download tar file from $ES_URL" >&2
+ exit 1
+ fi
+}
+
Added: incubator/whirr/trunk/services/elasticsearch/src/main/resources/whirr-elasticsearch-default.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/main/resources/whirr-elasticsearch-default.properties?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/main/resources/whirr-elasticsearch-default.properties (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/main/resources/whirr-elasticsearch-default.properties Tue Mar 29 12:13:27 2011
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# ElasticSearch defaults. The es. prefix is removed by Whirr.
+
+es.index.store.type=memory
+es.gateway.type=none
Added: incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilderTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilderTest.java?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilderTest.java (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/ElasticSearchConfigurationBuilderTest.java Tue Mar 29 12:13:27 2011
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.elasticsearch;
+
+import com.google.common.base.Predicate;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.commons.lang.StringUtils;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.ClusterSpec;
+import org.junit.Test;
+
+import java.net.InetAddress;
+import java.util.Set;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class ElasticSearchConfigurationBuilderTest {
+
+ @Test
+ public void testGenerateYamlConfig() {
+ Configuration defaults = new PropertiesConfiguration();
+
+ defaults.addProperty("cloud.aws.id", "a");
+ defaults.addProperty("cloud.aws.key", "b");
+ defaults.addProperty("index.store.type", "memory");
+
+ String content = StringUtils.join(
+ ElasticSearchConfigurationBuilder.asYamlLines(defaults), "\n");
+
+ assertThat(content, is("cloud:\n" +
+ " aws:\n" +
+ " id: a\n" +
+ " key: b\n" +
+ "index:\n" +
+ " store:\n" +
+ " type: memory"));
+ }
+
+ @Test
+ public void testDefaultConfigAwsEC2() throws Exception {
+ Configuration baseConfig = new PropertiesConfiguration();
+ baseConfig.addProperty("whirr.provider", "aws-ec2");
+ baseConfig.addProperty("es.plugins", "lang-javascript, lang-python");
+
+ ClusterSpec spec = ClusterSpec.withTemporaryKeys(baseConfig);
+ Configuration config = ElasticSearchConfigurationBuilder.buildConfig(spec, null);
+
+ assertThat(config.getStringArray("es.plugins"),
+ is(new String[]{"lang-javascript", "lang-python", "cloud-aws"}));
+ assertThat(config.getString("es.discovery.type"), is("ec2"));
+ }
+
+ @Test
+ public void testDefaultUnicastConfig() throws Exception {
+ Configuration baseConfig = new PropertiesConfiguration();
+ baseConfig.addProperty("whirr.provider", "cloudservers-us");
+
+ ClusterSpec spec = ClusterSpec.withTemporaryKeys(baseConfig);
+ Cluster cluster = mock(Cluster.class);
+
+ Set<Cluster.Instance> instances = Sets.newLinkedHashSet();
+ for(String host : Lists.newArrayList("10.0.0.1", "10.0.0.2")) {
+ Cluster.Instance instance = mock(Cluster.Instance.class);
+ when(instance.getPrivateAddress()).thenReturn(InetAddress.getByName(host));
+ instances.add(instance);
+ }
+ when(cluster.getInstancesMatching((Predicate<Cluster.Instance>)any()))
+ .thenReturn(instances);
+
+ Configuration config = ElasticSearchConfigurationBuilder.buildConfig(spec, cluster);
+ String content = StringUtils.join(
+ ElasticSearchConfigurationBuilder.asYamlLines(
+ config.subset(ElasticSearchConfigurationBuilder.ES_PREFIX)), "\n");
+
+ assertThat(content, is("index:\n" +
+ " store:\n" +
+ " type: memory\n" +
+ "gateway:\n" +
+ " type: none\n" +
+ "discovery:\n" +
+ " zen:\n" +
+ " ping:\n" +
+ " multicast:\n" +
+ " enabled: false\n" +
+ " unicast:\n" +
+ " hosts: [\"10.0.0.1:9300\", \"10.0.0.2:9300\"]"));
+ }
+
+ @Test
+ public void testOverrideDefaults() throws Exception {
+ Configuration baseConfig = new PropertiesConfiguration();
+ baseConfig.addProperty("whirr.provider", "aws-ec2");
+ baseConfig.addProperty("es.index.store.type", "fs");
+
+ ClusterSpec spec = ClusterSpec.withTemporaryKeys(baseConfig);
+ Configuration config = ElasticSearchConfigurationBuilder.buildConfig(spec, null);
+
+ assertThat(config.getString("es.index.store.type"), is("fs"));
+ }
+}
Added: incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/integration/ElasticSearchTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/integration/ElasticSearchTest.java?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/integration/ElasticSearchTest.java (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/test/java/org/apache/whirr/service/elasticsearch/integration/ElasticSearchTest.java Tue Mar 29 12:13:27 2011
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.whirr.service.elasticsearch.integration;
+
+import com.google.common.collect.Iterables;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.Service;
+import org.apache.whirr.service.elasticsearch.ElasticSearchHandler;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URL;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import static org.apache.whirr.service.RolePredicates.role;
+
+public class ElasticSearchTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ElasticSearchHandler.class);
+
+ private ClusterSpec clusterSpec;
+ private Service service;
+ private Cluster cluster;
+
+ @Before
+ public void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ config.addConfiguration(new PropertiesConfiguration("whirr-elasticsearch-test.properties"));
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ service = new Service();
+ cluster = service.launchCluster(clusterSpec);
+ }
+
+ @Test
+ public void testCheckNumberOfNodes() throws Exception {
+ for(int i = 0; i<20; i++) {
+ int nodes = getNumberOfNodes();
+ LOG.info("{}/{} nodes joined the elasticsearch cluster",
+ nodes, cluster.getInstances().size());
+ if (nodes == cluster.getInstances().size()) {
+ return;
+ }
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {}
+ }
+ throw new Exception("All nodes did not joined the cluster as expected");
+ }
+
+ private int getNumberOfNodes() throws Exception {
+ String healthInfo = getHealthInfo();
+ Pattern nodesPattern = Pattern.compile("\".*number_of_nodes\":(\\d+).*");
+ Matcher matcher = nodesPattern.matcher(healthInfo);
+ if (matcher.find()) {
+ return Integer.parseInt(matcher.group(1));
+ }
+ return 0;
+ }
+
+ private String getHealthInfo() throws Exception {
+ for(int i=0; i<20; i++) {
+ try {
+ Cluster.Instance instance = Iterables.get(
+ cluster.getInstancesMatching(role(ElasticSearchHandler.ROLE)), 0);
+ String address = instance.getPublicAddress().getHostAddress();
+
+ URL url = new URL(String.format("http://%s:9200/_cluster/health", address));
+ BufferedReader in = new BufferedReader(new InputStreamReader(url.openStream()));
+
+ StringBuilder builder = new StringBuilder();
+ String line;
+ while((line = in.readLine()) != null) {
+ builder.append(line);
+ }
+ in.close();
+ return builder.toString();
+
+ } catch(IOException e) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e1) {}
+ }
+ }
+ throw new Exception("Unable to get cluster health info.");
+ }
+
+ @After
+ public void tearDown() throws IOException, InterruptedException {
+ service.destroyCluster(clusterSpec);
+ }
+
+}
Added: incubator/whirr/trunk/services/elasticsearch/src/test/resources/whirr-elasticsearch-test.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/elasticsearch/src/test/resources/whirr-elasticsearch-test.properties?rev=1086564&view=auto
==============================================================================
--- incubator/whirr/trunk/services/elasticsearch/src/test/resources/whirr-elasticsearch-test.properties (added)
+++ incubator/whirr/trunk/services/elasticsearch/src/test/resources/whirr-elasticsearch-test.properties Tue Mar 29 12:13:27 2011
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=elasticsearchtest
+whirr.instance-templates=2 elasticsearch
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+whirr.hardware-min-ram=512
\ No newline at end of file
Modified: incubator/whirr/trunk/src/site/confluence/index.confluence
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/src/site/confluence/index.confluence?rev=1086564&r1=1086563&r2=1086564&view=diff
==============================================================================
--- incubator/whirr/trunk/src/site/confluence/index.confluence (original)
+++ incubator/whirr/trunk/src/site/confluence/index.confluence Tue Mar 29 12:13:27 2011
@@ -14,9 +14,9 @@ h2. Which services and cloud providers a
Whirr uses [jclouds|http://code.google.com/p/jclouds/] for provisioning, so in principle it should support all the cloud providers that jclouds supports. The following table shows
the cloud provider and service combinations that have been tested.
-||Cloud provider||Cassandra||Hadoop||ZooKeeper||HBase||
-|Amazon EC2|Yes|Yes|Yes|Yes|
-|Rackspace Cloud Servers|Yes|Yes|Yes|Yes|
+||Cloud provider||Cassandra||Hadoop||ZooKeeper||HBase||elasticsearch||
+|Amazon EC2|Yes|Yes|Yes|Yes|Yes|
+|Rackspace Cloud Servers|Yes|Yes|Yes|Yes|Yes|
h2. Download