You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by kv...@apache.org on 2012/03/29 10:31:59 UTC
svn commit: r1306750 - in /whirr/trunk: ./ cli/src/test/resources/ core/
core/src/main/java/org/apache/whirr/actions/
core/src/main/java/org/apache/whirr/service/
core/src/main/java/org/apache/whirr/template/
core/src/test/java/org/apache/whirr/service...
Author: kve
Date: Thu Mar 29 08:31:56 2012
New Revision: 1306750
URL: http://svn.apache.org/viewvc?rev=1306750&view=rev
Log:
Templating for configuration files (WHIRR-370)
Added:
whirr/trunk/core/src/main/java/org/apache/whirr/template/
whirr/trunk/core/src/main/java/org/apache/whirr/template/TemplateUtils.java
whirr/trunk/core/src/test/java/org/apache/whirr/template/
whirr/trunk/core/src/test/java/org/apache/whirr/template/TemplateUtilsTest.java
whirr/trunk/core/src/test/resources/template-test.txt.vm
whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-ganglia.properties.vm
whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-null.properties.vm
whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopDryRunTest.java
whirr/trunk/services/hbase/src/main/resources/hbase-metrics-ganglia.properties.vm
whirr/trunk/services/hbase/src/main/resources/hbase-metrics-null.properties.vm
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/cli/src/test/resources/log4j.xml
whirr/trunk/core/pom.xml
whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionEvent.java
whirr/trunk/core/src/test/java/org/apache/whirr/service/BaseServiceDryRunTest.java
whirr/trunk/core/src/test/resources/log4j.xml
whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml
whirr/trunk/pom.xml
whirr/trunk/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraServiceDryRunTest.java
whirr/trunk/services/cassandra/src/test/resources/log4j.xml
whirr/trunk/services/cdh-oldtests/src/test/resources/log4j.xml
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh
whirr/trunk/services/cdh/src/test/resources/log4j.xml
whirr/trunk/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java
whirr/trunk/services/chef/src/test/resources/log4j.xml
whirr/trunk/services/ganglia/src/test/resources/log4j.xml
whirr/trunk/services/hadoop/pom.xml
whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java
whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
whirr/trunk/services/hadoop/src/test/resources/log4j.xml
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseClusterActionHandler.java
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java
whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java
whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
whirr/trunk/services/hbase/src/test/resources/log4j.xml
whirr/trunk/services/mahout/src/test/resources/log4j.xml
whirr/trunk/services/pig/src/test/resources/log4j.xml
whirr/trunk/services/puppet/src/test/resources/log4j.xml
whirr/trunk/services/voldemort/src/test/resources/log4j.xml
whirr/trunk/services/yarn/src/test/resources/log4j.xml
whirr/trunk/services/zookeeper/src/test/resources/log4j.xml
Modified: whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Thu Mar 29 08:31:56 2012
@@ -13,6 +13,8 @@ Trunk (unreleased changes)
IMPROVEMENTS
+ WHIRR-370. Templating for configuration files (kve)
+
WHIRR-556. Improve the 'Running on $PROVIDER using $IDENTITY' message (kve)
WHIRR-514. Avoid hardcoded Whirr version number in karaf tests (kve)
Modified: whirr/trunk/cli/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/cli/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/cli/src/test/resources/log4j.xml (original)
+++ whirr/trunk/cli/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/core/pom.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/core/pom.xml (original)
+++ whirr/trunk/core/pom.xml Thu Mar 29 08:31:56 2012
@@ -137,6 +137,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ <version>${velocity.version}</version>
+ </dependency>
+ <dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<scope>test</scope>
Modified: whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
URL: http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java (original)
+++ whirr/trunk/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java Thu Mar 29 08:31:56 2012
@@ -18,52 +18,49 @@
package org.apache.whirr.actions;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
import static com.google.common.base.Preconditions.checkNotNull;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
import static com.google.common.base.Predicates.not;
-import static org.apache.whirr.RolePredicates.onlyRolesIn;
-import static org.jclouds.compute.options.RunScriptOptions.Builder.overrideLoginCredentials;
-
+import com.google.common.base.Throwables;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.UncheckedExecutionException;
import java.io.IOException;
import java.util.Collection;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
-
import javax.annotation.Nullable;
-
+import org.apache.velocity.app.VelocityEngine;
import org.apache.whirr.Cluster;
-import org.apache.whirr.Cluster.Instance;
+import static org.apache.whirr.Cluster.Instance;
import org.apache.whirr.ClusterAction;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.InstanceTemplate;
+import static org.apache.whirr.RolePredicates.onlyRolesIn;
import org.apache.whirr.service.ClusterActionEvent;
import org.apache.whirr.service.ClusterActionHandler;
import org.apache.whirr.service.FirewallManager;
import org.apache.whirr.service.jclouds.StatementBuilder;
+import org.apache.whirr.template.TemplateUtils;
import org.jclouds.compute.ComputeService;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.domain.ExecResponse;
import org.jclouds.compute.options.RunScriptOptions;
+import static org.jclouds.compute.options.RunScriptOptions.Builder.overrideLoginCredentials;
import org.jclouds.domain.LoginCredentials;
import org.jclouds.scriptbuilder.domain.Statement;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.base.Joiner;
-import com.google.common.base.Predicate;
-import com.google.common.base.Predicates;
-import com.google.common.base.Throwables;
-import com.google.common.cache.LoadingCache;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.common.util.concurrent.UncheckedExecutionException;
-
/**
* A {@link ClusterAction} that provides the base functionality for running
* scripts on instances in the cluster.
@@ -111,8 +108,10 @@ public abstract class ScriptBasedCluster
FirewallManager firewallManager = new FirewallManager(
computeServiceContext, clusterSpec, newCluster);
+ VelocityEngine velocityEngine = TemplateUtils.newVelocityEngine();
+
ClusterActionEvent event = new ClusterActionEvent(getAction(), clusterSpec,
- instanceTemplate, newCluster, statementBuilder, getCompute(), firewallManager);
+ instanceTemplate, newCluster, statementBuilder, getCompute(), firewallManager, velocityEngine);
eventMap.put(instanceTemplate, event);
for (String role : instanceTemplate.getRoles()) {
@@ -165,7 +164,7 @@ public abstract class ScriptBasedCluster
final RunScriptOptions options = overrideLoginCredentials(LoginCredentials.builder().user(clusterSpec.getClusterUser())
.privateKey(clusterSpec.getPrivateKey()).build());
- for (Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
+ for (Map.Entry<InstanceTemplate, ClusterActionEvent> entry : eventMap.entrySet()) {
if (shouldIgnoreInstanceTemplate(entry.getKey())) {
continue; // skip if not in the target
}
@@ -258,7 +257,7 @@ public abstract class ScriptBasedCluster
}
protected void eventSpecificActions(
- Entry<InstanceTemplate, ClusterActionEvent> entry) throws IOException {
+ Map.Entry<InstanceTemplate, ClusterActionEvent> entry) throws IOException {
}
protected void postRunScriptsActions(
Modified: whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionEvent.java
URL: http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionEvent.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionEvent.java (original)
+++ whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterActionEvent.java Thu Mar 29 08:31:56 2012
@@ -18,6 +18,8 @@
package org.apache.whirr.service;
+import org.apache.velocity.app.VelocityEngine;
+
import org.apache.whirr.Cluster;
import org.apache.whirr.ClusterSpec;
import org.apache.whirr.InstanceTemplate;
@@ -41,18 +43,19 @@ public class ClusterActionEvent {
new TemplateBuilderStrategy();
private FirewallManager firewallManager;
private Function<ClusterSpec, ComputeServiceContext> getCompute;
+ private VelocityEngine velocityEngine;
public ClusterActionEvent(String action, ClusterSpec clusterSpec,
InstanceTemplate instanceTemplate, Cluster cluster,
Function<ClusterSpec, ComputeServiceContext> getCompute,
- FirewallManager firewallManager) {
- this(action, clusterSpec, instanceTemplate, cluster, null, getCompute, firewallManager);
+ FirewallManager firewallManager, VelocityEngine velocityEngine) {
+ this(action, clusterSpec, instanceTemplate, cluster, null, getCompute, firewallManager, velocityEngine);
}
public ClusterActionEvent(String action, ClusterSpec clusterSpec,
InstanceTemplate instanceTemplate, Cluster cluster, StatementBuilder statementBuilder,
Function<ClusterSpec, ComputeServiceContext> getCompute,
- FirewallManager firewallManager) {
+ FirewallManager firewallManager, VelocityEngine velocityEngine) {
this.action = action;
this.clusterSpec = clusterSpec;
this.instanceTemplate = instanceTemplate;
@@ -60,6 +63,7 @@ public class ClusterActionEvent {
this.statementBuilder = statementBuilder;
this.getCompute = getCompute;
this.firewallManager = firewallManager;
+ this.velocityEngine = velocityEngine;
}
public Cluster getCluster() {
@@ -102,5 +106,9 @@ public class ClusterActionEvent {
public FirewallManager getFirewallManager() {
return firewallManager;
}
+
+ public VelocityEngine getTemplateEngine() {
+ return velocityEngine;
+ }
}
Added: whirr/trunk/core/src/main/java/org/apache/whirr/template/TemplateUtils.java
URL: http://svn.apache.org/viewvc/whirr/trunk/core/src/main/java/org/apache/whirr/template/TemplateUtils.java?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/core/src/main/java/org/apache/whirr/template/TemplateUtils.java (added)
+++ whirr/trunk/core/src/main/java/org/apache/whirr/template/TemplateUtils.java Thu Mar 29 08:31:56 2012
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.template;
+
+import java.util.Properties;
+import org.apache.velocity.runtime.RuntimeConstants;
+import org.apache.velocity.runtime.log.Log4JLogChute;
+import org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader;
+
+import java.io.StringWriter;
+import java.util.Arrays;
+import org.apache.velocity.Template;
+import org.apache.velocity.VelocityContext;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.RolePredicates;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+
+public class TemplateUtils {
+
+ public static Statement createFileFromTemplate(String path, VelocityEngine engine, String templateName, ClusterSpec clusterSpec, Cluster cluster) {
+ return Statements.createOrOverwriteFile(path, Arrays.asList(processTemplate(engine, templateName, clusterSpec, cluster).split("\n")));
+ }
+
+ public static String processTemplate(VelocityEngine engine, String templateName, ClusterSpec clusterSpec, Cluster cluster) {
+ VelocityContext context = new VelocityContext();
+ context.put("clusterSpec", clusterSpec);
+ context.put("cluster", cluster);
+ context.put("RolePredicates", RolePredicates.class);
+
+ Template template = engine.getTemplate(templateName);
+ StringWriter writer = new StringWriter();
+
+ template.merge(context, writer);
+
+ return writer.toString();
+ }
+
+ public static VelocityEngine newVelocityEngine() {
+ Properties defaults = new Properties();
+ defaults.setProperty(RuntimeConstants.RESOURCE_LOADER, "classpath");
+ defaults.setProperty("classpath.resource.loader.class", ClasspathResourceLoader.class.getName());
+ defaults.setProperty("runtime.log.logsystem.class", Log4JLogChute.class.getName());
+ defaults.setProperty("runtime.log.logsystem.log4j.logger", "org.apache.velocity");
+ return newVelocityEngine(defaults);
+ }
+
+ public static VelocityEngine newVelocityEngine(Properties properties) {
+ return new VelocityEngine(properties);
+ }
+
+
+}
Modified: whirr/trunk/core/src/test/java/org/apache/whirr/service/BaseServiceDryRunTest.java
URL: http://svn.apache.org/viewvc/whirr/trunk/core/src/test/java/org/apache/whirr/service/BaseServiceDryRunTest.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/core/src/test/java/org/apache/whirr/service/BaseServiceDryRunTest.java (original)
+++ whirr/trunk/core/src/test/java/org/apache/whirr/service/BaseServiceDryRunTest.java Thu Mar 29 08:31:56 2012
@@ -18,6 +18,8 @@
package org.apache.whirr.service;
+import com.google.common.base.Joiner;
+import java.util.Set;
import static junit.framework.Assert.assertTrue;
import static junit.framework.Assert.fail;
@@ -77,8 +79,8 @@ public abstract class BaseServiceDryRunT
*/
@Test
public void testBootstrapAndConfigure() throws Exception {
- ClusterSpec cookbookWithDefaultRecipe = newClusterSpecForProperties(ImmutableMap.of("whirr.instance-templates",
- "1 " + getRole()));
+ ClusterSpec cookbookWithDefaultRecipe = newClusterSpecForProperties(ImmutableMap.of(
+ "whirr.instance-templates", "1 " + Joiner.on("+").join(getInstanceRoles())));
DryRun dryRun = launchWithClusterSpec(cookbookWithDefaultRecipe);
assertScriptPredicateOnPhase(dryRun, "bootstrap", bootstrapPredicate());
@@ -89,7 +91,7 @@ public abstract class BaseServiceDryRunT
protected abstract Predicate<CharSequence> bootstrapPredicate();
- protected abstract String getRole();
+ protected abstract Set<String> getInstanceRoles();
protected void assertScriptPredicateOnPhase(DryRun dryRun, String phase, Predicate<CharSequence> predicate)
throws Exception {
Added: whirr/trunk/core/src/test/java/org/apache/whirr/template/TemplateUtilsTest.java
URL: http://svn.apache.org/viewvc/whirr/trunk/core/src/test/java/org/apache/whirr/template/TemplateUtilsTest.java?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/core/src/test/java/org/apache/whirr/template/TemplateUtilsTest.java (added)
+++ whirr/trunk/core/src/test/java/org/apache/whirr/template/TemplateUtilsTest.java Thu Mar 29 08:31:56 2012
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.template;
+
+import com.google.common.collect.Sets;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.velocity.app.VelocityEngine;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterSpec;
+import org.jclouds.domain.Credentials;
+import org.junit.Assert;
+import org.junit.Test;
+
+
+public class TemplateUtilsTest {
+
+ @Test
+ public void testProcessTemplate() throws Exception {
+ Credentials credentials = new Credentials("dummy", "dummy");
+ Cluster.Instance instance = new Cluster.Instance(credentials,
+ Sets.newHashSet("foo"), "127.0.0.1", "127.0.0.1", "id-0", null);
+
+ ClusterSpec clusterSpec = new ClusterSpec(new PropertiesConfiguration("whirr-core-test.properties"));
+ Cluster cluster = new Cluster(Sets.newHashSet(instance));
+ VelocityEngine ve = TemplateUtils.newVelocityEngine();
+
+ String result = TemplateUtils.processTemplate(ve, "template-test.txt.vm", clusterSpec, cluster);
+
+ Assert.assertEquals("instance ip: 127.0.0.1", result);
+ }
+
+}
Modified: whirr/trunk/core/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/core/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/core/src/test/resources/log4j.xml (original)
+++ whirr/trunk/core/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Added: whirr/trunk/core/src/test/resources/template-test.txt.vm
URL: http://svn.apache.org/viewvc/whirr/trunk/core/src/test/resources/template-test.txt.vm?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/core/src/test/resources/template-test.txt.vm (added)
+++ whirr/trunk/core/src/test/resources/template-test.txt.vm Thu Mar 29 08:31:56 2012
@@ -0,0 +1,13 @@
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+instance ip: $cluster.getInstancesMatching($RolePredicates.role("foo")).iterator().next().getPublicIp()
\ No newline at end of file
Modified: whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml (original)
+++ whirr/trunk/platforms/karaf/feature/src/main/resources/features.xml Thu Mar 29 08:31:56 2012
@@ -40,6 +40,7 @@
<bundle dependency='true'>mvn:commons-codec/commons-codec/${commons.codec.version}</bundle>
<bundle dependency='true'>mvn:commons-lang/commons-lang/${commons.lang.version}</bundle>
<bundle dependency="true">wrap:mvn:dnsjava/dnsjava/${dnsjava.version}</bundle>
+ <bundle dependency="true">mvn:org.apache.velocity/velocity/${velocity.version}</bundle>
<bundle>mvn:org.apache.whirr/whirr-core/${project.version}</bundle>
<bundle>mvn:org.apache.whirr/whirr-cli/${project.version}</bundle>
<bundle>mvn:org.apache.whirr.karaf/org.apache.whirr.karaf.commands/${project.version}</bundle>
Modified: whirr/trunk/pom.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/pom.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/pom.xml (original)
+++ whirr/trunk/pom.xml Thu Mar 29 08:31:56 2012
@@ -83,6 +83,7 @@
<slf4j.version>1.6.3</slf4j.version>
<voldermort.version>0.90</voldermort.version>
<zookeeper.version>3.3.1</zookeeper.version>
+ <velocity.version>1.7</velocity.version>
</properties>
<dependencyManagement>
Modified: whirr/trunk/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraServiceDryRunTest.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraServiceDryRunTest.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraServiceDryRunTest.java (original)
+++ whirr/trunk/services/cassandra/src/test/java/org/apache/whirr/service/cassandra/CassandraServiceDryRunTest.java Thu Mar 29 08:31:56 2012
@@ -20,6 +20,8 @@ package org.apache.whirr.service.cassand
import static com.google.common.base.Predicates.containsPattern;
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
import org.apache.whirr.service.BaseServiceDryRunTest;
import com.google.common.base.Predicate;
@@ -27,8 +29,8 @@ import com.google.common.base.Predicate;
public class CassandraServiceDryRunTest extends BaseServiceDryRunTest {
@Override
- protected String getRole() {
- return "cassandra";
+ protected Set<String> getInstanceRoles() {
+ return ImmutableSet.of("cassandra");
}
@Override
Modified: whirr/trunk/services/cassandra/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/cassandra/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/cassandra/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/cassandra/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/cdh-oldtests/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/cdh-oldtests/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/cdh-oldtests/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/cdh-oldtests/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
URL: http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh (original)
+++ whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh Thu Mar 29 08:31:56 2012
@@ -58,15 +58,7 @@ function configure_cdh_hadoop() {
# Copy generated configuration files in place
cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
cp /tmp/hadoop-env.sh $HADOOP_CONF_DIR
-
- # Expose /metrics URL endpoint
- cat > $HADOOP_CONF_DIR/hadoop-metrics.properties <<EOF
-# Exposes /metrics URL endpoint for metrics information.
-dfs.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
-mapred.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
-jvm.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
-rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
-EOF
+ cp /tmp/hadoop-metrics.properties $HADOOP_CONF_DIR
# Keep PID files in a non-temporary directory
HADOOP_PID_DIR=$(. /tmp/hadoop-env.sh; echo $HADOOP_PID_DIR)
Modified: whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh
URL: http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh (original)
+++ whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hbase.sh Thu Mar 29 08:31:56 2012
@@ -65,6 +65,7 @@ function configure_cdh_hbase() {
# Copy generated configuration files in place
cp /tmp/hbase-site.xml $HBASE_CONF_DIR
cp /tmp/hbase-env.sh $HBASE_CONF_DIR
+ cp /tmp/hbase-hadoop-metrics.properties $HBASE_CONF_DIR/hadoop-metrics.properties
# HBASE_PID_DIR should exist and be owned by hbase:hbase
HBASE_PID_DIR=$(. $HBASE_CONF_DIR/hbase-env.sh; echo $HBASE_PID_DIR)
@@ -84,19 +85,6 @@ function configure_cdh_hbase() {
ln -s /data/hbase/logs $HBASE_LOG_DIR
chown -R hbase:hbase $HBASE_LOG_DIR
- # configure hbase for ganglia
- cat > $HBASE_CONF_DIR/hadoop-metrics.properties <<EOF
-dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-dfs.period=10
-dfs.servers=$MASTER_HOST:8649
-hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-hbase.period=10
-hbase.servers=$MASTER_HOST:8649
-jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-jvm.period=10
-jvm.servers=$MASTER_HOST:8649
-EOF
-
# Now that the configuration is done, install the daemon packages
for role in $(echo "$ROLES" | tr "," "\n"); do
case $role in
Modified: whirr/trunk/services/cdh/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/cdh/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/cdh/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java (original)
+++ whirr/trunk/services/chef/src/test/java/org/apache/whirr/service/chef/ChefServiceDryRunTest.java Thu Mar 29 08:31:56 2012
@@ -21,6 +21,8 @@ package org.apache.whirr.service.chef;
import static com.google.common.base.Predicates.and;
import static com.google.common.base.Predicates.contains;
import static com.google.common.base.Predicates.containsPattern;
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
import static java.util.regex.Pattern.LITERAL;
import static java.util.regex.Pattern.compile;
@@ -35,8 +37,8 @@ import com.google.common.collect.Immutab
public class ChefServiceDryRunTest extends BaseServiceDryRunTest {
@Override
- protected String getRole() {
- return "chef:java";
+ protected Set<String> getInstanceRoles() {
+ return ImmutableSet.of("chef:java");
}
@Override
Modified: whirr/trunk/services/chef/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/chef/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/chef/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/chef/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/ganglia/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/ganglia/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/ganglia/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/ganglia/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/hadoop/pom.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/pom.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hadoop/pom.xml (original)
+++ whirr/trunk/services/hadoop/pom.xml Thu Mar 29 08:31:56 2012
@@ -53,6 +53,12 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-ganglia</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
Modified: whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java (original)
+++ whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopClusterActionHandler.java Thu Mar 29 08:31:56 2012
@@ -19,21 +19,23 @@
package org.apache.whirr.service.hadoop;
import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildCommon;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHadoopEnv;
import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHdfs;
import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildMapReduce;
-import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHadoopEnv;
import static org.jclouds.scriptbuilder.domain.Statements.call;
import com.google.common.base.Joiner;
-
import java.io.IOException;
-
+import java.util.Set;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.whirr.Cluster;
+import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.RolePredicates;
import org.apache.whirr.service.ClusterActionEvent;
import org.apache.whirr.service.ClusterActionHandlerSupport;
+import org.apache.whirr.template.TemplateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -103,11 +105,27 @@ public abstract class HadoopClusterActio
buildCommon("/tmp/core-site.xml", clusterSpec, cluster),
buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster),
buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster),
- buildHadoopEnv("/tmp/hadoop-env.sh", clusterSpec, cluster)
+ buildHadoopEnv("/tmp/hadoop-env.sh", clusterSpec, cluster),
+ TemplateUtils.createFileFromTemplate("/tmp/hadoop-metrics.properties", event.getTemplateEngine(), getMetricsTemplate(event, clusterSpec, cluster), clusterSpec, cluster)
);
+
} catch (ConfigurationException e) {
throw new IOException(e);
}
}
+ private String getMetricsTemplate(ClusterActionEvent event, ClusterSpec clusterSpec, Cluster cluster) {
+ Configuration conf = clusterSpec.getConfiguration();
+ if (conf.containsKey("hadoop-metrics.template")) {
+ return conf.getString("hadoop-metrics.template");
+ }
+
+ Set<Instance> gmetadInstances = cluster.getInstancesMatching(RolePredicates.role("ganglia-metad"));
+ if (!gmetadInstances.isEmpty()) {
+ return "hadoop-metrics-ganglia.properties.vm";
+ }
+
+ return "hadoop-metrics-null.properties.vm";
+ }
+
}
Modified: whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh (original)
+++ whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh Thu Mar 29 08:31:56 2012
@@ -50,6 +50,7 @@ function configure_hadoop() {
# Copy generated configuration files in place
cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
cp /tmp/hadoop-env.sh $HADOOP_CONF_DIR
+ cp /tmp/hadoop-metrics.properties $HADOOP_CONF_DIR
# Keep PID files in a non-temporary directory
HADOOP_PID_DIR=$(. /tmp/hadoop-env.sh; echo $HADOOP_PID_DIR)
@@ -68,7 +69,7 @@ function configure_hadoop() {
mkdir -p $(dirname $HADOOP_LOG_DIR)
ln -s /data/hadoop/logs $HADOOP_LOG_DIR
chown -R hadoop:hadoop $HADOOP_LOG_DIR
-
+
for role in $(echo "$ROLES" | tr "," "\n"); do
case $role in
hadoop-namenode)
Added: whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-ganglia.properties.vm
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-ganglia.properties.vm?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-ganglia.properties.vm (added)
+++ whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-ganglia.properties.vm Thu Mar 29 08:31:56 2012
@@ -0,0 +1,25 @@
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+#set( $GMETAD_HOST = $cluster.getInstancesMatching($RolePredicates.role("ganglia-metad")).iterator().next().getPrivateHostName() )
+dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+dfs.period=10
+dfs.servers=$GMETAD_HOST:8649
+mapred.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+mapred.period=10
+mapred.servers=$GMETAD_HOST:8649
+jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+jvm.period=10
+jvm.servers=$GMETAD_HOST:8649
+rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+rpc.period=10
+rpc.servers=$GMETAD_HOST:8649
Added: whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-null.properties.vm
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-null.properties.vm?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-null.properties.vm (added)
+++ whirr/trunk/services/hadoop/src/main/resources/hadoop-metrics-null.properties.vm Thu Mar 29 08:31:56 2012
@@ -0,0 +1,16 @@
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+dfs.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
+mapred.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
+jvm.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
+rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
\ No newline at end of file
Added: whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopDryRunTest.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopDryRunTest.java?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopDryRunTest.java (added)
+++ whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopDryRunTest.java Thu Mar 29 08:31:56 2012
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import com.google.common.base.Predicates;
+import static com.google.common.base.Predicates.and;
+import static com.google.common.base.Predicates.containsPattern;
+import com.google.common.collect.ImmutableSet;
+import java.util.Set;
+
+import com.google.common.base.Predicate;
+import org.apache.whirr.service.BaseServiceDryRunTest;
+
+public class HadoopDryRunTest extends BaseServiceDryRunTest {
+
+ @Override
+ protected Set<String> getInstanceRoles() {
+ return ImmutableSet.of("hadoop-namenode", "hadoop-jobtracker", "hadoop-datanode",
+ "hadoop-tasktracker", "ganglia-metad");
+ }
+
+ @Override
+ protected Predicate<CharSequence> configurePredicate() {
+ return and(
+ containsPattern("dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext31"),
+ containsPattern("dfs.servers=10.1.1.1")
+ );
+ }
+
+ @Override
+ protected Predicate<CharSequence> bootstrapPredicate() {
+ return Predicates.alwaysTrue();
+ }
+
+}
Modified: whirr/trunk/services/hadoop/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hadoop/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hadoop/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/hadoop/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseClusterActionHandler.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseClusterActionHandler.java (original)
+++ whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseClusterActionHandler.java Thu Mar 29 08:31:56 2012
@@ -18,6 +18,13 @@
package org.apache.whirr.service.hbase;
+import java.util.Set;
+import org.apache.whirr.RolePredicates;
+import org.apache.whirr.Cluster.Instance;
+
+import org.apache.whirr.Cluster;
+import org.apache.whirr.service.ClusterActionEvent;
+
import java.io.IOException;
import org.apache.commons.configuration.Configuration;
@@ -49,4 +56,20 @@ public abstract class HBaseClusterAction
return getConfigureFunction(config, "hbase", HBaseConstants.FUNCTION_CONFIGURE);
}
+ protected String getMetricsTemplate(ClusterActionEvent event,
+ ClusterSpec clusterSpec, Cluster cluster) {
+ Configuration conf = clusterSpec.getConfiguration();
+ if (conf.containsKey("hbase-metrics.template")) {
+ return conf.getString("hbase-metrics.template");
+ }
+
+ Set<Instance> gmetadInstances = cluster.getInstancesMatching(RolePredicates.role("ganglia-metad"));
+ if (!gmetadInstances.isEmpty()) {
+ return "hbase-metrics-ganglia.properties.vm";
+ }
+
+ return "hbase-metrics-null.properties.vm";
+ }
+
+
}
Modified: whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java (original)
+++ whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseMasterClusterActionHandler.java Thu Mar 29 08:31:56 2012
@@ -18,9 +18,19 @@
package org.apache.whirr.service.hbase;
+import static org.apache.whirr.RolePredicates.role;
+import static org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseEnv;
+import static org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseSite;
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
import com.google.common.io.Files;
+import java.io.File;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.util.Map.Entry;
+import java.util.Properties;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
import org.apache.whirr.Cluster;
@@ -30,20 +40,10 @@ import org.apache.whirr.service.ClusterA
import org.apache.whirr.service.FirewallManager.Rule;
import org.apache.whirr.service.hadoop.HadoopProxy;
import org.apache.whirr.service.zookeeper.ZooKeeperCluster;
+import org.apache.whirr.template.TemplateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.util.Map.Entry;
-import java.util.Properties;
-
-import static org.apache.whirr.RolePredicates.role;
-import static org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseEnv;
-import static org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseSite;
-import static org.jclouds.scriptbuilder.domain.Statements.call;
-
public class HBaseMasterClusterActionHandler extends HBaseClusterActionHandler {
private static final Logger LOG =
@@ -97,8 +97,10 @@ public class HBaseMasterClusterActionHan
try {
event.getStatementBuilder().addStatements(
buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster),
- buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster)
+ buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster),
+ TemplateUtils.createFileFromTemplate("/tmp/hbase-hadoop-metrics.properties", event.getTemplateEngine(), getMetricsTemplate(event, clusterSpec, cluster), clusterSpec, cluster)
);
+
} catch (ConfigurationException e) {
throw new IOException(e);
}
Modified: whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java (original)
+++ whirr/trunk/services/hbase/src/main/java/org/apache/whirr/service/hbase/HBaseRegionServerClusterActionHandler.java Thu Mar 29 08:31:56 2012
@@ -24,6 +24,8 @@ import static org.apache.whirr.service.h
import static org.apache.whirr.service.hbase.HBaseConfigurationBuilder.buildHBaseSite;
import static org.jclouds.scriptbuilder.domain.Statements.call;
+import org.apache.whirr.template.TemplateUtils;
+
import java.io.IOException;
import java.net.InetAddress;
@@ -86,7 +88,8 @@ public class HBaseRegionServerClusterAct
try {
event.getStatementBuilder().addStatements(
buildHBaseSite("/tmp/hbase-site.xml", clusterSpec, cluster),
- buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster)
+ buildHBaseEnv("/tmp/hbase-env.sh", clusterSpec, cluster),
+ TemplateUtils.createFileFromTemplate("/tmp/hbase-hadoop-metrics.properties", event.getTemplateEngine(), getMetricsTemplate(event, clusterSpec, cluster), clusterSpec, cluster)
);
} catch (ConfigurationException e) {
throw new IOException(e);
Modified: whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh (original)
+++ whirr/trunk/services/hbase/src/main/resources/functions/configure_hbase.sh Thu Mar 29 08:31:56 2012
@@ -79,6 +79,7 @@ function configure_hbase() {
# Copy generated configuration files in place
cp /tmp/hbase-site.xml $HBASE_CONF_DIR
cp /tmp/hbase-env.sh $HBASE_CONF_DIR
+ cp /tmp/hbase-hadoop-metrics.properties $HBASE_CONF_DIR/hadoop-metrics.properties
# HBASE_PID_DIR should exist and be owned by hadoop:hadoop
mkdir -p /var/run/hbase
@@ -96,19 +97,6 @@ function configure_hbase() {
ln -s $MOUNT/hbase/logs $HBASE_LOG_DIR
chown -R hadoop:hadoop $HBASE_LOG_DIR
- # configure hbase for ganglia
- cat > $HBASE_CONF_DIR/hadoop-metrics.properties <<EOF
-dfs.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-dfs.period=10
-dfs.servers=$MASTER_HOST:8649
-hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-hbase.period=10
-hbase.servers=$MASTER_HOST:8649
-jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext
-jvm.period=10
-jvm.servers=$MASTER_HOST:8649
-EOF
-
# update classpath to include hbase jars and config
# cat >> $HADOOP_HOME/conf/hadoop-env.sh <<EOF
#HADOOP_CLASSPATH="$HBASE_HOME/${HBASE_VERSION}.jar:$HBASE_HOME/lib/zookeeper-3.3.1.jar:$HBASE_CONF_DIR"
Added: whirr/trunk/services/hbase/src/main/resources/hbase-metrics-ganglia.properties.vm
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/resources/hbase-metrics-ganglia.properties.vm?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/services/hbase/src/main/resources/hbase-metrics-ganglia.properties.vm (added)
+++ whirr/trunk/services/hbase/src/main/resources/hbase-metrics-ganglia.properties.vm Thu Mar 29 08:31:56 2012
@@ -0,0 +1,22 @@
+#* Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*#
+#set( $GMETAD_HOST = $cluster.getInstancesMatching($RolePredicates.role("ganglia-metad")).iterator().next().getPrivateHostName() )
+hbase.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+hbase.period=10
+hbase.servers=$GMETAD_HOST:8649
+jvm.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+jvm.period=10
+jvm.servers=$GMETAD_HOST:8649
+rpc.class=org.apache.hadoop.metrics.ganglia.GangliaContext31
+rpc.period=10
+rpc.servers=$GMETAD_HOST:8649
Added: whirr/trunk/services/hbase/src/main/resources/hbase-metrics-null.properties.vm
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/main/resources/hbase-metrics-null.properties.vm?rev=1306750&view=auto
==============================================================================
--- whirr/trunk/services/hbase/src/main/resources/hbase-metrics-null.properties.vm (added)
+++ whirr/trunk/services/hbase/src/main/resources/hbase-metrics-null.properties.vm Thu Mar 29 08:31:56 2012
@@ -0,0 +1,15 @@
+## Licensed under the Apache License, Version 2.0 (the "License");
+## you may not use this file except in compliance with the License.
+## You may obtain a copy of the License at
+##
+## http://www.apache.org/licenses/LICENSE-2.0
+##
+## Unless required by applicable law or agreed to in writing, software
+## distributed under the License is distributed on an "AS IS" BASIS,
+## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+## See the License for the specific language governing permissions and
+## limitations under the License.
+##
+hbase.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
+jvm.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
+rpc.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
\ No newline at end of file
Modified: whirr/trunk/services/hbase/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/hbase/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/hbase/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/hbase/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/mahout/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/mahout/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/mahout/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/mahout/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/pig/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/pig/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/pig/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/pig/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/puppet/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/puppet/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/puppet/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/puppet/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/voldemort/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/voldemort/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/voldemort/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/voldemort/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/yarn/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/yarn/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/yarn/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/yarn/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />
Modified: whirr/trunk/services/zookeeper/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/trunk/services/zookeeper/src/test/resources/log4j.xml?rev=1306750&r1=1306749&r2=1306750&view=diff
==============================================================================
--- whirr/trunk/services/zookeeper/src/test/resources/log4j.xml (original)
+++ whirr/trunk/services/zookeeper/src/test/resources/log4j.xml Thu Mar 29 08:31:56 2012
@@ -179,6 +179,11 @@
<appender-ref ref="WHIRRFILE" />
</category>
+ <category name="org.apache.velocity">
+ <priority value="INFO" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
<category name="jclouds.headers">
<priority value="DEBUG" />
<appender-ref ref="ASYNCWIRE" />