You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by ab...@apache.org on 2013/03/25 17:12:42 UTC
[4/5] git commit: WHIRR-681. Enhance puppet service with an ability
to export cluster topology to the puppet code. Contributed by Roman
Shaposhnik.
WHIRR-681. Enhance puppet service with an ability to export cluster topology to the puppet code. Contributed by Roman Shaposhnik.
Project: http://git-wip-us.apache.org/repos/asf/whirr/repo
Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/9d3efedf
Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/9d3efedf
Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/9d3efedf
Branch: refs/heads/branch-0.8
Commit: 9d3efedfad1291dc8cb2f96d46ac2251d8e58662
Parents: 221f8ad
Author: Andrew Bayer <an...@gmail.com>
Authored: Mon Mar 11 11:09:01 2013 -0700
Committer: Andrew Bayer <an...@gmail.com>
Committed: Mon Mar 25 09:12:27 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../service/puppet/PuppetClusterActionHandler.java | 2 +-
.../whirr/service/puppet/PuppetConstants.java | 1 +
.../statements/CreateSitePpAndApplyRoles.java | 35 ++++++++++++++-
.../src/main/resources/functions/install_puppet.sh | 2 +-
.../statements/CreateSitePpAndApplyRolesTest.java | 13 +++++-
.../src/test/resources/nginx-with-attribs.txt | 6 +++
7 files changed, 58 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1a57f5a..a465421 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,9 @@ Release 0.8.2 (unreleased changes)
IMPROVEMENTS
+ WHIRR-681. Enhance puppet service with an ability to export
+ cluster topology to the puppet code. (Roman Shaposhnik via abayer)
+
WHIRR-661. Upgrade to Hadoop 1.0.3. (Steve Loughran via abayer)
WHIRR-634. Update private IP host file entry when required by
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
index 2fa69fd..a9841a5 100644
--- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
+++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
@@ -103,7 +103,7 @@ public class PuppetClusterActionHandler extends PuppetInstallClusterActionHandle
if (isLastPuppetRoleIn(event.getInstanceTemplate().getRoles()).apply(getRole())) {
Configuration config = event.getClusterSpec().getConfiguration();
Iterable<String> roles = RolesManagedByPuppet.INSTANCE.apply(event.getInstanceTemplate().getRoles());
- addStatement(event, new CreateSitePpAndApplyRoles(roles, config));
+ addStatement(event, new CreateSitePpAndApplyRoles(roles, event.getCluster().getInstances(), config));
}
}
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
index c6ccdaa..3aca26c 100644
--- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
+++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
@@ -30,6 +30,7 @@ public class PuppetConstants {
public static final String MODULES_DIR = "/etc/puppet/modules/";
public static final String SITE_PP_FILE_LOCATION = "/etc/puppet/manifests/site.pp";
+ public static final String CONF_PP_FILE_LOCATION = "/etc/puppet/manifests/extdata/common.csv";
public static final Pattern MODULE_KEY_PATTERN = Pattern.compile("^" + PUPPET + "\\.([^.]+)\\."
+ MODULE_SOURCE_SUBKEY + "$");
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
index 2a15a28..6b65af1 100644
--- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
+++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
@@ -20,15 +20,23 @@
package org.apache.whirr.service.puppet.statements;
import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.whirr.service.puppet.PuppetConstants.PUPPET;
import static org.apache.whirr.service.puppet.PuppetConstants.SITE_PP_FILE_LOCATION;
+import static org.apache.whirr.service.puppet.PuppetConstants.CONF_PP_FILE_LOCATION;
import static org.jclouds.scriptbuilder.domain.Statements.appendFile;
import static org.jclouds.scriptbuilder.domain.Statements.exec;
import java.util.Collection;
import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
import org.apache.whirr.service.puppet.Manifest;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.jclouds.scriptbuilder.domain.Statement;
@@ -44,10 +52,12 @@ import com.google.common.collect.ImmutableList.Builder;
public class CreateSitePpAndApplyRoles implements Statement {
private Iterable<String> roles;
+ private Iterable<Cluster.Instance> instances;
private Configuration config;
- public CreateSitePpAndApplyRoles(Iterable<String> roles, Configuration config) {
+ public CreateSitePpAndApplyRoles(Iterable<String> roles, Iterable<Cluster.Instance> instances, Configuration config) {
this.roles = checkNotNull(roles, "roles");
+ this.instances = checkNotNull(instances, "instances");
this.config = checkNotNull(config, "config");
}
@@ -64,8 +74,30 @@ public class CreateSitePpAndApplyRoles implements Statement {
Builder<Statement> statements = ImmutableList.<Statement> builder();
statements.add(Statements.rm(SITE_PP_FILE_LOCATION));
+ statements.add(Statements.rm(CONF_PP_FILE_LOCATION));
Builder<String> sitePp = ImmutableList.<String> builder();
+ Map<String, Set<String>> puppetRoles = Maps.newHashMap();
+ for (Cluster.Instance instance : instances) {
+ for (String role : instance.getRoles()) {
+ int firstColon = role.indexOf(':');
+ if (firstColon != -1 && role.substring(0, firstColon).equals(PUPPET)) {
+ String puppetClass = role.substring(firstColon + 1);
+ if (!puppetRoles.containsKey(puppetClass)) {
+ puppetRoles.put(puppetClass, Sets.<String>newHashSet());
+ }
+ puppetRoles.get(puppetClass).add(instance.getPrivateIp());
+ }
+ }
+ }
+
+ Builder<String> confPp = ImmutableList.<String> builder();
+ for (String puppetClass : puppetRoles.keySet()) {
+ confPp.add(puppetClass + "," + Joiner.on(',').join(puppetRoles.get(puppetClass)));
+ }
+
+ sitePp.add("$extlookup_datadir='/etc/puppet/manifests/extdata'");
+ sitePp.add("$extlookup_precedence = ['common']");
sitePp.add("node default {");
for (String role : roles) {
String manifestAttribPrefix = role.replaceAll(":+", ".");
@@ -79,6 +111,7 @@ public class CreateSitePpAndApplyRoles implements Statement {
}
sitePp.add("}");
+ statements.add(appendFile(CONF_PP_FILE_LOCATION, confPp.build()));
statements.add(appendFile(SITE_PP_FILE_LOCATION, sitePp.build()));
statements.add(exec("puppet apply " + SITE_PP_FILE_LOCATION));
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/resources/functions/install_puppet.sh
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/resources/functions/install_puppet.sh b/services/puppet/src/main/resources/functions/install_puppet.sh
index 5c18161..e01a152 100644
--- a/services/puppet/src/main/resources/functions/install_puppet.sh
+++ b/services/puppet/src/main/resources/functions/install_puppet.sh
@@ -27,7 +27,7 @@ function install_puppet_forced() {
# Setup the default module/fact path locations so we can populate them
# elsewhere
mkdir -p /etc/puppet/modules
- mkdir -p /etc/puppet/manifests
+ mkdir -p /etc/puppet/manifests/extdata
mkdir -p /usr/share/puppet/modules
mkdir -p /var/lib/puppet/lib/facter
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java b/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
index 3ff02b0..e404114 100644
--- a/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
+++ b/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
@@ -22,8 +22,11 @@ import static junit.framework.Assert.assertEquals;
import java.io.IOException;
+import com.google.common.collect.Sets;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.jclouds.domain.Credentials;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.junit.Test;
@@ -40,7 +43,15 @@ public class CreateSitePpAndApplyRolesTest {
conf.setProperty("puppet.nginx.module", "git://github.com/puppetlabs/puppetlabs-nginx.git");
conf.setProperty("nginx.server.hostname", "foohost");
- CreateSitePpAndApplyRoles nginx = new CreateSitePpAndApplyRoles(ImmutableSet.of("nginx::server"), conf);
+ CreateSitePpAndApplyRoles nginx = new CreateSitePpAndApplyRoles(ImmutableSet.of("nginx::server"),
+ ImmutableSet.of(new Cluster.Instance(
+ new Credentials("dummy", "dummy"),
+ Sets.newHashSet("puppet:nginx::server"),
+ "127.0.0.1",
+ "127.0.0.1",
+ "id-1",
+ null)),
+ conf);
assertEquals(CharStreams.toString(Resources.newReaderSupplier(Resources.getResource("nginx-with-attribs.txt"),
Charsets.UTF_8)), nginx.render(OsFamily.UNIX));
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/test/resources/nginx-with-attribs.txt
----------------------------------------------------------------------
diff --git a/services/puppet/src/test/resources/nginx-with-attribs.txt b/services/puppet/src/test/resources/nginx-with-attribs.txt
index ed53c47..0bd63da 100644
--- a/services/puppet/src/test/resources/nginx-with-attribs.txt
+++ b/services/puppet/src/test/resources/nginx-with-attribs.txt
@@ -1,5 +1,11 @@
rm /etc/puppet/manifests/site.pp
+rm /etc/puppet/manifests/extdata/common.csv
+cat >> /etc/puppet/manifests/extdata/common.csv <<-'END_OF_JCLOUDS_FILE'
+ nginx::server,127.0.0.1
+END_OF_JCLOUDS_FILE
cat >> /etc/puppet/manifests/site.pp <<-'END_OF_JCLOUDS_FILE'
+ $extlookup_datadir='/etc/puppet/manifests/extdata'
+ $extlookup_precedence = ['common']
node default {
class { 'nginx::server':
hostname => foohost,