You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by as...@apache.org on 2011/04/12 20:32:28 UTC
svn commit: r1091532 - in /incubator/whirr/trunk: ./
cli/src/test/java/org/apache/whirr/cli/command/
core/src/main/java/org/apache/whirr/cluster/actions/
core/src/main/java/org/apache/whirr/service/
core/src/test/java/org/apache/whirr/service/
Author: asavu
Date: Tue Apr 12 18:32:28 2011
New Revision: 1091532
URL: http://svn.apache.org/viewvc?rev=1091532&view=rev
Log:
WHIRR-278. Refactor ClusterSpec and extract InstanceTemplate class
Added:
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/InstanceTemplate.java
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyClusterCommandTest.java
incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/BootstrapClusterAction.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ConfigureClusterAction.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ScriptBasedClusterAction.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/Cluster.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/package-info.java
incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/ClusterSpecTest.java
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Tue Apr 12 18:32:28 2011
@@ -16,6 +16,8 @@ Trunk (unreleased changes)
WHIRR-275. Improve firewall API for services. (tomwhite)
+ WHIRR-278. Refactor ClusterSpec and extract InstanceTemplate class
+
BUG FIXES
WHIRR-253. ZooKeeper service should only authorize ingress to ZooKeeper
Modified: incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyClusterCommandTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyClusterCommandTest.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyClusterCommandTest.java (original)
+++ incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/DestroyClusterCommandTest.java Tue Apr 12 18:32:28 2011
@@ -95,7 +95,7 @@ public class DestroyClusterCommandTest {
Configuration conf = new PropertiesConfiguration();
conf.addProperty("whirr.version", "version-string");
- ClusterSpec expectedClusterSpec = ClusterSpec.withNoDefaults(conf);
+ ClusterSpec expectedClusterSpec = ClusterSpec.withTemporaryKeys(conf);
expectedClusterSpec.setServiceName("test-service");
expectedClusterSpec.setProvider("rackspace");
expectedClusterSpec.setIdentity("myusername");
Modified: incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java (original)
+++ incubator/whirr/trunk/cli/src/test/java/org/apache/whirr/cli/command/LaunchClusterCommandTest.java Tue Apr 12 18:32:28 2011
@@ -40,6 +40,7 @@ import org.apache.commons.configuration.
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.whirr.service.Cluster;
import org.apache.whirr.service.ClusterSpec;
+import org.apache.whirr.service.InstanceTemplate;
import org.apache.whirr.service.Service;
import org.apache.whirr.service.ServiceFactory;
import org.apache.whirr.ssh.KeyPair;
@@ -102,10 +103,10 @@ public class LaunchClusterCommandTest {
Configuration conf = new PropertiesConfiguration();
conf.addProperty("whirr.version", "version-string");
- ClusterSpec expectedClusterSpec = ClusterSpec.withNoDefaults(conf);
+ ClusterSpec expectedClusterSpec = ClusterSpec.withTemporaryKeys(conf);
expectedClusterSpec.setInstanceTemplates(Lists.newArrayList(
- new ClusterSpec.InstanceTemplate(1, ImmutableSet.of("role1", "role2")),
- new ClusterSpec.InstanceTemplate(2, ImmutableSet.of("role3"))
+ new InstanceTemplate(1, ImmutableSet.of("role1", "role2")),
+ new InstanceTemplate(2, ImmutableSet.of("role3"))
));
expectedClusterSpec.setServiceName("test-service");
expectedClusterSpec.setProvider("rackspace");
@@ -152,10 +153,10 @@ public class LaunchClusterCommandTest {
conf.addProperty("whirr.version", "version-string");
conf.addProperty("whirr.instance-templates-max-percent-failure", "60 dn+tt");
- ClusterSpec expectedClusterSpec = ClusterSpec.withNoDefaults(conf);
+ ClusterSpec expectedClusterSpec = ClusterSpec.withTemporaryKeys(conf);
expectedClusterSpec.setInstanceTemplates(Lists.newArrayList(
- new ClusterSpec.InstanceTemplate(1, 1, Sets.newHashSet("jt", "nn")),
- new ClusterSpec.InstanceTemplate(3, 2, Sets.newHashSet("dn", "tt"))
+ new InstanceTemplate(1, 1, Sets.newHashSet("jt", "nn")),
+ new InstanceTemplate(3, 2, Sets.newHashSet("dn", "tt"))
));
expectedClusterSpec.setServiceName("hadoop");
expectedClusterSpec.setProvider("ec2");
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/BootstrapClusterAction.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/BootstrapClusterAction.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/BootstrapClusterAction.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/BootstrapClusterAction.java Tue Apr 12 18:32:28 2011
@@ -50,7 +50,7 @@ import org.apache.whirr.service.Cluster.
import org.apache.whirr.service.ClusterActionEvent;
import org.apache.whirr.service.ClusterActionHandler;
import org.apache.whirr.service.ClusterSpec;
-import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
+import org.apache.whirr.service.InstanceTemplate;
import org.apache.whirr.service.ComputeServiceContextBuilder;
import org.apache.whirr.service.jclouds.StatementBuilder;
import org.apache.whirr.service.jclouds.TemplateBuilderStrategy;
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ConfigureClusterAction.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ConfigureClusterAction.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ConfigureClusterAction.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ConfigureClusterAction.java Tue Apr 12 18:32:28 2011
@@ -31,7 +31,7 @@ import org.apache.whirr.service.Cluster.
import org.apache.whirr.service.ClusterActionEvent;
import org.apache.whirr.service.ClusterActionHandler;
import org.apache.whirr.service.ClusterSpec;
-import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
+import org.apache.whirr.service.InstanceTemplate;
import org.apache.whirr.service.ComputeServiceContextBuilder;
import org.apache.whirr.service.RolePredicates;
import org.apache.whirr.service.jclouds.StatementBuilder;
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ScriptBasedClusterAction.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ScriptBasedClusterAction.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ScriptBasedClusterAction.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/cluster/actions/ScriptBasedClusterAction.java Tue Apr 12 18:32:28 2011
@@ -31,7 +31,7 @@ import org.apache.whirr.service.ClusterA
import org.apache.whirr.service.ClusterSpec;
import org.apache.whirr.service.ComputeServiceContextBuilder;
import org.apache.whirr.service.FirewallManager;
-import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
+import org.apache.whirr.service.InstanceTemplate;
import org.apache.whirr.service.jclouds.StatementBuilder;
import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.compute.ComputeServiceContextFactory;
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/Cluster.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/Cluster.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/Cluster.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/Cluster.java Tue Apr 12 18:32:28 2011
@@ -42,7 +42,7 @@ public class Cluster {
/**
* This class represents a real node running in a cluster. An instance has
* one or more roles.
- * @see org.apache.whirr.service.ClusterSpec.InstanceTemplate
+ * @see InstanceTemplate
*/
public static class Instance {
private final Credentials loginCredentials;
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/ClusterSpec.java Tue Apr 12 18:32:28 2011
@@ -21,14 +21,11 @@ package org.apache.whirr.service;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Objects;
import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -41,16 +38,14 @@ import com.jcraft.jsch.KeyPair;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.ConfigurationUtils;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.configuration.interpol.ConfigurationInterpolator;
import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.text.StrLookup;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.whirr.ssh.KeyPair.sameKeyPair;
@@ -76,9 +71,9 @@ public class ClusterSpec {
}
public enum Property {
- SERVICE_NAME(String.class, false, "(optional) The name of the " +
+ SERVICE_NAME(String.class, false, "(optional) The name of the " +
"service to use. E.g. hadoop."),
-
+
INSTANCE_TEMPLATES(String.class, false, "The number of instances " +
"to launch for each set of roles. E.g. 1 hadoop-namenode+" +
"hadoop-jobtracker, 10 hadoop-datanode+hadoop-tasktracker"),
@@ -174,151 +169,6 @@ public class ClusterSpec {
return description;
}
}
-
- /**
- * This class describes the type of instances that should be in the cluster.
- * This is done by specifying the number of instances in each role.
- */
- public static class InstanceTemplate {
- private static Map<String, String> aliases = new HashMap<String, String>();
- private static final Logger LOG = LoggerFactory.getLogger(InstanceTemplate.class);
-
- static {
- /*
- * WARNING: this is not a generic aliasing mechanism. This code
- * should be removed in the following releases and it's
- * used only temporary to deprecate short legacy role names.
- */
- aliases.put("nn", "hadoop-namenode");
- aliases.put("jt", "hadoop-jobtracker");
- aliases.put("dn", "hadoop-datanode");
- aliases.put("tt", "hadoop-tasktracker");
- aliases.put("zk", "zookeeper");
- }
-
- private Set<String> roles;
- private int numberOfInstances;
- private int minNumberOfInstances; // some instances may fail, at least a minimum number is required
-
- public InstanceTemplate(int numberOfInstances, String... roles) {
- this(numberOfInstances, numberOfInstances, Sets.newLinkedHashSet(Lists.newArrayList(roles)));
- }
-
- public InstanceTemplate(int numberOfInstances, Set<String> roles) {
- this(numberOfInstances, numberOfInstances, roles);
- }
-
- public InstanceTemplate(int numberOfInstances, int minNumberOfInstances, String... roles) {
- this(numberOfInstances, minNumberOfInstances, Sets.newLinkedHashSet(Lists.newArrayList(roles)));
- }
-
- public InstanceTemplate(int numberOfInstances, int minNumberOfInstances, Set<String> roles) {
- for (String role : roles) {
- checkArgument(!StringUtils.contains(role, " "),
- "Role '%s' may not contain space characters.", role);
- }
-
- this.roles = replaceAliases(roles);
- this.numberOfInstances = numberOfInstances;
- this.minNumberOfInstances = minNumberOfInstances;
- }
-
- private static Set<String> replaceAliases(Set<String> roles) {
- Set<String> newRoles = Sets.newLinkedHashSet();
- for(String role : roles) {
- if (aliases.containsKey(role)) {
- LOG.warn("Role name '{}' is deprecated, use '{}'",
- role, aliases.get(role));
- newRoles.add(aliases.get(role));
- } else {
- newRoles.add(role);
- }
- }
- return newRoles;
- }
-
- public Set<String> getRoles() {
- return roles;
- }
-
- public int getNumberOfInstances() {
- return numberOfInstances;
- }
-
- public int getMinNumberOfInstances() {
- return minNumberOfInstances;
- }
-
- public boolean equals(Object o) {
- if (o instanceof InstanceTemplate) {
- InstanceTemplate that = (InstanceTemplate) o;
- return Objects.equal(numberOfInstances, that.numberOfInstances)
- && Objects.equal(minNumberOfInstances, that.minNumberOfInstances)
- && Objects.equal(roles, that.roles);
- }
- return false;
- }
-
- public int hashCode() {
- return Objects.hashCode(numberOfInstances, minNumberOfInstances, roles);
- }
-
- public String toString() {
- return Objects.toStringHelper(this)
- .add("numberOfInstances", numberOfInstances)
- .add("minNumberOfInstances", minNumberOfInstances)
- .add("roles", roles)
- .toString();
- }
-
- public static Map<String, String> parse(String... strings) {
- Set<String> roles = Sets.newLinkedHashSet(Lists.newArrayList(strings));
- roles = replaceAliases(roles);
- Map<String, String> templates = Maps.newHashMap();
- for (String s : roles) {
- String[] parts = s.split(" ");
- checkArgument(parts.length == 2,
- "Invalid instance template syntax for '%s'. Does not match " +
- "'<number> <role1>+<role2>+<role3>...', e.g. '1 hadoop-namenode+hadoop-jobtracker'.", s);
- templates.put(parts[1], parts[0]);
- }
- return templates;
- }
-
- public static List<InstanceTemplate> parse(CompositeConfiguration cconf) {
- final String[] strings = cconf.getStringArray(Property.INSTANCE_TEMPLATES.getConfigName());
- Map<String, String> maxPercentFailures = parse(cconf.getStringArray(Property.INSTANCE_TEMPLATES_MAX_PERCENT_FAILURES.getConfigName()));
- Map<String, String> minInstances = parse(cconf.getStringArray(Property.INSTANCE_TEMPLATES_MINIMUM_NUMBER_OF_INSTANCES.getConfigName()));
- List<InstanceTemplate> templates = Lists.newArrayList();
- for (String s : strings) {
- String[] parts = s.split(" ");
- checkArgument(parts.length == 2,
- "Invalid instance template syntax for '%s'. Does not match " +
- "'<number> <role1>+<role2>+<role3>...', e.g. '1 hadoop-namenode+hadoop-jobtracker'.", s);
- int num = Integer.parseInt(parts[0]);
- int minNumberOfInstances = 0;
- final String maxPercentFail = maxPercentFailures.get(parts[1]);
- if (maxPercentFail != null) {
- // round up integer division (a + b -1) / b
- minNumberOfInstances = (Integer.parseInt(maxPercentFail) * num + 99) / 100;
- }
- String minNumberOfInst = minInstances.get(parts[1]);
- if (minNumberOfInst != null) {
- int minExplicitlySet = Integer.parseInt(minNumberOfInst);
- if (minNumberOfInstances > 0) { // maximum between two minims
- minNumberOfInstances = Math.max(minNumberOfInstances, minExplicitlySet);
- } else {
- minNumberOfInstances = minExplicitlySet;
- }
- }
- if (minNumberOfInstances == 0 || minNumberOfInstances > num) {
- minNumberOfInstances = num;
- }
- templates.add(new InstanceTemplate(num, minNumberOfInstances, parts[1].split("\\+")));
- }
- return templates;
- }
- }
private static final String DEFAULT_PROPERTIES = "whirr-default.properties";
@@ -361,25 +211,33 @@ public class ClusterSpec {
return new ClusterSpec(conf, false);
}
- private List<InstanceTemplate> instanceTemplates;
+ private String clusterName;
private String serviceName;
+
+ private String clusterUser;
+ private String loginUser;
+
+ private List<InstanceTemplate> instanceTemplates;
private int maxStartupRetries;
+
private String provider;
private String identity;
private String credential;
- private String clusterName;
+
private String privateKey;
private File privateKeyFile;
private String publicKey;
+
+ private String locationId;
private String imageId;
+
private String hardwareId;
private int hardwareMinRam;
- private String locationId;
+
private List<String> clientCidrs;
private String version;
private String runUrlBase;
- private String clusterUser;
-
+
private Configuration config;
public ClusterSpec() throws ConfigurationException {
@@ -391,31 +249,69 @@ public class ClusterSpec {
}
/**
- *
* @throws ConfigurationException if something is wrong
*/
- public ClusterSpec(Configuration config, boolean loadDefaults)
+ public ClusterSpec(Configuration userConfig, boolean loadDefaults)
throws ConfigurationException {
- CompositeConfiguration c = new CompositeConfiguration();
- c.addConfiguration(config);
if (loadDefaults) {
- c.addConfiguration(new PropertiesConfiguration(DEFAULT_PROPERTIES));
+ config = composeWithDefaults(userConfig);
+ } else {
+ config = ConfigurationUtils.cloneConfiguration(userConfig);
}
- setServiceName(c.getString(Property.SERVICE_NAME.getConfigName()));
- setInstanceTemplates(InstanceTemplate.parse(c));
- setMaxStartupRetries(c.getInt(Property.MAX_STARTUP_RETRIES.getConfigName(), 1));
- setProvider(c.getString(Property.PROVIDER.getConfigName()));
- setIdentity(c.getString(Property.IDENTITY.getConfigName()));
- setCredential(c.getString(Property.CREDENTIAL.getConfigName()));
- setClusterName(c.getString(Property.CLUSTER_NAME.getConfigName()));
+ setClusterName(getString(Property.CLUSTER_NAME));
+ setServiceName(getString(Property.SERVICE_NAME));
+
+ setLoginUser(getString(Property.LOGIN_USER));
+ setClusterUser(getString(Property.CLUSTER_USER));
+
+ setInstanceTemplates(InstanceTemplate.parse(config));
+ setMaxStartupRetries(getInt(Property.MAX_STARTUP_RETRIES, 1));
+
+ setProvider(getString(Property.PROVIDER));
+ setIdentity(getString(Property.IDENTITY));
+ setCredential(getString(Property.CREDENTIAL));
+
+ checkAndSetKeyPair();
+
+ setImageId(getString(Property.IMAGE_ID));
+ setHardwareId(getString(Property.HARDWARE_ID));
+ setHardwareMinRam(getInt(Property.HARDWARE_MIN_RAM, 1024));
+
+ setLocationId(getString(Property.LOCATION_ID));
+ setClientCidrs(getList(Property.CLIENT_CIDRS));
+
+ setVersion(getString(Property.VERSION));
+ setRunUrlBase(getString(Property.RUN_URL_BASE));
+ }
+
+ private String getString(Property key) {
+ return config.getString(key.getConfigName());
+ }
+
+ private int getInt(Property key, int defaultValue) {
+ return config.getInt(key.getConfigName(), defaultValue);
+ }
+ private List<String> getList(Property key) {
+ return config.getList(key.getConfigName());
+ }
+
+ private Configuration composeWithDefaults(Configuration userConfig)
+ throws ConfigurationException {
+ CompositeConfiguration composed = new CompositeConfiguration();
+ composed.addConfiguration(userConfig);
+ composed.addConfiguration(
+ new PropertiesConfiguration(DEFAULT_PROPERTIES));
+ return composed;
+ }
+
+ private void checkAndSetKeyPair() throws ConfigurationException {
try {
- String privateKeyPath = c.getString(
- Property.PRIVATE_KEY_FILE.getConfigName());
+ String privateKeyPath = getString(Property.PRIVATE_KEY_FILE);
- String publicKeyPath = c.getString(Property.PUBLIC_KEY_FILE.getConfigName());
+ String publicKeyPath = getString(Property.PUBLIC_KEY_FILE);
publicKeyPath = (publicKeyPath == null && privateKeyPath != null) ?
privateKeyPath + ".pub" : publicKeyPath;
if(privateKeyPath != null && publicKeyPath != null) {
@@ -440,97 +336,85 @@ public class ClusterSpec {
} catch (IOException e) {
throw new ConfigurationException("Error reading one of key file", e);
}
-
- setImageId(config.getString(Property.IMAGE_ID.getConfigName()));
- setHardwareId(config.getString(Property.HARDWARE_ID.getConfigName()));
- setHardwareMinRam(c.getInteger(Property.HARDWARE_MIN_RAM.getConfigName(), 1024));
- setLocationId(config.getString(Property.LOCATION_ID.getConfigName()));
- setClientCidrs(c.getList(Property.CLIENT_CIDRS.getConfigName()));
- setVersion(c.getString(Property.VERSION.getConfigName()));
- String runUrlBase = c.getString(Property.RUN_URL_BASE.getConfigName());
-
- if (runUrlBase == null && getVersion() != null) {
- try {
- runUrlBase = String.format("http://whirr.s3.amazonaws.com/%s/",
- URLEncoder.encode(getVersion(), "UTF-8"));
- } catch (UnsupportedEncodingException e) {
- throw new ConfigurationException(e);
- }
- }
- setRunUrlBase(runUrlBase);
-
- String loginUser = c.getString(Property.LOGIN_USER.getConfigName());
- if (loginUser != null) {
- // patch until jclouds 1.0-beta-10
- System.setProperty("whirr.login-user", loginUser);
- }
- clusterUser = c.getString(Property.CLUSTER_USER.getConfigName());
- this.config = c;
}
public List<InstanceTemplate> getInstanceTemplates() {
return instanceTemplates;
}
-
+
public InstanceTemplate getInstanceTemplate(final Set<String> roles) {
for (InstanceTemplate template : instanceTemplates) {
- if (roles.equals(template.roles)) {
+ if (roles.equals(template.getRoles())) {
return template;
}
}
return null;
}
-
+
public InstanceTemplate getInstanceTemplate(String... roles) {
return getInstanceTemplate(Sets.newLinkedHashSet(Lists.newArrayList(roles)));
}
-
- public String getServiceName() {
- return serviceName;
- }
+
public int getMaxStartupRetries() {
return maxStartupRetries;
}
+
public String getProvider() {
return provider;
}
+
public String getIdentity() {
return identity;
}
+
public String getCredential() {
return credential;
}
+
public String getClusterName() {
return clusterName;
}
+
+ public String getServiceName() {
+ return serviceName;
+ }
+
public String getPrivateKey() {
return privateKey;
}
+
public File getPrivateKeyFile() {
return privateKeyFile;
- }
+ }
+
public String getPublicKey() {
return publicKey;
}
+
public String getImageId() {
return imageId;
}
+
public String getHardwareId() {
return hardwareId;
}
+
public int getHardwareMinRam() {
return hardwareMinRam;
}
+
public String getLocationId() {
return locationId;
}
+
public List<String> getClientCidrs() {
return clientCidrs;
}
+
public String getVersion() {
return version;
}
- @Deprecated
+
public String getRunUrlBase() {
return runUrlBase;
}
@@ -539,15 +423,14 @@ public class ClusterSpec {
return clusterUser;
}
-
+ public String getLoginUser() {
+ return loginUser;
+ }
+
public void setInstanceTemplates(List<InstanceTemplate> instanceTemplates) {
this.instanceTemplates = instanceTemplates;
}
- public void setServiceName(String serviceName) {
- this.serviceName = serviceName;
- }
-
public void setMaxStartupRetries(int maxStartupRetries) {
this.maxStartupRetries = maxStartupRetries;
}
@@ -568,6 +451,10 @@ public class ClusterSpec {
this.clusterName = clusterName;
}
+ public void setServiceName(String serviceName) {
+ this.serviceName = serviceName;
+ }
+
/**
* The rsa public key which is authorized to login to your on the cloud nodes.
*
@@ -653,7 +540,6 @@ public class ClusterSpec {
this.version = version;
}
- @Deprecated
public void setRunUrlBase(String runUrlBase) {
this.runUrlBase = runUrlBase;
}
@@ -662,6 +548,14 @@ public class ClusterSpec {
this.clusterUser = user;
}
+ public void setLoginUser(String user) {
+ loginUser = config.getString(Property.LOGIN_USER.getConfigName());
+ if (loginUser != null) {
+ // patch until jclouds 1.0-beta-10
+ System.setProperty("whirr.login-user", loginUser);
+ }
+ }
+
public Configuration getConfiguration() {
return config;
}
@@ -691,12 +585,14 @@ public class ClusterSpec {
if (o instanceof ClusterSpec) {
ClusterSpec that = (ClusterSpec) o;
return Objects.equal(instanceTemplates, that.instanceTemplates)
- && Objects.equal(serviceName, that.serviceName)
&& Objects.equal(maxStartupRetries, that.maxStartupRetries)
&& Objects.equal(provider, that.provider)
&& Objects.equal(identity, that.identity)
&& Objects.equal(credential, that.credential)
&& Objects.equal(clusterName, that.clusterName)
+ && Objects.equal(serviceName, that.serviceName)
+ && Objects.equal(clusterUser, that.clusterUser)
+ && Objects.equal(loginUser, that.loginUser)
&& Objects.equal(imageId, that.imageId)
&& Objects.equal(hardwareId, that.hardwareId)
&& Objects.equal(hardwareMinRam, that.hardwareMinRam)
@@ -709,21 +605,23 @@ public class ClusterSpec {
}
public int hashCode() {
- return Objects.hashCode(instanceTemplates, serviceName,
- maxStartupRetries, provider, identity, credential, clusterName, publicKey,
- privateKey, imageId, hardwareId, locationId, clientCidrs, version,
- runUrlBase);
+ return Objects.hashCode(instanceTemplates, maxStartupRetries, provider,
+ identity, credential, clusterName, serviceName, clusterUser, loginUser,
+ publicKey, privateKey, imageId, hardwareId, locationId, clientCidrs,
+ version, runUrlBase);
}
public String toString() {
return Objects.toStringHelper(this)
.add("instanceTemplates", instanceTemplates)
- .add("serviceName", serviceName)
.add("maxStartupRetries", maxStartupRetries)
.add("provider", provider)
.add("identity", identity)
.add("credential", credential)
.add("clusterName", clusterName)
+ .add("serviceName", serviceName)
+ .add("clusterUser", clusterUser)
+ .add("loginUser", loginUser)
.add("publicKey", publicKey)
.add("privateKey", privateKey)
.add("imageId", imageId)
@@ -734,5 +632,4 @@ public class ClusterSpec {
.add("version", version)
.toString();
}
-
}
Added: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/InstanceTemplate.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/InstanceTemplate.java?rev=1091532&view=auto
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/InstanceTemplate.java (added)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/InstanceTemplate.java Tue Apr 12 18:32:28 2011
@@ -0,0 +1,163 @@
+package org.apache.whirr.service;
+
+import com.google.common.base.Objects;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+/**
+ * This class describes the type of instances that should be in the cluster.
+ * This is done by specifying the number of instances in each role.
+ */
+public class InstanceTemplate {
+ private static Map<String, String> aliases = new HashMap<String, String>();
+ private static final Logger LOG = LoggerFactory.getLogger(InstanceTemplate.class);
+
+ static {
+ /*
+ * WARNING: this is not a generic aliasing mechanism. This code
+ * should be removed in the following releases and it's
+ * used only temporary to deprecate short legacy role names.
+ */
+ aliases.put("nn", "hadoop-namenode");
+ aliases.put("jt", "hadoop-jobtracker");
+ aliases.put("dn", "hadoop-datanode");
+ aliases.put("tt", "hadoop-tasktracker");
+ aliases.put("zk", "zookeeper");
+ }
+
+ private Set<String> roles;
+ private int numberOfInstances;
+ private int minNumberOfInstances; // some instances may fail, at least a minimum number is required
+
+ public InstanceTemplate(int numberOfInstances, String... roles) {
+ this(numberOfInstances, numberOfInstances, Sets.newLinkedHashSet(Lists.newArrayList(roles)));
+ }
+
+ public InstanceTemplate(int numberOfInstances, Set<String> roles) {
+ this(numberOfInstances, numberOfInstances, roles);
+ }
+
+ public InstanceTemplate(int numberOfInstances, int minNumberOfInstances, String... roles) {
+ this(numberOfInstances, minNumberOfInstances, Sets.newLinkedHashSet(Lists.newArrayList(roles)));
+ }
+
+ public InstanceTemplate(int numberOfInstances, int minNumberOfInstances, Set<String> roles) {
+ for (String role : roles) {
+ checkArgument(!StringUtils.contains(role, " "),
+ "Role '%s' may not contain space characters.", role);
+ }
+
+ this.roles = replaceAliases(roles);
+ this.numberOfInstances = numberOfInstances;
+ this.minNumberOfInstances = minNumberOfInstances;
+ }
+
+ private static Set<String> replaceAliases(Set<String> roles) {
+ Set<String> newRoles = Sets.newLinkedHashSet();
+ for(String role : roles) {
+ if (aliases.containsKey(role)) {
+ LOG.warn("Role name '{}' is deprecated, use '{}'",
+ role, aliases.get(role));
+ newRoles.add(aliases.get(role));
+ } else {
+ newRoles.add(role);
+ }
+ }
+ return newRoles;
+ }
+
+ public Set<String> getRoles() {
+ return roles;
+ }
+
+ public int getNumberOfInstances() {
+ return numberOfInstances;
+ }
+
+ public int getMinNumberOfInstances() {
+ return minNumberOfInstances;
+ }
+
+ public boolean equals(Object o) {
+ if (o instanceof InstanceTemplate) {
+ InstanceTemplate that = (InstanceTemplate) o;
+ return Objects.equal(numberOfInstances, that.numberOfInstances)
+ && Objects.equal(minNumberOfInstances, that.minNumberOfInstances)
+ && Objects.equal(roles, that.roles);
+ }
+ return false;
+ }
+
+ public int hashCode() {
+ return Objects.hashCode(numberOfInstances, minNumberOfInstances, roles);
+ }
+
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("numberOfInstances", numberOfInstances)
+ .add("minNumberOfInstances", minNumberOfInstances)
+ .add("roles", roles)
+ .toString();
+ }
+
+ public static Map<String, String> parse(String... strings) {
+ Set<String> roles = Sets.newLinkedHashSet(Lists.newArrayList(strings));
+ roles = replaceAliases(roles);
+ Map<String, String> templates = Maps.newHashMap();
+ for (String s : roles) {
+ String[] parts = s.split(" ");
+ checkArgument(parts.length == 2,
+ "Invalid instance template syntax for '%s'. Does not match " +
+ "'<number> <role1>+<role2>+<role3>...', e.g. '1 hadoop-namenode+hadoop-jobtracker'.", s);
+ templates.put(parts[1], parts[0]);
+ }
+ return templates;
+ }
+
+ public static List<InstanceTemplate> parse(Configuration cconf) {
+ final String[] strings = cconf.getStringArray(ClusterSpec.Property.INSTANCE_TEMPLATES.getConfigName());
+ Map<String, String> maxPercentFailures = parse(cconf.getStringArray(ClusterSpec.Property.INSTANCE_TEMPLATES_MAX_PERCENT_FAILURES.getConfigName()));
+ Map<String, String> minInstances = parse(cconf.getStringArray(ClusterSpec.Property.INSTANCE_TEMPLATES_MINIMUM_NUMBER_OF_INSTANCES.getConfigName()));
+ List<InstanceTemplate> templates = Lists.newArrayList();
+ for (String s : strings) {
+ String[] parts = s.split(" ");
+ checkArgument(parts.length == 2,
+ "Invalid instance template syntax for '%s'. Does not match " +
+ "'<number> <role1>+<role2>+<role3>...', e.g. '1 hadoop-namenode+hadoop-jobtracker'.", s);
+ int num = Integer.parseInt(parts[0]);
+ int minNumberOfInstances = 0;
+ final String maxPercentFail = maxPercentFailures.get(parts[1]);
+ if (maxPercentFail != null) {
+ // round up integer division (a + b -1) / b
+ minNumberOfInstances = (Integer.parseInt(maxPercentFail) * num + 99) / 100;
+ }
+ String minNumberOfInst = minInstances.get(parts[1]);
+ if (minNumberOfInst != null) {
+ int minExplicitlySet = Integer.parseInt(minNumberOfInst);
+ if (minNumberOfInstances > 0) { // maximum between two minims
+ minNumberOfInstances = Math.max(minNumberOfInstances, minExplicitlySet);
+ } else {
+ minNumberOfInstances = minExplicitlySet;
+ }
+ }
+ if (minNumberOfInstances == 0 || minNumberOfInstances > num) {
+ minNumberOfInstances = num;
+ }
+ templates.add(new InstanceTemplate(num, minNumberOfInstances, parts[1].split("\\+")));
+ }
+ return templates;
+ }
+}
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/package-info.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/package-info.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/package-info.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/package-info.java Tue Apr 12 18:32:28 2011
@@ -33,7 +33,7 @@
* required.
* </p>
* <p>
- * An <i>instance template</i> ({@link org.apache.whirr.service.ClusterSpec.InstanceTemplate}) is a specification of the role sets and
+ * An <i>instance template</i> ({@link InstanceTemplate}) is a specification of the role sets and
* cardinalities that make up a cluster. For example,
* <tt>1 role-a+role-b,4 role-c</tt>
* specifies a cluster in which one node is in roles <tt>role-a</tt> and
Modified: incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/ClusterSpecTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/ClusterSpecTest.java?rev=1091532&r1=1091531&r2=1091532&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/ClusterSpecTest.java (original)
+++ incubator/whirr/trunk/core/src/test/java/org/apache/whirr/service/ClusterSpecTest.java Tue Apr 12 18:32:28 2011
@@ -43,7 +43,6 @@ import org.apache.commons.configuration.
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.io.IOUtils;
-import org.apache.whirr.service.ClusterSpec.InstanceTemplate;
import org.apache.whirr.ssh.KeyPair;
import org.junit.Assert;
import org.junit.Test;
@@ -54,8 +53,9 @@ public class ClusterSpecTest {
public void testDefaultsAreSet()
throws ConfigurationException, JSchException, IOException {
ClusterSpec spec = ClusterSpec.withTemporaryKeys();
- assertThat(spec.getRunUrlBase(),
- startsWith("http://whirr.s3.amazonaws.com/"));
+ assertThat(spec.getClusterUser(),
+ is(System.getProperty("user.name")));
+ assertThat(spec.getMaxStartupRetries(), is(1));
}
@Test
@@ -79,16 +79,6 @@ public class ClusterSpecTest {
}
@Test
- public void testVersionInRunUrlbaseIsUrlEncoded()
- throws ConfigurationException, JSchException, IOException {
- Configuration conf = new PropertiesConfiguration();
- conf.setProperty(ClusterSpec.Property.VERSION.getConfigName(), "0.1.0+1");
- ClusterSpec spec = ClusterSpec.withNoDefaults(conf);
- assertThat(spec.getRunUrlBase(),
- is("http://whirr.s3.amazonaws.com/0.1.0%2B1/"));
- }
-
- @Test
public void testGetConfigurationForKeysWithPrefix()
throws ConfigurationException, JSchException, IOException {
Configuration conf = new PropertiesConfiguration();