You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@brooklyn.apache.org by ri...@apache.org on 2014/05/29 14:55:13 UTC
[2/5] git commit: many minor usability enhancements for chef,
as shown by new simple example mysql-chef.yaml
many minor usability enhancements for chef, as shown by new simple example mysql-chef.yaml
- catch occasional NPE
- don't use ruby `absolute_path` as it is not in some common (older) versions
- better messages on errors
- respond to more flags in simple yaml
- install recipes from local machine (using classpath or file:/// URL's)
- misc tweaks to deploy/archive/install (cc @grkvlt) to run as tasks for traceability
Project: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/commit/41a8a092
Tree: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/tree/41a8a092
Diff: http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/diff/41a8a092
Branch: refs/heads/master
Commit: 41a8a0928e2963a50bbfd0d1105180e992cd8769
Parents: 9b04960
Author: Alex Heneveld <al...@cloudsoftcorp.com>
Authored: Thu May 29 00:12:17 2014 +0100
Committer: Alex Heneveld <al...@cloudsoftcorp.com>
Committed: Thu May 29 03:44:29 2014 +0100
----------------------------------------------------------------------
.../location/basic/SshMachineLocation.java | 43 ++++----
.../java/brooklyn/util/config/ConfigBag.java | 2 +
.../java/brooklyn/util/file/ArchiveTasks.java | 34 +++++++
.../java/brooklyn/util/file/ArchiveUtils.java | 48 +++++++--
.../java/brooklyn/util/task/ssh/SshTasks.java | 26 +++++
.../brooklyn/entity/chef/ChefBashCommands.java | 100 ++++++++++---------
.../java/brooklyn/entity/chef/ChefConfig.java | 1 +
.../java/brooklyn/entity/chef/ChefEntity.java | 2 +-
.../brooklyn/entity/chef/ChefEntityImpl.java | 6 ++
.../entity/chef/ChefLifecycleEffectorTasks.java | 2 +
.../brooklyn/entity/chef/ChefSoloTasks.java | 48 +--------
.../java/brooklyn/entity/chef/ChefTasks.java | 64 +++++++++---
usage/camp/src/test/resources/mysql-chef.yaml | 26 +++++
.../brooklyn/util/collections/MutableList.java | 14 ++-
.../src/main/java/brooklyn/util/net/Urls.java | 14 +++
.../main/java/brooklyn/util/text/Strings.java | 21 +++-
.../test/java/brooklyn/util/net/UrlsTest.java | 10 ++
17 files changed, 323 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java b/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java
index cfb61c8..4adee00 100644
--- a/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java
+++ b/core/src/main/java/brooklyn/location/basic/SshMachineLocation.java
@@ -1,6 +1,7 @@
package brooklyn.location.basic;
import static brooklyn.util.GroovyJavaMethods.truth;
+import groovy.lang.Closure;
import java.io.Closeable;
import java.io.File;
@@ -22,29 +23,12 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+
import javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import com.google.common.base.Function;
-import com.google.common.base.Objects;
-import com.google.common.base.Preconditions;
-import com.google.common.base.Predicate;
-import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import com.google.common.net.HostAndPort;
-
import brooklyn.config.BrooklynLogging;
import brooklyn.config.ConfigKey;
import brooklyn.config.ConfigKey.HasConfigKey;
@@ -88,7 +72,24 @@ import brooklyn.util.task.system.internal.ExecWithLoggingHelpers;
import brooklyn.util.task.system.internal.ExecWithLoggingHelpers.ExecRunner;
import brooklyn.util.text.Strings;
import brooklyn.util.time.Duration;
-import groovy.lang.Closure;
+
+import com.google.common.base.Function;
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.base.Throwables;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+import com.google.common.net.HostAndPort;
/**
* Operations on a machine that is accessible via ssh.
@@ -667,6 +668,7 @@ public class SshMachineLocation extends AbstractLocation implements MachineLocat
*
* TODO allow s3://bucket/file URIs for AWS S3 resources
* TODO use PAX-URL style URIs for maven artifacts
+ * TODO use subtasks here for greater visibility?; deprecate in favour of SshTasks.installFromUrl?
*
* @param utils A {@link ResourceUtils} that can resolve the source URLs
* @param url The source URL to be installed
@@ -687,7 +689,8 @@ public class SshMachineLocation extends AbstractLocation implements MachineLocat
Map<String, ?> sshProps = MutableMap.<String, Object>builder().putAll(props).put("out", outO).put("err", outE).build();
int result = execScript(sshProps, "copying remote resource "+url+" to server", ImmutableList.of(
BashCommands.INSTALL_CURL, // TODO should hold the 'installing' mutex
- "curl "+url+" -L --silent --insecure --show-error --fail --connect-timeout 60 --max-time 600 --retry 5 -o "+destPath));
+ "mkdir -p `dirname '"+destPath+"'`",
+ "curl "+url+" -L --silent --insecure --show-error --fail --connect-timeout 60 --max-time 600 --retry 5 -o '"+destPath+"'"));
sgsO.close();
sgsE.close();
if (result != 0) {
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/core/src/main/java/brooklyn/util/config/ConfigBag.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/config/ConfigBag.java b/core/src/main/java/brooklyn/util/config/ConfigBag.java
index 7ab3609..6a01bc4 100644
--- a/core/src/main/java/brooklyn/util/config/ConfigBag.java
+++ b/core/src/main/java/brooklyn/util/config/ConfigBag.java
@@ -174,6 +174,8 @@ public class ConfigBag {
}
public ConfigBag putIfAbsent(Map<?, ?> propertiesToSet) {
+ if (propertiesToSet==null)
+ return this;
for (Map.Entry<?, ?> entry: propertiesToSet.entrySet()) {
Object key = entry.getKey();
if (key instanceof HasConfigKey<?>)
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/core/src/main/java/brooklyn/util/file/ArchiveTasks.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/file/ArchiveTasks.java b/core/src/main/java/brooklyn/util/file/ArchiveTasks.java
new file mode 100644
index 0000000..5d64aab
--- /dev/null
+++ b/core/src/main/java/brooklyn/util/file/ArchiveTasks.java
@@ -0,0 +1,34 @@
+package brooklyn.util.file;
+
+import java.util.Map;
+
+import brooklyn.location.basic.SshMachineLocation;
+import brooklyn.management.TaskAdaptable;
+import brooklyn.management.TaskFactory;
+import brooklyn.util.ResourceUtils;
+import brooklyn.util.net.Urls;
+import brooklyn.util.task.Tasks;
+
+public class ArchiveTasks {
+
+ /** as {@link #deploy(ResourceUtils, Map, String, SshMachineLocation, String, String, String)} with the most common parameters */
+ public static TaskFactory<?> deploy(final ResourceUtils optionalResolver, final String archiveUrl, final SshMachineLocation machine, final String destDir) {
+ return deploy(optionalResolver, null, archiveUrl, machine, destDir, false, null, null);
+ }
+
+ /** returns a task which installs and unpacks the given archive, as per {@link ArchiveUtils#deploy(ResourceUtils, Map, String, SshMachineLocation, String, String, String)} */
+ public static TaskFactory<?> deploy(final ResourceUtils resolver, final Map<String, ?> props, final String archiveUrl, final SshMachineLocation machine, final String destDir, final boolean keepArchiveAfterDeploy, final String tmpDir, final String destFile) {
+ return new TaskFactory<TaskAdaptable<?>>() {
+ @Override
+ public TaskAdaptable<?> newTask() {
+ return Tasks.<Void>builder().name("deploying "+Urls.getFilename(archiveUrl)).description("installing "+archiveUrl+" and unpacking to "+destDir).body(new Runnable() {
+ @Override
+ public void run() {
+ ArchiveUtils.deploy(resolver, props, archiveUrl, machine, destDir, keepArchiveAfterDeploy, tmpDir, destFile);
+ }
+ }).build();
+ }
+ };
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/core/src/main/java/brooklyn/util/file/ArchiveUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/file/ArchiveUtils.java b/core/src/main/java/brooklyn/util/file/ArchiveUtils.java
index f832d78..9f7b7ec 100644
--- a/core/src/main/java/brooklyn/util/file/ArchiveUtils.java
+++ b/core/src/main/java/brooklyn/util/file/ArchiveUtils.java
@@ -29,16 +29,21 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import brooklyn.location.basic.SshMachineLocation;
+import brooklyn.util.ResourceUtils;
+import brooklyn.util.collections.MutableList;
import brooklyn.util.collections.MutableMap;
import brooklyn.util.exceptions.Exceptions;
import brooklyn.util.javalang.StackTraceSimplifier;
import brooklyn.util.net.Urls;
import brooklyn.util.os.Os;
import brooklyn.util.ssh.BashCommands;
+import brooklyn.util.task.DynamicTasks;
import brooklyn.util.task.Tasks;
import brooklyn.util.task.ssh.SshTasks;
+import brooklyn.util.text.Strings;
import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
import com.google.common.io.Files;
public class ArchiveUtils {
@@ -126,6 +131,11 @@ public class ArchiveUtils {
* @see #extractCommands(String, String)
*/
public static List<String> extractCommands(String fileName, String sourceDir, String targetDir, boolean extractJar) {
+ return extractCommands(fileName, sourceDir, targetDir, extractJar, true);
+ }
+
+ /** as {@link #extractCommands(String, String, String, boolean)}, but also with option to keep the original */
+ public static List<String> extractCommands(String fileName, String sourceDir, String targetDir, boolean extractJar, boolean keepOriginal) {
List<String> commands = new LinkedList<String>();
commands.add("cd " + targetDir);
String sourcePath = Os.mergePathsUnix(sourceDir, fileName);
@@ -150,9 +160,12 @@ public class ArchiveUtils {
break;
}
case UNKNOWN:
- commands.add("cp " + sourcePath + " " + targetDir);
+ if (!sourcePath.equals(Urls.mergePaths(targetDir, fileName)))
+ commands.add("cp " + sourcePath + " " + targetDir);
break;
}
+ if (!keepOriginal && !commands.isEmpty())
+ commands.add("rm "+sourcePath);
return commands;
}
@@ -212,7 +225,10 @@ public class ArchiveUtils {
public static void deploy(Map<String, ?> props, String archiveUrl, SshMachineLocation machine, String destDir, String destFile) {
deploy(props, archiveUrl, machine, destDir, destDir, destFile);
}
-
+ public static void deploy(Map<String, ?> props, String archiveUrl, SshMachineLocation machine, String tmpDir, String destDir, String destFile) {
+ deploy(null, props, archiveUrl, machine, destDir, true, tmpDir, destFile);
+ }
+
/**
* Deploys an archive file to a remote machine and extracts the contents.
* <p>
@@ -224,17 +240,30 @@ public class ArchiveUtils {
* @see #deploy(Map, String, SshMachineLocation, String, String, String)
* @see #install(SshMachineLocation, String, String, int)
*/
- public static void deploy(Map<String, ?> props, String archiveUrl, SshMachineLocation machine, String tmpDir, String destDir, String destFile) {
- String destPath = Os.mergePaths(tmpDir, destFile);
+ public static void deploy(ResourceUtils resolver, Map<String, ?> props, String archiveUrl, SshMachineLocation machine, String destDir, boolean keepArchiveAfterUnpacking, String optionalTmpDir, String optionalDestFile) {
+ if (optionalDestFile==null) optionalDestFile = Urls.getFilename(Preconditions.checkNotNull(archiveUrl, "archiveUrl"));
+ if (Strings.isBlank(optionalDestFile))
+ throw new IllegalStateException("Not given filename and cannot infer archive type from '"+archiveUrl+"'");
+ if (optionalTmpDir==null) optionalTmpDir=Preconditions.checkNotNull(destDir, "destDir");
+ if (props==null) props = MutableMap.of();
+ String destPath = Os.mergePaths(optionalTmpDir, optionalDestFile);
// Use the location mutex to prevent package manager locking issues
try {
machine.acquireMutex("installing", "installing archive");
- int result = install(props, machine, archiveUrl, destPath, NUM_RETRIES_FOR_COPYING);
+ int result = install(resolver, props, machine, archiveUrl, destPath, NUM_RETRIES_FOR_COPYING);
if (result != 0) {
throw new IllegalStateException(format("Unable to install archive %s to %s", archiveUrl, machine));
}
- result = machine.execCommands(props, "extracting content", extractCommands(destFile, tmpDir, destDir, false));
+
+ // extract, now using task if available
+ MutableList<String> commands = MutableList.copyOf(installCommands(optionalDestFile))
+ .appendAll(extractCommands(optionalDestFile, optionalTmpDir, destDir, false, keepArchiveAfterUnpacking));
+ if (DynamicTasks.getTaskQueuingContext()!=null) {
+ result = DynamicTasks.queue(SshTasks.newSshExecTaskFactory(machine, commands.toArray(new String[0])).summary("extracting archive").requiringExitCodeZero()).get();
+ } else {
+ result = machine.execCommands(props, "extracting content", commands);
+ }
if (result != 0) {
throw new IllegalStateException(format("Failed to expand archive %s on %s", archiveUrl, machine));
}
@@ -261,6 +290,11 @@ public class ArchiveUtils {
* @see SshMachineLocation#installTo(Map, String, String)
*/
public static int install(Map<String, ?> props, SshMachineLocation machine, String urlToInstall, String target, int numAttempts) {
+ return install(null, props, machine, urlToInstall, target, numAttempts);
+ }
+
+ public static int install(ResourceUtils resolver, Map<String, ?> props, SshMachineLocation machine, String urlToInstall, String target, int numAttempts) {
+ if (resolver==null) resolver = ResourceUtils.create(machine);
Exception lastError = null;
int retriesRemaining = numAttempts;
int attemptNum = 0;
@@ -269,7 +303,7 @@ public class ArchiveUtils {
try {
Tasks.setBlockingDetails("Installing "+urlToInstall+" at "+machine);
// TODO would be nice to have this in a task (and the things within it!)
- return machine.installTo(props, urlToInstall, target);
+ return machine.installTo(resolver, props, urlToInstall, target);
} catch (Exception e) {
Exceptions.propagateIfFatal(e);
lastError = e;
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/core/src/main/java/brooklyn/util/task/ssh/SshTasks.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/brooklyn/util/task/ssh/SshTasks.java b/core/src/main/java/brooklyn/util/task/ssh/SshTasks.java
index de1dd14..d0b92a5 100644
--- a/core/src/main/java/brooklyn/util/task/ssh/SshTasks.java
+++ b/core/src/main/java/brooklyn/util/task/ssh/SshTasks.java
@@ -18,8 +18,12 @@ import brooklyn.location.basic.LocationInternal;
import brooklyn.location.basic.SshMachineLocation;
import brooklyn.management.ManagementContext;
import brooklyn.management.Task;
+import brooklyn.management.TaskAdaptable;
+import brooklyn.management.TaskFactory;
+import brooklyn.util.ResourceUtils;
import brooklyn.util.config.ConfigBag;
import brooklyn.util.internal.ssh.SshTool;
+import brooklyn.util.net.Urls;
import brooklyn.util.ssh.BashCommands;
import brooklyn.util.stream.Streams;
import brooklyn.util.task.Tasks;
@@ -31,6 +35,7 @@ import brooklyn.util.text.Strings;
import com.google.common.annotations.Beta;
import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
/**
@@ -147,4 +152,25 @@ public class SshTasks {
};
}
+ /** task to install a file given a url, where the url is resolved remotely first then locally */
+ public static TaskFactory<?> installFromUrl(final SshMachineLocation location, final String url, final String destPath) {
+ return installFromUrl(ResourceUtils.create(SshTasks.class), ImmutableMap.<String,Object>of(), location, url, destPath);
+ }
+ /** task to install a file given a url, where the url is resolved remotely first then locally */
+ public static TaskFactory<?> installFromUrl(final ResourceUtils utils, final Map<String, ?> props, final SshMachineLocation location, final String url, final String destPath) {
+ return new TaskFactory<TaskAdaptable<?>>() {
+ @Override
+ public TaskAdaptable<?> newTask() {
+ return Tasks.<Void>builder().name("installing "+Urls.getFilename(url)).description("installing "+url+" to "+destPath).body(new Runnable() {
+ @Override
+ public void run() {
+ int result = location.installTo(utils, props, url, destPath);
+ if (result!=0)
+ throw new IllegalStateException("Failed to install '"+url+"' to '"+destPath+"' at "+location+": exit code "+result);
+ }
+ }).build();
+ }
+ };
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/software/base/src/main/java/brooklyn/entity/chef/ChefBashCommands.java
----------------------------------------------------------------------
diff --git a/software/base/src/main/java/brooklyn/entity/chef/ChefBashCommands.java b/software/base/src/main/java/brooklyn/entity/chef/ChefBashCommands.java
index e008976..40b0f18 100644
--- a/software/base/src/main/java/brooklyn/entity/chef/ChefBashCommands.java
+++ b/software/base/src/main/java/brooklyn/entity/chef/ChefBashCommands.java
@@ -5,17 +5,9 @@ import static brooklyn.util.ssh.BashCommands.INSTALL_TAR;
import static brooklyn.util.ssh.BashCommands.INSTALL_UNZIP;
import static brooklyn.util.ssh.BashCommands.downloadToStdout;
import static brooklyn.util.ssh.BashCommands.sudo;
-
-import javax.annotation.Nullable;
-
-import org.apache.commons.io.FilenameUtils;
-
import brooklyn.util.ssh.BashCommands;
-import brooklyn.util.text.Identifiers;
-import brooklyn.util.text.Strings;
import com.google.common.annotations.Beta;
-import com.google.common.io.Files;
/** BASH commands useful for setting up Chef */
@Beta
@@ -28,46 +20,56 @@ public class ChefBashCommands {
INSTALL_UNZIP,
"( "+downloadToStdout("https://www.opscode.com/chef/install.sh") + " | " + sudo("bash")+" )");
- /** this assumes the download is an archive containing a single directory on the root which will be renamed to "cookbookName";
- * if that directory already has the correct name cookbookName can be null,
- * but if e.g. taking from a github tarball it will typically be of the form cookbookName-master/
- * hence the renaming */
- // TODO support installing from classpath, and using the repository (tie in with those methods)
- public static final String downloadAndExpandCookbook(String source, @Nullable String cookbookName, boolean force) {
- String dl = downloadAndExpandCookbook(source);
- if (cookbookName==null) return dl;
- String tmpName = "tmp-"+Strings.makeValidFilename(cookbookName)+"-"+Identifiers.makeRandomId(4);
- String installCmd = BashCommands.chain("mkdir "+tmpName, "cd "+tmpName, dl,
- BashCommands.requireTest("`ls | wc -w` -eq 1",
- "The downloaded archive must contain exactly one directory; contained"),
- "COOKBOOK_EXPANDED_DIR=`ls`",
- "mv $COOKBOOK_EXPANDED_DIR '../"+cookbookName+"'",
- "cd ..",
- "rm -rf "+tmpName);
- if (!force) return BashCommands.alternatives("ls "+cookbookName, installCmd);
- else return BashCommands.alternatives("rm -rf "+cookbookName, installCmd);
- }
-
- /** as {@link #downloadAndExpandCookbook(String, String)} with no cookbook name */
- public static final String downloadAndExpandCookbook(String source) {
-// curl -f -L https://github.com/opscode-cookbooks/postgresql/archive/master.tar.gz | tar xvz
- String ext = Files.getFileExtension(source);
- if ("tar".equalsIgnoreCase(ext))
- return downloadToStdout(source) + " | tar xv";
- if ("tgz".equalsIgnoreCase(ext) || source.toLowerCase().endsWith(".tar.gz"))
- return downloadToStdout(source) + " | tar xvz";
-
- String target = FilenameUtils.getName(source);
- if (target==null) target = ""; else target = target.trim();
- target += "_"+Strings.makeRandomId(4);
-
- if ("zip".equalsIgnoreCase(ext) || "tar.gz".equalsIgnoreCase(ext))
- return BashCommands.chain(
- BashCommands.commandToDownloadUrlAs(source, target),
- "unzip "+target,
- "rm "+target);
-
- throw new UnsupportedOperationException("No way to expand "+source+" (yet)");
- }
+ // TODO replaced by tasks
+// /** this assumes the download is an archive containing a single directory on the root which will be renamed to "cookbookName";
+// * if that directory already has the correct name cookbookName can be null,
+// * but if e.g. taking from a github tarball it will typically be of the form cookbookName-master/
+// * hence the renaming */
+// // TODO support installing from classpath, and using the repository (tie in with those methods)
+// public static final String downloadAndExpandCookbook(String cookbookArchiveUrl, @Nullable String cookbookName, boolean force) {
+// String dl = downloadAndExpandCookbook(cookbookArchiveUrl);
+// if (cookbookName==null) return dl;
+// XXX;
+// String privateTmpDirContainingUnpackedCookbook = "tmp-"+Strings.makeValidFilename(cookbookName)+"-"+Identifiers.makeRandomId(4);
+// String installCmd = BashCommands.chain("mkdir "+privateTmpDirContainingUnpackedCookbook, "cd "+privateTmpDirContainingUnpackedCookbook, dl,
+// BashCommands.requireTest("`ls | wc -w` -eq 1",
+// "The downloaded archive must contain exactly one directory; contained"),
+// "COOKBOOK_EXPANDED_DIR=`ls`",
+// "mv $COOKBOOK_EXPANDED_DIR '../"+cookbookName+"'",
+// "cd ..",
+// "rm -rf "+privateTmpDirContainingUnpackedCookbook);
+// if (!force) return BashCommands.alternatives("ls "+cookbookName, installCmd);
+// else return BashCommands.alternatives("rm -rf "+cookbookName, installCmd);
+// }
+//
+// /** as {@link #downloadAndExpandCookbook(String, String)} with no cookbook name */
+// // TODO deprecate
+// public static final String downloadAndExpandCookbook(String cookbookArchiveUrl) {
+//// curl -f -L https://github.com/opscode-cookbooks/postgresql/archive/master.tar.gz | tar xvz
+// String ext = Files.getFileExtension(cookbookArchiveUrl);
+// if ("tar".equalsIgnoreCase(ext))
+// return downloadToStdout(cookbookArchiveUrl) + " | tar xv";
+// if ("tgz".equalsIgnoreCase(ext) || cookbookArchiveUrl.toLowerCase().endsWith(".tar.gz"))
+// return downloadToStdout(cookbookArchiveUrl) + " | tar xvz";
+//
+// String target = FilenameUtils.getName(cookbookArchiveUrl);
+// if (target==null) target = ""; else target = target.trim();
+// target += "_"+Strings.makeRandomId(4);
+//
+// if ("zip".equalsIgnoreCase(ext) || "tar.gz".equalsIgnoreCase(ext))
+// return BashCommands.chain(
+// BashCommands.commandToDownloadUrlAs(cookbookArchiveUrl, target),
+// "unzip "+target,
+// "rm "+target);
+//
+// // TODO if it's a local dir, automatically pack it
+// throw new UnsupportedOperationException("No way to install cookbooks in format "+cookbookArchiveUrl+" (yet) -- use tgz, tar, or zip");
+// }
+//
+// public static String renameDownloadedCookbook(String privateTmpDirContainingUnpackedCookbook, String cookbook, boolean force) {
+// XXX;
+// return null;
+// }
+//
}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/software/base/src/main/java/brooklyn/entity/chef/ChefConfig.java
----------------------------------------------------------------------
diff --git a/software/base/src/main/java/brooklyn/entity/chef/ChefConfig.java b/software/base/src/main/java/brooklyn/entity/chef/ChefConfig.java
index f67d51c..c0ba0aa 100644
--- a/software/base/src/main/java/brooklyn/entity/chef/ChefConfig.java
+++ b/software/base/src/main/java/brooklyn/entity/chef/ChefConfig.java
@@ -32,6 +32,7 @@ public interface ChefConfig {
/** typically set from spec, to customize the launch part of the start effector */
public static final SetConfigKey<String> CHEF_LAUNCH_RUN_LIST = new SetConfigKey<String>(String.class, "brooklyn.chef.launch.runList");
/** typically set from spec, to customize the launch part of the start effector */
+ @SetFromFlag("launch_attributes")
public static final MapConfigKey<Object> CHEF_LAUNCH_ATTRIBUTES = new MapConfigKey<Object>(Object.class, "brooklyn.chef.launch.attributes");
public static enum ChefModes {
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/software/base/src/main/java/brooklyn/entity/chef/ChefEntity.java
----------------------------------------------------------------------
diff --git a/software/base/src/main/java/brooklyn/entity/chef/ChefEntity.java b/software/base/src/main/java/brooklyn/entity/chef/ChefEntity.java
index f3a0e9f..4f8e532 100644
--- a/software/base/src/main/java/brooklyn/entity/chef/ChefEntity.java
+++ b/software/base/src/main/java/brooklyn/entity/chef/ChefEntity.java
@@ -4,5 +4,5 @@ import brooklyn.entity.basic.SoftwareProcess;
import brooklyn.entity.proxying.ImplementedBy;
@ImplementedBy(ChefEntityImpl.class)
-public interface ChefEntity extends SoftwareProcess {
+public interface ChefEntity extends SoftwareProcess, ChefConfig {
}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/software/base/src/main/java/brooklyn/entity/chef/ChefEntityImpl.java
----------------------------------------------------------------------
diff --git a/software/base/src/main/java/brooklyn/entity/chef/ChefEntityImpl.java b/software/base/src/main/java/brooklyn/entity/chef/ChefEntityImpl.java
index e3288e1..b204e73 100644
--- a/software/base/src/main/java/brooklyn/entity/chef/ChefEntityImpl.java
+++ b/software/base/src/main/java/brooklyn/entity/chef/ChefEntityImpl.java
@@ -1,12 +1,18 @@
package brooklyn.entity.chef;
import brooklyn.entity.basic.EffectorStartableImpl;
+import brooklyn.util.text.Strings;
public class ChefEntityImpl extends EffectorStartableImpl implements ChefEntity {
public void init() {
+ String primaryName = getConfig(CHEF_COOKBOOK_PRIMARY_NAME);
+ if (!Strings.isBlank(primaryName)) setDefaultDisplayName(primaryName+" (chef)");
+
super.init();
new ChefLifecycleEffectorTasks().attachLifecycleEffectors(this);
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/software/base/src/main/java/brooklyn/entity/chef/ChefLifecycleEffectorTasks.java
----------------------------------------------------------------------
diff --git a/software/base/src/main/java/brooklyn/entity/chef/ChefLifecycleEffectorTasks.java b/software/base/src/main/java/brooklyn/entity/chef/ChefLifecycleEffectorTasks.java
index 5634067..eecd2ba 100644
--- a/software/base/src/main/java/brooklyn/entity/chef/ChefLifecycleEffectorTasks.java
+++ b/software/base/src/main/java/brooklyn/entity/chef/ChefLifecycleEffectorTasks.java
@@ -9,6 +9,7 @@ import org.slf4j.LoggerFactory;
import brooklyn.entity.Entity;
import brooklyn.entity.basic.Attributes;
import brooklyn.entity.basic.Lifecycle;
+import brooklyn.entity.basic.SoftwareProcess;
import brooklyn.entity.software.MachineLifecycleEffectorTasks;
import brooklyn.entity.software.SshEffectorTasks;
import brooklyn.location.MachineLocation;
@@ -206,6 +207,7 @@ public class ChefLifecycleEffectorTasks extends MachineLifecycleEffectorTasks im
if (!result) {
log.warn("No way to check whether "+entity()+" is running; assuming yes");
}
+ entity().setAttribute(SoftwareProcess.SERVICE_UP, true);
}
protected boolean tryCheckStartPid() {
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/software/base/src/main/java/brooklyn/entity/chef/ChefSoloTasks.java
----------------------------------------------------------------------
diff --git a/software/base/src/main/java/brooklyn/entity/chef/ChefSoloTasks.java b/software/base/src/main/java/brooklyn/entity/chef/ChefSoloTasks.java
index d384534..5239d68 100644
--- a/software/base/src/main/java/brooklyn/entity/chef/ChefSoloTasks.java
+++ b/software/base/src/main/java/brooklyn/entity/chef/ChefSoloTasks.java
@@ -2,20 +2,11 @@ package brooklyn.entity.chef;
import java.util.Map;
-import brooklyn.entity.Entity;
-import brooklyn.entity.effector.EffectorTasks;
import brooklyn.entity.software.SshEffectorTasks;
import brooklyn.management.TaskFactory;
-import brooklyn.util.collections.MutableMap;
-import brooklyn.util.net.Urls;
import brooklyn.util.ssh.BashCommands;
-import brooklyn.util.task.DynamicTasks;
-import brooklyn.util.task.Tasks;
import com.google.common.annotations.Beta;
-import com.google.common.collect.ImmutableList;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
@Beta
public class ChefSoloTasks {
@@ -28,23 +19,11 @@ public class ChefSoloTasks {
}
public static TaskFactory<?> installCookbooks(final String chefDirectory, final Map<String,String> cookbooksAndUrls, final boolean force) {
- return Tasks.<Void>builder().name("install cookbooks").body(
- new Runnable() {
- public void run() {
- Entity e = EffectorTasks.findEntity();
- if (cookbooksAndUrls==null)
- throw new IllegalStateException("No cookbooks defined to install at "+e);
- for (String cookbook: cookbooksAndUrls.keySet())
- DynamicTasks.queue(installCookbook(chefDirectory, cookbook, cookbooksAndUrls.get(cookbook), force));
- }
- }).buildFactory();
+ return ChefTasks.installCookbooks(chefDirectory, cookbooksAndUrls, force);
}
- public static TaskFactory<?> installCookbook(String chefDirectory, String cookbook, String url, boolean force) {
- // TODO if it's server, try knife first
- // TODO support downloads from classpath / local server
- return SshEffectorTasks.ssh(cdAndRun(chefDirectory, ChefBashCommands.downloadAndExpandCookbook(url, cookbook, force))).
- summary("install cookbook "+cookbook).requiringExitCodeZero();
+ public static TaskFactory<?> installCookbook(String chefDirectory, String cookbookName, String cookbookArchiveUrl, boolean force) {
+ return ChefTasks.installCookbook(chefDirectory, cookbookName, cookbookArchiveUrl, force);
}
protected static String cdAndRun(String targetDirectory, String command) {
@@ -55,26 +34,7 @@ public class ChefSoloTasks {
public static TaskFactory<?> buildChefFile(String runDirectory, String chefDirectory, String phase, Iterable<? extends String> runList,
Map<String, Object> optionalAttributes) {
- // TODO if it's server, try knife first
- // TODO configure add'l properties
- String phaseRb =
- "root = File.absolute_path(File.dirname(__FILE__))\n"+
- "\n"+
- "file_cache_path root\n"+
-// "cookbook_path root + '/cookbooks'\n";
- "cookbook_path '"+chefDirectory+"'\n";
-
- Map<String,Object> phaseJsonMap = MutableMap.of();
- if (optionalAttributes!=null)
- phaseJsonMap.putAll(optionalAttributes);
- if (runList!=null)
- phaseJsonMap.put("run_list", ImmutableList.copyOf(runList));
- Gson json = new GsonBuilder().create();
- String phaseJson = json.toJson(phaseJsonMap);
-
- return Tasks.sequential("build chef files for "+phase,
- SshEffectorTasks.put(Urls.mergePaths(runDirectory)+"/"+phase+".rb").contents(phaseRb).createDirectory(),
- SshEffectorTasks.put(Urls.mergePaths(runDirectory)+"/"+phase+".json").contents(phaseJson));
+ return ChefTasks.buildChefFile(runDirectory, chefDirectory, phase, runList, optionalAttributes);
}
public static TaskFactory<?> runChef(String runDir, String phase) {
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/software/base/src/main/java/brooklyn/entity/chef/ChefTasks.java
----------------------------------------------------------------------
diff --git a/software/base/src/main/java/brooklyn/entity/chef/ChefTasks.java b/software/base/src/main/java/brooklyn/entity/chef/ChefTasks.java
index c797c11..4f2efc1 100644
--- a/software/base/src/main/java/brooklyn/entity/chef/ChefTasks.java
+++ b/software/base/src/main/java/brooklyn/entity/chef/ChefTasks.java
@@ -5,12 +5,17 @@ import java.util.Map;
import brooklyn.entity.Entity;
import brooklyn.entity.effector.EffectorTasks;
import brooklyn.entity.software.SshEffectorTasks;
+import brooklyn.management.TaskAdaptable;
import brooklyn.management.TaskFactory;
import brooklyn.util.collections.MutableMap;
+import brooklyn.util.file.ArchiveTasks;
import brooklyn.util.net.Urls;
import brooklyn.util.ssh.BashCommands;
import brooklyn.util.task.DynamicTasks;
+import brooklyn.util.task.TaskBuilder;
import brooklyn.util.task.Tasks;
+import brooklyn.util.text.Identifiers;
+import brooklyn.util.text.Strings;
import com.google.common.annotations.Beta;
import com.google.common.collect.ImmutableList;
@@ -28,7 +33,7 @@ public class ChefTasks {
}
public static TaskFactory<?> installCookbooks(final String chefDirectory, final Map<String,String> cookbooksAndUrls, final boolean force) {
- return Tasks.<Void>builder().name("install cookbooks").body(
+ return Tasks.<Void>builder().name("install "+(cookbooksAndUrls==null ? "0" : cookbooksAndUrls.size())+" cookbook"+Strings.s(cookbooksAndUrls)).body(
new Runnable() {
public void run() {
Entity e = EffectorTasks.findEntity();
@@ -40,16 +45,46 @@ public class ChefTasks {
}).buildFactory();
}
- public static TaskFactory<?> installCookbook(String chefDirectory, String cookbook, String url, boolean force) {
- // TODO if it's server, try knife first
- // TODO support downloads from classpath / local server
- return SshEffectorTasks.ssh(cdAndRun(chefDirectory, ChefBashCommands.downloadAndExpandCookbook(url, cookbook, force))).
- summary("install cookbook "+cookbook).requiringExitCodeZero();
+ public static TaskFactory<?> installCookbook(final String chefDirectory, final String cookbookName, final String cookbookArchiveUrl, final boolean force) {
+ return new TaskFactory<TaskAdaptable<?>>() {
+ @Override
+ public TaskAdaptable<?> newTask() {
+ TaskBuilder<Void> tb = Tasks.<Void>builder().name("install cookbook "+cookbookName);
+
+ String cookbookDir = Urls.mergePaths(chefDirectory, cookbookName);
+ String privateTmpDirContainingUnpackedCookbook =
+ Urls.mergePaths(chefDirectory, "tmp-"+Strings.makeValidFilename(cookbookName)+"-"+Identifiers.makeRandomId(4));
+
+ // TODO - skip the install earlier if it exists and isn't forced
+// if (!force) {
+// // in builder.body, check
+// // "ls "+cookbookDir
+// // and stop if it's zero
+// // remove reference to 'force' below
+// }
+
+ tb.add(ArchiveTasks.deploy(null, cookbookArchiveUrl, EffectorTasks.findSshMachine(), privateTmpDirContainingUnpackedCookbook).newTask());
+
+ String installCmd = BashCommands.chain(
+ "cd "+privateTmpDirContainingUnpackedCookbook,
+ "COOKBOOK_EXPANDED_DIR=`ls`",
+ BashCommands.requireTest("`ls | wc -w` -eq 1",
+ "The deployed archive "+cookbookArchiveUrl+" must contain exactly one directory"),
+ "mv $COOKBOOK_EXPANDED_DIR '../"+cookbookName+"'",
+ "cd ..",
+ "rm -rf '"+privateTmpDirContainingUnpackedCookbook+"'");
+
+ installCmd = force ? BashCommands.alternatives("rm -rf "+cookbookDir, installCmd) : BashCommands.alternatives("ls "+cookbookDir+" > /dev/null 2> /dev/null", installCmd);
+ tb.add(SshEffectorTasks.ssh(installCmd).summary("renaming cookbook dir").requiringExitCodeZero().newTask());
+
+ return tb.build();
+ }
+ };
}
protected static String cdAndRun(String targetDirectory, String command) {
- return BashCommands.chain("mkdir -p "+targetDirectory,
- "cd "+targetDirectory,
+ return BashCommands.chain("mkdir -p '"+targetDirectory+"'",
+ "cd '"+targetDirectory+"'",
command);
}
@@ -58,11 +93,14 @@ public class ChefTasks {
// TODO if it's server, try knife first
// TODO configure add'l properties
String phaseRb =
- "root = File.absolute_path(File.dirname(__FILE__))\n"+
- "\n"+
- "file_cache_path root\n"+
-// "cookbook_path root + '/cookbooks'\n";
- "cookbook_path '"+chefDirectory+"'\n";
+ "root = "
+ + "'"+runDirectory+"'"
+ // recommended alternate to runDir is the following, but it is not available in some rubies
+ //+ File.absolute_path(File.dirname(__FILE__))"+
+ + "\n"+
+ "file_cache_path root\n"+
+// "cookbook_path root + '/cookbooks'\n";
+ "cookbook_path '"+chefDirectory+"'\n";
Map<String,Object> phaseJsonMap = MutableMap.of();
if (optionalAttributes!=null)
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/usage/camp/src/test/resources/mysql-chef.yaml
----------------------------------------------------------------------
diff --git a/usage/camp/src/test/resources/mysql-chef.yaml b/usage/camp/src/test/resources/mysql-chef.yaml
new file mode 100644
index 0000000..cec45fb
--- /dev/null
+++ b/usage/camp/src/test/resources/mysql-chef.yaml
@@ -0,0 +1,26 @@
+name: chef-mysql-sample
+services:
+- type: chef:mysql
+
+ cookbook_urls:
+ # the standard cookbooks should be packaged as tarballs, unless you have knife installed
+ mysql: file:///tmp/brooklyn-chef/cookbooks/mysql.tgz
+ openssl: file:///tmp/brooklyn-chef/cookbooks/openssl.tgz
+
+ launch_attributes:
+ mysql:
+ # these attrs are required by the mysql cookbook under node['mysql']
+ # (many others are supported and can also be passed here)
+ server_debian_password: p4ssw0rd
+ server_root_password: p4ssw0rd
+ server_repl_password: p4ssw0rd
+
+ # how to determine if the process is running and how to kill it
+ # (supported options are `service_name` and `pid_file`; typically pick one)
+ service_name: mysqld
+ #pid_file: /var/run/mysqld/mysqld.pid
+
+location:
+ byon:
+ hosts: 192.168.33.12
+ user: vagrant
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/utils/common/src/main/java/brooklyn/util/collections/MutableList.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/brooklyn/util/collections/MutableList.java b/utils/common/src/main/java/brooklyn/util/collections/MutableList.java
index 420d845..8cffc53 100644
--- a/utils/common/src/main/java/brooklyn/util/collections/MutableList.java
+++ b/utils/common/src/main/java/brooklyn/util/collections/MutableList.java
@@ -162,14 +162,22 @@ public class MutableList<V> extends ArrayList<V> {
for (V item: items) add(item);
return this;
}
+ /** as {@link List#addAll(Collection)} but fluent style */
+ public MutableList<V> appendAll(Iterator<? extends V> items) {
+ addAll(items);
+ return this;
+ }
public boolean addAll(Iterable<? extends V> setToAdd) {
// copy of parent, but accepting Iterable and null
if (setToAdd==null) return false;
+ return addAll(setToAdd.iterator());
+ }
+ public boolean addAll(Iterator<? extends V> setToAdd) {
+ if (setToAdd==null) return false;
boolean modified = false;
- Iterator<? extends V> e = setToAdd.iterator();
- while (e.hasNext()) {
- if (add(e.next()))
+ while (setToAdd.hasNext()) {
+ if (add(setToAdd.next()))
modified = true;
}
return modified;
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/utils/common/src/main/java/brooklyn/util/net/Urls.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/brooklyn/util/net/Urls.java b/utils/common/src/main/java/brooklyn/util/net/Urls.java
index a6e4569..74d6f16 100644
--- a/utils/common/src/main/java/brooklyn/util/net/Urls.java
+++ b/utils/common/src/main/java/brooklyn/util/net/Urls.java
@@ -9,6 +9,8 @@ import java.net.URLEncoder;
import javax.annotation.Nullable;
+import brooklyn.util.text.Strings;
+
import com.google.common.base.Function;
import com.google.common.base.Throwables;
@@ -143,6 +145,18 @@ public class Urls {
}
}
+ /** return the last segment of the given url before any '?', typically its name */
+ public static String getFilename(String url) {
+ if (url==null) return null;
+ if (getProtocol(url)!=null) {
+ int firstQ = url.indexOf('?');
+ if (firstQ>=0)
+ url = url.substring(0, firstQ);
+ }
+ url = Strings.removeAllFromEnd(url, "/");
+ return url.substring(url.lastIndexOf('/')+1);
+ }
+
public static boolean isDirectory(String fileUrl) {
File file;
if (isUrlWithProtocol(fileUrl)) {
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/utils/common/src/main/java/brooklyn/util/text/Strings.java
----------------------------------------------------------------------
diff --git a/utils/common/src/main/java/brooklyn/util/text/Strings.java b/utils/common/src/main/java/brooklyn/util/text/Strings.java
index c74c450..2428546 100644
--- a/utils/common/src/main/java/brooklyn/util/text/Strings.java
+++ b/utils/common/src/main/java/brooklyn/util/text/Strings.java
@@ -7,6 +7,7 @@ import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Iterator;
import java.util.Map;
import java.util.StringTokenizer;
@@ -664,6 +665,24 @@ public class Strings {
public static String s(int count) {
return count==1 ? "" : "s";
}
+ /** as {@link #s(int)} based on size of argument */
+ public static String s(@Nullable Map<?,?> map) {
+ if (map==null) return "s";
+ return s(map.size());
+ }
+ /** as {@link #s(int)} based on size of argument */
+ public static String s(Iterable<?> iter) {
+ if (iter==null) return "s";
+ return s(iter.iterator());
+ }
+ /** as {@link #s(int)} based on size of argument */
+ public static String s(Iterator<?> iter) {
+ if (iter==null) return "s";
+ if (!iter.hasNext()) return "s";
+ iter.next();
+ if (!iter.hasNext()) return "";
+ return "s";
+ }
/** converts a map of any objects to a map of strings, preserving nulls and invoking toString where needed */
public static Map<String, String> toStringMap(Map<?,?> map) {
@@ -721,5 +740,5 @@ public class Strings {
else
return Collections.list(new StringTokenizer(phrase)).size();
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-brooklyn/blob/41a8a092/utils/common/src/test/java/brooklyn/util/net/UrlsTest.java
----------------------------------------------------------------------
diff --git a/utils/common/src/test/java/brooklyn/util/net/UrlsTest.java b/utils/common/src/test/java/brooklyn/util/net/UrlsTest.java
index e374602..6630e40 100644
--- a/utils/common/src/test/java/brooklyn/util/net/UrlsTest.java
+++ b/utils/common/src/test/java/brooklyn/util/net/UrlsTest.java
@@ -33,5 +33,15 @@ public class UrlsTest {
Assert.assertFalse(Urls.isUrlWithProtocol("1:/"));
Assert.assertFalse(Urls.isUrlWithProtocol(null));
}
+
+ @Test
+ public void testGetFilename() {
+ assertEquals(Urls.getFilename("http://somewhere.com/path/to/file.txt"), "file.txt");
+ assertEquals(Urls.getFilename("http://somewhere.com/path/to/dir/"), "dir");
+ assertEquals(Urls.getFilename("http://somewhere.com/path/to/file.txt?with/optional/suffice"), "file.txt");
+ assertEquals(Urls.getFilename("filewith?.txt"), "filewith?.txt");
+ assertEquals(Urls.getFilename(""), "");
+ assertEquals(Urls.getFilename(null), null);
+ }
}