You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by ab...@apache.org on 2013/03/25 17:12:39 UTC
[1/5] git commit: WHIRR-660. Provide useful error message/exception
if empty instance templates
Updated Branches:
refs/heads/branch-0.8 9f3ce7b89 -> 265d98ea5
WHIRR-660. Provide useful error message/exception if empty instance templates
Project: http://git-wip-us.apache.org/repos/asf/whirr/repo
Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/ff134d26
Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/ff134d26
Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/ff134d26
Branch: refs/heads/branch-0.8
Commit: ff134d2695af58bd6569347e0126d5a749a43587
Parents: 9f3ce7b
Author: Andrew Bayer <an...@gmail.com>
Authored: Mon Mar 11 10:48:03 2013 -0700
Committer: Andrew Bayer <an...@gmail.com>
Committed: Mon Mar 25 09:12:08 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 ++
.../whirr/actions/ScriptBasedClusterAction.java | 4 +++
.../actions/ScriptBasedClusterActionTest.java | 21 +++++++++++++++
3 files changed, 28 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/whirr/blob/ff134d26/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 65dab4d..28c28fd 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,9 @@ Release 0.8.2 (unreleased changes)
IMPROVEMENTS
+ WHIRR-660. Provide useful message if whirr.instance-templates is
+ empty or not provided. (abayer)
+
WHIRR-672. Allow eager caching of instance hostname based on pre-provision
instance metadata. (Graham Gear via tomwhite)
http://git-wip-us.apache.org/repos/asf/whirr/blob/ff134d26/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java b/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
index 865b5cc..b2a522f 100644
--- a/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
+++ b/core/src/main/java/org/apache/whirr/actions/ScriptBasedClusterAction.java
@@ -95,6 +95,10 @@ public abstract class ScriptBasedClusterAction extends ClusterAction {
public Cluster execute(ClusterSpec clusterSpec, Cluster cluster)
throws IOException, InterruptedException {
+ if (clusterSpec.getInstanceTemplates().size() == 0) {
+ throw new IllegalArgumentException("No instance templates specified.");
+ }
+
Map<InstanceTemplate, ClusterActionEvent> eventMap = Maps.newHashMap();
Cluster newCluster = cluster;
for (InstanceTemplate instanceTemplate : clusterSpec.getInstanceTemplates()) {
http://git-wip-us.apache.org/repos/asf/whirr/blob/ff134d26/core/src/test/java/org/apache/whirr/actions/ScriptBasedClusterActionTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/whirr/actions/ScriptBasedClusterActionTest.java b/core/src/test/java/org/apache/whirr/actions/ScriptBasedClusterActionTest.java
index f992dec..dc3ab79 100644
--- a/core/src/test/java/org/apache/whirr/actions/ScriptBasedClusterActionTest.java
+++ b/core/src/test/java/org/apache/whirr/actions/ScriptBasedClusterActionTest.java
@@ -117,6 +117,27 @@ public abstract class ScriptBasedClusterActionTest<T extends ScriptBasedClusterA
return InstanceTemplate.builder().numberOfInstance(1).roles(roles).build();
}
+ @Test(expected = IllegalArgumentException.class)
+ public void testEmptyInstanceTemplates() throws Exception {
+ T action = newClusterActionInstance(EMPTYSET, EMPTYSET);
+ DryRun dryRun = getDryRunForAction(action).reset();
+
+ ClusterSpec tempSpec = ClusterSpec.withTemporaryKeys();
+
+ tempSpec.setClusterName("test-cluster-for-script-exection");
+ tempSpec.setProvider("stub");
+ tempSpec.setIdentity("dummy");
+ tempSpec.setStateStore("none");
+
+ ClusterController controller = new ClusterController();
+ Cluster tempCluster = controller.launchCluster(tempSpec);
+
+
+ action.execute(tempSpec, tempCluster);
+
+ List<StatementOnNode> executions = dryRun.getTotallyOrderedExecutions();
+ }
+
@Test
public void testActionIsExecutedOnAllRelevantNodes() throws Exception {
T action = newClusterActionInstance(EMPTYSET, EMPTYSET);
[3/5] git commit: WHIRR-661. Upgrade to Hadoop 1.0.3. Contributed by
Steve Loughran.
Posted by ab...@apache.org.
WHIRR-661. Upgrade to Hadoop 1.0.3. Contributed by Steve Loughran.
Project: http://git-wip-us.apache.org/repos/asf/whirr/repo
Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/221f8ad4
Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/221f8ad4
Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/221f8ad4
Branch: refs/heads/branch-0.8
Commit: 221f8ad4d29a09886ef785784a51fc0300d49e26
Parents: 13b76ae
Author: Andrew Bayer <an...@gmail.com>
Authored: Mon Mar 11 11:06:21 2013 -0700
Committer: Andrew Bayer <an...@gmail.com>
Committed: Mon Mar 25 09:12:19 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 ++
examples/pom.xml | 2 +-
pom.xml | 2 +-
recipes/hadoop.properties | 2 +-
recipes/hbase-0.90.properties | 8 ++++----
recipes/pig.properties | 3 ---
.../main/resources/whirr-hadoop-default.properties | 2 +-
services/hama/pom.xml | 2 +-
.../main/resources/whirr-hama-default.properties | 1 -
.../src/test/resources/whirr-hama-test.properties | 1 -
.../whirr-hbase-0.90-singlenode-test.properties | 1 -
.../resources/whirr-hbase-0.90-test.properties | 1 -
.../main/resources/whirr-hbase-default.properties | 1 -
.../src/test/resources/whirr-pig-test.properties | 2 --
services/yarn/pom.xml | 2 +-
15 files changed, 12 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index cb0fe01..1a57f5a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,8 @@ Release 0.8.2 (unreleased changes)
IMPROVEMENTS
+ WHIRR-661. Upgrade to Hadoop 1.0.3. (Steve Loughran via abayer)
+
WHIRR-634. Update private IP host file entry when required by
AUTO_HOSTNAME_SUFFIX. (Graham Gear via abayer)
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/examples/pom.xml
----------------------------------------------------------------------
diff --git a/examples/pom.xml b/examples/pom.xml
index 076a9ce..eaa54eb 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -93,7 +93,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
- <version>0.20.2</version>
+ <version>${hadoop.version}</version>
</dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 8026e5c..498af2d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -68,7 +68,7 @@
<common-ssl.version>0.3.11</common-ssl.version>
<dnsjava.version>2.1.1</dnsjava.version>
<guava.version>13.0</guava.version>
- <hadoop.version>0.20.205.0</hadoop.version>
+ <hadoop.version>1.0.3</hadoop.version>
<hamcrest.version>1.1</hamcrest.version>
<hama.version>0.4.0-incubating</hama.version>
<hbase.version>0.94.1</hbase.version>
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/recipes/hadoop.properties
----------------------------------------------------------------------
diff --git a/recipes/hadoop.properties b/recipes/hadoop.properties
index d0542a0..9a60222 100644
--- a/recipes/hadoop.properties
+++ b/recipes/hadoop.properties
@@ -93,5 +93,5 @@ whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker,5 hadoop-datanode+h
# Expert: specify the version of Hadoop to install.
#
-#whirr.hadoop.version=0.20.2
+#whirr.hadoop.version=1.0.3
#whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/recipes/hbase-0.90.properties
----------------------------------------------------------------------
diff --git a/recipes/hbase-0.90.properties b/recipes/hbase-0.90.properties
index 6578d3a..bdd7300 100644
--- a/recipes/hbase-0.90.properties
+++ b/recipes/hbase-0.90.properties
@@ -69,10 +69,10 @@ hbase-site.dfs.replication=2
whirr.hbase.tarball.url=http://apache.cu.be/hbase/hbase-0.90.3/hbase-0.90.3.tar.gz
# The Hadoop version to use. See http://hbase.apache.org/book/hadoop.html
-# The default Hadoop version used by Whirr does not fulfill the HBase 0.90.x requirements.
-# Whirr will replace the hadoop-core jar in HBase with the one from the actually installed Hadoop.
-# This example uses Cloudera's CDH3.
-whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u1.tar.gz
+
+#whirr.hadoop.version=1.0.3
+#whirr.hadoop.tarball.url=http://archive.apache.org/dist/hadoop/core/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
+
# Options for the hbase master & regionserver processes
#hbase-env.HBASE_MASTER_OPTS=-Xms1000m -Xmx1000m -Xmn256m -XX:+UseConcMarkSweepGC -XX:+AggressiveOpts -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -Xloggc:/data/hbase/logs/hbase-master-gc.log
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/recipes/pig.properties
----------------------------------------------------------------------
diff --git a/recipes/pig.properties b/recipes/pig.properties
index c310038..e0b483e 100644
--- a/recipes/pig.properties
+++ b/recipes/pig.properties
@@ -28,9 +28,6 @@ whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+pig-client,1 hadoop
whirr.pig.version=0.9.2
whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
-whirr.hadoop.version=0.20.205.0
-whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
-
# Setup your cloud credentials by copying conf/credentials.sample
# to ~/.whirr/credentials and editing as needed
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/hadoop/src/main/resources/whirr-hadoop-default.properties
----------------------------------------------------------------------
diff --git a/services/hadoop/src/main/resources/whirr-hadoop-default.properties b/services/hadoop/src/main/resources/whirr-hadoop-default.properties
index c4837d1..446251e 100644
--- a/services/hadoop/src/main/resources/whirr-hadoop-default.properties
+++ b/services/hadoop/src/main/resources/whirr-hadoop-default.properties
@@ -16,7 +16,7 @@
# limitations under the License.
#
-whirr.hadoop.version=0.20.205.0
+whirr.hadoop.version=1.0.3
whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
# Hadoop defaults. The first part of the key is removed by whirr.
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/hama/pom.xml
----------------------------------------------------------------------
diff --git a/services/hama/pom.xml b/services/hama/pom.xml
index f2fd7a7..1d3f207 100644
--- a/services/hama/pom.xml
+++ b/services/hama/pom.xml
@@ -29,7 +29,7 @@
<version>0.8.1</version>
<name>Apache Whirr Hama</name>
<properties>
- <hadoop.version>0.20.2</hadoop.version>
+ <hadoop.version>1.0.3</hadoop.version>
<zookeeper.version>3.3.3</zookeeper.version>
<osgi.import>
!org.apache.whirr.service.hama*,
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/hama/src/main/resources/whirr-hama-default.properties
----------------------------------------------------------------------
diff --git a/services/hama/src/main/resources/whirr-hama-default.properties b/services/hama/src/main/resources/whirr-hama-default.properties
index ea8759a..02fdd15 100644
--- a/services/hama/src/main/resources/whirr-hama-default.properties
+++ b/services/hama/src/main/resources/whirr-hama-default.properties
@@ -10,5 +10,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-whirr.hadoop.version=0.20.2
whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/hama/src/test/resources/whirr-hama-test.properties
----------------------------------------------------------------------
diff --git a/services/hama/src/test/resources/whirr-hama-test.properties b/services/hama/src/test/resources/whirr-hama-test.properties
index 22112b8..0137f16 100644
--- a/services/hama/src/test/resources/whirr-hama-test.properties
+++ b/services/hama/src/test/resources/whirr-hama-test.properties
@@ -15,7 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-whirr.hadoop.version=0.20.2
whirr.cluster-name=hama-itest-${sys:whirr.test.provider}-${sys:user.name}
whirr.instance-templates=1 zookeeper,1 hadoop-namenode+hadoop-jobtracker+hama-master,1 hadoop-datanode+hadoop-tasktracker+hama-groomserver
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-singlenode-test.properties
----------------------------------------------------------------------
diff --git a/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-singlenode-test.properties b/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-singlenode-test.properties
index 3224386..910dc40 100644
--- a/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-singlenode-test.properties
+++ b/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-singlenode-test.properties
@@ -25,4 +25,3 @@ whirr.credential=${sys:whirr.test.credential}
whirr.hardware-min-ram=4096
whirr.hbase.tarball.url=http://apache.osuosl.org/hbase/hbase-0.90.5/hbase-0.90.5.tar.gz
-whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u0.tar.gz
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-test.properties
----------------------------------------------------------------------
diff --git a/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-test.properties b/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-test.properties
index d0b707f..f3adeee 100644
--- a/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-test.properties
+++ b/services/hbase-oldtests/src/test/resources/whirr-hbase-0.90-test.properties
@@ -23,6 +23,5 @@ whirr.identity=${sys:whirr.test.identity}
whirr.credential=${sys:whirr.test.credential}
whirr.hbase.tarball.url=http://apache.osuosl.org/hbase/hbase-0.90.5/hbase-0.90.5.tar.gz
-whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u0.tar.gz
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/hbase/src/main/resources/whirr-hbase-default.properties
----------------------------------------------------------------------
diff --git a/services/hbase/src/main/resources/whirr-hbase-default.properties b/services/hbase/src/main/resources/whirr-hbase-default.properties
index 9df1c99..5bd2a4a 100644
--- a/services/hbase/src/main/resources/whirr-hbase-default.properties
+++ b/services/hbase/src/main/resources/whirr-hbase-default.properties
@@ -11,7 +11,6 @@
# limitations under the License.
whirr.hbase.tarball.url=http://archive.apache.org/dist/hbase/hbase-0.90.3/hbase-0.90.3.tar.gz
-whirr.hadoop.tarball.url=http://archive.cloudera.com/cdh/3/hadoop-0.20.2-cdh3u0.tar.gz
hbase-site.hbase.cluster.distributed=true
hbase-site.hbase.regionserver.handler.count=100
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/pig/src/test/resources/whirr-pig-test.properties
----------------------------------------------------------------------
diff --git a/services/pig/src/test/resources/whirr-pig-test.properties b/services/pig/src/test/resources/whirr-pig-test.properties
index a1769a8..4d4671a 100644
--- a/services/pig/src/test/resources/whirr-pig-test.properties
+++ b/services/pig/src/test/resources/whirr-pig-test.properties
@@ -26,5 +26,3 @@ whirr.credential=${sys:whirr.test.credential}
whirr.pig.version=0.9.2
whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
-whirr.hadoop.version=0.20.205.0
-whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
http://git-wip-us.apache.org/repos/asf/whirr/blob/221f8ad4/services/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/services/yarn/pom.xml b/services/yarn/pom.xml
index 7055946..64fd8a8 100644
--- a/services/yarn/pom.xml
+++ b/services/yarn/pom.xml
@@ -29,7 +29,7 @@
<version>0.8.1</version>
<name>Apache Whirr YARN</name>
<properties>
- <hadoop.version>0.20.2</hadoop.version>
+ <hadoop.version>1.0.3</hadoop.version>
<osgi.import>
!org.apache.whirr.service.yarn*,
org.apache.commons.configuration*;version="[1.6,2)",
[2/5] git commit: WHIRR-634. Update private IP host file entry when
required by AUTO_HOSTNAME_SUFFIX. Contributed by Graham Gear.
Posted by ab...@apache.org.
WHIRR-634. Update private IP host file entry when required by AUTO_HOSTNAME_SUFFIX. Contributed by Graham Gear.
Project: http://git-wip-us.apache.org/repos/asf/whirr/repo
Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/13b76ae7
Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/13b76ae7
Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/13b76ae7
Branch: refs/heads/branch-0.8
Commit: 13b76ae7cd429732774c926ffcc248fd83ca0054
Parents: ff134d2
Author: Andrew Bayer <an...@gmail.com>
Authored: Mon Mar 11 10:57:12 2013 -0700
Committer: Andrew Bayer <an...@gmail.com>
Committed: Mon Mar 25 09:12:13 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../resources/functions/configure_hostnames.sh | 4 ++++
2 files changed, 7 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/whirr/blob/13b76ae7/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 28c28fd..cb0fe01 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,9 @@ Release 0.8.2 (unreleased changes)
IMPROVEMENTS
+ WHIRR-634. Update private IP host file entry when required by
+ AUTO_HOSTNAME_SUFFIX. (Graham Gear via abayer)
+
WHIRR-660. Provide useful message if whirr.instance-templates is
empty or not provided. (abayer)
http://git-wip-us.apache.org/repos/asf/whirr/blob/13b76ae7/core/src/main/resources/functions/configure_hostnames.sh
----------------------------------------------------------------------
diff --git a/core/src/main/resources/functions/configure_hostnames.sh b/core/src/main/resources/functions/configure_hostnames.sh
index 60709a0..07374c7 100644
--- a/core/src/main/resources/functions/configure_hostnames.sh
+++ b/core/src/main/resources/functions/configure_hostnames.sh
@@ -20,6 +20,7 @@ function configure_hostnames() {
if [ ! -z $AUTO_HOSTNAME_SUFFIX ]; then
PUBLIC_IP=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
+ PRIVATE_IP=`/sbin/ifconfig eth1 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
HOSTNAME=${AUTO_HOSTNAME_PREFIX}`echo $PUBLIC_IP | tr . -`${AUTO_HOSTNAME_SUFFIX}
if [ -f /etc/hostname ]; then
echo $HOSTNAME > /etc/hostname
@@ -28,6 +29,9 @@ function configure_hostnames() {
sed -i -e "s/HOSTNAME=.*/HOSTNAME=$HOSTNAME/" /etc/sysconfig/network
fi
sed -i -e "s/$PUBLIC_IP.*/$PUBLIC_IP $HOSTNAME/" /etc/hosts
+ if [ ! -z $PRIVATE_IP ]; then
+ sed -i -e "s/$PRIVATE_IP.*/$PRIVATE_IP private.$HOSTNAME/" /etc/hosts
+ fi
set +e
if [ -f /etc/init.d/hostname ]; then
/etc/init.d/hostname restart
[4/5] git commit: WHIRR-681. Enhance puppet service with an ability
to export cluster topology to the puppet code. Contributed by Roman
Shaposhnik.
Posted by ab...@apache.org.
WHIRR-681. Enhance puppet service with an ability to export cluster topology to the puppet code. Contributed by Roman Shaposhnik.
Project: http://git-wip-us.apache.org/repos/asf/whirr/repo
Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/9d3efedf
Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/9d3efedf
Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/9d3efedf
Branch: refs/heads/branch-0.8
Commit: 9d3efedfad1291dc8cb2f96d46ac2251d8e58662
Parents: 221f8ad
Author: Andrew Bayer <an...@gmail.com>
Authored: Mon Mar 11 11:09:01 2013 -0700
Committer: Andrew Bayer <an...@gmail.com>
Committed: Mon Mar 25 09:12:27 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../service/puppet/PuppetClusterActionHandler.java | 2 +-
.../whirr/service/puppet/PuppetConstants.java | 1 +
.../statements/CreateSitePpAndApplyRoles.java | 35 ++++++++++++++-
.../src/main/resources/functions/install_puppet.sh | 2 +-
.../statements/CreateSitePpAndApplyRolesTest.java | 13 +++++-
.../src/test/resources/nginx-with-attribs.txt | 6 +++
7 files changed, 58 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 1a57f5a..a465421 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,9 @@ Release 0.8.2 (unreleased changes)
IMPROVEMENTS
+ WHIRR-681. Enhance puppet service with an ability to export
+ cluster topology to the puppet code. (Roman Shaposhnik via abayer)
+
WHIRR-661. Upgrade to Hadoop 1.0.3. (Steve Loughran via abayer)
WHIRR-634. Update private IP host file entry when required by
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
index 2fa69fd..a9841a5 100644
--- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
+++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetClusterActionHandler.java
@@ -103,7 +103,7 @@ public class PuppetClusterActionHandler extends PuppetInstallClusterActionHandle
if (isLastPuppetRoleIn(event.getInstanceTemplate().getRoles()).apply(getRole())) {
Configuration config = event.getClusterSpec().getConfiguration();
Iterable<String> roles = RolesManagedByPuppet.INSTANCE.apply(event.getInstanceTemplate().getRoles());
- addStatement(event, new CreateSitePpAndApplyRoles(roles, config));
+ addStatement(event, new CreateSitePpAndApplyRoles(roles, event.getCluster().getInstances(), config));
}
}
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
index c6ccdaa..3aca26c 100644
--- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
+++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/PuppetConstants.java
@@ -30,6 +30,7 @@ public class PuppetConstants {
public static final String MODULES_DIR = "/etc/puppet/modules/";
public static final String SITE_PP_FILE_LOCATION = "/etc/puppet/manifests/site.pp";
+ public static final String CONF_PP_FILE_LOCATION = "/etc/puppet/manifests/extdata/common.csv";
public static final Pattern MODULE_KEY_PATTERN = Pattern.compile("^" + PUPPET + "\\.([^.]+)\\."
+ MODULE_SOURCE_SUBKEY + "$");
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java b/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
index 2a15a28..6b65af1 100644
--- a/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
+++ b/services/puppet/src/main/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRoles.java
@@ -20,15 +20,23 @@
package org.apache.whirr.service.puppet.statements;
import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.whirr.service.puppet.PuppetConstants.PUPPET;
import static org.apache.whirr.service.puppet.PuppetConstants.SITE_PP_FILE_LOCATION;
+import static org.apache.whirr.service.puppet.PuppetConstants.CONF_PP_FILE_LOCATION;
import static org.jclouds.scriptbuilder.domain.Statements.appendFile;
import static org.jclouds.scriptbuilder.domain.Statements.exec;
import java.util.Collection;
import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import com.google.common.base.Joiner;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
import org.apache.whirr.service.puppet.Manifest;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.jclouds.scriptbuilder.domain.Statement;
@@ -44,10 +52,12 @@ import com.google.common.collect.ImmutableList.Builder;
public class CreateSitePpAndApplyRoles implements Statement {
private Iterable<String> roles;
+ private Iterable<Cluster.Instance> instances;
private Configuration config;
- public CreateSitePpAndApplyRoles(Iterable<String> roles, Configuration config) {
+ public CreateSitePpAndApplyRoles(Iterable<String> roles, Iterable<Cluster.Instance> instances, Configuration config) {
this.roles = checkNotNull(roles, "roles");
+ this.instances = checkNotNull(instances, "instances");
this.config = checkNotNull(config, "config");
}
@@ -64,8 +74,30 @@ public class CreateSitePpAndApplyRoles implements Statement {
Builder<Statement> statements = ImmutableList.<Statement> builder();
statements.add(Statements.rm(SITE_PP_FILE_LOCATION));
+ statements.add(Statements.rm(CONF_PP_FILE_LOCATION));
Builder<String> sitePp = ImmutableList.<String> builder();
+ Map<String, Set<String>> puppetRoles = Maps.newHashMap();
+ for (Cluster.Instance instance : instances) {
+ for (String role : instance.getRoles()) {
+ int firstColon = role.indexOf(':');
+ if (firstColon != -1 && role.substring(0, firstColon).equals(PUPPET)) {
+ String puppetClass = role.substring(firstColon + 1);
+ if (!puppetRoles.containsKey(puppetClass)) {
+ puppetRoles.put(puppetClass, Sets.<String>newHashSet());
+ }
+ puppetRoles.get(puppetClass).add(instance.getPrivateIp());
+ }
+ }
+ }
+
+ Builder<String> confPp = ImmutableList.<String> builder();
+ for (String puppetClass : puppetRoles.keySet()) {
+ confPp.add(puppetClass + "," + Joiner.on(',').join(puppetRoles.get(puppetClass)));
+ }
+
+ sitePp.add("$extlookup_datadir='/etc/puppet/manifests/extdata'");
+ sitePp.add("$extlookup_precedence = ['common']");
sitePp.add("node default {");
for (String role : roles) {
String manifestAttribPrefix = role.replaceAll(":+", ".");
@@ -79,6 +111,7 @@ public class CreateSitePpAndApplyRoles implements Statement {
}
sitePp.add("}");
+ statements.add(appendFile(CONF_PP_FILE_LOCATION, confPp.build()));
statements.add(appendFile(SITE_PP_FILE_LOCATION, sitePp.build()));
statements.add(exec("puppet apply " + SITE_PP_FILE_LOCATION));
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/main/resources/functions/install_puppet.sh
----------------------------------------------------------------------
diff --git a/services/puppet/src/main/resources/functions/install_puppet.sh b/services/puppet/src/main/resources/functions/install_puppet.sh
index 5c18161..e01a152 100644
--- a/services/puppet/src/main/resources/functions/install_puppet.sh
+++ b/services/puppet/src/main/resources/functions/install_puppet.sh
@@ -27,7 +27,7 @@ function install_puppet_forced() {
# Setup the default module/fact path locations so we can populate them
# elsewhere
mkdir -p /etc/puppet/modules
- mkdir -p /etc/puppet/manifests
+ mkdir -p /etc/puppet/manifests/extdata
mkdir -p /usr/share/puppet/modules
mkdir -p /var/lib/puppet/lib/facter
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
----------------------------------------------------------------------
diff --git a/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java b/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
index 3ff02b0..e404114 100644
--- a/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
+++ b/services/puppet/src/test/java/org/apache/whirr/service/puppet/statements/CreateSitePpAndApplyRolesTest.java
@@ -22,8 +22,11 @@ import static junit.framework.Assert.assertEquals;
import java.io.IOException;
+import com.google.common.collect.Sets;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.jclouds.domain.Credentials;
import org.jclouds.scriptbuilder.domain.OsFamily;
import org.junit.Test;
@@ -40,7 +43,15 @@ public class CreateSitePpAndApplyRolesTest {
conf.setProperty("puppet.nginx.module", "git://github.com/puppetlabs/puppetlabs-nginx.git");
conf.setProperty("nginx.server.hostname", "foohost");
- CreateSitePpAndApplyRoles nginx = new CreateSitePpAndApplyRoles(ImmutableSet.of("nginx::server"), conf);
+ CreateSitePpAndApplyRoles nginx = new CreateSitePpAndApplyRoles(ImmutableSet.of("nginx::server"),
+ ImmutableSet.of(new Cluster.Instance(
+ new Credentials("dummy", "dummy"),
+ Sets.newHashSet("puppet:nginx::server"),
+ "127.0.0.1",
+ "127.0.0.1",
+ "id-1",
+ null)),
+ conf);
assertEquals(CharStreams.toString(Resources.newReaderSupplier(Resources.getResource("nginx-with-attribs.txt"),
Charsets.UTF_8)), nginx.render(OsFamily.UNIX));
http://git-wip-us.apache.org/repos/asf/whirr/blob/9d3efedf/services/puppet/src/test/resources/nginx-with-attribs.txt
----------------------------------------------------------------------
diff --git a/services/puppet/src/test/resources/nginx-with-attribs.txt b/services/puppet/src/test/resources/nginx-with-attribs.txt
index ed53c47..0bd63da 100644
--- a/services/puppet/src/test/resources/nginx-with-attribs.txt
+++ b/services/puppet/src/test/resources/nginx-with-attribs.txt
@@ -1,5 +1,11 @@
rm /etc/puppet/manifests/site.pp
+rm /etc/puppet/manifests/extdata/common.csv
+cat >> /etc/puppet/manifests/extdata/common.csv <<-'END_OF_JCLOUDS_FILE'
+ nginx::server,127.0.0.1
+END_OF_JCLOUDS_FILE
cat >> /etc/puppet/manifests/site.pp <<-'END_OF_JCLOUDS_FILE'
+ $extlookup_datadir='/etc/puppet/manifests/extdata'
+ $extlookup_precedence = ['common']
node default {
class { 'nginx::server':
hostname => foohost,
[5/5] git commit: WHIRR-711. Add security group support for OpenStack.
Posted by ab...@apache.org.
WHIRR-711. Add security group support for OpenStack.
Project: http://git-wip-us.apache.org/repos/asf/whirr/repo
Commit: http://git-wip-us.apache.org/repos/asf/whirr/commit/265d98ea
Tree: http://git-wip-us.apache.org/repos/asf/whirr/tree/265d98ea
Diff: http://git-wip-us.apache.org/repos/asf/whirr/diff/265d98ea
Branch: refs/heads/branch-0.8
Commit: 265d98ea506b1490fbec5e4872304d9969c0716e
Parents: 9d3efed
Author: Andrew Bayer <an...@gmail.com>
Authored: Sun Mar 24 15:03:53 2013 -0700
Committer: Andrew Bayer <an...@gmail.com>
Committed: Mon Mar 25 09:12:33 2013 -0700
----------------------------------------------------------------------
CHANGES.txt | 2 +
.../org/apache/whirr/service/FirewallManager.java | 44 +++++++++++++++
2 files changed, 46 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/whirr/blob/265d98ea/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index a465421..171098a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -4,6 +4,8 @@ Release 0.8.2 (unreleased changes)
IMPROVEMENTS
+ WHIRR-711. Add security group support for OpenStack. (abayer)
+
WHIRR-681. Enhance puppet service with an ability to export
cluster topology to the puppet code. (Roman Shaposhnik via abayer)
http://git-wip-us.apache.org/repos/asf/whirr/blob/265d98ea/core/src/main/java/org/apache/whirr/service/FirewallManager.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/whirr/service/FirewallManager.java b/core/src/main/java/org/apache/whirr/service/FirewallManager.java
index e3d53df..d9f780f 100644
--- a/core/src/main/java/org/apache/whirr/service/FirewallManager.java
+++ b/core/src/main/java/org/apache/whirr/service/FirewallManager.java
@@ -34,11 +34,16 @@ import org.jclouds.compute.ComputeServiceContext;
import org.jclouds.ec2.EC2ApiMetadata;
import org.jclouds.ec2.EC2Client;
import org.jclouds.ec2.domain.IpProtocol;
+import org.jclouds.openstack.nova.v2_0.NovaApiMetadata;
+import org.jclouds.openstack.nova.v2_0.domain.Ingress;
+import org.jclouds.openstack.nova.v2_0.domain.SecurityGroup;
+import org.jclouds.openstack.nova.v2_0.extensions.SecurityGroupApi;
import org.jclouds.javax.annotation.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
+import com.google.common.base.Optional;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
@@ -220,6 +225,45 @@ public class FirewallManager {
}
}
}
+ } else if (NovaApiMetadata.CONTEXT_TOKEN.isAssignableFrom(computeServiceContext.getBackendType())) {
+ // This code (or something like it) may be added to jclouds (see
+ // http://code.google.com/p/jclouds/issues/detail?id=336).
+ // Until then we need this temporary workaround.
+ Optional<? extends SecurityGroupApi> securityGroupApi = computeServiceContext.unwrap(NovaApiMetadata.CONTEXT_TOKEN)
+ .getApi()
+ .getSecurityGroupExtensionForZone(clusterSpec.getTemplate().getLocationId());
+
+ if (securityGroupApi.isPresent()) {
+ final String groupName = "jclouds-" + clusterSpec.getClusterName();
+ Optional<? extends SecurityGroup> group = securityGroupApi.get().list().firstMatch(new Predicate<SecurityGroup>() {
+ @Override
+ public boolean apply(SecurityGroup secGrp) {
+ return secGrp.getName().equals(groupName);
+ }
+ });
+
+ if (group.isPresent()) {
+ for (String cidr : cidrs) {
+ for (int port : ports) {
+ try {
+ securityGroupApi.get().createRuleAllowingCidrBlock(group.get().getId(),
+ Ingress.builder()
+ .ipProtocol(org.jclouds.openstack.nova.v2_0.domain.IpProtocol.TCP)
+ .fromPort(port).toPort(port).build(),
+ cidr);
+
+ } catch(IllegalStateException e) {
+ LOG.warn(e.getMessage());
+ /* ignore, it means that this permission was already granted */
+ }
+ }
+ }
+ } else {
+ LOG.warn("Expected security group " + groupName + " does not exist.");
+ }
+ } else {
+ LOG.warn("OpenStack security group extension not available for this cloud.");
+ }
}
}
}