You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by as...@apache.org on 2012/03/09 16:53:02 UTC
svn commit: r1298892 - in /whirr/branches/branch-0.7: ./ cli/ recipes/
services/pig/ services/pig/src/ services/pig/src/main/
services/pig/src/main/java/ services/pig/src/main/java/org/
services/pig/src/main/java/org/apache/ services/pig/src/main/java/...
Author: asavu
Date: Fri Mar 9 15:53:01 2012
New Revision: 1298892
URL: http://svn.apache.org/viewvc?rev=1298892&view=rev
Log:
WHIRR-531. Add Pig as a service (Marco Didonna, Andrei Savu)
Added:
whirr/branches/branch-0.7/recipes/pig-ec2.properties
whirr/branches/branch-0.7/services/pig/ (with props)
whirr/branches/branch-0.7/services/pig/pom.xml
whirr/branches/branch-0.7/services/pig/src/
whirr/branches/branch-0.7/services/pig/src/main/
whirr/branches/branch-0.7/services/pig/src/main/java/
whirr/branches/branch-0.7/services/pig/src/main/java/org/
whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/
whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/
whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/service/
whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/service/pig/
whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java
whirr/branches/branch-0.7/services/pig/src/main/resources/
whirr/branches/branch-0.7/services/pig/src/main/resources/META-INF/
whirr/branches/branch-0.7/services/pig/src/main/resources/META-INF/services/
whirr/branches/branch-0.7/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
whirr/branches/branch-0.7/services/pig/src/main/resources/functions/
whirr/branches/branch-0.7/services/pig/src/main/resources/functions/configure_pig_client.sh
whirr/branches/branch-0.7/services/pig/src/main/resources/whirr-pig-default.properties
whirr/branches/branch-0.7/services/pig/src/test/
whirr/branches/branch-0.7/services/pig/src/test/java/
whirr/branches/branch-0.7/services/pig/src/test/java/org/
whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/
whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/
whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/
whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/pig/
whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/pig/integration/
whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java
whirr/branches/branch-0.7/services/pig/src/test/resources/
whirr/branches/branch-0.7/services/pig/src/test/resources/log4j.xml
whirr/branches/branch-0.7/services/pig/src/test/resources/whirr-pig-test.properties
Modified:
whirr/branches/branch-0.7/CHANGES.txt
whirr/branches/branch-0.7/cli/pom.xml
whirr/branches/branch-0.7/pom.xml
Modified: whirr/branches/branch-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/CHANGES.txt?rev=1298892&r1=1298891&r2=1298892&view=diff
==============================================================================
--- whirr/branches/branch-0.7/CHANGES.txt (original)
+++ whirr/branches/branch-0.7/CHANGES.txt Fri Mar 9 15:53:01 2012
@@ -3,6 +3,8 @@ Apache Whirr Change Log
Release 0.7.2
NEW FEATURES
+
+ WHIRR-531. Add Pig as a service (Marco Didonna, Andrei Savu)
IMPROVEMENTS
Modified: whirr/branches/branch-0.7/cli/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/cli/pom.xml?rev=1298892&r1=1298891&r2=1298892&view=diff
==============================================================================
--- whirr/branches/branch-0.7/cli/pom.xml (original)
+++ whirr/branches/branch-0.7/cli/pom.xml Fri Mar 9 15:53:01 2012
@@ -61,6 +61,11 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>whirr-pig</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>whirr-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
Modified: whirr/branches/branch-0.7/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/pom.xml?rev=1298892&r1=1298891&r2=1298892&view=diff
==============================================================================
--- whirr/branches/branch-0.7/pom.xml (original)
+++ whirr/branches/branch-0.7/pom.xml Fri Mar 9 15:53:01 2012
@@ -51,6 +51,7 @@
<module>services/puppet</module>
<module>services/chef</module>
<module>services/mahout</module>
+ <module>services/pig</module>
</modules>
<properties>
Added: whirr/branches/branch-0.7/recipes/pig-ec2.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/recipes/pig-ec2.properties?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/recipes/pig-ec2.properties (added)
+++ whirr/branches/branch-0.7/recipes/pig-ec2.properties Fri Mar 9 15:53:01 2012
@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Deploy an Apache Hadoop cluster with Pig
+
+# Read the Configuration Guide for more info:
+# http://whirr.apache.org/docs/latest/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=pig
+
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+pig-client,1 hadoop-datanode+hadoop-tasktracker
+
+whirr.pig.version=0.9.2
+whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
+
+whirr.hadoop.version=0.20.205.0
+whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+#
+# Customise instance types & AMI for Amazon EC2
+#
+
+# Use a specific instance type. See http://aws.amazon.com/ec2/instance-types/
+# whirr.hardware-id=c1.xlarge
+
+# Ubuntu 10.04 LTS Lucid. See http://cloud.ubuntu.com/ami/
+# whirr.image-id=us-east-1/ami-35de095c
+
+# You can also specify the spot instance price: http://aws.amazon.com/ec2/spot-instances/
+# whirr.aws-ec2-spot-price=0.15
+
+#
+# Or for Rackspace Cloud
+#
+
+# The size of the instance to use. See http://www.rackspacecloud.com/cloud_hosting_products/servers/faq/
+# id 3: 1GB, 1 virtual core
+# id 4: 2GB, 2 virtual cores
+# id 5: 4GB, 2 virtual cores
+# id 6: 8GB, 4 virtual cores
+# id 7: 15.5GB, 4 virtual cores
+# whirr.hardware-id=6
+# Ubuntu 10.04 LTS Lucid
+# whirr.image-id=49
+
+#
+# By default use the user system SSH keys. Override them here.
+#
+
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+
Propchange: whirr/branches/branch-0.7/services/pig/
------------------------------------------------------------------------------
--- svn:ignore (added)
+++ svn:ignore Fri Mar 9 15:53:01 2012
@@ -0,0 +1 @@
+target
Added: whirr/branches/branch-0.7/services/pig/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/pom.xml?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/pom.xml (added)
+++ whirr/branches/branch-0.7/services/pig/pom.xml Fri Mar 9 15:53:01 2012
@@ -0,0 +1,79 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.7.1</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-pig</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Whirr Pig</name>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
Added: whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java (added)
+++ whirr/branches/branch-0.7/services/pig/src/main/java/org/apache/whirr/service/pig/PigClientClusterActionHandler.java Fri Mar 9 15:53:01 2012
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.pig;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+
+import java.io.IOException;
+
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+/**
+ * Pig cluster action handler which configures Pig by unpacking a binary tarball and
+ * setting PIG_HOME and PATH environment variables.
+ */
+public class PigClientClusterActionHandler extends ClusterActionHandlerSupport {
+
+ public final static String PIG_CLIENT_ROLE = "pig-client";
+
+ final static String PIG_DEFAULT_PROPERTIES = "whirr-pig-default.properties";
+
+ final static String PIG_TAR_URL = "whirr.pig.tarball.url";
+
+ final static String PIG_CLIENT_SCRIPT = "configure_pig_client";
+
+ final static String URL_FLAG = "-u";
+
+ @Override
+ public String getRole() {
+ return PIG_CLIENT_ROLE;
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) throws IOException, InterruptedException {
+ Configuration conf = getConfiguration(event.getClusterSpec(), PIG_DEFAULT_PROPERTIES);
+
+ String pigTarball = prepareRemoteFileUrl(event, conf.getString(PIG_TAR_URL));
+
+ addStatement(event, call(PIG_CLIENT_SCRIPT, URL_FLAG, pigTarball));
+ }
+}
Added: whirr/branches/branch-0.7/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler (added)
+++ whirr/branches/branch-0.7/services/pig/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler Fri Mar 9 15:53:01 2012
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.whirr.service.pig.PigClientClusterActionHandler
+
Added: whirr/branches/branch-0.7/services/pig/src/main/resources/functions/configure_pig_client.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/src/main/resources/functions/configure_pig_client.sh?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/src/main/resources/functions/configure_pig_client.sh (added)
+++ whirr/branches/branch-0.7/services/pig/src/main/resources/functions/configure_pig_client.sh Fri Mar 9 15:53:01 2012
@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function configure_pig_client() {
+
+ local OPTARG
+
+ PIG_TAR_URL=
+ while getopts "u:" OPTION; do
+ case $OPTION in
+ u)
+ PIG_TAR_URL="$OPTARG"
+ ;;
+ esac
+ done
+
+ PIG_HOME=/usr/local/$(basename $PIG_TAR_URL .tar.gz)
+
+ install_tarball $PIG_TAR_URL
+ ln -s $PIG_HOME /usr/local/pig
+
+ if [ -f /etc/profile ]; then
+ echo "export PIG_HOME=$PIG_HOME" >> /etc/profile
+ echo 'export PATH=$PIG_HOME/bin:$PATH' >> /etc/profile
+ echo 'export PIG_CLASSPATH=/etc/hadoop/conf' >> /etc/profile
+ fi
+ if [ -f /etc/bashrc ]; then
+ echo "export PIG_HOME=$PIG_HOME" >> /etc/bashrc
+ echo 'export PATH=$PIG_HOME/bin:$PATH' >> /etc/bashrc
+ echo 'export PIG_CLASSPATH=/etc/hadoop/conf' >> /etc/bashrc
+ fi
+ if [ -f /etc/skel/.bashrc ]; then
+ echo "export PIG_HOME=$PIG_HOME" >> /etc/skel/.bashrc
+ echo 'export PATH=$PIG_HOME/bin:$PATH' >> /etc/skel/.bashrc
+ echo 'export PIG_CLASSPATH=/etc/hadoop/conf' >> /etc/skel/.bashrc
+ fi
+
+
+}
+
Added: whirr/branches/branch-0.7/services/pig/src/main/resources/whirr-pig-default.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/src/main/resources/whirr-pig-default.properties?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/src/main/resources/whirr-pig-default.properties (added)
+++ whirr/branches/branch-0.7/services/pig/src/main/resources/whirr-pig-default.properties Fri Mar 9 15:53:01 2012
@@ -0,0 +1,14 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+whirr.pig.version=0.9.2
+whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
\ No newline at end of file
Added: whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java (added)
+++ whirr/branches/branch-0.7/services/pig/src/test/java/org/apache/whirr/service/pig/integration/PigServiceTest.java Fri Mar 9 15:53:01 2012
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.pig.integration;
+
+import com.google.common.base.Predicate;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.state.ClusterStateStoreFactory;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static com.google.common.base.Predicates.alwaysTrue;
+import static com.google.common.base.Predicates.and;
+import static com.google.common.collect.Sets.newHashSet;
+import static junit.framework.Assert.failNotEquals;
+import static org.apache.whirr.RolePredicates.anyRoleIn;
+import static org.apache.whirr.service.pig.PigClientClusterActionHandler.PIG_CLIENT_ROLE;
+import static org.jclouds.compute.predicates.NodePredicates.withIds;
+
+/**
+ * Install the pig binary distribution.
+ */
+public class PigServiceTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PigServiceTest.class);
+
+ private static ClusterSpec clusterSpec;
+ private static ClusterController controller;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(new PropertiesConfiguration("whirr-pig-test.properties"));
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+ controller.launchCluster(clusterSpec);
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException, InterruptedException {
+ if (controller != null) {
+ controller.destroyCluster(clusterSpec);
+ }
+ }
+
+ @Test
+ public void testPigBin() throws Exception {
+ Statement binPig = Statements.exec(". /etc/profile && pig -e fs -ls /");
+
+ Cluster.Instance pigInstance = findPigInstance();
+ Predicate<NodeMetadata> pigClientRole = and(alwaysTrue(), withIds(pigInstance.getId()));
+
+ Map<? extends NodeMetadata, ExecResponse> responses = controller.runScriptOnNodesMatching(clusterSpec, pigClientRole, binPig);
+
+ LOG.info("Responses for Statement: " + binPig);
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry : responses.entrySet()) {
+ LOG.info("Node[" + entry.getKey().getId() + "]: " + entry.getValue());
+ }
+
+ assertResponsesContain(responses, binPig, "/hadoop");
+ }
+
+ public static void assertResponsesContain(Map<? extends NodeMetadata, ExecResponse> responses, Statement statement, String text) {
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry : responses.entrySet()) {
+ if (!entry.getValue().getOutput().contains(text)) {
+ failNotEquals("Node: " + entry.getKey().getId()
+ + " failed to execute the command: " + statement
+ + " as could not find expected text", text, entry.getValue());
+ }
+ }
+ }
+
+ private Cluster.Instance findPigInstance() throws IOException {
+ Cluster cluster = new ClusterStateStoreFactory().create(clusterSpec).load();
+ return cluster.getInstanceMatching(anyRoleIn(newHashSet(PIG_CLIENT_ROLE)));
+ }
+}
Added: whirr/branches/branch-0.7/services/pig/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/src/test/resources/log4j.xml?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/src/test/resources/log4j.xml (added)
+++ whirr/branches/branch-0.7/services/pig/src/test/resources/log4j.xml Fri Mar 9 15:53:01 2012
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+ <!--
+ For more configuration infromation and examples see the Apache
+ Log4j website: http://logging.apache.org/log4j/
+ -->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
+ debug="false">
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <param name="Threshold" value="INFO" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-wire.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-blobstore.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+
+ <!-- A time/date based rolling appender -->
+ <appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-compute.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WHIRRFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/whirr.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="SSHFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-ssh.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="COMPUTEFILE" />
+ </appender>
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="FILE" />
+ </appender>
+
+ <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="WIREFILE" />
+ </appender>
+
+ <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="BLOBSTOREFILE" />
+ </appender>
+
+ <appender name="ASYNCSSH" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="SSHFILE" />
+ </appender>
+ <!-- ================ -->
+ <!-- Limit categories -->
+ <!-- ================ -->
+ <category name="jclouds.blobstore">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCBLOBSTORE" />
+ </category>
+
+ <category name="org.jclouds">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNC" />
+ </category>
+
+ <category name="org.apache.whirr">
+ <priority value="DEBUG" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
+ <category name="jclouds.headers">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+ <category name="jclouds.compute">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCCOMPUTE" />
+ </category>
+
+ <category name="jclouds.ssh">
+ <priority value="TRACE" />
+ <appender-ref ref="ASYNCSSH" />
+ </category>
+
+ <category name="jclouds.wire">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category><!--
+
+ <category name="jclouds.signature">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+
+
+ --><!-- ======================= -->
+ <!-- Setup the Root category -->
+ <!-- ======================= -->
+
+ <root>
+ <priority value="WARN" />
+ <appender-ref ref="CONSOLE" />
+ </root>
+
+</log4j:configuration>
Added: whirr/branches/branch-0.7/services/pig/src/test/resources/whirr-pig-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/pig/src/test/resources/whirr-pig-test.properties?rev=1298892&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/pig/src/test/resources/whirr-pig-test.properties (added)
+++ whirr/branches/branch-0.7/services/pig/src/test/resources/whirr-pig-test.properties Fri Mar 9 15:53:01 2012
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+whirr.cluster-name=pig-itest-${sys:whirr.test.provider}-${sys:user.name}
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+pig-client,1 hadoop-datanode+hadoop-tasktracker
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.pig.version=0.9.2
+whirr.pig.tarball.url=http://apache.osuosl.org/pig/pig-${whirr.pig.version}/pig-${whirr.pig.version}.tar.gz
+
+whirr.hadoop.version=0.20.205.0
+whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz