You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by to...@apache.org on 2012/03/15 18:37:24 UTC
svn commit: r1301112 [1/3] - in /whirr/branches/branch-0.7: ./ cli/ recipes/
services/cdh-oldtests/ services/cdh-oldtests/src/
services/cdh-oldtests/src/test/ services/cdh-oldtests/src/test/java/
services/cdh-oldtests/src/test/java/org/ services/cdh-ol...
Author: tomwhite
Date: Thu Mar 15 17:37:22 2012
New Revision: 1301112
URL: http://svn.apache.org/viewvc?rev=1301112&view=rev
Log:
WHIRR-391. Write a YARN (Hadoop MR2) service for Whirr.
Added:
whirr/branches/branch-0.7/cli/pom.xml.orig
whirr/branches/branch-0.7/recipes/hadoop-yarn-cdh-ec2.properties (with props)
whirr/branches/branch-0.7/recipes/hadoop-yarn-ec2.properties (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/
whirr/branches/branch-0.7/services/cdh-oldtests/pom.xml (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/src/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HBaseServiceTest.java (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HadoopServiceTest.java (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3ZooKeeperServiceTest.java (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/log4j.xml (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hadoop-cdh3-test.properties (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hbase-cdh3-test.properties (with props)
whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-zookeeper-cdh3-test.properties (with props)
whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_yarn.sh (with props)
whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_mr_jobhistory.sh (with props)
whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_yarn.sh (with props)
whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhYarnServiceTest.java (with props)
whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-cdh-test.properties (with props)
whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties (with props)
whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hbase-cdh-test.properties (with props)
whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-zookeeper-cdh-test.properties (with props)
whirr/branches/branch-0.7/services/yarn/
whirr/branches/branch-0.7/services/yarn/pom.xml (with props)
whirr/branches/branch-0.7/services/yarn/src/
whirr/branches/branch-0.7/services/yarn/src/main/
whirr/branches/branch-0.7/services/yarn/src/main/java/
whirr/branches/branch-0.7/services/yarn/src/main/java/org/
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java (with props)
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java (with props)
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnHandler.java (with props)
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java (with props)
whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java (with props)
whirr/branches/branch-0.7/services/yarn/src/main/resources/
whirr/branches/branch-0.7/services/yarn/src/main/resources/META-INF/
whirr/branches/branch-0.7/services/yarn/src/main/resources/META-INF/services/
whirr/branches/branch-0.7/services/yarn/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/
whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh (with props)
whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_yarn.sh (with props)
whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/install_hadoop_mr2.sh (with props)
whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_mr_jobhistory.sh (with props)
whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_yarn.sh (with props)
whirr/branches/branch-0.7/services/yarn/src/main/resources/whirr-yarn-default.properties (with props)
whirr/branches/branch-0.7/services/yarn/src/test/
whirr/branches/branch-0.7/services/yarn/src/test/java/
whirr/branches/branch-0.7/services/yarn/src/test/java/org/
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/AbstractHadoopServiceTest.java (with props)
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/HadoopServiceController.java (with props)
whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/YarnServiceTest.java (with props)
whirr/branches/branch-0.7/services/yarn/src/test/resources/
whirr/branches/branch-0.7/services/yarn/src/test/resources/log4j.xml (with props)
whirr/branches/branch-0.7/services/yarn/src/test/resources/whirr-yarn-test.properties (with props)
Removed:
whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-test.properties
whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hbase-test.properties
whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-zookeeper-test.properties
Modified:
whirr/branches/branch-0.7/CHANGES.txt
whirr/branches/branch-0.7/cli/pom.xml
whirr/branches/branch-0.7/pom.xml
whirr/branches/branch-0.7/services/cdh/pom.xml
whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_zookeeper.sh
whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_hadoop.sh
whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_zookeeper.sh
whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java
whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java
whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhZooKeeperServiceTest.java
whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopCluster.java
whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
whirr/branches/branch-0.7/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/integration/HadoopServiceController.java
Modified: whirr/branches/branch-0.7/CHANGES.txt
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/CHANGES.txt?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/CHANGES.txt (original)
+++ whirr/branches/branch-0.7/CHANGES.txt Thu Mar 15 17:37:22 2012
@@ -6,6 +6,8 @@ Release 0.7.2
WHIRR-531. Add Pig as a service (Marco Didonna, Andrei Savu)
+ WHIRR-391. Write a YARN (Hadoop MR2) service for Whirr. (tomwhite)
+
IMPROVEMENTS
WHIRR-457. Upgrade to jclouds 1.2.2 (Andrei Savu)
Modified: whirr/branches/branch-0.7/cli/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/cli/pom.xml?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/cli/pom.xml (original)
+++ whirr/branches/branch-0.7/cli/pom.xml Thu Mar 15 17:37:22 2012
@@ -66,6 +66,11 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>whirr-yarn</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>whirr-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
@@ -95,6 +100,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-yarn</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
Added: whirr/branches/branch-0.7/cli/pom.xml.orig
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/cli/pom.xml.orig?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/cli/pom.xml.orig (added)
+++ whirr/branches/branch-0.7/cli/pom.xml.orig Thu Mar 15 17:37:22 2012
@@ -0,0 +1,179 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.7.1</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-cli</artifactId>
+ <packaging>jar</packaging>
+ <version>0.7.1</version>
+ <name>Apache Whirr CLI</name>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-cassandra</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-cdh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-ganglia</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-mahout</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-pig</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-zookeeper</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-elasticsearch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hbase</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hama</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-puppet</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-chef</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.jopt-simple</groupId>
+ <artifactId>jopt-simple</artifactId>
+ <version>3.2</version>
+ </dependency>
+ </dependencies>
+ <build>
+ <resources>
+ <resource>
+ <directory>${project.basedir}/src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <!-- Copy dependencies to lib directory so that bin/whirr works in place -->
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>install</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <outputDirectory>${project.build.directory}/lib</outputDirectory>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <profile>
+ <id>voldemort</id>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-voldemort</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+ </profile>
+ </profiles>
+</project>
Modified: whirr/branches/branch-0.7/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/pom.xml?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/pom.xml (original)
+++ whirr/branches/branch-0.7/pom.xml Thu Mar 15 17:37:22 2012
@@ -42,8 +42,10 @@
<module>examples</module>
<module>services/cassandra</module>
<module>services/cdh</module>
+ <module>services/cdh-oldtests</module>
<module>services/ganglia</module>
<module>services/hadoop</module>
+ <module>services/yarn</module>
<module>services/zookeeper</module>
<module>services/hbase</module>
<module>services/elasticsearch</module>
Added: whirr/branches/branch-0.7/recipes/hadoop-yarn-cdh-ec2.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/recipes/hadoop-yarn-cdh-ec2.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/recipes/hadoop-yarn-cdh-ec2.properties (added)
+++ whirr/branches/branch-0.7/recipes/hadoop-yarn-cdh-ec2.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Hadoop Cluster on AWS EC2
+#
+
+# Read the Configuration Guide for more info:
+# http://whirr.apache.org/docs/latest/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=hadoop-yarn
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=1 hadoop-namenode+yarn-resourcemanager+mapreduce-historyserver,1 hadoop-datanode+yarn-nodemanager
+
+# We need to use modified scripts for the installation since it has changed
+# significantly since 0.20.x
+whirr.java.install-function=install_oracle_jdk6
+whirr.hadoop.install-function=install_cdh_hadoop
+whirr.hadoop.configure-function=configure_cdh_hadoop
+whirr.yarn.configure-function=configure_cdh_yarn
+whirr.yarn.start-function=start_cdh_yarn
+whirr.mr_jobhistory.start-function=start_cdh_mr_jobhistory
+whirr.env.repo=cdh4
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/
+whirr.hardware-id=m1.large
+# Ubuntu 10.04 LTS Lucid. See http://alestic.com/
+#whirr.image-id=us-east-1/ami-da0cf8b3
+# RightImage CentOS_5.4_x64_v4.4.10
+whirr.image-id=us-east-1/ami-ccb35ea5
+# If you choose a different location, make sure whirr.image-id is updated too
+whirr.location-id=us-east-1
+
+# You can also specify the spot instance price
+# http://aws.amazon.com/ec2/spot-instances/
+# whirr.aws-ec2-spot-price=0.15
+
+# By default use the user system SSH keys. Override them here.
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+# Expert: override Hadoop properties by setting properties with the prefix
+# hadoop-common, hadoop-hdfs, hadoop-mapreduce to set Common, HDFS, MapReduce
+# site properties, respectively. The prefix is removed by Whirr, so that for
+# example, setting
+# hadoop-common.fs.trash.interval=1440
+# will result in fs.trash.interval being set to 1440 in core-site.xml.
+
+hadoop-mapreduce.mapreduce.framework.name=yarn
+hadoop-common.ipc.client.connect.max.retries=100
Propchange: whirr/branches/branch-0.7/recipes/hadoop-yarn-cdh-ec2.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/recipes/hadoop-yarn-ec2.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/recipes/hadoop-yarn-ec2.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/recipes/hadoop-yarn-ec2.properties (added)
+++ whirr/branches/branch-0.7/recipes/hadoop-yarn-ec2.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,71 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Hadoop Cluster on AWS EC2
+#
+
+# Read the Configuration Guide for more info:
+# http://whirr.apache.org/docs/latest/configuration-guide.html
+
+# Change the cluster name here
+whirr.cluster-name=hadoop-yarn
+
+# Change the number of machines in the cluster here
+whirr.instance-templates=1 hadoop-namenode+yarn-resourcemanager+mapreduce-historyserver,1 hadoop-datanode+yarn-nodemanager
+
+# We need to use modified scripts for the installation since it has changed
+# significantly since 0.20.x
+whirr.hadoop.install-function=install_hadoop_mr2
+whirr.hadoop.configure-function=configure_hadoop_mr2
+
+whirr.java.install-function=install_openjdk
+hadoop-env.JAVA_HOME=/usr/lib/jvm/java-6-openjdk
+
+# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables.
+whirr.provider=aws-ec2
+whirr.identity=${env:AWS_ACCESS_KEY_ID}
+whirr.credential=${env:AWS_SECRET_ACCESS_KEY}
+
+# The size of the instance to use. See http://aws.amazon.com/ec2/instance-types/
+whirr.hardware-id=m1.large
+# Ubuntu 10.04 LTS Lucid. See http://alestic.com/
+#whirr.image-id=us-east-1/ami-da0cf8b3
+# If you choose a different location, make sure whirr.image-id is updated too
+#whirr.location-id=us-east-1
+
+# You can also specify the spot instance price
+# http://aws.amazon.com/ec2/spot-instances/
+# whirr.aws-ec2-spot-price=0.15
+
+# By default use the user system SSH keys. Override them here.
+# whirr.private-key-file=${sys:user.home}/.ssh/id_rsa
+# whirr.public-key-file=${whirr.private-key-file}.pub
+
+# Expert: override Hadoop properties by setting properties with the prefix
+# hadoop-common, hadoop-hdfs, hadoop-mapreduce to set Common, HDFS, MapReduce
+# site properties, respectively. The prefix is removed by Whirr, so that for
+# example, setting
+# hadoop-common.fs.trash.interval=1440
+# will result in fs.trash.interval being set to 1440 in core-site.xml.
+
+hadoop-mapreduce.mapreduce.framework.name=yarn
+hadoop-common.ipc.client.connect.max.retries=100
+
+# Expert: specify the version of Hadoop to install.
+whirr.hadoop.version=0.23.1
+whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
\ No newline at end of file
Propchange: whirr/branches/branch-0.7/recipes/hadoop-yarn-ec2.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/pom.xml?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/pom.xml (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/pom.xml Thu Mar 15 17:37:22 2012
@@ -0,0 +1,154 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.7.1</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-cdh-oldtests</artifactId>
+ <packaging>jar</packaging>
+ <version>0.7.1</version>
+ <name>Apache Whirr CDH Old Tests</name>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-cdh</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hbase</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hbase</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-zookeeper</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ </dependency>
+ <dependency>
+ <!-- look at https://repository.cloudera.com/artifactory/libs-release-local/org/apache/hadoop/hadoop-core/
+ to find version info for other cdh REPO versions, same for hbase and zookeeper -->
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <!-- <version>0.20.2-cdh3u0</version> -->
+ <!-- <version>0.20.2-cdh3u1</version> -->
+ <version>0.20.2-cdh3u2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase</artifactId>
+ <!-- <version>0.90.1-cdh3u0</version> -->
+ <!-- <version>0.90.3-cdh3u1</version> -->
+ <version>0.90.4-cdh3u2</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <!-- <version>3.3.3-cdh3u0</version> -->
+ <!-- <version>3.3.3-cdh3u1</version> -->
+ <version>3.3.3-cdh3u2</version>
+ <scope>test</scope>
+ </dependency>
+ <!-- The following are missing CDH dependencies -->
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/pom.xml
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HBaseServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HBaseServiceTest.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HBaseServiceTest.java (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HBaseServiceTest.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.cdh.integration;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+
+import com.google.common.collect.Lists;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor;
+import org.apache.hadoop.hbase.thrift.generated.Hbase;
+import org.apache.hadoop.hbase.thrift.generated.Mutation;
+import org.apache.hadoop.hbase.thrift.generated.TRowResult;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.whirr.service.hbase.integration.HBaseServiceController;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class Cdh3HBaseServiceTest {
+
+ private static final byte[] FIRST = Bytes.toBytes("");
+ private static final byte[] TABLE = Bytes.toBytes("testtable");
+ private static final byte[] ROW = Bytes.toBytes("testRow");
+ private static final byte[] FAMILY1 = Bytes.toBytes("testFamily1");
+ private static final byte[] FAMILY2 = Bytes.toBytes("testFamily2");
+ private static final byte[] COLUMN = Bytes.toBytes("testFamily1:testColumn");
+ private static final byte[] VALUE = Bytes.toBytes("testValue");
+
+ private static HBaseServiceController controller =
+ HBaseServiceController.getInstance("whirr-hbase-cdh3-test.properties");
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ controller.ensureClusterRunning();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ controller.shutdown();
+ }
+
+ @Test
+ public void test() throws Exception {
+ ArrayList<ColumnDescriptor> columns = new ArrayList<ColumnDescriptor>();
+ ColumnDescriptor cd = new ColumnDescriptor();
+ cd.name = FAMILY1;
+ columns.add(cd);
+ cd = new ColumnDescriptor();
+ cd.name = FAMILY2;
+ columns.add(cd);
+
+ Hbase.Client client = controller.getThriftClient();
+ client.createTable(TABLE, columns);
+
+ ArrayList<Mutation> mutations = new ArrayList<Mutation>();
+ mutations.add(new Mutation(false, COLUMN, VALUE));
+ client.mutateRow(TABLE, ROW, mutations);
+
+ int scan1 = client.scannerOpen(TABLE, FIRST, Lists.newArrayList(FAMILY1));
+ List<TRowResult> rows = client.scannerGet(scan1);
+ assertThat(rows.size(), is(1));
+ assertThat(Bytes.toString(rows.get(0).getRow()), is("testRow"));
+ assertTrue("No more rows", client.scannerGet(scan1).isEmpty());
+ client.scannerClose(scan1);
+
+ int scan2 = client.scannerOpen(TABLE, FIRST, Lists.newArrayList(FAMILY2));
+ assertTrue("No more rows", client.scannerGet(scan2).isEmpty());
+ client.scannerClose(scan2);
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HBaseServiceTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HadoopServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HadoopServiceTest.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HadoopServiceTest.java (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HadoopServiceTest.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,196 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.cdh.integration;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNull;
+import static junit.framework.Assert.failNotEquals;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.lib.LongSumReducer;
+import org.apache.hadoop.mapred.lib.TokenCountMapper;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.hadoop.HadoopProxy;
+import org.jclouds.compute.domain.ExecResponse;
+import org.jclouds.compute.domain.NodeMetadata;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Cdh3HadoopServiceTest {
+
+ private static final Logger LOG = LoggerFactory
+ .getLogger(Cdh3HadoopServiceTest.class);
+
+ private final static Predicate<NodeMetadata> ALL = Predicates.alwaysTrue();
+
+ protected static ClusterSpec clusterSpec;
+ protected static ClusterController controller;
+ protected static HadoopProxy proxy;
+ protected static Cluster cluster;
+
+ protected static String getPropertiesFilename() {
+ return "whirr-hadoop-cdh3-test.properties";
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(new PropertiesConfiguration(getPropertiesFilename()));
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+
+ cluster = controller.launchCluster(clusterSpec);
+ proxy = new HadoopProxy(clusterSpec, cluster);
+ proxy.start();
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException, InterruptedException {
+ if (proxy != null) {
+ proxy.stop();
+ }
+ controller.destroyCluster(clusterSpec);
+ }
+
+ @Test
+ public void testVersion() throws Exception {
+ Statement checkVersion = Statements.exec("ls /etc/alternatives/hadoop-lib");
+ Map<? extends NodeMetadata, ExecResponse> responses =
+ controller.runScriptOnNodesMatching(clusterSpec, ALL, checkVersion);
+
+ printResponses(checkVersion, responses);
+ assertResponsesContain(responses, checkVersion, "cdh3");
+ }
+
+ @Test
+ public void testJobExecution() throws Exception {
+ Configuration conf = getConfiguration();
+
+ JobConf job = new JobConf(conf, Cdh3HadoopServiceTest.class);
+ JobClient client = new JobClient(job);
+ waitForTaskTrackers(client);
+
+ checkHadoop(conf, client, job);
+ }
+
+ protected void checkHadoop(Configuration conf, JobClient client, JobConf job) throws Exception {
+ FileSystem fs = FileSystem.get(conf);
+
+ OutputStream os = fs.create(new Path("input"));
+ Writer wr = new OutputStreamWriter(os);
+ wr.write("b a\n");
+ wr.close();
+
+ job.setMapperClass(TokenCountMapper.class);
+ job.setReducerClass(LongSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(LongWritable.class);
+ FileInputFormat.setInputPaths(job, new Path("input"));
+ FileOutputFormat.setOutputPath(job, new Path("output"));
+
+ JobClient.runJob(job);
+
+ FSDataInputStream in = fs.open(new Path("output/part-00000"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ assertEquals("a\t1", reader.readLine());
+ assertEquals("b\t1", reader.readLine());
+ assertNull(reader.readLine());
+ reader.close();
+ }
+
+ private Configuration getConfiguration() {
+ Configuration conf = new Configuration();
+ for (Entry<Object, Object> entry : cluster.getConfiguration().entrySet()) {
+ conf.set(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return conf;
+ }
+
+ private static void waitForTaskTrackers(JobClient client) throws IOException {
+ while (true) {
+ ClusterStatus clusterStatus = client.getClusterStatus();
+ int taskTrackerCount = clusterStatus.getTaskTrackers();
+ if (taskTrackerCount > 0) {
+ break;
+ }
+ try {
+ System.out.print(".");
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ }
+
+ private static void assertResponsesContain(
+ Map<? extends NodeMetadata, ExecResponse> responses, Statement statement,
+ String text) {
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry : responses
+ .entrySet()) {
+ if (!entry.getValue().getOutput().contains(text)) {
+ failNotEquals("Node: " + entry.getKey().getId()
+ + " failed to execute the command: " + statement
+ + " as could not find expected text", text, entry.getValue());
+ }
+ }
+ }
+
+ public static void printResponses(Statement statement,
+ Map<? extends NodeMetadata, ExecResponse> responses) {
+ LOG.info("Responses for Statement: " + statement);
+ for (Map.Entry<? extends NodeMetadata, ExecResponse> entry : responses
+ .entrySet()) {
+ LOG.info("Node[" + entry.getKey().getId() + "]: " + entry.getValue());
+ }
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3HadoopServiceTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3ZooKeeperServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3ZooKeeperServiceTest.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3ZooKeeperServiceTest.java (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3ZooKeeperServiceTest.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,114 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.cdh.integration;
+
+import static junit.framework.Assert.assertEquals;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.zookeeper.ZooKeeperCluster;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.Watcher.Event.KeeperState;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooKeeper;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class Cdh3ZooKeeperServiceTest {
+
+ private ClusterSpec clusterSpec;
+ private Cluster cluster;
+ private ClusterController controller;
+ private String hosts;
+
+ @Before
+ public void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(new PropertiesConfiguration("whirr-zookeeper-cdh3-test.properties"));
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+
+ cluster = controller.launchCluster(clusterSpec);
+ hosts = ZooKeeperCluster.getHosts(cluster);
+ }
+
+ @Test
+ public void test() throws Exception {
+ class ConnectionWatcher implements Watcher {
+
+ private ZooKeeper zk;
+ private CountDownLatch latch = new CountDownLatch(1);
+
+ public void connect(String hosts) throws IOException, InterruptedException {
+ zk = new ZooKeeper(hosts, 5000, this);
+ latch.await();
+ }
+
+ public ZooKeeper getZooKeeper() {
+ return zk;
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getState() == KeeperState.SyncConnected) {
+ latch.countDown();
+ }
+ }
+
+ public void close() throws InterruptedException {
+ if (zk != null) {
+ zk.close();
+ }
+ }
+
+ }
+
+ String path = "/data";
+ String data = "Hello";
+ ConnectionWatcher watcher = new ConnectionWatcher();
+ watcher.connect(hosts);
+ watcher.getZooKeeper().create(path, data.getBytes(), Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ watcher.close();
+
+ watcher = new ConnectionWatcher();
+ watcher.connect(hosts);
+ byte[] actualData = watcher.getZooKeeper().getData(path, false, null);
+ assertEquals(data, new String(actualData));
+ watcher.close();
+ }
+
+ @After
+ public void tearDown() throws IOException, InterruptedException {
+ controller.destroyCluster(clusterSpec);
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/java/org/apache/whirr/service/cdh/integration/Cdh3ZooKeeperServiceTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/log4j.xml?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/log4j.xml (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/log4j.xml Thu Mar 15 17:37:22 2012
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+ <!--
+ For more configuration infromation and examples see the Apache
+ Log4j website: http://logging.apache.org/log4j/
+ -->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
+ debug="false">
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <param name="Threshold" value="INFO" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-wire.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-blobstore.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+
+ <!-- A time/date based rolling appender -->
+ <appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-compute.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WHIRRFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/whirr.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="SSHFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-ssh.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="COMPUTEFILE" />
+ </appender>
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="FILE" />
+ </appender>
+
+ <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="WIREFILE" />
+ </appender>
+
+ <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="BLOBSTOREFILE" />
+ </appender>
+
+ <appender name="ASYNCSSH" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="SSHFILE" />
+ </appender>
+ <!-- ================ -->
+ <!-- Limit categories -->
+ <!-- ================ -->
+ <category name="jclouds.blobstore">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCBLOBSTORE" />
+ </category>
+
+ <category name="org.jclouds">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNC" />
+ </category>
+
+ <category name="org.apache.whirr">
+ <priority value="DEBUG" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
+ <category name="jclouds.headers">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+ <category name="jclouds.compute">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCCOMPUTE" />
+ </category>
+
+ <category name="jclouds.ssh">
+ <priority value="TRACE" />
+ <appender-ref ref="ASYNCSSH" />
+ </category>
+
+ <category name="jclouds.wire">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category><!--
+
+ <category name="jclouds.signature">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+
+
+ --><!-- ======================= -->
+ <!-- Setup the Root category -->
+ <!-- ======================= -->
+
+ <root>
+ <priority value="WARN" />
+ <appender-ref ref="CONSOLE" />
+ </root>
+
+</log4j:configuration>
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/log4j.xml
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hadoop-cdh3-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hadoop-cdh3-test.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hadoop-cdh3-test.properties (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hadoop-cdh3-test.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=hadoop-cdh3-itest-${sys:whirr.test.provider}-${sys:user.name}
+
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker,1 hadoop-datanode+hadoop-tasktracker
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.env.repo=cdh3u2
+
+whirr.hadoop.install-function=install_cdh_hadoop
+whirr.hadoop.configure-function=configure_cdh_hadoop
+
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hadoop-cdh3-test.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hbase-cdh3-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hbase-cdh3-test.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hbase-cdh3-test.properties (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hbase-cdh3-test.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=hbase-cdh3-itest-${sys:whirr.test.provider}-${sys:user.name}
+
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+zookeeper+hbase-master+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+whirr.hardware-min-ram=2048
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.env.repo=cdh3u2
+
+whirr.hadoop.install-function=install_cdh_hadoop
+whirr.hadoop.configure-function=configure_cdh_hadoop
+
+whirr.hbase.install-function=install_cdh_hbase
+whirr.hbase.configure-function=configure_cdh_hbase
+
+whirr.zookeeper.install-function=install_cdh_zookeeper
+whirr.zookeeper.configure-function=configure_cdh_zookeeper
+
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-hbase-cdh3-test.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-zookeeper-cdh3-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-zookeeper-cdh3-test.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-zookeeper-cdh3-test.properties (added)
+++ whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-zookeeper-cdh3-test.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=zookeeper-cdh3-itest-${sys:whirr.test.provider}-${sys:user.name}
+whirr.instance-templates=2 zookeeper
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.env.repo=cdh3u2
+
+whirr.zookeeper.install-function=install_cdh_zookeeper
+whirr.zookeeper.configure-function=configure_cdh_zookeeper
+
+# Waiting for http://code.google.com/p/jclouds/issues/detail?id=746
+# whirr.hardware-min-ram=512
+
Propchange: whirr/branches/branch-0.7/services/cdh-oldtests/src/test/resources/whirr-zookeeper-cdh3-test.properties
------------------------------------------------------------------------------
svn:eol-style = native
Modified: whirr/branches/branch-0.7/services/cdh/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/pom.xml?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/pom.xml (original)
+++ whirr/branches/branch-0.7/services/cdh/pom.xml Thu Mar 15 17:37:22 2012
@@ -60,6 +60,11 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>whirr-yarn</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>whirr-zookeeper</artifactId>
<version>${project.version}</version>
</dependency>
@@ -98,40 +103,27 @@
<artifactId>dnsjava</artifactId>
</dependency>
<dependency>
- <!-- look at https://repository.cloudera.com/artifactory/libs-release-local/org/apache/hadoop/hadoop-core/
- to find version info for other cdh REPO versions, same for hbase and zookeeper -->
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-core</artifactId>
- <!-- <version>0.20.2-cdh3u0</version> -->
- <!-- <version>0.20.2-cdh3u1</version> -->
- <version>0.20.2-cdh3u2</version>
+ <artifactId>hadoop-common</artifactId>
+ <version>0.23.0-cdh4b1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>0.23.0-cdh4b1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase</artifactId>
- <!-- <version>0.90.1-cdh3u0</version> -->
- <!-- <version>0.90.3-cdh3u1</version> -->
- <version>0.90.4-cdh3u2</version>
+ <version>0.92.0-cdh4b1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <!-- <version>3.3.3-cdh3u0</version> -->
- <!-- <version>3.3.3-cdh3u1</version> -->
- <version>3.3.3-cdh3u2</version>
- <scope>test</scope>
- </dependency>
- <!-- The following are missing CDH dependencies -->
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-core-asl</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
+ <version>3.4.1-cdh4b1</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -146,4 +138,83 @@
</plugin>
</plugins>
</build>
+ <profiles>
+ <profile>
+ <!-- Use with -Dit.test=CdhHadoopServiceTest -->
+ <id>cdh4-mr1</id>
+ <activation>
+ <property>
+ <name>mr1</name>
+ <value>true</value>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <!-- look at https://repository.cloudera.com/artifactory/libs-release-local/org/apache/hadoop/
+ to find version info for other cdh REPO versions, same for hbase and zookeeper -->
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <version>0.23.0-mr1-cdh4b1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <testExcludes>
+ <exclude>**/CdhHBaseServiceTest.java</exclude><!-- TODO: remove when WHIRR-525 is done -->
+ <exclude>**/CdhYarnServiceTest.java</exclude>
+ </testExcludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <!-- Use with -Dit.test=CdhYarnServiceTest -->
+ <id>cdh4-mr2</id>
+ <activation>
+ <property>
+ <name>!mr1</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <version>0.23.0-cdh4b1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>0.23.0-cdh4b1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <version>0.23.0-cdh4b1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <testExcludes>
+ <exclude>**/CdhHBaseServiceTest.java</exclude><!-- TODO: remove when WHIRR-525 is done -->
+ <exclude>**/CdhHadoopServiceTest.java</exclude>
+ </testExcludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
</project>
Modified: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh (original)
+++ whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh Thu Mar 15 17:37:22 2012
@@ -35,9 +35,19 @@ function configure_cdh_hadoop() {
;;
esac
- REPO=${REPO:-cdh3}
- HADOOP=hadoop-${HADOOP_VERSION:-0.20}
- HADOOP_CONF_DIR=/etc/$HADOOP/conf.dist
+ REPO=${REPO:-cdh4}
+ CDH_MAJOR_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9]\).*/\1/')
+ if [ $CDH_MAJOR_VERSION = "4" ]; then
+ HADOOP=hadoop
+ HADOOP_CONF_DIR=/etc/$HADOOP/conf.dist
+ HDFS_PACKAGE_PREFIX=hadoop-hdfs
+ MAPREDUCE_PACKAGE_PREFIX=hadoop-0.20-mapreduce
+ else
+ HADOOP=hadoop-${HADOOP_VERSION:-0.20}
+ HADOOP_CONF_DIR=/etc/$HADOOP/conf.dist
+ HDFS_PACKAGE_PREFIX=hadoop-${HADOOP_VERSION:-0.20}
+ MAPREDUCE_PACKAGE_PREFIX=hadoop-${HADOOP_VERSION:-0.20}
+ fi
mkdir -p /data/hadoop
chgrp hadoop /data/hadoop
@@ -85,16 +95,16 @@ EOF
start_namenode
;;
hadoop-secondarynamenode)
- start_hadoop_daemon secondarynamenode
+ start_hadoop_daemon $HDFS_PACKAGE_PREFIX-secondarynamenode
;;
hadoop-jobtracker)
- start_hadoop_daemon jobtracker
+ start_hadoop_daemon $MAPREDUCE_PACKAGE_PREFIX-jobtracker
;;
hadoop-datanode)
- start_hadoop_daemon datanode
+ start_hadoop_daemon $HDFS_PACKAGE_PREFIX-datanode
;;
hadoop-tasktracker)
- start_hadoop_daemon tasktracker
+ start_hadoop_daemon $MAPREDUCE_PACKAGE_PREFIX-tasktracker
;;
esac
done
@@ -105,18 +115,18 @@ EOF
function start_namenode() {
if which dpkg &> /dev/null; then
- retry_apt_get -y install $HADOOP-namenode
+ retry_apt_get -y install $HDFS_PACKAGE_PREFIX-namenode
AS_HDFS="su -s /bin/bash - hdfs -c"
# Format HDFS
[ ! -e /data/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
elif which rpm &> /dev/null; then
- retry_yum install -y $HADOOP-namenode
+ retry_yum install -y $HDFS_PACKAGE_PREFIX-namenode
AS_HDFS="/sbin/runuser -s /bin/bash - hdfs -c"
# Format HDFS
[ ! -e /data/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
fi
- service $HADOOP-namenode start
+ service $HDFS_PACKAGE_PREFIX-namenode start
$AS_HDFS "$HADOOP dfsadmin -safemode wait"
$AS_HDFS "/usr/bin/$HADOOP fs -mkdir /user"
@@ -140,10 +150,10 @@ function start_namenode() {
function start_hadoop_daemon() {
daemon=$1
if which dpkg &> /dev/null; then
- retry_apt_get -y install $HADOOP-$daemon
+ retry_apt_get -y install $daemon
elif which rpm &> /dev/null; then
- retry_yum install -y $HADOOP-$daemon
+ retry_yum install -y $daemon
fi
- service $HADOOP-$daemon start
+ service $daemon start
}
Added: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_yarn.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_yarn.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_yarn.sh (added)
+++ whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_yarn.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -x
+
+function configure_cdh_yarn() {
+ cp /tmp/yarn-site.xml /etc/hadoop/conf
+}
Propchange: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_yarn.sh
------------------------------------------------------------------------------
svn:eol-style = native
Modified: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_zookeeper.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_zookeeper.sh?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_zookeeper.sh (original)
+++ whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/configure_cdh_zookeeper.sh Thu Mar 15 17:37:22 2012
@@ -18,8 +18,16 @@ function configure_cdh_zookeeper() {
local OPTIND
local OPTARG
+ REPO=${REPO:-cdh4}
+ CDH_MAJOR_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9]\).*/\1/')
+ ZOOKEEPER_SERVER_PACKAGE=hadoop-zookeeper-server
myid_file=/var/log/zookeeper/txlog/myid
config_file=/etc/zookeeper/zoo.cfg
+
+ if [ $CDH_MAJOR_VERSION = "4" ]; then
+ ZOOKEEPER_SERVER_PACKAGE=zookeeper-server
+ config_file=/etc/zookeeper/conf/zoo.cfg
+ fi
cat > $config_file <<EOF
# The number of milliseconds of each tick
@@ -56,14 +64,14 @@ EOF
# Now that it's configured, install daemon package
if which dpkg &> /dev/null; then
- retry_apt_get -y install hadoop-zookeeper-server
+ retry_apt_get -y install $ZOOKEEPER_SERVER_PACKAGE
elif which rpm &> /dev/null; then
- retry_yum install -y hadoop-zookeeper-server
+ retry_yum install -y $ZOOKEEPER_SERVER_PACKAGE
fi
# Start ZooKeeper
# For DEB, the service is already started as part of the daemon package installation
if which rpm &> /dev/null; then
- service hadoop-zookeeper-server start
+ service $ZOOKEEPER_SERVER_PACKAGE start
fi
}
Modified: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_hadoop.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_hadoop.sh?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_hadoop.sh (original)
+++ whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_hadoop.sh Thu Mar 15 17:37:22 2012
@@ -16,23 +16,35 @@
#
set -x
function register_cloudera_repo() {
+ CDH_MAJOR_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9]\).*/\1/')
+ CDH_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9][0-9]*\)/\1/')
if which dpkg &> /dev/null; then
cat > /etc/apt/sources.list.d/cloudera.list <<EOF
deb http://archive.cloudera.com/debian lucid-$REPO contrib
deb-src http://archive.cloudera.com/debian lucid-$REPO contrib
EOF
curl -s http://archive.cloudera.com/debian/archive.key | apt-key add -
- retry_apt_get update
+ retry_apt_get -y update
elif which rpm &> /dev/null; then
- rm -f /etc/yum.repos.d/cloudera.repo
- REPO_NUMBER=`echo $REPO | sed -e 's/cdh\([0-9][0-9]*\)/\1/'`
- cat > /etc/yum.repos.d/cloudera-$REPO.repo <<EOF
+ rm -f /etc/yum.repos.d/cloudera*.repo
+ if [ $CDH_MAJOR_VERSION = "4" ]; then
+ cat > /etc/yum.repos.d/cloudera-cdh4.repo <<EOF
+[cloudera-cdh4]
+name=Cloudera's Distribution for Hadoop, Version 4
+baseurl=http://archive.cloudera.com/cdh4/redhat/5/x86_64/cdh/4/
+http://repos.jenkins.sf.cloudera.com/cdh4-nightly/redhat/5/x86_64/cdh/4/
+gpgkey = http://archive.cloudera.com/cdh4/redhat/5/x86_64/cdh/RPM-GPG-KEY-cloudera
+gpgcheck = 1
+EOF
+ else
+ cat > /etc/yum.repos.d/cloudera-$REPO.repo <<EOF
[cloudera-$REPO]
-name=Cloudera's Distribution for Hadoop, Version $REPO_NUMBER
-mirrorlist=http://archive.cloudera.com/redhat/cdh/$REPO_NUMBER/mirrors
+name=Cloudera's Distribution for Hadoop, Version $CDH_VERSION
+mirrorlist=http://archive.cloudera.com/redhat/cdh/$CDH_VERSION/mirrors
gpgkey = http://archive.cloudera.com/redhat/cdh/RPM-GPG-KEY-cloudera
gpgcheck = 0
EOF
+ fi
retry_yum update -y yum
fi
}
@@ -41,19 +53,32 @@ function install_cdh_hadoop() {
local OPTIND
local OPTARG
- REPO=${REPO:-cdh3}
+ REPO=${REPO:-cdh4}
+ MAPREDUCE_VERSION=${MAPREDUCE_VERSION:-1}
HADOOP=hadoop-${HADOOP_VERSION:-0.20}
HADOOP_CONF_DIR=/etc/$HADOOP/conf.dist
+ HADOOP_PACKAGE=hadoop-${HADOOP_VERSION:-0.20}
+
+ CDH_MAJOR_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9]\).*/\1/')
+ if [ $CDH_MAJOR_VERSION = "4" ]; then
+ HADOOP=hadoop
+ HADOOP_CONF_DIR=/etc/$HADOOP/conf.dist
+ if [ $MAPREDUCE_VERSION = "1" ]; then
+ HADOOP_PACKAGE=hadoop-0.20-mapreduce
+ else
+ HADOOP_PACKAGE=hadoop-mapreduce
+ fi
+ fi
register_cloudera_repo
if which dpkg &> /dev/null; then
retry_apt_get update
- retry_apt_get -y install $HADOOP
+ retry_apt_get -y install $HADOOP_PACKAGE
cp -r /etc/$HADOOP/conf.empty $HADOOP_CONF_DIR
update-alternatives --install /etc/$HADOOP/conf $HADOOP-conf $HADOOP_CONF_DIR 90
elif which rpm &> /dev/null; then
- retry_yum install -y $HADOOP
+ retry_yum install -y $HADOOP_PACKAGE
cp -r /etc/$HADOOP/conf.empty $HADOOP_CONF_DIR
alternatives --install /etc/$HADOOP/conf $HADOOP-conf $HADOOP_CONF_DIR 90
fi
Modified: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_zookeeper.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_zookeeper.sh?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_zookeeper.sh (original)
+++ whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/install_cdh_zookeeper.sh Thu Mar 15 17:37:22 2012
@@ -15,6 +15,8 @@
# limitations under the License.
#
function register_cloudera_repo() {
+ CDH_MAJOR_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9]\).*/\1/')
+ CDH_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9][0-9]*\)/\1/')
if which dpkg &> /dev/null; then
cat > /etc/apt/sources.list.d/cloudera.list <<EOF
deb http://archive.cloudera.com/debian lucid-$REPO contrib
@@ -23,15 +25,25 @@ EOF
curl -s http://archive.cloudera.com/debian/archive.key | apt-key add -
retry_apt_get -y update
elif which rpm &> /dev/null; then
- rm -f /etc/yum.repos.d/cloudera.repo
- REPO_NUMBER=`echo $REPO | sed -e 's/cdh\([0-9][0-9]*\)/\1/'`
- cat > /etc/yum.repos.d/cloudera-$REPO.repo <<EOF
+ rm -f /etc/yum.repos.d/cloudera*.repo
+ if [ $CDH_MAJOR_VERSION = "4" ]; then
+ cat > /etc/yum.repos.d/cloudera-cdh4.repo <<EOF
+[cloudera-cdh4]
+name=Cloudera's Distribution for Hadoop, Version 4
+baseurl=http://archive.cloudera.com/cdh4/redhat/5/x86_64/cdh/4/
+http://repos.jenkins.sf.cloudera.com/cdh4-nightly/redhat/5/x86_64/cdh/4/
+gpgkey = http://archive.cloudera.com/cdh4/redhat/5/x86_64/cdh/RPM-GPG-KEY-cloudera
+gpgcheck = 1
+EOF
+ else
+ cat > /etc/yum.repos.d/cloudera-$REPO.repo <<EOF
[cloudera-$REPO]
-name=Cloudera's Distribution for Hadoop, Version $REPO_NUMBER
-mirrorlist=http://archive.cloudera.com/redhat/cdh/$REPO_NUMBER/mirrors
+name=Cloudera's Distribution for Hadoop, Version $CDH_VERSION
+mirrorlist=http://archive.cloudera.com/redhat/cdh/$CDH_VERSION/mirrors
gpgkey = http://archive.cloudera.com/redhat/cdh/RPM-GPG-KEY-cloudera
gpgcheck = 0
EOF
+ fi
retry_yum update -y yum
fi
}
@@ -49,19 +61,26 @@ function install_cdh_zookeeper() {
;;
esac
- REPO=${REPO:-cdh3}
+ REPO=${REPO:-cdh4}
ZOOKEEPER_HOME=/usr/lib/zookeeper
ZK_CONF_DIR=/etc/zookeeper
ZK_LOG_DIR=/var/log/zookeeper
ZK_DATA_DIR=$ZK_LOG_DIR/txlog
+
+ CDH_MAJOR_VERSION=$(echo $REPO | sed -e 's/cdh\([0-9]\).*/\1/')
+ ZOOKEEPER_PACKAGE=hadoop-zookeeper
+ if [ $CDH_MAJOR_VERSION = "4" ]; then
+ ZOOKEEPER_PACKAGE=zookeeper
+ ZK_CONF_DIR=/etc/zookeeper/conf
+ fi
register_cloudera_repo
if which dpkg &> /dev/null; then
retry_apt_get update
- retry_apt_get -y install hadoop-zookeeper
+ retry_apt_get -y install $ZOOKEEPER_PACKAGE
elif which rpm &> /dev/null; then
- retry_yum install -y hadoop-zookeeper
+ retry_yum install -y $ZOOKEEPER_PACKAGE
fi
echo "export ZOOKEEPER_HOME=$ZOOKEEPER_HOME" >> /etc/profile
Added: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_mr_jobhistory.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_mr_jobhistory.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_mr_jobhistory.sh (added)
+++ whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_mr_jobhistory.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function start_cdh_mr_jobhistory() {
+ if which dpkg &> /dev/null; then
+ retry_apt_get -y install hadoop-mapreduce-historyserver
+ elif which rpm &> /dev/null; then
+ retry_yum install -y hadoop-mapreduce-historyserver
+ fi
+ service hadoop-mapreduce-historyserver start
+}
Propchange: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_mr_jobhistory.sh
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_yarn.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_yarn.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_yarn.sh (added)
+++ whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_yarn.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function start_cdh_yarn() {
+ daemon=$1
+ if [[ $1 == 'nodemanager' ]]; then
+ sleep 60 # give RM a chance to start in absence of WHIRR-221
+ fi
+ if which dpkg &> /dev/null; then
+ retry_apt_get -y install hadoop-yarn-$daemon
+ elif which rpm &> /dev/null; then
+ retry_yum install -y hadoop-yarn-$daemon
+ fi
+ service hadoop-yarn-$daemon start
+}
Propchange: whirr/branches/branch-0.7/services/cdh/src/main/resources/functions/start_cdh_yarn.sh
------------------------------------------------------------------------------
svn:eol-style = native
Modified: whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java (original)
+++ whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHBaseServiceTest.java Thu Mar 15 17:37:22 2012
@@ -48,7 +48,7 @@ public class CdhHBaseServiceTest {
private static final byte[] VALUE = Bytes.toBytes("testValue");
private static HBaseServiceController controller =
- HBaseServiceController.getInstance("whirr-hbase-test.properties");
+ HBaseServiceController.getInstance("whirr-hbase-cdh-test.properties");
@BeforeClass
public static void setUp() throws Exception {
Modified: whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java (original)
+++ whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhHadoopServiceTest.java Thu Mar 15 17:37:22 2012
@@ -75,7 +75,7 @@ public class CdhHadoopServiceTest {
protected static Cluster cluster;
protected static String getPropertiesFilename() {
- return "whirr-hadoop-test.properties";
+ return "whirr-hadoop-cdh-test.properties";
}
@BeforeClass
@@ -108,7 +108,7 @@ public class CdhHadoopServiceTest {
controller.runScriptOnNodesMatching(clusterSpec, ALL, checkVersion);
printResponses(checkVersion, responses);
- assertResponsesContain(responses, checkVersion, "cdh3u2");
+ assertResponsesContain(responses, checkVersion, "cdh4");
}
@Test