You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by to...@apache.org on 2012/03/15 18:37:24 UTC
svn commit: r1301112 [2/3] - in /whirr/branches/branch-0.7: ./ cli/ recipes/
services/cdh-oldtests/ services/cdh-oldtests/src/
services/cdh-oldtests/src/test/ services/cdh-oldtests/src/test/java/
services/cdh-oldtests/src/test/java/org/ services/cdh-ol...
Added: whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhYarnServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhYarnServiceTest.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhYarnServiceTest.java (added)
+++ whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhYarnServiceTest.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.cdh.integration;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.hadoop.HadoopProxy;
+
+import org.junit.BeforeClass;
+
+public class CdhYarnServiceTest extends CdhHadoopServiceTest {
+
+ protected static String getPropertiesFilename() {
+ return "whirr-hadoop-mr2-cdh-test.properties";
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(new PropertiesConfiguration(getPropertiesFilename()));
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+
+ cluster = controller.launchCluster(clusterSpec);
+ proxy = new HadoopProxy(clusterSpec, cluster);
+ proxy.start();
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhYarnServiceTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhZooKeeperServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhZooKeeperServiceTest.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhZooKeeperServiceTest.java (original)
+++ whirr/branches/branch-0.7/services/cdh/src/test/java/org/apache/whirr/service/cdh/integration/CdhZooKeeperServiceTest.java Thu Mar 15 17:37:22 2012
@@ -52,7 +52,7 @@ public class CdhZooKeeperServiceTest {
if (System.getProperty("config") != null) {
config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
}
- config.addConfiguration(new PropertiesConfiguration("whirr-zookeeper-test.properties"));
+ config.addConfiguration(new PropertiesConfiguration("whirr-zookeeper-cdh-test.properties"));
clusterSpec = ClusterSpec.withTemporaryKeys(config);
controller = new ClusterController();
Added: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-cdh-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-cdh-test.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-cdh-test.properties (added)
+++ whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-cdh-test.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=hadoop-cdh-itest-${sys:whirr.test.provider}-${sys:user.name}
+
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker,1 hadoop-datanode+hadoop-tasktracker
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.java.install-function=install_oracle_jdk6
+whirr.hadoop.install-function=install_cdh_hadoop
+whirr.hadoop.configure-function=configure_cdh_hadoop
+whirr.env.repo=cdh4
+
+# CDH4b1 only has RPMs available (not debs) so use Centos
+whirr.hardware-id=m1.large
+# RightImage CentOS_5.4_x64_v4.4.10
+whirr.image-id=us-east-1/ami-ccb35ea5
+whirr.location-id=us-east-1
Propchange: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-cdh-test.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties (added)
+++ whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=hadoop-mr2-cdh-itest-${sys:whirr.test.provider}-${sys:user.name}
+
+whirr.instance-templates=1 hadoop-namenode+yarn-resourcemanager+mapreduce-historyserver,1 hadoop-datanode+yarn-nodemanager
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.java.install-function=install_oracle_jdk6
+whirr.hadoop.install-function=install_cdh_hadoop
+whirr.hadoop.configure-function=configure_cdh_hadoop
+whirr.yarn.configure-function=configure_cdh_yarn
+whirr.yarn.start-function=start_cdh_yarn
+whirr.mr_jobhistory.start-function=start_cdh_mr_jobhistory
+whirr.env.repo=cdh4
+whirr.env.mapreduce_version=2
+
+hadoop-mapreduce.mapreduce.framework.name=yarn
+hadoop-common.ipc.client.connect.max.retries=100
+
+# CDH4b1 only has RPMs available (not debs) so use Centos
+whirr.hardware-id=m1.large
+# RightImage CentOS_5.4_x64_v4.4.10
+whirr.image-id=us-east-1/ami-ccb35ea5
+whirr.location-id=us-east-1
+
Propchange: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hbase-cdh-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hbase-cdh-test.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hbase-cdh-test.properties (added)
+++ whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hbase-cdh-test.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=hbase-cdh-itest-${sys:whirr.test.provider}-${sys:user.name}
+
+whirr.instance-templates=1 hadoop-namenode+hadoop-jobtracker+zookeeper+hbase-master+hbase-thriftserver,1 hadoop-datanode+hadoop-tasktracker+hbase-regionserver
+whirr.hardware-min-ram=2048
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.env.repo=cdh4
+
+whirr.java.install-function=install_oracle_jdk6
+
+whirr.hadoop.install-function=install_cdh_hadoop
+whirr.hadoop.configure-function=configure_cdh_hadoop
+
+whirr.hbase.install-function=install_cdh_hbase
+whirr.hbase.configure-function=configure_cdh_hbase
+
+whirr.zookeeper.install-function=install_cdh_zookeeper
+whirr.zookeeper.configure-function=configure_cdh_zookeeper
+
+# CDH4b1 only has RPMs available (not debs) so use Centos
+whirr.hardware-id=m1.large
+# RightImage CentOS_5.4_x64_v4.4.10
+whirr.image-id=us-east-1/ami-ccb35ea5
+whirr.location-id=us-east-1
\ No newline at end of file
Propchange: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-hbase-cdh-test.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-zookeeper-cdh-test.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-zookeeper-cdh-test.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-zookeeper-cdh-test.properties (added)
+++ whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-zookeeper-cdh-test.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+whirr.cluster-name=zookeeper-cdh-itest-${sys:whirr.test.provider}-${sys:user.name}
+whirr.instance-templates=2 zookeeper
+
+whirr.provider=${sys:whirr.test.provider}
+whirr.identity=${sys:whirr.test.identity}
+whirr.credential=${sys:whirr.test.credential}
+
+whirr.env.repo=cdh4
+
+whirr.zookeeper.install-function=install_cdh_zookeeper
+whirr.zookeeper.configure-function=configure_cdh_zookeeper
+
+# Waiting for http://code.google.com/p/jclouds/issues/detail?id=746
+# whirr.hardware-min-ram=512
+
+# CDH4b1 only has RPMs available (not debs) so use Centos
+whirr.hardware-id=m1.large
+# RightImage CentOS_5.4_x64_v4.4.10
+whirr.image-id=us-east-1/ami-ccb35ea5
+whirr.location-id=us-east-1
+
Propchange: whirr/branches/branch-0.7/services/cdh/src/test/resources/whirr-zookeeper-cdh-test.properties
------------------------------------------------------------------------------
svn:eol-style = native
Modified: whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopCluster.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopCluster.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopCluster.java (original)
+++ whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopCluster.java Thu Mar 15 17:37:22 2012
@@ -18,10 +18,14 @@
package org.apache.whirr.service.hadoop;
+import com.google.common.collect.Iterables;
+
import java.io.IOException;
import java.net.InetAddress;
+import java.util.Set;
import org.apache.whirr.Cluster;
+import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.RolePredicates;
public class HadoopCluster {
@@ -41,14 +45,20 @@ public class HadoopCluster {
RolePredicates.role(HadoopNameNodeClusterActionHandler.ROLE))
.getPrivateAddress();
}
+ private static Instance getJobTracker(Cluster cluster) {
+ Set<Instance> jobtracker = cluster.getInstancesMatching(
+ RolePredicates.role(HadoopJobTrackerClusterActionHandler.ROLE));
+ if (jobtracker.isEmpty()) {
+ return null;
+ }
+ return Iterables.getOnlyElement(jobtracker);
+ }
public static InetAddress getJobTrackerPublicAddress(Cluster cluster) throws IOException {
- return cluster.getInstanceMatching(
- RolePredicates.role(HadoopJobTrackerClusterActionHandler.ROLE))
- .getPublicAddress();
+ Instance jt = getJobTracker(cluster);
+ return jt == null ? null : jt.getPublicAddress();
}
public static InetAddress getJobTrackerPrivateAddress(Cluster cluster) throws IOException {
- return cluster.getInstanceMatching(
- RolePredicates.role(HadoopJobTrackerClusterActionHandler.ROLE))
- .getPrivateAddress();
+ Instance jt = getJobTracker(cluster);
+ return jt == null ? null : jt.getPrivateAddress();
}
}
Modified: whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java (original)
+++ whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java Thu Mar 15 17:37:22 2012
@@ -132,10 +132,13 @@ public class HadoopConfigurationBuilder
}
}
- Instance jobtracker = cluster
- .getInstanceMatching(role(HadoopJobTrackerClusterActionHandler.ROLE));
- config.setProperty("mapred.job.tracker", String.format("%s:8021",
- jobtracker.getPublicHostName()));
+ Set<Instance> jobtracker = cluster
+ .getInstancesMatching(role(HadoopJobTrackerClusterActionHandler.ROLE));
+ if (!jobtracker.isEmpty()) {
+ config.setProperty("mapred.job.tracker", String.format("%s:8021",
+ Iterables.getOnlyElement(jobtracker).getPublicHostName()));
+ }
+
return config;
}
Modified: whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java (original)
+++ whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java Thu Mar 15 17:37:22 2012
@@ -19,14 +19,23 @@
package org.apache.whirr.service.hadoop;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+
import org.apache.commons.configuration.AbstractConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.StringUtils;
import org.jclouds.scriptbuilder.domain.Statement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.io.IOException;
import java.util.Iterator;
import java.util.List;
+import java.util.Properties;
+import java.util.Map.Entry;
import static org.jclouds.scriptbuilder.domain.Statements.createOrOverwriteFile;
@@ -35,6 +44,9 @@ import static org.jclouds.scriptbuilder.
*/
public class HadoopConfigurationConverter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HadoopConfigurationConverter.class);
+
private static final String FINAL_SUFFIX = ".final";
@VisibleForTesting
@@ -97,4 +109,29 @@ public class HadoopConfigurationConverte
return createOrOverwriteFile(path, asEnvironmentVariablesLines(config));
}
+ private static CharSequence generateHadoopConfigurationFile(Properties config) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<?xml version=\"1.0\"?>\n");
+ sb.append("<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n");
+ sb.append("<configuration>\n");
+ for (Entry<Object, Object> entry : config.entrySet()) {
+ sb.append(" <property>\n");
+ sb.append(" <name>").append(entry.getKey()).append("</name>\n");
+ sb.append(" <value>").append(entry.getValue()).append("</value>\n");
+ sb.append(" </property>\n");
+ }
+ sb.append("</configuration>\n");
+ return sb;
+ }
+
+ public static void createClientSideHadoopSiteFile(File file, Properties config) {
+ try {
+ Files.write(generateHadoopConfigurationFile(config), file,
+ Charsets.UTF_8);
+ LOG.info("Wrote file {}", file);
+ } catch (IOException e) {
+ LOG.error("Problem writing file {}", file, e);
+ }
+ }
+
}
Modified: whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java (original)
+++ whirr/branches/branch-0.7/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java Thu Mar 15 17:37:22 2012
@@ -27,7 +27,6 @@ import com.google.common.io.Files;
import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
-import java.util.Map.Entry;
import java.util.Properties;
import org.apache.whirr.Cluster;
@@ -84,7 +83,10 @@ public class HadoopNameNodeClusterAction
Properties config = createClientSideProperties(clusterSpec, namenodePublicAddress, jobtrackerPublicAddress);
createClientSideHadoopSiteFile(clusterSpec, config);
createProxyScript(clusterSpec, cluster);
- event.setCluster(new Cluster(cluster.getInstances(), config));
+ Properties combined = new Properties();
+ combined.putAll(cluster.getConfiguration());
+ combined.putAll(config);
+ event.setCluster(new Cluster(cluster.getInstances(), combined));
}
private Properties createClientSideProperties(ClusterSpec clusterSpec,
@@ -92,9 +94,13 @@ public class HadoopNameNodeClusterAction
Properties config = new Properties();
config.setProperty("hadoop.job.ugi", "root,root");
config.setProperty("fs.default.name", String.format("hdfs://%s:8020/", namenode.getHostName()));
- config.setProperty("mapred.job.tracker", String.format("%s:8021", jobtracker.getHostName()));
+ if (jobtracker != null) {
+ config.setProperty("mapred.job.tracker", String.format("%s:8021", jobtracker.getHostName()));
+ }
config.setProperty("hadoop.socks.server", "localhost:6666");
config.setProperty("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.net.SocksSocketFactory");
+ // See https://issues.apache.org/jira/browse/HDFS-3068
+ config.setProperty("dfs.client.use.legacy.blockreader", "true");
if (clusterSpec.getProvider().endsWith("ec2")) {
config.setProperty("fs.s3.awsAccessKeyId", clusterSpec.getIdentity());
config.setProperty("fs.s3.awsSecretAccessKey", clusterSpec.getCredential());
@@ -107,13 +113,7 @@ public class HadoopNameNodeClusterAction
private void createClientSideHadoopSiteFile(ClusterSpec clusterSpec, Properties config) {
File configDir = getConfigDir(clusterSpec);
File hadoopSiteFile = new File(configDir, "hadoop-site.xml");
- try {
- Files.write(generateHadoopConfigurationFile(config), hadoopSiteFile,
- Charsets.UTF_8);
- LOG.info("Wrote Hadoop site file {}", hadoopSiteFile);
- } catch (IOException e) {
- LOG.error("Problem writing Hadoop site file {}", hadoopSiteFile, e);
- }
+ HadoopConfigurationConverter.createClientSideHadoopSiteFile(hadoopSiteFile, config);
}
private File getConfigDir(ClusterSpec clusterSpec) {
@@ -124,21 +124,6 @@ public class HadoopNameNodeClusterAction
return configDir;
}
- private CharSequence generateHadoopConfigurationFile(Properties config) {
- StringBuilder sb = new StringBuilder();
- sb.append("<?xml version=\"1.0\"?>\n");
- sb.append("<?xml-stylesheet type=\"text/xsl\" href=\"configuration.xsl\"?>\n");
- sb.append("<configuration>\n");
- for (Entry<Object, Object> entry : config.entrySet()) {
- sb.append(" <property>\n");
- sb.append(" <name>").append(entry.getKey()).append("</name>\n");
- sb.append(" <value>").append(entry.getValue()).append("</value>\n");
- sb.append(" </property>\n");
- }
- sb.append("</configuration>\n");
- return sb;
- }
-
private void createProxyScript(ClusterSpec clusterSpec, Cluster cluster) {
File configDir = getConfigDir(clusterSpec);
File hadoopProxyFile = new File(configDir, "hadoop-proxy.sh");
Modified: whirr/branches/branch-0.7/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/integration/HadoopServiceController.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/integration/HadoopServiceController.java?rev=1301112&r1=1301111&r2=1301112&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/integration/HadoopServiceController.java (original)
+++ whirr/branches/branch-0.7/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/integration/HadoopServiceController.java Thu Mar 15 17:37:22 2012
@@ -49,7 +49,7 @@ public class HadoopServiceController {
new HashMap<String, HadoopServiceController>();
public static HadoopServiceController getInstance(String config) throws ConfigurationException {
- if (!INSTANCES.containsKey("config")) {
+ if (!INSTANCES.containsKey(config)) {
PropertiesConfiguration configuration = new PropertiesConfiguration(config);
INSTANCES.put(config, new HadoopServiceController(configuration));
}
Added: whirr/branches/branch-0.7/services/yarn/pom.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/pom.xml?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/pom.xml (added)
+++ whirr/branches/branch-0.7/services/yarn/pom.xml Thu Mar 15 17:37:22 2012
@@ -0,0 +1,97 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr</artifactId>
+ <version>0.7.1</version>
+ <relativePath>../../pom.xml</relativePath>
+ </parent>
+ <groupId>org.apache.whirr</groupId>
+ <artifactId>whirr-yarn</artifactId>
+ <packaging>jar</packaging>
+ <version>0.7.1</version>
+ <name>Apache Whirr YARN</name>
+ <properties>
+ <hadoop.version>0.20.2</hadoop.version>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-core</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>whirr-hadoop</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-configuration</groupId>
+ <artifactId>commons-configuration</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>dnsjava</groupId>
+ <artifactId>dnsjava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>0.23.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
Propchange: whirr/branches/branch-0.7/services/yarn/pom.xml
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn;
+
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+import java.io.IOException;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.ClusterActionEvent;
+
+public class MapReduceJobHistoryServerHandler extends YarnHandler {
+
+ public static final String ROLE = "mapreduce-historyserver";
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ protected void beforeConfigure(ClusterActionEvent event) throws IOException,
+ InterruptedException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Configuration conf = getConfiguration(clusterSpec);
+ addStatement(event, call(getStartFunction(conf, "mr_jobhistory", "start_mr_jobhistory"), "historyserver"));
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/MapReduceJobHistoryServerHandler.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn;
+
+import static org.apache.whirr.RolePredicates.role;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.Cluster.Instance;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.hadoop.HadoopConfigurationConverter;
+import org.jclouds.scriptbuilder.domain.Statement;
+
+public class YarnConfigurationBuilder {
+
+ private static final String WHIRR_YARN_DEFAULT_PROPERTIES =
+ "whirr-yarn-default.properties";
+
+ private static Configuration build(ClusterSpec clusterSpec, Cluster cluster,
+ Configuration defaults, String prefix)
+ throws ConfigurationException {
+ CompositeConfiguration config = new CompositeConfiguration();
+ Configuration sub = clusterSpec.getConfigurationForKeysWithPrefix(prefix);
+ config.addConfiguration(sub.subset(prefix)); // remove prefix
+ config.addConfiguration(defaults.subset(prefix));
+ return config;
+ }
+
+ public static Statement build(String path, ClusterSpec clusterSpec,
+ Cluster cluster, String role) throws ConfigurationException, IOException {
+ Configuration config = buildConfiguration(clusterSpec, cluster, role,
+ new PropertiesConfiguration(WHIRR_YARN_DEFAULT_PROPERTIES));
+ return HadoopConfigurationConverter.asCreateXmlConfigurationFileStatement(path, config);
+ }
+
+ @VisibleForTesting
+ static Configuration buildConfiguration(ClusterSpec clusterSpec,
+ Cluster cluster, String role, Configuration defaults) throws ConfigurationException,
+ IOException {
+ Configuration config = build(clusterSpec, cluster, defaults,
+ "hadoop-yarn");
+
+ Instance resourceManager = cluster
+ .getInstanceMatching(role(YarnResourceManagerHandler.ROLE));
+ String resourceManagerPrivateAddress =
+ resourceManager.getPrivateAddress().getHostName();
+ config.setProperty("yarn.resourcemanager.address",
+ String.format("%s:8040", resourceManagerPrivateAddress));
+ config.setProperty("yarn.resourcemanager.scheduler.address",
+ String.format("%s:8030", resourceManagerPrivateAddress));
+ config.setProperty("yarn.resourcemanager.resource-tracker.address",
+ String.format("%s:8025", resourceManagerPrivateAddress));
+ return config;
+ }
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnHandler.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnHandler.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnHandler.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnHandler.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn;
+
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+import java.io.IOException;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.ClusterActionHandlerSupport;
+
+public abstract class YarnHandler extends ClusterActionHandlerSupport {
+
+ protected Configuration getConfiguration(
+ ClusterSpec clusterSpec) throws IOException {
+ return getConfiguration(clusterSpec, "whirr-yarn-default.properties");
+ }
+
+ protected String getInstallFunction(Configuration config) {
+ return getInstallFunction(config, "hadoop", "install_hadoop_mr2");
+ }
+
+ protected String getConfigureFunction(Configuration config) {
+ return getConfigureFunction(config, "yarn", "configure_yarn");
+ }
+
+ protected String getStartFunction(Configuration config) {
+ return getStartFunction(config, "yarn", "start_yarn");
+ }
+
+ @Override
+ protected void beforeBootstrap(ClusterActionEvent event) throws IOException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Configuration conf = getConfiguration(clusterSpec);
+
+ addStatement(event, call("configure_hostnames"));
+
+ addStatement(event, call(getInstallFunction(conf, "java", "install_openjdk")));
+ addStatement(event, call("install_tarball"));
+
+ String installFn = getInstallFunction(conf);
+ String tarball = prepareRemoteFileUrl(event,
+ conf.getString("whirr.hadoop.tarball.url"));
+ addStatement(event, call(installFn, "-u", tarball));
+ }
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnHandler.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn;
+
+import static org.apache.whirr.RolePredicates.role;
+import static org.apache.whirr.service.yarn.YarnConfigurationBuilder.build;
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+import java.io.IOException;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.Cluster.Instance;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.FirewallManager.Rule;
+
+public class YarnNodeManagerHandler extends YarnHandler {
+
+ public static final String ROLE = "yarn-nodemanager";
+
+ public static final int NODE_MANAGER_WEB_UI_PORT = 9999;
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ protected void beforeConfigure(ClusterActionEvent event) throws IOException,
+ InterruptedException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Configuration conf = getConfiguration(clusterSpec);
+ Cluster cluster = event.getCluster();
+
+ Instance nodeManager = cluster.getInstanceMatching(role(ROLE));
+ event.getFirewallManager().addRules(
+ Rule.create()
+ .destination(nodeManager)
+ .ports(NODE_MANAGER_WEB_UI_PORT)
+ );
+
+ try {
+ event.getStatementBuilder().addStatements(
+ build("/tmp/yarn-site.xml", clusterSpec, cluster, ROLE)
+ );
+ } catch (ConfigurationException e) {
+ throw new IOException(e);
+ }
+
+ addStatement(event, call(getConfigureFunction(conf)));
+ addStatement(event, call(getStartFunction(conf), "nodemanager"));
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnNodeManagerHandler.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn;
+
+import static org.apache.whirr.RolePredicates.role;
+import static org.apache.whirr.service.yarn.YarnConfigurationBuilder.build;
+import static org.jclouds.scriptbuilder.domain.Statements.call;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.Cluster.Instance;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.ClusterActionEvent;
+import org.apache.whirr.service.FirewallManager.Rule;
+import org.apache.whirr.service.hadoop.HadoopConfigurationConverter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class YarnResourceManagerHandler extends YarnHandler {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(YarnResourceManagerHandler.class);
+
+ public static final String ROLE = "yarn-resourcemanager";
+
+ public static final int RESOURCE_MANAGER_RPC_PORT = 8040;
+ public static final int RESOURCE_MANAGER_WEB_UI_PORT = 8088;
+
+ @Override
+ public String getRole() {
+ return ROLE;
+ }
+
+ @Override
+ protected void beforeConfigure(ClusterActionEvent event) throws IOException,
+ InterruptedException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Configuration conf = getConfiguration(clusterSpec);
+ Cluster cluster = event.getCluster();
+
+ Instance resourceManager = cluster.getInstanceMatching(role(ROLE));
+ event.getFirewallManager().addRules(
+ Rule.create()
+ .destination(resourceManager)
+ .ports(RESOURCE_MANAGER_RPC_PORT, RESOURCE_MANAGER_WEB_UI_PORT),
+ Rule.create()
+ .source(resourceManager.getPublicIp())
+ .destination(resourceManager)
+ .ports(RESOURCE_MANAGER_RPC_PORT)
+ );
+
+ try {
+ event.getStatementBuilder().addStatements(
+ build("/tmp/yarn-site.xml", clusterSpec, cluster, ROLE)
+ );
+ } catch (ConfigurationException e) {
+ throw new IOException(e);
+ }
+
+ addStatement(event, call(getConfigureFunction(conf)));
+ addStatement(event, call(getStartFunction(conf), "resourcemanager"));
+ }
+
+ @Override
+ protected void afterConfigure(ClusterActionEvent event) throws IOException {
+ ClusterSpec clusterSpec = event.getClusterSpec();
+ Cluster cluster = event.getCluster();
+ Instance resourceManager = cluster.getInstanceMatching(role(ROLE));
+ LOG.info("Resource manager web UI available at http://{}:{}",
+ resourceManager.getPublicHostName(), RESOURCE_MANAGER_WEB_UI_PORT);
+
+ Properties mrConfig = createClientSideMapReduceProperties(clusterSpec);
+ createClientSideMapReduceSiteFile(clusterSpec, mrConfig);
+
+ Properties yarnConfig = createClientSideYarnProperties(clusterSpec, resourceManager);
+ createClientSideYarnSiteFile(clusterSpec, yarnConfig);
+
+ Properties combined = new Properties();
+ combined.putAll(cluster.getConfiguration());
+ combined.putAll(mrConfig);
+ combined.putAll(yarnConfig);
+ event.setCluster(new Cluster(cluster.getInstances(), combined));
+ }
+
+ private Properties createClientSideMapReduceProperties(ClusterSpec clusterSpec) throws IOException {
+ Properties config = new Properties();
+ config.setProperty("mapreduce.framework.name", "yarn");
+ config.setProperty("mapreduce.jobhistory.address", "");
+ return config;
+ }
+
+ private Properties createClientSideYarnProperties(ClusterSpec clusterSpec, Instance resourceManager) throws IOException {
+ Properties config = new Properties();
+ config.setProperty("yarn.resourcemanager.address",
+ String.format("%s:8040", resourceManager.getPublicHostName()));
+ config.setProperty("yarn.resourcemanager.scheduler.address",
+ String.format("%s:8030", resourceManager.getPublicHostName()));
+ config.setProperty("yarn.resourcemanager.resource-tracker.address",
+ String.format("%s:8025", resourceManager.getPublicHostName()));
+ config.setProperty("yarn.app.mapreduce.am.staging-dir", "/user");
+ config.setProperty("yarn.application.classpath",
+ "/etc/hadoop/conf.dist," +
+ "/usr/lib/hadoop/*," +
+ "/usr/lib/hadoop/lib/*," +
+ "$HADOOP_CONF_DIR," +
+ "$HADOOP_COMMON_HOME/share/hadoop/common/*," +
+ "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*," +
+ "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*," +
+ "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*," +
+ "$YARN_HOME/share/hadoop/mapreduce/*," +
+ "$YARN_HOME/share/hadoop/mapreduce/lib/*");
+ return config;
+ }
+
+ private void createClientSideMapReduceSiteFile(ClusterSpec clusterSpec, Properties config) {
+ File configDir = getConfigDir(clusterSpec);
+ File hadoopSiteFile = new File(configDir, "mapred-site.xml");
+ HadoopConfigurationConverter.createClientSideHadoopSiteFile(hadoopSiteFile, config);
+ }
+
+ private void createClientSideYarnSiteFile(ClusterSpec clusterSpec, Properties config) {
+ File configDir = getConfigDir(clusterSpec);
+ File hadoopSiteFile = new File(configDir, "yarn-site.xml");
+ HadoopConfigurationConverter.createClientSideHadoopSiteFile(hadoopSiteFile, config);
+ }
+
+
+ private File getConfigDir(ClusterSpec clusterSpec) {
+ File configDir = new File(new File(System.getProperty("user.home")),
+ ".whirr");
+ configDir = new File(configDir, clusterSpec.getClusterName());
+ configDir.mkdirs();
+ return configDir;
+ }
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/resources/META-INF/services/org.apache.whirr.service.ClusterActionHandler Thu Mar 15 17:37:22 2012
@@ -0,0 +1,14 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+org.apache.whirr.service.yarn.MapReduceJobHistoryServerHandler
+org.apache.whirr.service.yarn.YarnNodeManagerHandler
+org.apache.whirr.service.yarn.YarnResourceManagerHandler
Added: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,138 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -x
+
+function configure_hadoop_mr2() {
+ local OPTIND
+ local OPTARG
+
+ if [ "$CONFIGURE_HADOOP_DONE" == "1" ]; then
+ echo "Hadoop is already configured."
+ return;
+ fi
+
+ ROLES=$1
+ shift
+
+ case $CLOUD_PROVIDER in
+ ec2 | aws-ec2 )
+ # Alias /mnt as /data
+ ln -s /mnt /data
+ ;;
+ *)
+ ;;
+ esac
+
+ HADOOP_COMMON_HOME=$HADOOP_HOME
+ HADOOP_HDFS_HOME=$HADOOP_HOME
+ HADOOP_HOME=/usr/local/hadoop
+ HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
+
+ mkdir -p /data/hadoop
+ chown hadoop:hadoop /data/hadoop
+ if [ ! -e /data/tmp ]; then
+ mkdir /data/tmp
+ chmod a+rwxt /data/tmp
+ fi
+ mkdir /etc/hadoop
+ ln -s $HADOOP_CONF_DIR /etc/hadoop/conf
+
+ # Copy generated configuration files in place
+ cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
+
+ # Keep PID files in a non-temporary directory
+# sed -i -e "s|# export HADOOP_PID_DIR=.*|export HADOOP_PID_DIR=/var/run/hadoop|" \
+# $HADOOP_CONF_DIR/hadoop-env.sh
+# mkdir -p /var/run/hadoop
+# chown -R hadoop:hadoop /var/run/hadoop
+
+ # Set SSH options within the cluster
+# sed -i -e 's|# export HADOOP_SSH_OPTS=.*|export HADOOP_SSH_OPTS="-o StrictHostKeyChecking=no"|' \
+# $HADOOP_CONF_DIR/hadoop-env.sh
+
+ # Disable IPv6
+# sed -i -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"|' \
+# $HADOOP_CONF_DIR/hadoop-env.sh
+
+ # Hadoop logs should be on the /data partition
+# sed -i -e 's|# export HADOOP_LOG_DIR=.*|export HADOOP_LOG_DIR=/var/log/hadoop/logs|' \
+# $HADOOP_CONF_DIR/hadoop-env.sh
+# rm -rf /var/log/hadoop
+# mkdir /data/hadoop/logs
+# chown hadoop:hadoop /data/hadoop/logs
+# ln -s /data/hadoop/logs /var/log/hadoop
+# chown -R hadoop:hadoop /var/log/hadoop
+
+ for role in $(echo "$ROLES" | tr "," "\n"); do
+ case $role in
+ hadoop-namenode)
+ start_namenode
+ ;;
+ hadoop-secondarynamenode)
+ start_hadoop_daemon secondarynamenode
+ ;;
+ hadoop-jobtracker)
+ start_hadoop_daemon jobtracker
+ ;;
+ hadoop-datanode)
+ start_hadoop_daemon datanode
+ ;;
+ hadoop-tasktracker)
+ start_hadoop_daemon tasktracker
+ ;;
+ esac
+ done
+
+ CONFIGURE_HADOOP_DONE=1
+
+}
+
+function start_namenode() {
+ if which dpkg &> /dev/null; then
+ AS_HADOOP="su -s /bin/bash - hadoop -c"
+ elif which rpm &> /dev/null; then
+ AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
+ fi
+
+ # Format HDFS
+ [ ! -e /data/hadoop/hdfs ] && $AS_HADOOP "$HADOOP_HOME/bin/hadoop namenode -format"
+
+ $AS_HADOOP "$HADOOP_HOME/sbin/hadoop-daemon.sh start namenode"
+
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop dfsadmin -safemode wait"
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -mkdir /user"
+ # The following is questionable, as it allows a user to delete another user
+ # It's needed to allow users to create their own user directories
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod +w /user"
+
+ # Create temporary directory for Pig and Hive in HDFS
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -mkdir /tmp"
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod +w /tmp"
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -mkdir /user/hive/warehouse"
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod +w /user/hive/warehouse"
+
+}
+
+function start_hadoop_daemon() {
+ if which dpkg &> /dev/null; then
+ AS_HADOOP="su -s /bin/bash - hadoop -c"
+ elif which rpm &> /dev/null; then
+ AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
+ fi
+ $AS_HADOOP "$HADOOP_HOME/sbin/hadoop-daemon.sh start $1"
+}
+
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_yarn.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_yarn.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_yarn.sh (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_yarn.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -x
+
+function configure_yarn() {
+ HADOOP_HOME=/usr/local/hadoop
+ HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
+ export YARN_HOME=$HADOOP_HOME
+ export YARN_CONF_DIR=$HADOOP_CONF_DIR
+
+ cp /tmp/yarn-site.xml $YARN_CONF_DIR
+
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/configure_yarn.sh
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/install_hadoop_mr2.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/install_hadoop_mr2.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/install_hadoop_mr2.sh (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/install_hadoop_mr2.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -x
+
+function update_repo() {
+ if which dpkg &> /dev/null; then
+ sudo apt-get update
+ elif which rpm &> /dev/null; then
+ yum update -y yum
+ fi
+}
+
+function install_hadoop_mr2() {
+ local OPTIND
+ local OPTARG
+
+ if [ "$INSTALL_HADOOP_DONE" == "1" ]; then
+ echo "Hadoop is already installed."
+ return;
+ fi
+
+ HADOOP_TAR_URL=
+ while getopts "u:" OPTION; do
+ case $OPTION in
+ u)
+ HADOOP_TAR_URL="$OPTARG"
+ ;;
+ esac
+ done
+
+ HADOOP_HOME=/usr/local/$(basename $HADOOP_TAR_URL .tar.gz)
+ HADOOP_COMMON_HOME=$HADOOP_HOME
+ HADOOP_HDFS_HOME=$HADOOP_HOME
+ HADOOP_MAPRED_HOME=$HADOOP_HOME
+ YARN_HOME=$HADOOP_HOME
+
+ update_repo
+
+ if ! id hadoop &> /dev/null; then
+ useradd -m hadoop
+ fi
+
+ install_tarball $HADOOP_TAR_URL
+ ln -s $HADOOP_HOME /usr/local/hadoop
+ chown hadoop:hadoop $HADOOP_HOME
+
+ cat >> /etc/profile <<EOF
+export HADOOP_HOME=$HADOOP_HOME
+export HADOOP_COMMON_HOME=$HADOOP_COMMON_HOME
+export HADOOP_HDFS_HOME=$HADOOP_HDFS_HOME
+export HADOOP_MAPRED_HOME=$HADOOP_MAPRED_HOME
+export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
+export YARN_HOME=$YARN_HOME
+export YARN_CONF_DIR=$YARN_HOME/conf
+export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$PATH
+EOF
+
+ INSTALL_HADOOP_DONE=1
+
+}
+
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/install_hadoop_mr2.sh
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_mr_jobhistory.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_mr_jobhistory.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_mr_jobhistory.sh (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_mr_jobhistory.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function start_mr_jobhistory() {
+ if which dpkg &> /dev/null; then
+ AS_HADOOP="su -s /bin/bash - hadoop -c"
+ elif which rpm &> /dev/null; then
+ AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
+ fi
+ $AS_HADOOP "$HADOOP_HOME/sbin/mr-jobhistory-daemon.sh start $1"
+}
\ No newline at end of file
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_mr_jobhistory.sh
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_yarn.sh
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_yarn.sh?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_yarn.sh (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_yarn.sh Thu Mar 15 17:37:22 2012
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+function start_yarn() {
+ if which dpkg &> /dev/null; then
+ AS_HADOOP="su -s /bin/bash - hadoop -c"
+ elif which rpm &> /dev/null; then
+ AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
+ fi
+ if [[ $1 == 'nodemanager' ]]; then
+ sleep 60 # give RM a chance to start in absence of WHIRR-221
+ fi
+ $AS_HADOOP "$HADOOP_HOME/sbin/yarn-daemon.sh start $1"
+}
\ No newline at end of file
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/resources/functions/start_yarn.sh
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/main/resources/whirr-yarn-default.properties
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/main/resources/whirr-yarn-default.properties?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/main/resources/whirr-yarn-default.properties (added)
+++ whirr/branches/branch-0.7/services/yarn/src/main/resources/whirr-yarn-default.properties Thu Mar 15 17:37:22 2012
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+whirr.hadoop.version=0.23.1
+whirr.hadoop.tarball.url=http://apache.osuosl.org/hadoop/common/hadoop-${whirr.hadoop.version}/hadoop-${whirr.hadoop.version}.tar.gz
+
+# Hadoop defaults. The first part of the key (hadoop-yarn.) is removed by whirr.
+
+hadoop-yarn.yarn.nodemanager.log-dirs=/tmp/nm-logs
+hadoop-yarn.yarn.nodemanager.remote-app-log-dir=/tmp/nm-remote-app-logs
+
+hadoop-yarn.yarn.nodemanager.aux-services=mapreduce.shuffle
+hadoop-yarn.yarn.nodemanager.aux-services.mapreduce.shuffle.class=org.apache.hadoop.mapred.ShuffleHandler
+
+hadoop-yarn.yarn.app.mapreduce.am.staging-dir=/user
+
+hadoop-mapreduce.mapreduce.framework.name=yarn
+hadoop-common.ipc.client.connect.max.retries=100
+
Propchange: whirr/branches/branch-0.7/services/yarn/src/main/resources/whirr-yarn-default.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/AbstractHadoopServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/AbstractHadoopServiceTest.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/AbstractHadoopServiceTest.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/AbstractHadoopServiceTest.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn.integration;
+
+import static junit.framework.Assert.assertEquals;
+import static junit.framework.Assert.assertNull;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.lib.LongSumReducer;
+import org.apache.hadoop.mapred.lib.TokenCountMapper;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+/**
+ * Subclasses should implement @BeforeClass static void setUp()
+ * to set the controller.
+ */
+public abstract class AbstractHadoopServiceTest {
+
+ protected static HadoopServiceController controller;
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ controller.shutdown();
+ }
+
+ public AbstractHadoopServiceTest() {
+ super();
+ }
+
+ @Test
+ public void test() throws Exception {
+ Configuration conf = controller.getConfiguration();
+ JobConf job = new JobConf(conf, AbstractHadoopServiceTest.class);
+
+ FileSystem fs = FileSystem.get(conf);
+
+ OutputStream os = fs.create(new Path("input"));
+ Writer wr = new OutputStreamWriter(os);
+ wr.write("b a\n");
+ wr.close();
+
+ job.setMapperClass(TokenCountMapper.class);
+ job.setReducerClass(LongSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(LongWritable.class);
+ FileInputFormat.setInputPaths(job, new Path("input"));
+ FileOutputFormat.setOutputPath(job, new Path("output"));
+
+ JobClient.runJob(job);
+
+ FSDataInputStream in = fs.open(new Path("output/part-00000"));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in));
+ assertEquals("a\t1", reader.readLine());
+ assertEquals("b\t1", reader.readLine());
+ assertNull(reader.readLine());
+ reader.close();
+
+ }
+
+ @Test
+ public void testExistsTemporaryFolderAndHiveWarehouse() throws Exception {
+ Configuration conf = controller.getConfiguration();
+ FileSystem fs = FileSystem.get(conf);
+
+ assertThat(fs.exists(new Path("/tmp")), is(true));
+ assertThat(fs.exists(new Path("/user/hive/warehouse")), is(true));
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/AbstractHadoopServiceTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/HadoopServiceController.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/HadoopServiceController.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/HadoopServiceController.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/HadoopServiceController.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn.integration;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.protocol.FSConstants;
+import org.apache.hadoop.mapred.ClusterStatus;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.whirr.Cluster;
+import org.apache.whirr.ClusterController;
+import org.apache.whirr.ClusterSpec;
+import org.apache.whirr.service.hadoop.HadoopProxy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class HadoopServiceController {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HadoopServiceController.class);
+
+ private static final Map<String, HadoopServiceController> INSTANCES =
+ new HashMap<String, HadoopServiceController>();
+
+ public static HadoopServiceController getInstance(String config) throws ConfigurationException {
+ if (!INSTANCES.containsKey(config)) {
+ PropertiesConfiguration configuration = new PropertiesConfiguration(config);
+ INSTANCES.put(config, new HadoopServiceController(configuration));
+ }
+ return INSTANCES.get(config);
+ }
+
+ private org.apache.commons.configuration.Configuration configuration;
+
+ private boolean running;
+ private ClusterSpec clusterSpec;
+ private ClusterController controller;
+ private HadoopProxy proxy;
+ private Cluster cluster;
+
+ private HadoopServiceController(org.apache.commons.configuration.Configuration configuration) {
+ this.configuration = configuration;
+ }
+
+ public synchronized boolean ensureClusterRunning() throws Exception {
+ if (running) {
+ LOG.info("Cluster already running.");
+ return false;
+ } else {
+ startup();
+ return true;
+ }
+ }
+
+ public synchronized void startup() throws Exception {
+ LOG.info("Starting up cluster...");
+ CompositeConfiguration config = new CompositeConfiguration();
+ if (System.getProperty("config") != null) {
+ config.addConfiguration(new PropertiesConfiguration(System.getProperty("config")));
+ }
+ config.addConfiguration(configuration);
+ clusterSpec = ClusterSpec.withTemporaryKeys(config);
+ controller = new ClusterController();
+
+ cluster = controller.launchCluster(clusterSpec);
+ proxy = new HadoopProxy(clusterSpec, cluster);
+ proxy.start();
+
+ Configuration conf = getConfiguration();
+ JobConf job = new JobConf(conf, HadoopServiceController.class);
+ JobClient client = new JobClient(job);
+ waitToExitSafeMode(client);
+ waitForTaskTrackers(client);
+ running = true;
+ }
+
+ public Cluster getCluster() {
+ return cluster;
+ }
+
+ public Configuration getConfiguration() {
+ Configuration conf = new Configuration();
+ for (Entry<Object, Object> entry : cluster.getConfiguration().entrySet()) {
+ conf.set(entry.getKey().toString(), entry.getValue().toString());
+ }
+ return conf;
+ }
+
+ public JobConf getJobConf() {
+ return new JobConf(getConfiguration());
+ }
+
+ private static void waitToExitSafeMode(JobClient client) throws IOException {
+ LOG.info("Waiting to exit safe mode...");
+ FileSystem fs = client.getFs();
+ DistributedFileSystem dfs = (DistributedFileSystem) fs;
+ boolean inSafeMode = true;
+ while (inSafeMode) {
+ inSafeMode = dfs.setSafeMode(FSConstants.SafeModeAction.SAFEMODE_GET);
+ try {
+ System.out.print(".");
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ LOG.info("Exited safe mode");
+ }
+
+ private static void waitForTaskTrackers(JobClient client) throws IOException {
+ LOG.info("Waiting for tasktrackers...");
+ while (true) {
+ ClusterStatus clusterStatus = client.getClusterStatus();
+ int taskTrackerCount = clusterStatus.getTaskTrackers();
+ if (taskTrackerCount > 0) {
+ LOG.info("{} tasktrackers reported in. Continuing.", taskTrackerCount);
+ break;
+ }
+ try {
+ System.out.print(".");
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ }
+
+ public synchronized void shutdown() throws IOException, InterruptedException {
+ LOG.info("Shutting down cluster...");
+ if (proxy != null) {
+ proxy.stop();
+ }
+ controller.destroyCluster(clusterSpec);
+ running = false;
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/HadoopServiceController.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/YarnServiceTest.java
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/YarnServiceTest.java?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/YarnServiceTest.java (added)
+++ whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/YarnServiceTest.java Thu Mar 15 17:37:22 2012
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.yarn.integration;
+
+import org.junit.BeforeClass;
+
+public class YarnServiceTest extends AbstractHadoopServiceTest {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ controller = HadoopServiceController.getInstance("whirr-yarn-test.properties");
+ controller.ensureClusterRunning();
+ }
+
+}
Propchange: whirr/branches/branch-0.7/services/yarn/src/test/java/org/apache/whirr/service/yarn/integration/YarnServiceTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: whirr/branches/branch-0.7/services/yarn/src/test/resources/log4j.xml
URL: http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/yarn/src/test/resources/log4j.xml?rev=1301112&view=auto
==============================================================================
--- whirr/branches/branch-0.7/services/yarn/src/test/resources/log4j.xml (added)
+++ whirr/branches/branch-0.7/services/yarn/src/test/resources/log4j.xml Thu Mar 15 17:37:22 2012
@@ -0,0 +1,216 @@
+<?xml version="1.0" encoding="UTF-8"?>
+ <!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+
+ <!--
+ For more configuration infromation and examples see the Apache
+ Log4j website: http://logging.apache.org/log4j/
+ -->
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"
+ debug="false">
+
+ <appender name="CONSOLE" class="org.apache.log4j.ConsoleAppender">
+ <param name="Threshold" value="INFO" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n"/>
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WIREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-wire.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="FILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="BLOBSTOREFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-blobstore.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+
+ <!-- A time/date based rolling appender -->
+ <appender name="COMPUTEFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-compute.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="WHIRRFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/whirr.log" />
+ <param name="Append" value="true" />
+
+ <!-- Rollover at midnight each day -->
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+
+ <param name="Threshold" value="TRACE" />
+
+ <layout class="org.apache.log4j.PatternLayout">
+ <!-- The default pattern: Date Priority [Category] Message\n -->
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+
+ <!--
+ The full pattern: Date MS Priority [Category]
+ (Thread:NDC) Message\n <param name="ConversionPattern"
+ value="%d %-5r %-5p [%c] (%t:%x) %m%n"/>
+ -->
+ </layout>
+ </appender>
+
+ <!-- A time/date based rolling appender -->
+ <appender name="SSHFILE" class="org.apache.log4j.DailyRollingFileAppender">
+ <param name="File" value="target/test-data/jclouds-ssh.log" />
+ <param name="Append" value="true" />
+ <param name="DatePattern" value="'.'yyyy-MM-dd" />
+ <param name="Threshold" value="TRACE" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%d %-5p [%c] (%t) %m%n" />
+ </layout>
+ </appender>
+
+ <appender name="ASYNCCOMPUTE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="COMPUTEFILE" />
+ </appender>
+ <appender name="ASYNC" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="FILE" />
+ </appender>
+
+ <appender name="ASYNCWIRE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="WIREFILE" />
+ </appender>
+
+ <appender name="ASYNCBLOBSTORE" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="BLOBSTOREFILE" />
+ </appender>
+
+ <appender name="ASYNCSSH" class="org.apache.log4j.AsyncAppender">
+ <appender-ref ref="SSHFILE" />
+ </appender>
+ <!-- ================ -->
+ <!-- Limit categories -->
+ <!-- ================ -->
+ <category name="jclouds.blobstore">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCBLOBSTORE" />
+ </category>
+
+ <category name="org.jclouds">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNC" />
+ </category>
+
+ <category name="org.apache.whirr">
+ <priority value="DEBUG" />
+ <appender-ref ref="WHIRRFILE" />
+ </category>
+
+ <category name="jclouds.headers">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+ <category name="jclouds.compute">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCCOMPUTE" />
+ </category>
+
+ <category name="jclouds.ssh">
+ <priority value="TRACE" />
+ <appender-ref ref="ASYNCSSH" />
+ </category>
+
+ <category name="jclouds.wire">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category><!--
+
+ <category name="jclouds.signature">
+ <priority value="DEBUG" />
+ <appender-ref ref="ASYNCWIRE" />
+ </category>
+
+
+ --><!-- ======================= -->
+ <!-- Setup the Root category -->
+ <!-- ======================= -->
+
+ <root>
+ <priority value="WARN" />
+ <appender-ref ref="CONSOLE" />
+ </root>
+
+</log4j:configuration>