You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by to...@apache.org on 2011/02/10 19:42:30 UTC
svn commit: r1069518 - in /incubator/whirr/trunk: ./
core/src/main/java/org/apache/whirr/service/jclouds/
core/src/main/resources/functions/ services/cdh/src/main/resources/functions/
services/hadoop/src/main/java/org/apache/whirr/service/hadoop/ servi...
Author: tomwhite
Date: Thu Feb 10 18:42:29 2011
New Revision: 1069518
URL: http://svn.apache.org/viewvc?rev=1069518&view=rev
Log:
WHIRR-55. Users should be able to override an arbitrary Hadoop property before launch.
Added:
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java (with props)
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java (with props)
incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties (with props)
incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java (with props)
incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java (with props)
Modified:
incubator/whirr/trunk/CHANGES.txt
incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh
incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Thu Feb 10 18:42:29 2011
@@ -36,6 +36,9 @@ Trunk (unreleased changes)
WHIRR-167. Improve bootstrapping and configuration to be able to
isolate and repair or evict failing nodes on EC2 (Tibor Kiss via asavu)
+ WHIRR-55. Users should be able to override an arbitrary Hadoop property
+ before launch. (tomwhite)
+
BUG FIXES
WHIRR-170. Instances should be started in the order specified in
Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java Thu Feb 10 18:42:29 2011
@@ -36,6 +36,12 @@ public class StatementBuilder implements
}
}
+ public void addStatements(Statement... statements) {
+ for (Statement statement : statements) {
+ addStatement(statement);
+ }
+ }
+
@Override
public Iterable<String> functionDependecies(OsFamily family) {
List<String> functions = Lists.newArrayList();
Modified: incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh (original)
+++ incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh Thu Feb 10 18:42:29 2011
@@ -13,7 +13,7 @@ function configure_hostnames() {
done
case $CLOUD_PROVIDER in
- cloudservers)
+ cloudservers | cloudservers-uk | cloudservers-us )
if which dpkg &> /dev/null; then
PRIVATE_IP=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
HOSTNAME=`echo $PRIVATE_IP | tr . -`.static.cloud-ips.com
Modified: incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh (original)
+++ incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh Thu Feb 10 18:42:29 2011
@@ -5,17 +5,9 @@ function configure_cdh_hadoop() {
ROLES=$1
shift
- NN_HOST=
- JT_HOST=
CLOUD_PROVIDER=
- while getopts "n:j:c:" OPTION; do
+ while getopts "c:" OPTION; do
case $OPTION in
- n)
- NN_HOST="$OPTARG"
- ;;
- j)
- JT_HOST="$OPTARG"
- ;;
c)
CLOUD_PROVIDER="$OPTARG"
;;
@@ -24,266 +16,27 @@ function configure_cdh_hadoop() {
case $CLOUD_PROVIDER in
ec2 | aws-ec2 )
- # Use public hostname for EC2
- SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname`
+ # Alias /mnt as /data
+ ln -s /mnt /data
;;
*)
- SELF_HOST=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
;;
esac
REPO=${REPO:-cdh3}
HADOOP=hadoop-${HADOOP_VERSION:-0.20}
HADOOP_CONF_DIR=/etc/$HADOOP/conf.dist
- for role in $(echo "$ROLES" | tr "," "\n"); do
- case $role in
- hadoop-namenode)
- NN_HOST=$SELF_HOST
- ;;
- hadoop-jobtracker)
- JT_HOST=$SELF_HOST
- ;;
- esac
- done
- case $CLOUD_PROVIDER in
- ec2)
- MOUNT=/mnt
- ;;
- *)
- MOUNT=/data
- ;;
- esac
- FIRST_MOUNT=$MOUNT
- DFS_NAME_DIR=$MOUNT/hadoop/hdfs/name
- FS_CHECKPOINT_DIR=$MOUNT/hadoop/hdfs/secondary
- DFS_DATA_DIR=$MOUNT/hadoop/hdfs/data
- MAPRED_LOCAL_DIR=$MOUNT/hadoop/mapred/local
- MAX_MAP_TASKS=2
- MAX_REDUCE_TASKS=1
- CHILD_MAP_OPTS=-Xmx550m
- CHILD_REDUCE_OPTS=-Xmx550m
- CHILD_MAP_ULIMIT=1126400
- CHILD_REDUCE_ULIMIT=1126400
- TMP_DIR=$MOUNT/tmp/hadoop-\${user.name}
-
- mkdir -p $MOUNT/hadoop
- chgrp hadoop $MOUNT/hadoop
- chmod g+w $MOUNT/hadoop
- mkdir $MOUNT/tmp
- chmod a+rwxt $MOUNT/tmp
-
- ##############################################################################
- # Modify this section to customize your Hadoop cluster.
- ##############################################################################
- cat > $HADOOP_CONF_DIR/hadoop-site.xml <<EOF
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
- <name>dfs.block.size</name>
- <value>134217728</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.data.dir</name>
- <value>$DFS_DATA_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.datanode.handler.count</name>
- <value>3</value>
- <final>true</final>
-</property>
-<!--property>
- <name>dfs.hosts</name>
- <value>$HADOOP_CONF_DIR/dfs.hosts</value>
- <final>true</final>
-</property-->
-<!--property>
- <name>dfs.hosts.exclude</name>
- <value>$HADOOP_CONF_DIR/dfs.hosts.exclude</value>
- <final>true</final>
-</property-->
-<property>
- <name>dfs.name.dir</name>
- <value>$DFS_NAME_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.namenode.handler.count</name>
- <value>5</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.permissions</name>
- <value>true</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.replication</name>
- <value>$DFS_REPLICATION</value>
-</property>
-<property>
- <name>fs.checkpoint.dir</name>
- <value>$FS_CHECKPOINT_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>fs.default.name</name>
- <value>hdfs://$NN_HOST:8020/</value>
-</property>
-<property>
- <name>fs.trash.interval</name>
- <value>1440</value>
- <final>true</final>
-</property>
-<property>
- <name>hadoop.tmp.dir</name>
- <value>$MOUNT/tmp/hadoop-\${user.name}</value>
- <final>true</final>
-</property>
-<property>
- <name>io.file.buffer.size</name>
- <value>65536</value>
-</property>
-<property>
- <name>mapred.map.child.java.opts</name>
- <value>$CHILD_MAP_OPTS</value>
-</property>
-<property>
- <name>mapred.reduce.child.java.opts</name>
- <value>$CHILD_REDUCE_OPTS</value>
-</property>
-<property>
- <name>mapred.map.child.ulimit</name>
- <value>$CHILD_MAP_ULIMIT</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.reduce.child.ulimit</name>
- <value>$CHILD_REDUCE_ULIMIT</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.job.tracker</name>
- <value>$JT_HOST:8021</value>
-</property>
-<property>
- <name>mapred.job.tracker.handler.count</name>
- <value>5</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.local.dir</name>
- <value>$MAPRED_LOCAL_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.map.tasks.speculative.execution</name>
- <value>true</value>
-</property>
-<property>
- <name>mapred.reduce.parallel.copies</name>
- <value>10</value>
-</property>
-<property>
- <name>mapred.reduce.tasks</name>
- <value>10</value>
-</property>
-<property>
- <name>mapred.reduce.tasks.speculative.execution</name>
- <value>false</value>
-</property>
-<property>
- <name>mapred.submit.replication</name>
- <value>10</value>
-</property>
-<property>
- <name>mapred.system.dir</name>
- <value>/hadoop/system/mapred</value>
-</property>
-<property>
- <name>mapreduce.jobtracker.staging.root.dir</name>
- <value>/user</value>
-</property>
-<property>
- <name>mapred.tasktracker.map.tasks.maximum</name>
- <value>$MAX_MAP_TASKS</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.tasktracker.reduce.tasks.maximum</name>
- <value>$MAX_REDUCE_TASKS</value>
- <final>true</final>
-</property>
-<property>
- <name>tasktracker.http.threads</name>
- <value>46</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
-</property>
-<property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
-</property>
-<property>
- <name>hadoop.rpc.socket.factory.class.default</name>
- <value>org.apache.hadoop.net.StandardSocketFactory</value>
- <final>true</final>
-</property>
-<property>
- <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
- <value></value>
- <final>true</final>
-</property>
-<property>
- <name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
- <value></value>
- <final>true</final>
-</property>
-<property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec</value>
-</property>
-<!-- Hue configuration -->
-<property>
- <name>dfs.namenode.plugins</name>
- <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
- <description>Comma-separated list of namenode plug-ins to be activated.
- </description>
-</property>
-<property>
- <name>dfs.datanode.plugins</name>
- <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
- <description>Comma-separated list of datanode plug-ins to be activated.
- </description>
-</property>
-<property>
- <name>dfs.thrift.address</name>
- <value>0.0.0.0:9090</value>
-</property>
-<property>
- <name>jobtracker.thrift.address</name>
- <value>0.0.0.0:9290</value>
-</property>
-<property>
- <name>mapred.jobtracker.plugins</name>
- <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
- <description>Comma-separated list of jobtracker plug-ins to be activated.</description>
-</property>
-</configuration>
-EOF
+ mkdir -p /data/hadoop
+ chgrp hadoop /data/hadoop
+ chmod g+w /data/hadoop
+ mkdir /data/tmp
+ chmod a+rwxt /data/tmp
+
+ # Copy generated configuration files in place
+ cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
-# Expose /metrics URL endpoint
+ # Expose /metrics URL endpoint
cat > $HADOOP_CONF_DIR/hadoop-metrics.properties <<EOF
# Exposes /metrics URL endpoint for metrics information.
dfs.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
@@ -300,12 +53,12 @@ EOF
sed -i -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"|' \
$HADOOP_CONF_DIR/hadoop-env.sh
- # Hadoop logs should be on the /mnt partition
+ # Hadoop logs should be on the /data partition
rm -rf /var/log/hadoop-0.20
- mkdir -p $MOUNT/hadoop/logs
- chmod g+w $MOUNT/hadoop/logs
- chgrp -R hadoop $MOUNT/hadoop/logs
- ln -s $MOUNT/hadoop/logs /var/log/hadoop-0.20
+ mkdir -p /data/hadoop/logs
+ chmod g+w /data/hadoop/logs
+ chgrp -R hadoop /data/hadoop/logs
+ ln -s /data/hadoop/logs /var/log/hadoop-0.20
chgrp -R hadoop /var/log/hadoop /var/log/hadoop-0.20
for role in $(echo "$ROLES" | tr "," "\n"); do
@@ -334,12 +87,12 @@ function start_namenode() {
apt-get -y install $HADOOP-namenode
AS_HDFS="su -s /bin/bash - hdfs -c"
# Format HDFS
- [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
+ [ ! -e /data/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
elif which rpm &> /dev/null; then
yum install -y $HADOOP-namenode
AS_HDFS="/sbin/runuser -s /bin/bash - hdfs -c"
# Format HDFS
- [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
+ [ ! -e /data/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
fi
service $HADOOP-namenode start
Added: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import static org.apache.whirr.service.RolePredicates.role;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.net.DnsUtil;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.Cluster.Instance;
+import org.apache.whirr.service.ClusterSpec;
+import org.jclouds.scriptbuilder.domain.Statement;
+
+public class HadoopConfigurationBuilder {
+
+ private static final String WHIRR_HADOOP_DEFAULT_PROPERTIES =
+ "whirr-hadoop-default.properties";
+
+ private static Configuration build(ClusterSpec clusterSpec, Cluster cluster,
+ Configuration defaults, String prefix)
+ throws ConfigurationException {
+ CompositeConfiguration config = new CompositeConfiguration();
+ Configuration sub = clusterSpec.getConfigurationForKeysWithPrefix(prefix);
+ config.addConfiguration(sub.subset(prefix)); // remove prefix
+ config.addConfiguration(defaults.subset(prefix));
+ return config;
+ }
+
+ public static Statement buildCommon(String path, ClusterSpec clusterSpec,
+ Cluster cluster) throws ConfigurationException, IOException {
+ Configuration config = buildCommonConfiguration(clusterSpec, cluster,
+ new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
+ return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+ }
+
+ public static Statement buildHdfs(String path, ClusterSpec clusterSpec,
+ Cluster cluster) throws ConfigurationException, IOException {
+ Configuration config = buildHdfsConfiguration(clusterSpec, cluster,
+ new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
+ return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+ }
+
+ public static Statement buildMapReduce(String path, ClusterSpec clusterSpec,
+ Cluster cluster) throws ConfigurationException, IOException {
+ Configuration config = buildMapReduceConfiguration(clusterSpec, cluster,
+ new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
+ return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+ }
+
+ @VisibleForTesting
+ static Configuration buildCommonConfiguration(ClusterSpec clusterSpec,
+ Cluster cluster, Configuration defaults) throws ConfigurationException, IOException {
+ Configuration config = build(clusterSpec, cluster, defaults,
+ "hadoop-common");
+
+ Instance namenode = cluster
+ .getInstanceMatching(role(HadoopNameNodeClusterActionHandler.ROLE));
+ config.setProperty("fs.default.name", String.format("hdfs://%s:8020/",
+ DnsUtil.resolveAddress(namenode.getPublicAddress().getHostAddress())));
+ return config;
+ }
+
+ @VisibleForTesting
+ static Configuration buildHdfsConfiguration(ClusterSpec clusterSpec,
+ Cluster cluster, Configuration defaults) throws ConfigurationException {
+ return build(clusterSpec, cluster, defaults, "hadoop-hdfs");
+ }
+
+ @VisibleForTesting
+ static Configuration buildMapReduceConfiguration(ClusterSpec clusterSpec,
+ Cluster cluster, Configuration defaults) throws ConfigurationException, IOException {
+ Configuration config = build(clusterSpec, cluster, defaults,
+ "hadoop-mapreduce");
+
+ Instance jobtracker = cluster
+ .getInstanceMatching(role(HadoopJobTrackerClusterActionHandler.ROLE));
+ config.setProperty("mapred.job.tracker", String.format("%s:8021",
+ DnsUtil.resolveAddress(jobtracker.getPublicAddress().getHostAddress())));
+ return config;
+ }
+
+}
Propchange: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.configuration.Configuration;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+
+/**
+ * Helper class to convert between Hadoop configuration representations.
+ */
+public class HadoopConfigurationConverter {
+
+ private static final String FINAL_SUFFIX = ".final";
+
+ @VisibleForTesting
+ static List<String> asLinesInFile(Configuration hadoopConfig) {
+ List<String> lines = Lists.newArrayList();
+ lines.add("<configuration>");
+ for (@SuppressWarnings("unchecked")
+ Iterator<String> it = hadoopConfig.getKeys(); it.hasNext(); ) {
+ String key = it.next();
+ if (key.endsWith(FINAL_SUFFIX)) {
+ continue;
+ }
+ Object value = hadoopConfig.getProperty(key);
+ lines.add(" <property>");
+ lines.add(String.format(" <name>%s</name>", key));
+ lines.add(String.format(" <value>%s</value>", value));
+ String finalValue = hadoopConfig.getString(key + FINAL_SUFFIX);
+ if (finalValue != null) {
+ lines.add(String.format(" <final>%s</final>", finalValue));
+ }
+ lines.add(" </property>");
+ }
+ lines.add("</configuration>");
+ return lines;
+ }
+
+ public static Statement asCreateFileStatement(String path,
+ Configuration hadoopConfig) {
+ return Statements.appendFile(path, asLinesInFile(hadoopConfig));
+ }
+
+}
Propchange: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java Thu Feb 10 18:42:29 2011
@@ -19,11 +19,15 @@
package org.apache.whirr.service.hadoop;
import static org.apache.whirr.service.RolePredicates.role;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildCommon;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHdfs;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildMapReduce;
import static org.jclouds.scriptbuilder.domain.Statements.call;
import java.io.IOException;
import java.net.InetAddress;
+import org.apache.commons.configuration.ConfigurationException;
import org.apache.whirr.net.DnsUtil;
import org.apache.whirr.service.Cluster;
import org.apache.whirr.service.Cluster.Instance;
@@ -61,13 +65,21 @@ public class HadoopDataNodeClusterAction
role(HadoopNameNodeClusterActionHandler.ROLE));
InetAddress namenodePublicAddress = instance.getPublicAddress();
InetAddress jobtrackerPublicAddress = namenodePublicAddress;
+
+ try {
+ event.getStatementBuilder().addStatements(
+ buildCommon("/tmp/core-site.xml", clusterSpec, cluster),
+ buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster),
+ buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster)
+ );
+ } catch (ConfigurationException e) {
+ throw new IOException(e);
+ }
String hadoopConfigureFunction = clusterSpec.getConfiguration().getString(
"whirr.hadoop-configure-function", "configure_hadoop");
addStatement(event, call(hadoopConfigureFunction,
"hadoop-datanode,hadoop-tasktracker",
- "-n", DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
- "-j", DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
"-c", clusterSpec.getProvider()
));
}
Modified: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java Thu Feb 10 18:42:29 2011
@@ -19,6 +19,9 @@
package org.apache.whirr.service.hadoop;
import static org.apache.whirr.service.RolePredicates.role;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildCommon;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHdfs;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildMapReduce;
import static org.jclouds.scriptbuilder.domain.Statements.call;
import com.google.common.base.Charsets;
@@ -31,6 +34,7 @@ import java.net.InetAddress;
import java.util.Map.Entry;
import java.util.Properties;
+import org.apache.commons.configuration.ConfigurationException;
import org.apache.whirr.net.DnsUtil;
import org.apache.whirr.service.Cluster;
import org.apache.whirr.service.Cluster.Instance;
@@ -50,7 +54,6 @@ public class HadoopNameNodeClusterAction
public static final String ROLE = "hadoop-namenode";
- public static final int WEB_PORT = 80;
public static final int NAMENODE_PORT = 8020;
public static final int JOBTRACKER_PORT = 8021;
public static final int NAMENODE_WEB_UI_PORT = 50070;
@@ -85,8 +88,6 @@ public class HadoopNameNodeClusterAction
ComputeServiceContext computeServiceContext =
ComputeServiceContextBuilder.build(clusterSpec);
FirewallSettings.authorizeIngress(computeServiceContext, instance, clusterSpec,
- WEB_PORT);
- FirewallSettings.authorizeIngress(computeServiceContext, instance, clusterSpec,
NAMENODE_WEB_UI_PORT);
FirewallSettings.authorizeIngress(computeServiceContext, instance, clusterSpec,
JOBTRACKER_WEB_UI_PORT);
@@ -101,12 +102,20 @@ public class HadoopNameNodeClusterAction
jobtrackerPublicAddress.getHostAddress(), JOBTRACKER_PORT);
}
+ try {
+ event.getStatementBuilder().addStatements(
+ buildCommon("/tmp/core-site.xml", clusterSpec, cluster),
+ buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster),
+ buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster)
+ );
+ } catch (ConfigurationException e) {
+ throw new IOException(e);
+ }
+
String hadoopConfigureFunction = clusterSpec.getConfiguration().getString(
"whirr.hadoop-configure-function", "configure_hadoop");
addStatement(event, call(hadoopConfigureFunction,
"hadoop-namenode,hadoop-jobtracker",
- "-n", DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
- "-j", DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
"-c", clusterSpec.getProvider()
));
}
@@ -123,8 +132,12 @@ public class HadoopNameNodeClusterAction
InetAddress namenodePublicAddress = instance.getPublicAddress();
InetAddress jobtrackerPublicAddress = namenodePublicAddress;
- LOG.info("Web UI available at http://{}",
- DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()));
+ LOG.info("Namenode web UI available at http://{}:{}",
+ DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
+ NAMENODE_WEB_UI_PORT);
+ LOG.info("Jobtracker web UI available at http://{}:{}",
+ DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
+ JOBTRACKER_WEB_UI_PORT);
Properties config = createClientSideProperties(clusterSpec, namenodePublicAddress, jobtrackerPublicAddress);
createClientSideHadoopSiteFile(clusterSpec, config);
createProxyScript(clusterSpec, cluster);
Modified: incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh Thu Feb 10 18:42:29 2011
@@ -5,17 +5,9 @@ function configure_hadoop() {
ROLES=$1
shift
- NN_HOST=
- JT_HOST=
CLOUD_PROVIDER=
- while getopts "n:j:c:" OPTION; do
+ while getopts "c:" OPTION; do
case $OPTION in
- n)
- NN_HOST="$OPTARG"
- ;;
- j)
- JT_HOST="$OPTARG"
- ;;
c)
CLOUD_PROVIDER="$OPTARG"
;;
@@ -24,11 +16,10 @@ function configure_hadoop() {
case $CLOUD_PROVIDER in
ec2 | aws-ec2 )
- # Use public hostname for EC2
- SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname`
+ # Alias /mnt as /data
+ ln -s /mnt /data
;;
*)
- SELF_HOST=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
;;
esac
@@ -36,203 +27,17 @@ function configure_hadoop() {
HADOOP_HOME=/usr/local/hadoop-$HADOOP_VERSION
HADOOP_CONF_DIR=$HADOOP_HOME/conf
- case $CLOUD_PROVIDER in
- ec2 | aws-ec2 )
- MOUNT=/mnt
- ;;
- *)
- MOUNT=/data
- ;;
- esac
- FIRST_MOUNT=$MOUNT
- DFS_NAME_DIR=$MOUNT/hadoop/hdfs/name
- FS_CHECKPOINT_DIR=$MOUNT/hadoop/hdfs/secondary
- DFS_DATA_DIR=$MOUNT/hadoop/hdfs/data
- MAPRED_LOCAL_DIR=$MOUNT/hadoop/mapred/local
- MAX_MAP_TASKS=2
- MAX_REDUCE_TASKS=1
- CHILD_OPTS=-Xmx550m
- CHILD_ULIMIT=1126400
-
- mkdir -p $MOUNT/hadoop
- chown hadoop:hadoop $MOUNT/hadoop
- if [ ! -e $MOUNT/tmp ]; then
- mkdir $MOUNT/tmp
- chmod a+rwxt $MOUNT/tmp
+ mkdir -p /data/hadoop
+ chown hadoop:hadoop /data/hadoop
+ if [ ! -e /data/tmp ]; then
+ mkdir /data/tmp
+ chmod a+rwxt /data/tmp
fi
mkdir /etc/hadoop
ln -s $HADOOP_CONF_DIR /etc/hadoop/conf
- ##############################################################################
- # Modify this section to customize your Hadoop cluster.
- ##############################################################################
- cat > $HADOOP_CONF_DIR/hadoop-site.xml <<EOF
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
- <name>dfs.block.size</name>
- <value>134217728</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.data.dir</name>
- <value>$DFS_DATA_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.datanode.du.reserved</name>
- <value>1073741824</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.datanode.handler.count</name>
- <value>3</value>
- <final>true</final>
-</property>
-<!--property>
- <name>dfs.hosts</name>
- <value>$HADOOP_CONF_DIR/dfs.hosts</value>
- <final>true</final>
-</property-->
-<!--property>
- <name>dfs.hosts.exclude</name>
- <value>$HADOOP_CONF_DIR/dfs.hosts.exclude</value>
- <final>true</final>
-</property-->
-<property>
- <name>dfs.name.dir</name>
- <value>$DFS_NAME_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.namenode.handler.count</name>
- <value>5</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.permissions</name>
- <value>true</value>
- <final>true</final>
-</property>
-<property>
- <name>dfs.replication</name>
- <value>$DFS_REPLICATION</value>
-</property>
-<property>
- <name>fs.checkpoint.dir</name>
- <value>$FS_CHECKPOINT_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>fs.default.name</name>
- <value>hdfs://$NN_HOST:8020/</value>
-</property>
-<property>
- <name>fs.trash.interval</name>
- <value>1440</value>
- <final>true</final>
-</property>
-<property>
- <name>hadoop.tmp.dir</name>
- <value>$MOUNT/tmp/hadoop-\${user.name}</value>
- <final>true</final>
-</property>
-<property>
- <name>io.file.buffer.size</name>
- <value>65536</value>
-</property>
-<property>
- <name>mapred.child.java.opts</name>
- <value>$CHILD_OPTS</value>
-</property>
-<property>
- <name>mapred.child.ulimit</name>
- <value>$CHILD_ULIMIT</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.job.tracker</name>
- <value>$JT_HOST:8021</value>
-</property>
-<property>
- <name>mapred.job.tracker.handler.count</name>
- <value>5</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.local.dir</name>
- <value>$MAPRED_LOCAL_DIR</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.map.tasks.speculative.execution</name>
- <value>true</value>
-</property>
-<property>
- <name>mapred.reduce.parallel.copies</name>
- <value>10</value>
-</property>
-<property>
- <name>mapred.reduce.tasks</name>
- <value>10</value>
-</property>
-<property>
- <name>mapred.reduce.tasks.speculative.execution</name>
- <value>false</value>
-</property>
-<property>
- <name>mapred.submit.replication</name>
- <value>10</value>
-</property>
-<property>
- <name>mapred.system.dir</name>
- <value>/hadoop/system/mapred</value>
-</property>
-<property>
- <name>mapred.tasktracker.map.tasks.maximum</name>
- <value>$MAX_MAP_TASKS</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.tasktracker.reduce.tasks.maximum</name>
- <value>$MAX_REDUCE_TASKS</value>
- <final>true</final>
-</property>
-<property>
- <name>tasktracker.http.threads</name>
- <value>46</value>
- <final>true</final>
-</property>
-<property>
- <name>mapred.compress.map.output</name>
- <value>true</value>
-</property>
-<property>
- <name>mapred.output.compression.type</name>
- <value>BLOCK</value>
-</property>
-<property>
- <name>hadoop.rpc.socket.factory.class.default</name>
- <value>org.apache.hadoop.net.StandardSocketFactory</value>
- <final>true</final>
-</property>
-<property>
- <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
- <value></value>
- <final>true</final>
-</property>
-<property>
- <name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
- <value></value>
- <final>true</final>
-</property>
-<property>
- <name>io.compression.codecs</name>
- <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec</value>
-</property>
-</configuration>
-EOF
+ # Copy generated configuration files in place
+ cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
# Keep PID files in a non-temporary directory
sed -i -e "s|# export HADOOP_PID_DIR=.*|export HADOOP_PID_DIR=/var/run/hadoop|" \
@@ -248,19 +53,18 @@ EOF
sed -i -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"|' \
$HADOOP_CONF_DIR/hadoop-env.sh
- # Hadoop logs should be on the /mnt partition
+ # Hadoop logs should be on the /data partition
sed -i -e 's|# export HADOOP_LOG_DIR=.*|export HADOOP_LOG_DIR=/var/log/hadoop/logs|' \
$HADOOP_CONF_DIR/hadoop-env.sh
rm -rf /var/log/hadoop
- mkdir $MOUNT/hadoop/logs
- chown hadoop:hadoop $MOUNT/hadoop/logs
- ln -s $MOUNT/hadoop/logs /var/log/hadoop
+ mkdir /data/hadoop/logs
+ chown hadoop:hadoop /data/hadoop/logs
+ ln -s /data/hadoop/logs /var/log/hadoop
chown -R hadoop:hadoop /var/log/hadoop
for role in $(echo "$ROLES" | tr "," "\n"); do
case $role in
hadoop-namenode)
- setup_web
start_namenode
;;
hadoop-secondarynamenode)
@@ -280,47 +84,6 @@ EOF
}
-# Sets up small website on cluster.
-function setup_web() {
-
- if which dpkg &> /dev/null; then
- apt-get -y install thttpd
- WWW_BASE=/var/www
- elif which rpm &> /dev/null; then
- # enable EPEL for thttpd package
- rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-4.noarch.rpm
- sed -i -e 's/enabled=1/enabled=0/' /etc/yum.repos.d/epel.repo
- yum install -y --enablerepo=epel thttpd
- chkconfig --add thttpd
- WWW_BASE=/var/www/thttpd/html
- fi
-
- cat > $WWW_BASE/index.html << END
-<html>
-<head>
-<title>Hadoop Cloud Cluster</title>
-</head>
-<body>
-<h1>Hadoop Cloud Cluster</h1>
-To browse the cluster you need to have a proxy configured.
-Start the proxy with <tt>hadoop-cloud proxy <cluster_name></tt>,
-and point your browser to
-<a href="http://apache-hadoop-ec2.s3.amazonaws.com/proxy.pac">this Proxy
-Auto-Configuration (PAC)</a> file. To manage multiple proxy configurations,
-you may wish to use
-<a href="https://addons.mozilla.org/en-US/firefox/addon/2464">FoxyProxy</a>.
-<ul>
-<li><a href="http://$NN_HOST:50070/">NameNode</a>
-<li><a href="http://$JT_HOST:50030/">JobTracker</a>
-</ul>
-</body>
-</html>
-END
-
- service thttpd restart
-
-}
-
function start_namenode() {
if which dpkg &> /dev/null; then
AS_HADOOP="su -s /bin/bash - hadoop -c"
@@ -329,7 +92,7 @@ function start_namenode() {
fi
# Format HDFS
- [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HADOOP "$HADOOP_HOME/bin/hadoop namenode -format"
+ [ ! -e /data/hadoop/hdfs ] && $AS_HADOOP "$HADOOP_HOME/bin/hadoop namenode -format"
$AS_HADOOP "$HADOOP_HOME/bin/hadoop-daemon.sh start namenode"
Added: incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties (added)
+++ incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties Thu Feb 10 18:42:29 2011
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Hadoop defaults. The first part of the key is removed by whirr.
+
+# Common
+hadoop-common.hadoop.tmp.dir=/data/tmp/hadoop-\${user.name}
+hadoop-common.io.file.buffer.size=65536
+hadoop-common.hadoop.rpc.socket.factory.class.default=org.apache.hadoop.net.StandardSocketFactory
+hadoop-common.hadoop.rpc.socket.factory.class.default.final=true
+hadoop-common.hadoop.rpc.socket.factory.class.ClientProtocol=
+hadoop-common.hadoop.rpc.socket.factory.class.JobSubmissionProtocol=
+hadoop-common.fs.trash.interval=1440
+
+# HDFS
+hadoop-hdfs.dfs.block.size=134217728
+hadoop-hdfs.dfs.data.dir=/data/hadoop/hdfs/data
+hadoop-hdfs.dfs.datanode.du.reserved=1073741824
+hadoop-hdfs.dfs.name.dir=/data/hadoop/hdfs/name
+hadoop-hdfs.fs.checkpoint.dir=/data/hadoop/hdfs/secondary
+
+# MR
+hadoop-mapreduce.mapred.local.dir=/data/hadoop/mapred/local
+hadoop-mapreduce.mapred.map.tasks.speculative.execution=true
+hadoop-mapreduce.mapred.reduce.tasks.speculative.execution=false
+hadoop-mapreduce.mapred.system.dir=/hadoop/system/mapred
+hadoop-mapreduce.mapreduce.jobtracker.staging.root.dir=/user
+hadoop-mapreduce.mapred.compress.map.output=true
+hadoop-mapreduce.mapred.output.compression.type=BLOCK
+hadoop-mapreduce.mapred.child.java.opts=-Xmx550m
+hadoop-mapreduce.mapred.child.ulimit=1126400
+hadoop-mapreduce.mapred.tasktracker.map.tasks.maximum=2
+hadoop-mapreduce.mapred.tasktracker.reduce.tasks.maximum=1
+hadoop-mapreduce.mapred.reduce.tasks=10
Propchange: incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+
+import java.net.InetAddress;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.Cluster.Instance;
+import org.apache.whirr.service.ClusterSpec;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.jclouds.domain.Credentials;
+import org.junit.Before;
+import org.junit.Test;
+
+public class HadoopConfigurationBuilderTest {
+
+ static class RegexMatcher extends BaseMatcher {
+ private final String regex;
+
+ public RegexMatcher(String regex) {
+ this.regex = regex;
+ }
+
+ public boolean matches(Object o) {
+ return ((String) o).matches(regex);
+ }
+
+ public void describeTo(Description description) {
+ description.appendText("matches regex=");
+ }
+
+ }
+
+ static RegexMatcher matches(String regex) {
+ return new RegexMatcher(regex);
+ }
+
+ private Configuration defaults;
+ private ClusterSpec clusterSpec;
+ private Cluster cluster;
+
+ @Before
+ public void setUp() throws Exception {
+ defaults = new PropertiesConfiguration();
+ defaults.addProperty("hadoop-common.p1", "common1");
+ defaults.addProperty("hadoop-common.p2", "common2");
+ defaults.addProperty("hadoop-hdfs.p1", "hdfs1");
+ defaults.addProperty("hadoop-mapreduce.p1", "mapred1");
+
+ clusterSpec = ClusterSpec.withTemporaryKeys();
+ InetAddress publicAddress = InetAddress.getLocalHost();
+ InetAddress privateAddress = publicAddress;
+ Instance master = new Instance(new Credentials("", ""),
+ Sets.newHashSet(HadoopNameNodeClusterActionHandler.ROLE,
+ HadoopJobTrackerClusterActionHandler.ROLE),
+ publicAddress, privateAddress, "id");
+ cluster = new Cluster(Sets.newHashSet(master));
+ }
+
+ @Test
+ public void testCommon() throws Exception {
+ Configuration conf = HadoopConfigurationBuilder.buildCommonConfiguration(
+ clusterSpec, cluster, defaults);
+ assertThat(Iterators.size(conf.getKeys()), is(3));
+ assertThat(conf.getString("p1"), is("common1"));
+ assertThat(conf.getString("p2"), is("common2"));
+ assertThat(conf.getString("fs.default.name"), matches("hdfs://.+:8020/"));
+ }
+
+ @Test
+ public void testOverrides() throws Exception {
+ Configuration overrides = new PropertiesConfiguration();
+ overrides.addProperty("hadoop-common.p1", "overridden1");
+ overrides.addProperty("hadoop-common.p2", "overridden2");
+ overrides.addProperty("hadoop-common.fs.default.name", "not-overridden");
+ clusterSpec = ClusterSpec.withNoDefaults(overrides);
+ Configuration conf = HadoopConfigurationBuilder.buildCommonConfiguration(
+ clusterSpec, cluster, defaults);
+ assertThat(Iterators.size(conf.getKeys()), is(3));
+ assertThat(conf.getString("p1"), is("overridden1"));
+ assertThat(conf.getString("p2"), is("overridden2"));
+ assertThat("Can't override dynamically set properties",
+ conf.getString("fs.default.name"), matches("hdfs://.+:8020/"));
+ }
+
+ @Test
+ public void testHdfs() throws Exception {
+ Configuration conf = HadoopConfigurationBuilder.buildHdfsConfiguration(
+ clusterSpec, cluster, defaults);
+ assertThat(Iterators.size(conf.getKeys()), is(1));
+ assertThat(conf.getString("p1"), is("hdfs1"));
+ }
+
+ @Test
+ public void testMapReduce() throws Exception {
+ Configuration conf = HadoopConfigurationBuilder
+ .buildMapReduceConfiguration(clusterSpec, cluster, defaults);
+ assertThat(Iterators.size(conf.getKeys()), is(2));
+ assertThat(conf.getString("p1"), is("mapred1"));
+ assertThat(conf.getString("mapred.job.tracker"), matches(".+:8021"));
+ }
+
+}
Propchange: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.junit.Test;
+
+public class HadoopConfigurationConverterTest {
+
+ @Test
+ public void testConversion() {
+ Configuration conf = new PropertiesConfiguration();
+ conf.setProperty("p1", "v1");
+ conf.setProperty("p2", "v2");
+ List<String> lines = HadoopConfigurationConverter.asLinesInFile(conf);
+ assertThat(lines, is((List<String>) Lists.newArrayList(
+ "<configuration>",
+ " <property>",
+ " <name>p1</name>",
+ " <value>v1</value>",
+ " </property>",
+ " <property>",
+ " <name>p2</name>",
+ " <value>v2</value>",
+ " </property>",
+ "</configuration>"
+ )));
+ }
+
+ @Test
+ public void testFinal() {
+ Configuration conf = new PropertiesConfiguration();
+ conf.setProperty("p1", "v1");
+ conf.setProperty("p2", "v2");
+ conf.setProperty("p2.final", "true");
+ List<String> lines = HadoopConfigurationConverter.asLinesInFile(conf);
+ assertThat(lines, is((List<String>) Lists.newArrayList(
+ "<configuration>",
+ " <property>",
+ " <name>p1</name>",
+ " <value>v1</value>",
+ " </property>",
+ " <property>",
+ " <name>p2</name>",
+ " <value>v2</value>",
+ " <final>true</final>",
+ " </property>",
+ "</configuration>"
+ )));
+ }
+
+}
Propchange: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
------------------------------------------------------------------------------
svn:eol-style = native