You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@whirr.apache.org by to...@apache.org on 2011/02/10 19:42:30 UTC

svn commit: r1069518 - in /incubator/whirr/trunk: ./ core/src/main/java/org/apache/whirr/service/jclouds/ core/src/main/resources/functions/ services/cdh/src/main/resources/functions/ services/hadoop/src/main/java/org/apache/whirr/service/hadoop/ servi...

Author: tomwhite
Date: Thu Feb 10 18:42:29 2011
New Revision: 1069518

URL: http://svn.apache.org/viewvc?rev=1069518&view=rev
Log:
WHIRR-55. Users should be able to override an arbitrary Hadoop property before launch.

Added:
    incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java   (with props)
    incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java   (with props)
    incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties   (with props)
    incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java   (with props)
    incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java   (with props)
Modified:
    incubator/whirr/trunk/CHANGES.txt
    incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
    incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh
    incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
    incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
    incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
    incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh

Modified: incubator/whirr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/CHANGES.txt?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/CHANGES.txt (original)
+++ incubator/whirr/trunk/CHANGES.txt Thu Feb 10 18:42:29 2011
@@ -36,6 +36,9 @@ Trunk (unreleased changes)
     WHIRR-167. Improve bootstrapping and configuration to be able to 
     isolate and repair or evict failing nodes on EC2 (Tibor Kiss via asavu)
 
+    WHIRR-55. Users should be able to override an arbitrary Hadoop property
+    before launch. (tomwhite)
+
   BUG FIXES
 
     WHIRR-170. Instances should be started in the order specified in 

Modified: incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java (original)
+++ incubator/whirr/trunk/core/src/main/java/org/apache/whirr/service/jclouds/StatementBuilder.java Thu Feb 10 18:42:29 2011
@@ -36,6 +36,12 @@ public class StatementBuilder implements
     }
   }
   
+  public void addStatements(Statement... statements) {
+    for (Statement statement : statements) {
+      addStatement(statement);
+    }
+  }
+  
   @Override
   public Iterable<String> functionDependecies(OsFamily family) {
      List<String> functions = Lists.newArrayList();

Modified: incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh (original)
+++ incubator/whirr/trunk/core/src/main/resources/functions/configure_hostnames.sh Thu Feb 10 18:42:29 2011
@@ -13,7 +13,7 @@ function configure_hostnames() {
   done
   
   case $CLOUD_PROVIDER in
-    cloudservers)
+    cloudservers | cloudservers-uk | cloudservers-us )
       if which dpkg &> /dev/null; then
         PRIVATE_IP=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
         HOSTNAME=`echo $PRIVATE_IP | tr . -`.static.cloud-ips.com

Modified: incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh (original)
+++ incubator/whirr/trunk/services/cdh/src/main/resources/functions/configure_cdh_hadoop.sh Thu Feb 10 18:42:29 2011
@@ -5,17 +5,9 @@ function configure_cdh_hadoop() {
   ROLES=$1
   shift
   
-  NN_HOST=
-  JT_HOST=
   CLOUD_PROVIDER=
-  while getopts "n:j:c:" OPTION; do
+  while getopts "c:" OPTION; do
     case $OPTION in
-    n)
-      NN_HOST="$OPTARG"
-      ;;
-    j)
-      JT_HOST="$OPTARG"
-      ;;
     c)
       CLOUD_PROVIDER="$OPTARG"
       ;;
@@ -24,266 +16,27 @@ function configure_cdh_hadoop() {
   
   case $CLOUD_PROVIDER in
     ec2 | aws-ec2 )
-      # Use public hostname for EC2
-      SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname`
+      # Alias /mnt as /data
+      ln -s /mnt /data
       ;;
     *)
-      SELF_HOST=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
       ;;
   esac
   
   REPO=${REPO:-cdh3}
   HADOOP=hadoop-${HADOOP_VERSION:-0.20}
   HADOOP_CONF_DIR=/etc/$HADOOP/conf.dist
-  for role in $(echo "$ROLES" | tr "," "\n"); do
-    case $role in
-    hadoop-namenode)
-      NN_HOST=$SELF_HOST
-      ;;
-    hadoop-jobtracker)
-      JT_HOST=$SELF_HOST
-      ;;
-    esac
-  done
   
-  case $CLOUD_PROVIDER in
-  ec2)
-    MOUNT=/mnt
-    ;;
-  *)
-    MOUNT=/data
-    ;;
-  esac
-  FIRST_MOUNT=$MOUNT
-  DFS_NAME_DIR=$MOUNT/hadoop/hdfs/name
-  FS_CHECKPOINT_DIR=$MOUNT/hadoop/hdfs/secondary
-  DFS_DATA_DIR=$MOUNT/hadoop/hdfs/data
-  MAPRED_LOCAL_DIR=$MOUNT/hadoop/mapred/local
-  MAX_MAP_TASKS=2
-  MAX_REDUCE_TASKS=1
-  CHILD_MAP_OPTS=-Xmx550m
-  CHILD_REDUCE_OPTS=-Xmx550m
-  CHILD_MAP_ULIMIT=1126400
-  CHILD_REDUCE_ULIMIT=1126400
-  TMP_DIR=$MOUNT/tmp/hadoop-\${user.name}
-
-  mkdir -p $MOUNT/hadoop
-  chgrp hadoop $MOUNT/hadoop
-  chmod g+w $MOUNT/hadoop
-  mkdir $MOUNT/tmp
-  chmod a+rwxt $MOUNT/tmp
-
-  ##############################################################################
-  # Modify this section to customize your Hadoop cluster.
-  ##############################################################################
-  cat > $HADOOP_CONF_DIR/hadoop-site.xml <<EOF
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>dfs.block.size</name>
-  <value>134217728</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.data.dir</name>
-  <value>$DFS_DATA_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.datanode.du.reserved</name>
-  <value>1073741824</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.datanode.handler.count</name>
-  <value>3</value>
-  <final>true</final>
-</property>
-<!--property>
-  <name>dfs.hosts</name>
-  <value>$HADOOP_CONF_DIR/dfs.hosts</value>
-  <final>true</final>
-</property-->
-<!--property>
-  <name>dfs.hosts.exclude</name>
-  <value>$HADOOP_CONF_DIR/dfs.hosts.exclude</value>
-  <final>true</final>
-</property-->
-<property>
-  <name>dfs.name.dir</name>
-  <value>$DFS_NAME_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.namenode.handler.count</name>
-  <value>5</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.permissions</name>
-  <value>true</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.replication</name>
-  <value>$DFS_REPLICATION</value>
-</property>
-<property>
-  <name>fs.checkpoint.dir</name>
-  <value>$FS_CHECKPOINT_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>fs.default.name</name>
-  <value>hdfs://$NN_HOST:8020/</value>
-</property>
-<property>
-  <name>fs.trash.interval</name>
-  <value>1440</value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>$MOUNT/tmp/hadoop-\${user.name}</value>
-  <final>true</final>
-</property>
-<property>
-  <name>io.file.buffer.size</name>
-  <value>65536</value>
-</property>
-<property>
-  <name>mapred.map.child.java.opts</name>
-  <value>$CHILD_MAP_OPTS</value>
-</property>
-<property>
-  <name>mapred.reduce.child.java.opts</name>
-  <value>$CHILD_REDUCE_OPTS</value>
-</property>
-<property>
-  <name>mapred.map.child.ulimit</name>
-  <value>$CHILD_MAP_ULIMIT</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.reduce.child.ulimit</name>
-  <value>$CHILD_REDUCE_ULIMIT</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.job.tracker</name>
-  <value>$JT_HOST:8021</value>
-</property>
-<property>
-  <name>mapred.job.tracker.handler.count</name>
-  <value>5</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.local.dir</name>
-  <value>$MAPRED_LOCAL_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.map.tasks.speculative.execution</name>
-  <value>true</value>
-</property>
-<property>
-  <name>mapred.reduce.parallel.copies</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.reduce.tasks</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.reduce.tasks.speculative.execution</name>
-  <value>false</value>
-</property>
-<property>
-  <name>mapred.submit.replication</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.system.dir</name>
-  <value>/hadoop/system/mapred</value>
-</property>
-<property>
-  <name>mapreduce.jobtracker.staging.root.dir</name>
-  <value>/user</value>
-</property>
-<property>
-  <name>mapred.tasktracker.map.tasks.maximum</name>
-  <value>$MAX_MAP_TASKS</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.tasktracker.reduce.tasks.maximum</name>
-  <value>$MAX_REDUCE_TASKS</value>
-  <final>true</final>
-</property>
-<property>
-  <name>tasktracker.http.threads</name>
-  <value>46</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.compress.map.output</name>
-  <value>true</value>
-</property>
-<property>
-  <name>mapred.output.compression.type</name>
-  <value>BLOCK</value>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.default</name>
-  <value>org.apache.hadoop.net.StandardSocketFactory</value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
-  <value></value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
-  <value></value>
-  <final>true</final>
-</property>
-<property>
-  <name>io.compression.codecs</name>
-  <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec</value>
-</property>
-<!-- Hue configuration -->
-<property>
-  <name>dfs.namenode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.NamenodePlugin</value>
-  <description>Comma-separated list of namenode plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>dfs.datanode.plugins</name>
-  <value>org.apache.hadoop.thriftfs.DatanodePlugin</value>
-  <description>Comma-separated list of datanode plug-ins to be activated.
-  </description>
-</property>
-<property>
-  <name>dfs.thrift.address</name>
-  <value>0.0.0.0:9090</value>
-</property>
-<property>
-  <name>jobtracker.thrift.address</name>
-  <value>0.0.0.0:9290</value>
-</property>
-<property>
-  <name>mapred.jobtracker.plugins</name>
-  <value>org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin</value>
-  <description>Comma-separated list of jobtracker plug-ins to be activated.</description>
-</property>
-</configuration>
-EOF
+  mkdir -p /data/hadoop
+  chgrp hadoop /data/hadoop
+  chmod g+w /data/hadoop
+  mkdir /data/tmp
+  chmod a+rwxt /data/tmp
+
+  # Copy generated configuration files in place
+  cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
 
-# Expose /metrics URL endpoint
+  # Expose /metrics URL endpoint
   cat > $HADOOP_CONF_DIR/hadoop-metrics.properties <<EOF
 # Exposes /metrics URL endpoint for metrics information.
 dfs.class=org.apache.hadoop.metrics.spi.NoEmitMetricsContext
@@ -300,12 +53,12 @@ EOF
   sed -i -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"|' \
     $HADOOP_CONF_DIR/hadoop-env.sh
 
-  # Hadoop logs should be on the /mnt partition
+  # Hadoop logs should be on the /data partition
   rm -rf /var/log/hadoop-0.20
-  mkdir -p $MOUNT/hadoop/logs
-  chmod g+w $MOUNT/hadoop/logs
-  chgrp -R hadoop $MOUNT/hadoop/logs
-  ln -s $MOUNT/hadoop/logs /var/log/hadoop-0.20
+  mkdir -p /data/hadoop/logs
+  chmod g+w /data/hadoop/logs
+  chgrp -R hadoop /data/hadoop/logs
+  ln -s /data/hadoop/logs /var/log/hadoop-0.20
   chgrp -R hadoop /var/log/hadoop /var/log/hadoop-0.20
 
   for role in $(echo "$ROLES" | tr "," "\n"); do
@@ -334,12 +87,12 @@ function start_namenode() {
     apt-get -y install $HADOOP-namenode
     AS_HDFS="su -s /bin/bash - hdfs -c"
     # Format HDFS
-    [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
+    [ ! -e /data/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
   elif which rpm &> /dev/null; then
     yum install -y $HADOOP-namenode
     AS_HDFS="/sbin/runuser -s /bin/bash - hdfs -c"
     # Format HDFS
-    [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
+    [ ! -e /data/hadoop/hdfs ] && $AS_HDFS "$HADOOP namenode -format"
   fi
 
   service $HADOOP-namenode start

Added: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import static org.apache.whirr.service.RolePredicates.role;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import java.io.IOException;
+
+import org.apache.commons.configuration.CompositeConfiguration;
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.net.DnsUtil;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.Cluster.Instance;
+import org.apache.whirr.service.ClusterSpec;
+import org.jclouds.scriptbuilder.domain.Statement;
+
+public class HadoopConfigurationBuilder {
+  
+  private static final String WHIRR_HADOOP_DEFAULT_PROPERTIES =
+    "whirr-hadoop-default.properties";
+
+  private static Configuration build(ClusterSpec clusterSpec, Cluster cluster,
+      Configuration defaults, String prefix)
+      throws ConfigurationException {
+    CompositeConfiguration config = new CompositeConfiguration();
+    Configuration sub = clusterSpec.getConfigurationForKeysWithPrefix(prefix);
+    config.addConfiguration(sub.subset(prefix)); // remove prefix
+    config.addConfiguration(defaults.subset(prefix));
+    return config;
+  }
+
+  public static Statement buildCommon(String path, ClusterSpec clusterSpec,
+      Cluster cluster) throws ConfigurationException, IOException {
+    Configuration config = buildCommonConfiguration(clusterSpec, cluster,
+        new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
+    return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+  }
+  
+  public static Statement buildHdfs(String path, ClusterSpec clusterSpec,
+      Cluster cluster) throws ConfigurationException, IOException {
+    Configuration config = buildHdfsConfiguration(clusterSpec, cluster,
+        new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
+    return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+  }
+  
+  public static Statement buildMapReduce(String path, ClusterSpec clusterSpec,
+      Cluster cluster) throws ConfigurationException, IOException {
+    Configuration config = buildMapReduceConfiguration(clusterSpec, cluster,
+        new PropertiesConfiguration(WHIRR_HADOOP_DEFAULT_PROPERTIES));
+    return HadoopConfigurationConverter.asCreateFileStatement(path, config);
+  }
+  
+  @VisibleForTesting
+  static Configuration buildCommonConfiguration(ClusterSpec clusterSpec,
+      Cluster cluster, Configuration defaults) throws ConfigurationException, IOException {
+    Configuration config = build(clusterSpec, cluster, defaults,
+        "hadoop-common");
+
+    Instance namenode = cluster
+        .getInstanceMatching(role(HadoopNameNodeClusterActionHandler.ROLE));
+    config.setProperty("fs.default.name", String.format("hdfs://%s:8020/",
+        DnsUtil.resolveAddress(namenode.getPublicAddress().getHostAddress())));
+    return config;
+  }
+  
+  @VisibleForTesting
+  static Configuration buildHdfsConfiguration(ClusterSpec clusterSpec,
+      Cluster cluster, Configuration defaults) throws ConfigurationException {
+    return build(clusterSpec, cluster, defaults, "hadoop-hdfs");
+  }
+
+  @VisibleForTesting
+  static Configuration buildMapReduceConfiguration(ClusterSpec clusterSpec,
+      Cluster cluster, Configuration defaults) throws ConfigurationException, IOException {
+    Configuration config = build(clusterSpec, cluster, defaults,
+        "hadoop-mapreduce");
+
+    Instance jobtracker = cluster
+        .getInstanceMatching(role(HadoopJobTrackerClusterActionHandler.ROLE));
+    config.setProperty("mapred.job.tracker", String.format("%s:8021",
+        DnsUtil.resolveAddress(jobtracker.getPublicAddress().getHostAddress())));
+    return config;
+  }
+
+}

Propchange: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilder.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.configuration.Configuration;
+import org.jclouds.scriptbuilder.domain.Statement;
+import org.jclouds.scriptbuilder.domain.Statements;
+
+/**
+ * Helper class to convert between Hadoop configuration representations.
+ */
+public class HadoopConfigurationConverter {
+  
+  private static final String FINAL_SUFFIX = ".final";
+
+  @VisibleForTesting
+  static List<String> asLinesInFile(Configuration hadoopConfig) {
+    List<String> lines = Lists.newArrayList();
+    lines.add("<configuration>");
+    for (@SuppressWarnings("unchecked")
+        Iterator<String> it = hadoopConfig.getKeys(); it.hasNext(); ) {
+      String key = it.next();
+      if (key.endsWith(FINAL_SUFFIX)) {
+        continue;
+      }
+      Object value = hadoopConfig.getProperty(key);
+      lines.add("  <property>");
+      lines.add(String.format("    <name>%s</name>", key));
+      lines.add(String.format("    <value>%s</value>", value));
+      String finalValue = hadoopConfig.getString(key + FINAL_SUFFIX);
+      if (finalValue != null) {
+        lines.add(String.format("    <final>%s</final>", finalValue));
+      }
+      lines.add("  </property>");
+    }
+    lines.add("</configuration>");
+    return lines;
+  }
+  
+  public static Statement asCreateFileStatement(String path, 
+      Configuration hadoopConfig) {
+    return Statements.appendFile(path, asLinesInFile(hadoopConfig));
+  }
+
+}

Propchange: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverter.java
------------------------------------------------------------------------------
    svn:eol-style = native

Modified: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopDataNodeClusterActionHandler.java Thu Feb 10 18:42:29 2011
@@ -19,11 +19,15 @@
 package org.apache.whirr.service.hadoop;
 
 import static org.apache.whirr.service.RolePredicates.role;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildCommon;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHdfs;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildMapReduce;
 import static org.jclouds.scriptbuilder.domain.Statements.call;
 
 import java.io.IOException;
 import java.net.InetAddress;
 
+import org.apache.commons.configuration.ConfigurationException;
 import org.apache.whirr.net.DnsUtil;
 import org.apache.whirr.service.Cluster;
 import org.apache.whirr.service.Cluster.Instance;
@@ -61,13 +65,21 @@ public class HadoopDataNodeClusterAction
         role(HadoopNameNodeClusterActionHandler.ROLE));
     InetAddress namenodePublicAddress = instance.getPublicAddress();
     InetAddress jobtrackerPublicAddress = namenodePublicAddress;
+    
+    try {
+      event.getStatementBuilder().addStatements(
+        buildCommon("/tmp/core-site.xml", clusterSpec, cluster),
+        buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster),
+        buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster)
+      );
+    } catch (ConfigurationException e) {
+      throw new IOException(e);
+    }
 
     String hadoopConfigureFunction = clusterSpec.getConfiguration().getString(
         "whirr.hadoop-configure-function", "configure_hadoop");
     addStatement(event, call(hadoopConfigureFunction,
         "hadoop-datanode,hadoop-tasktracker",
-        "-n", DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
-        "-j", DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
         "-c", clusterSpec.getProvider()
     ));
   }

Modified: incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/java/org/apache/whirr/service/hadoop/HadoopNameNodeClusterActionHandler.java Thu Feb 10 18:42:29 2011
@@ -19,6 +19,9 @@
 package org.apache.whirr.service.hadoop;
 
 import static org.apache.whirr.service.RolePredicates.role;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildCommon;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildHdfs;
+import static org.apache.whirr.service.hadoop.HadoopConfigurationBuilder.buildMapReduce;
 import static org.jclouds.scriptbuilder.domain.Statements.call;
 
 import com.google.common.base.Charsets;
@@ -31,6 +34,7 @@ import java.net.InetAddress;
 import java.util.Map.Entry;
 import java.util.Properties;
 
+import org.apache.commons.configuration.ConfigurationException;
 import org.apache.whirr.net.DnsUtil;
 import org.apache.whirr.service.Cluster;
 import org.apache.whirr.service.Cluster.Instance;
@@ -50,7 +54,6 @@ public class HadoopNameNodeClusterAction
   
   public static final String ROLE = "hadoop-namenode";
   
-  public static final int WEB_PORT = 80;
   public static final int NAMENODE_PORT = 8020;
   public static final int JOBTRACKER_PORT = 8021;
   public static final int NAMENODE_WEB_UI_PORT = 50070;
@@ -85,8 +88,6 @@ public class HadoopNameNodeClusterAction
     ComputeServiceContext computeServiceContext =
       ComputeServiceContextBuilder.build(clusterSpec);
     FirewallSettings.authorizeIngress(computeServiceContext, instance, clusterSpec,
-        WEB_PORT);
-    FirewallSettings.authorizeIngress(computeServiceContext, instance, clusterSpec,
         NAMENODE_WEB_UI_PORT);
     FirewallSettings.authorizeIngress(computeServiceContext, instance, clusterSpec,
         JOBTRACKER_WEB_UI_PORT);
@@ -101,12 +102,20 @@ public class HadoopNameNodeClusterAction
           jobtrackerPublicAddress.getHostAddress(), JOBTRACKER_PORT);
     }
     
+    try {
+      event.getStatementBuilder().addStatements(
+        buildCommon("/tmp/core-site.xml", clusterSpec, cluster),
+        buildHdfs("/tmp/hdfs-site.xml", clusterSpec, cluster),
+        buildMapReduce("/tmp/mapred-site.xml", clusterSpec, cluster)
+      );
+    } catch (ConfigurationException e) {
+      throw new IOException(e);
+    }
+    
     String hadoopConfigureFunction = clusterSpec.getConfiguration().getString(
         "whirr.hadoop-configure-function", "configure_hadoop");
     addStatement(event, call(hadoopConfigureFunction,
         "hadoop-namenode,hadoop-jobtracker",
-        "-n", DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
-        "-j", DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
         "-c", clusterSpec.getProvider()
     ));
   }
@@ -123,8 +132,12 @@ public class HadoopNameNodeClusterAction
     InetAddress namenodePublicAddress = instance.getPublicAddress();
     InetAddress jobtrackerPublicAddress = namenodePublicAddress;
 
-    LOG.info("Web UI available at http://{}",
-      DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()));
+    LOG.info("Namenode web UI available at http://{}:{}",
+      DnsUtil.resolveAddress(namenodePublicAddress.getHostAddress()),
+      NAMENODE_WEB_UI_PORT);
+    LOG.info("Jobtracker web UI available at http://{}:{}",
+      DnsUtil.resolveAddress(jobtrackerPublicAddress.getHostAddress()),
+      JOBTRACKER_WEB_UI_PORT);
     Properties config = createClientSideProperties(clusterSpec, namenodePublicAddress, jobtrackerPublicAddress);
     createClientSideHadoopSiteFile(clusterSpec, config);
     createProxyScript(clusterSpec, cluster);

Modified: incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh?rev=1069518&r1=1069517&r2=1069518&view=diff
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh (original)
+++ incubator/whirr/trunk/services/hadoop/src/main/resources/functions/configure_hadoop.sh Thu Feb 10 18:42:29 2011
@@ -5,17 +5,9 @@ function configure_hadoop() {
   ROLES=$1
   shift
   
-  NN_HOST=
-  JT_HOST=
   CLOUD_PROVIDER=
-  while getopts "n:j:c:" OPTION; do
+  while getopts "c:" OPTION; do
     case $OPTION in
-    n)
-      NN_HOST="$OPTARG"
-      ;;
-    j)
-      JT_HOST="$OPTARG"
-      ;;
     c)
       CLOUD_PROVIDER="$OPTARG"
       ;;
@@ -24,11 +16,10 @@ function configure_hadoop() {
   
   case $CLOUD_PROVIDER in
     ec2 | aws-ec2 )
-      # Use public hostname for EC2
-      SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname`
+      # Alias /mnt as /data
+      ln -s /mnt /data
       ;;
     *)
-      SELF_HOST=`/sbin/ifconfig eth0 | grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'`
       ;;
   esac
   
@@ -36,203 +27,17 @@ function configure_hadoop() {
   HADOOP_HOME=/usr/local/hadoop-$HADOOP_VERSION
   HADOOP_CONF_DIR=$HADOOP_HOME/conf
 
-  case $CLOUD_PROVIDER in
-  ec2 | aws-ec2 )
-    MOUNT=/mnt
-    ;;
-  *)
-    MOUNT=/data
-    ;;
-  esac
-  FIRST_MOUNT=$MOUNT
-  DFS_NAME_DIR=$MOUNT/hadoop/hdfs/name
-  FS_CHECKPOINT_DIR=$MOUNT/hadoop/hdfs/secondary
-  DFS_DATA_DIR=$MOUNT/hadoop/hdfs/data
-  MAPRED_LOCAL_DIR=$MOUNT/hadoop/mapred/local
-  MAX_MAP_TASKS=2
-  MAX_REDUCE_TASKS=1
-  CHILD_OPTS=-Xmx550m
-  CHILD_ULIMIT=1126400
-
-  mkdir -p $MOUNT/hadoop
-  chown hadoop:hadoop $MOUNT/hadoop
-  if [ ! -e $MOUNT/tmp ]; then
-    mkdir $MOUNT/tmp
-    chmod a+rwxt $MOUNT/tmp
+  mkdir -p /data/hadoop
+  chown hadoop:hadoop /data/hadoop
+  if [ ! -e /data/tmp ]; then
+    mkdir /data/tmp
+    chmod a+rwxt /data/tmp
   fi
   mkdir /etc/hadoop
   ln -s $HADOOP_CONF_DIR /etc/hadoop/conf
 
-  ##############################################################################
-  # Modify this section to customize your Hadoop cluster.
-  ##############################################################################
-  cat > $HADOOP_CONF_DIR/hadoop-site.xml <<EOF
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>dfs.block.size</name>
-  <value>134217728</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.data.dir</name>
-  <value>$DFS_DATA_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.datanode.du.reserved</name>
-  <value>1073741824</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.datanode.handler.count</name>
-  <value>3</value>
-  <final>true</final>
-</property>
-<!--property>
-  <name>dfs.hosts</name>
-  <value>$HADOOP_CONF_DIR/dfs.hosts</value>
-  <final>true</final>
-</property-->
-<!--property>
-  <name>dfs.hosts.exclude</name>
-  <value>$HADOOP_CONF_DIR/dfs.hosts.exclude</value>
-  <final>true</final>
-</property-->
-<property>
-  <name>dfs.name.dir</name>
-  <value>$DFS_NAME_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.namenode.handler.count</name>
-  <value>5</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.permissions</name>
-  <value>true</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.replication</name>
-  <value>$DFS_REPLICATION</value>
-</property>
-<property>
-  <name>fs.checkpoint.dir</name>
-  <value>$FS_CHECKPOINT_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>fs.default.name</name>
-  <value>hdfs://$NN_HOST:8020/</value>
-</property>
-<property>
-  <name>fs.trash.interval</name>
-  <value>1440</value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>$MOUNT/tmp/hadoop-\${user.name}</value>
-  <final>true</final>
-</property>
-<property>
-  <name>io.file.buffer.size</name>
-  <value>65536</value>
-</property>
-<property>
-  <name>mapred.child.java.opts</name>
-  <value>$CHILD_OPTS</value>
-</property>
-<property>
-  <name>mapred.child.ulimit</name>
-  <value>$CHILD_ULIMIT</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.job.tracker</name>
-  <value>$JT_HOST:8021</value>
-</property>
-<property>
-  <name>mapred.job.tracker.handler.count</name>
-  <value>5</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.local.dir</name>
-  <value>$MAPRED_LOCAL_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.map.tasks.speculative.execution</name>
-  <value>true</value>
-</property>
-<property>
-  <name>mapred.reduce.parallel.copies</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.reduce.tasks</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.reduce.tasks.speculative.execution</name>
-  <value>false</value>
-</property>
-<property>
-  <name>mapred.submit.replication</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.system.dir</name>
-  <value>/hadoop/system/mapred</value>
-</property>
-<property>
-  <name>mapred.tasktracker.map.tasks.maximum</name>
-  <value>$MAX_MAP_TASKS</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.tasktracker.reduce.tasks.maximum</name>
-  <value>$MAX_REDUCE_TASKS</value>
-  <final>true</final>
-</property>
-<property>
-  <name>tasktracker.http.threads</name>
-  <value>46</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.compress.map.output</name>
-  <value>true</value>
-</property>
-<property>
-  <name>mapred.output.compression.type</name>
-  <value>BLOCK</value>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.default</name>
-  <value>org.apache.hadoop.net.StandardSocketFactory</value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
-  <value></value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
-  <value></value>
-  <final>true</final>
-</property>
-<property>
-  <name>io.compression.codecs</name>
-  <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec</value>
-</property>
-</configuration>
-EOF
+  # Copy generated configuration files in place
+  cp /tmp/{core,hdfs,mapred}-site.xml $HADOOP_CONF_DIR
 
   # Keep PID files in a non-temporary directory
   sed -i -e "s|# export HADOOP_PID_DIR=.*|export HADOOP_PID_DIR=/var/run/hadoop|" \
@@ -248,19 +53,18 @@ EOF
   sed -i -e 's|# export HADOOP_OPTS=.*|export HADOOP_OPTS="-Djava.net.preferIPv4Stack=true"|' \
     $HADOOP_CONF_DIR/hadoop-env.sh
 
-  # Hadoop logs should be on the /mnt partition
+  # Hadoop logs should be on the /data partition
   sed -i -e 's|# export HADOOP_LOG_DIR=.*|export HADOOP_LOG_DIR=/var/log/hadoop/logs|' \
     $HADOOP_CONF_DIR/hadoop-env.sh
   rm -rf /var/log/hadoop
-  mkdir $MOUNT/hadoop/logs
-  chown hadoop:hadoop $MOUNT/hadoop/logs
-  ln -s $MOUNT/hadoop/logs /var/log/hadoop
+  mkdir /data/hadoop/logs
+  chown hadoop:hadoop /data/hadoop/logs
+  ln -s /data/hadoop/logs /var/log/hadoop
   chown -R hadoop:hadoop /var/log/hadoop
   
   for role in $(echo "$ROLES" | tr "," "\n"); do
     case $role in
     hadoop-namenode)
-      setup_web
       start_namenode
       ;;
     hadoop-secondarynamenode)
@@ -280,47 +84,6 @@ EOF
 
 }
 
-# Sets up small website on cluster.
-function setup_web() {
-
-  if which dpkg &> /dev/null; then
-    apt-get -y install thttpd
-    WWW_BASE=/var/www
-  elif which rpm &> /dev/null; then
-    # enable EPEL for thttpd package
-    rpm -Uvh http://download.fedora.redhat.com/pub/epel/5/i386/epel-release-5-4.noarch.rpm
-    sed -i -e 's/enabled=1/enabled=0/' /etc/yum.repos.d/epel.repo
-    yum install -y --enablerepo=epel thttpd
-    chkconfig --add thttpd
-    WWW_BASE=/var/www/thttpd/html
-  fi
-
-  cat > $WWW_BASE/index.html << END
-<html>
-<head>
-<title>Hadoop Cloud Cluster</title>
-</head>
-<body>
-<h1>Hadoop Cloud Cluster</h1>
-To browse the cluster you need to have a proxy configured.
-Start the proxy with <tt>hadoop-cloud proxy &lt;cluster_name&gt;</tt>,
-and point your browser to
-<a href="http://apache-hadoop-ec2.s3.amazonaws.com/proxy.pac">this Proxy
-Auto-Configuration (PAC)</a> file.  To manage multiple proxy configurations,
-you may wish to use
-<a href="https://addons.mozilla.org/en-US/firefox/addon/2464">FoxyProxy</a>.
-<ul>
-<li><a href="http://$NN_HOST:50070/">NameNode</a>
-<li><a href="http://$JT_HOST:50030/">JobTracker</a>
-</ul>
-</body>
-</html>
-END
-
-  service thttpd restart
-
-}
-
 function start_namenode() {
   if which dpkg &> /dev/null; then
     AS_HADOOP="su -s /bin/bash - hadoop -c"
@@ -329,7 +92,7 @@ function start_namenode() {
   fi
 
   # Format HDFS
-  [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HADOOP "$HADOOP_HOME/bin/hadoop namenode -format"
+  [ ! -e /data/hadoop/hdfs ] && $AS_HADOOP "$HADOOP_HOME/bin/hadoop namenode -format"
 
   $AS_HADOOP "$HADOOP_HOME/bin/hadoop-daemon.sh start namenode"
 

Added: incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties (added)
+++ incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties Thu Feb 10 18:42:29 2011
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Hadoop defaults. The first part of the key is removed by whirr.
+
+# Common
+hadoop-common.hadoop.tmp.dir=/data/tmp/hadoop-\${user.name}
+hadoop-common.io.file.buffer.size=65536
+hadoop-common.hadoop.rpc.socket.factory.class.default=org.apache.hadoop.net.StandardSocketFactory
+hadoop-common.hadoop.rpc.socket.factory.class.default.final=true
+hadoop-common.hadoop.rpc.socket.factory.class.ClientProtocol=
+hadoop-common.hadoop.rpc.socket.factory.class.JobSubmissionProtocol=
+hadoop-common.fs.trash.interval=1440
+
+# HDFS
+hadoop-hdfs.dfs.block.size=134217728
+hadoop-hdfs.dfs.data.dir=/data/hadoop/hdfs/data
+hadoop-hdfs.dfs.datanode.du.reserved=1073741824
+hadoop-hdfs.dfs.name.dir=/data/hadoop/hdfs/name
+hadoop-hdfs.fs.checkpoint.dir=/data/hadoop/hdfs/secondary
+
+# MR
+hadoop-mapreduce.mapred.local.dir=/data/hadoop/mapred/local
+hadoop-mapreduce.mapred.map.tasks.speculative.execution=true
+hadoop-mapreduce.mapred.reduce.tasks.speculative.execution=false
+hadoop-mapreduce.mapred.system.dir=/hadoop/system/mapred
+hadoop-mapreduce.mapreduce.jobtracker.staging.root.dir=/user
+hadoop-mapreduce.mapred.compress.map.output=true
+hadoop-mapreduce.mapred.output.compression.type=BLOCK
+hadoop-mapreduce.mapred.child.java.opts=-Xmx550m
+hadoop-mapreduce.mapred.child.ulimit=1126400
+hadoop-mapreduce.mapred.tasktracker.map.tasks.maximum=2
+hadoop-mapreduce.mapred.tasktracker.reduce.tasks.maximum=1
+hadoop-mapreduce.mapred.reduce.tasks=10

Propchange: incubator/whirr/trunk/services/hadoop/src/main/resources/whirr-hadoop-default.properties
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Sets;
+
+import java.net.InetAddress;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.apache.whirr.service.Cluster;
+import org.apache.whirr.service.Cluster.Instance;
+import org.apache.whirr.service.ClusterSpec;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.jclouds.domain.Credentials;
+import org.junit.Before;
+import org.junit.Test;
+
+public class HadoopConfigurationBuilderTest {
+  
+  static class RegexMatcher extends BaseMatcher {
+    private final String regex;
+
+    public RegexMatcher(String regex) {
+      this.regex = regex;
+    }
+
+    public boolean matches(Object o) {
+      return ((String) o).matches(regex);
+    }
+
+    public void describeTo(Description description) {
+      description.appendText("matches regex=");
+    }
+
+  }
+  
+  static RegexMatcher matches(String regex) {
+    return new RegexMatcher(regex);
+  }
+  
+  private Configuration defaults;
+  private ClusterSpec clusterSpec;
+  private Cluster cluster;
+  
+  @Before
+  public void setUp() throws Exception {
+    defaults = new PropertiesConfiguration();
+    defaults.addProperty("hadoop-common.p1", "common1");
+    defaults.addProperty("hadoop-common.p2", "common2");
+    defaults.addProperty("hadoop-hdfs.p1", "hdfs1");
+    defaults.addProperty("hadoop-mapreduce.p1", "mapred1");
+
+    clusterSpec = ClusterSpec.withTemporaryKeys();
+    InetAddress publicAddress = InetAddress.getLocalHost();
+    InetAddress privateAddress = publicAddress;
+    Instance master = new Instance(new Credentials("", ""),
+        Sets.newHashSet(HadoopNameNodeClusterActionHandler.ROLE,
+            HadoopJobTrackerClusterActionHandler.ROLE),
+        publicAddress, privateAddress, "id");
+    cluster = new Cluster(Sets.newHashSet(master));
+  }
+
+  @Test
+  public void testCommon() throws Exception {
+    Configuration conf = HadoopConfigurationBuilder.buildCommonConfiguration(
+        clusterSpec, cluster, defaults);
+    assertThat(Iterators.size(conf.getKeys()), is(3));
+    assertThat(conf.getString("p1"), is("common1"));
+    assertThat(conf.getString("p2"), is("common2"));
+    assertThat(conf.getString("fs.default.name"), matches("hdfs://.+:8020/"));
+  }
+  
+  @Test
+  public void testOverrides() throws Exception {
+    Configuration overrides = new PropertiesConfiguration();
+    overrides.addProperty("hadoop-common.p1", "overridden1");
+    overrides.addProperty("hadoop-common.p2", "overridden2");
+    overrides.addProperty("hadoop-common.fs.default.name", "not-overridden");
+    clusterSpec = ClusterSpec.withNoDefaults(overrides);
+    Configuration conf = HadoopConfigurationBuilder.buildCommonConfiguration(
+        clusterSpec, cluster, defaults);
+    assertThat(Iterators.size(conf.getKeys()), is(3));
+    assertThat(conf.getString("p1"), is("overridden1"));
+    assertThat(conf.getString("p2"), is("overridden2"));
+    assertThat("Can't override dynamically set properties",
+        conf.getString("fs.default.name"), matches("hdfs://.+:8020/"));
+  }
+
+  @Test
+  public void testHdfs() throws Exception {
+    Configuration conf = HadoopConfigurationBuilder.buildHdfsConfiguration(
+        clusterSpec, cluster, defaults);
+    assertThat(Iterators.size(conf.getKeys()), is(1));
+    assertThat(conf.getString("p1"), is("hdfs1"));
+  }
+
+  @Test
+  public void testMapReduce() throws Exception {
+    Configuration conf = HadoopConfigurationBuilder
+        .buildMapReduceConfiguration(clusterSpec, cluster, defaults);
+    assertThat(Iterators.size(conf.getKeys()), is(2));
+    assertThat(conf.getString("p1"), is("mapred1"));
+    assertThat(conf.getString("mapred.job.tracker"), matches(".+:8021"));
+  }
+
+}

Propchange: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationBuilderTest.java
------------------------------------------------------------------------------
    svn:eol-style = native

Added: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
URL: http://svn.apache.org/viewvc/incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java?rev=1069518&view=auto
==============================================================================
--- incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java (added)
+++ incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java Thu Feb 10 18:42:29 2011
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.whirr.service.hadoop;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+import org.apache.commons.configuration.Configuration;
+import org.apache.commons.configuration.PropertiesConfiguration;
+import org.junit.Test;
+
+public class HadoopConfigurationConverterTest {
+
+  @Test
+  public void testConversion() {
+    Configuration conf = new PropertiesConfiguration();
+    conf.setProperty("p1", "v1");
+    conf.setProperty("p2", "v2");
+    List<String> lines = HadoopConfigurationConverter.asLinesInFile(conf);
+    assertThat(lines, is((List<String>) Lists.newArrayList(
+        "<configuration>",
+        "  <property>",
+        "    <name>p1</name>",
+        "    <value>v1</value>",
+        "  </property>",
+        "  <property>",
+        "    <name>p2</name>",
+        "    <value>v2</value>",
+        "  </property>",
+        "</configuration>"
+    )));
+  }
+
+  @Test
+  public void testFinal() {
+    Configuration conf = new PropertiesConfiguration();
+    conf.setProperty("p1", "v1");
+    conf.setProperty("p2", "v2");
+    conf.setProperty("p2.final", "true");
+    List<String> lines = HadoopConfigurationConverter.asLinesInFile(conf);
+    assertThat(lines, is((List<String>) Lists.newArrayList(
+        "<configuration>",
+        "  <property>",
+        "    <name>p1</name>",
+        "    <value>v1</value>",
+        "  </property>",
+        "  <property>",
+        "    <name>p2</name>",
+        "    <value>v2</value>",
+        "    <final>true</final>",
+        "  </property>",
+        "</configuration>"
+    )));
+  }
+
+}

Propchange: incubator/whirr/trunk/services/hadoop/src/test/java/org/apache/whirr/service/hadoop/HadoopConfigurationConverterTest.java
------------------------------------------------------------------------------
    svn:eol-style = native