You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2012/03/28 03:02:07 UTC
svn commit: r1306090 - in
/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet: manifests/
modules/hadoop-oozie/templates/ modules/hadoop/manifests/
modules/hadoop/templates/ modules/nfs/ modules/nfs/manifests/
modules/nfs/manifests/client/
Author: rvs
Date: Wed Mar 28 01:02:06 2012
New Revision: 1306090
URL: http://svn.apache.org/viewvc?rev=1306090&view=rev
Log:
BIGTOP-495. [puppet] Add support for HA namenodes (Patrick Taylor Ramsey via rvs)
Added:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/init.pp
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/site.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp?rev=1306090&r1=1306089&r2=1306090&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp Wed Mar 28 01:02:06 2012
@@ -16,13 +16,30 @@
class hadoop_cluster_node {
require bigtop_util
- $hadoop_namenode_host = $hadoop_head_node
+ $hadoop_head_node = extlookup("hadoop_head_node")
+ $standby_head_node = extlookup("standby_head_node", "")
+ $hadoop_gateway_node = extlookup("hadoop_gateway_node", $hadoop_head_node)
+
+ $hadoop_ha = $standby_head_node ? {
+ "" => disabled,
+ default => enabled,
+ }
+
+ $hadoop_namenode_host = $hadoop_ha ? {
+ "enabled" => [ $hadoop_head_node, $standby_head_node ],
+ default => $hadoop_head_node,
+ }
$hadoop_namenode_port = extlookup("hadoop_namenode_port", "17020")
$hadoop_namenode_thrift_port = extlookup("hadoop_namenode_thrift_port", "10090")
$hadoop_dfs_namenode_plugins = extlookup("hadoop_dfs_namenode_plugins", "")
$hadoop_dfs_datanode_plugins = extlookup("hadoop_dfs_datanode_plugins", "")
# $hadoop_dfs_namenode_plugins="org.apache.hadoop.thriftfs.NamenodePlugin"
# $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
+ $hadoop_ha_nameservice_id = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
+ $hadoop_namenode_uri = $hadoop_ha ? {
+ "enabled" => "hdfs://${hadoop_ha_nameservice_id}:8020",
+ default => "hdfs://$hadoop_namenode_host:$hadoop_namenode_port",
+ }
$hadoop_rm_host = $hadoop_head_node
$hadoop_rt_port = extlookup("hadoop_rt_port", "8025")
@@ -42,11 +59,11 @@ class hadoop_cluster_node {
# $hadoop_mapred_jobtracker_plugins="org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin"
# $hadoop_mapred_tasktracker_plugins="org.apache.hadoop.mapred.TaskTrackerCmonInst"
- $hadoop_core_proxyusers = { oozie => { groups => 'root,hadoop,jenkins,oozie,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" },
- httpfs => { groups => 'root,hadoop,jenkins,oozie,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" } }
+ $hadoop_core_proxyusers = { oozie => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" },
+ httpfs => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" } }
$hbase_relative_rootdir = extlookup("hadoop_hbase_rootdir", "/hbase")
- $hadoop_hbase_rootdir = "hdfs://$hadoop_namenode_host:$hadoop_namenode_port/$hbase_relative_rootdir"
+ $hadoop_hbase_rootdir = "$hadoop_namenode_uri$hbase_relative_rootdir"
$hadoop_hbase_zookeeper_quorum = $hadoop_head_node
$hbase_heap_size = extlookup("hbase_heap_size", "1024")
@@ -69,12 +86,14 @@ class hadoop_cluster_node {
}
}
+
class hadoop_worker_node inherits hadoop_cluster_node {
hadoop::datanode { "datanode":
namenode_host => $hadoop_namenode_host,
namenode_port => $hadoop_namenode_port,
dirs => $hdfs_data_dirs,
auth => $hadoop_security_authentication,
+ ha => $hadoop_ha,
}
hadoop::nodemanager { "nodemanager":
@@ -114,12 +133,15 @@ class hadoop_head_node inherits hadoop_c
dirs => $namenode_data_dirs,
# thrift_port => $hadoop_namenode_thrift_port,
auth => $hadoop_security_authentication,
+ ha => $hadoop_ha,
}
- hadoop::secondarynamenode { "secondary namenode":
- namenode_host => $hadoop_namenode_host,
- namenode_port => $hadoop_namenode_port,
- auth => $hadoop_security_authentication,
+ if ($hadoop_ha != "enabled") {
+ hadoop::secondarynamenode { "secondary namenode":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ auth => $hadoop_security_authentication,
+ }
}
hadoop::resourcemanager { "resourcemanager":
@@ -176,7 +198,18 @@ class hadoop_head_node inherits hadoop_c
}
}
-class hadoop_gateway_node inherits hadoop_head_node {
+class standby_head_node inherits hadoop_worker_node {
+ hadoop::namenode { "namenode":
+ host => $hadoop_namenode_host,
+ port => $hadoop_namenode_port,
+ dirs => $namenode_data_dirs,
+ # thrift_port => $hadoop_namenode_thrift_port,
+ auth => $hadoop_security_authentication,
+ ha => $hadoop_ha,
+ }
+}
+
+class hadoop_gateway_node inherits hadoop_cluster_node {
hadoop::client { "hadoop client":
namenode_host => $hadoop_namenode_host,
namenode_port => $hadoop_namenode_port,
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/site.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/site.pp?rev=1306090&r1=1306089&r2=1306090&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/site.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/site.pp Wed Mar 28 01:02:06 2012
@@ -32,21 +32,27 @@ package { "jdk":
ensure => "installed",
}
-# $hadoop_head_node="beefy.my.org"
-# $hadoop_security_authentication="kerberos"
-
import "cluster.pp"
node default {
- # Fails if hadoop_head_node is unset
- if (!$::hadoop_head_node) {
- $hadoop_head_node = extlookup("hadoop_head_node")
+ $hadoop_head_node = extlookup("hadoop_head_node")
+ $standby_head_node = extlookup("standby_head_node", "")
+ $hadoop_gateway_node = extlookup("hadoop_gateway_node", $hadoop_head_node)
+
+ case $::fqdn {
+ $hadoop_head_node: {
+ include hadoop_head_node
+ }
+ $standby_head_node: {
+ include standby_head_node
+ }
+ default: {
+ include hadoop_worker_node
+ }
}
- if ($hadoop_head_node == $fqdn) {
+ if ($hadoop_gateway_node == $::fqdn) {
include hadoop_gateway_node
- } else {
- include hadoop_worker_node
}
}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml?rev=1306090&r1=1306089&r2=1306090&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml Wed Mar 28 01:02:06 2012
@@ -1,25 +1,42 @@
<?xml version="1.0"?>
<!--
- Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
+ limitations under the License.
-->
-
<configuration>
-<!--
- Refer to the oozie-default.xml file for the complete list of
- Oozie configuration properties and their default values.
--->
+ <!--
+ Refer to the oozie-default.xml file for the complete list of
+ Oozie configuration properties and their default values.
+ -->
+
+ <property>
+ <name>oozie.service.ActionService.executor.ext.classes</name>
+ <value>
+ org.apache.oozie.action.email.EmailActionExecutor,
+ org.apache.oozie.action.hadoop.HiveActionExecutor,
+ org.apache.oozie.action.hadoop.ShellActionExecutor,
+ org.apache.oozie.action.hadoop.SqoopActionExecutor
+ </value>
+ </property>
+
+ <property>
+ <name>oozie.service.SchemaService.wf.ext.schemas</name>
+ <value>shell-action-0.1.xsd,email-action-0.1.xsd,hive-action-0.2.xsd,sqoop-action-0.2.xsd,ssh-action-0.1.xsd</value>
+ </property>
<property>
<name>oozie.system.id</name>
@@ -103,8 +120,8 @@
</property>
<property>
- <name>oozie.service.StoreService.create.db.schema</name>
- <value>true</value>
+ <name>oozie.service.JPAService.create.db.schema</name>
+ <value>false</value>
<description>
Creates Oozie DB.
@@ -114,7 +131,7 @@
</property>
<property>
- <name>oozie.service.StoreService.jdbc.driver</name>
+ <name>oozie.service.JPAService.jdbc.driver</name>
<value>org.apache.derby.jdbc.EmbeddedDriver</value>
<description>
JDBC driver class.
@@ -122,7 +139,7 @@
</property>
<property>
- <name>oozie.service.StoreService.jdbc.url</name>
+ <name>oozie.service.JPAService.jdbc.url</name>
<value>jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true</value>
<description>
JDBC URL.
@@ -130,7 +147,7 @@
</property>
<property>
- <name>oozie.service.StoreService.jdbc.username</name>
+ <name>oozie.service.JPAService.jdbc.username</name>
<value>sa</value>
<description>
DB user name.
@@ -138,21 +155,18 @@
</property>
<property>
- <name>oozie.service.StoreService.jdbc.password</name>
+ <name>oozie.service.JPAService.jdbc.password</name>
<value> </value>
<description>
DB user password.
IMPORTANT: if password is emtpy leave a 1 space string, the service trims the value,
if empty Configuration assumes it is NULL.
-
- IMPORTANT: if the StoreServicePasswordService is active, it will reset this value with the value given in
- the console.
</description>
</property>
<property>
- <name>oozie.service.StoreService.pool.max.active.conn</name>
+ <name>oozie.service.JPAService.pool.max.active.conn</name>
<value>10</value>
<description>
Max number of connections.
@@ -208,6 +222,19 @@
</property>
<property>
+ <name>oozie.service.HadoopAccessorService.hadoop.configurations</name>
+ <value>*=/etc/hadoop/conf</value>
+ <description>
+ Comma separated AUTHORITY=HADOOP_CONF_DIR, where AUTHORITY is the HOST:PORT of
+ the Hadoop service (JobTracker, HDFS). The wildcard '*' configuration is
+ used when there is no exact match for an authority. The HADOOP_CONF_DIR contains
+ the relevant Hadoop *-site.xml files. If the path is relative is looked within
+ the Oozie configuration directory; though the path can be absolute (i.e. to point
+ to Hadoop client conf/ directories in the local filesystem.
+ </description>
+ </property>
+
+ <property>
<name>oozie.service.WorkflowAppService.system.libpath</name>
<value>/user/${user.name}/share/lib</value>
<description>
@@ -294,5 +321,44 @@
</description>
</property>
-</configuration>
+ <!-- Proxyuser Configuration -->
+ <!--
+
+ <property>
+ <name>oozie.service.ProxyUserService.proxyuser.#USER#.hosts</name>
+ <value>*</value>
+ <description>
+ List of hosts the '#USER#' user is allowed to perform 'doAs'
+ operations.
+
+ The '#USER#' must be replaced with the username o the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard or a list of hostnames.
+
+ For multiple users copy this property and replace the user name
+ in the property name.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.ProxyUserService.proxyuser.#USER#.groups</name>
+ <value>*</value>
+ <description>
+ List of groups the '#USER#' user is allowed to impersonate users
+ from to perform 'doAs' operations.
+
+ The '#USER#' must be replaced with the username o the user who is
+ allowed to perform 'doAs' operations.
+
+ The value can be the '*' wildcard or a list of groups.
+
+ For multiple users copy this property and replace the user name
+ in the property name.
+ </description>
+ </property>
+
+ -->
+
+</configuration>
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1306090&r1=1306089&r2=1306090&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Wed Mar 28 01:02:06 2012
@@ -75,6 +75,14 @@ class hadoop {
}
class common-hdfs inherits common {
+ if ($auth == "kerberos" and $ha == "enabled") {
+ fail("High-availability secure clusters are not currently supported")
+ }
+
+ if ($ha == 'enabled') {
+ $nameservice_id = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
+ }
+
package { "hadoop-hdfs":
ensure => latest,
require => [Package["jdk"], Package["hadoop"]],
@@ -111,13 +119,21 @@ class hadoop {
}
}
- define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple", $dirs = ["/tmp/data"]) {
+ define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple", $dirs = ["/tmp/data"], $ha = 'disabled') {
- $hadoop_namenode_host = $namenode_host
- $hadoop_namenode_port = $namenode_port
- $hadoop_datanode_port = $port
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_datanode_port = $port
$hadoop_security_authentication = $auth
+ if ($ha == 'enabled') {
+ # Needed by hdfs-site.xml
+ $sshfence_keydir = "/usr/lib/hadoop/.ssh"
+ $sshfence_keypath = "$sshfence_keydir/id_sshfence"
+ $sshfence_user = extlookup("hadoop_ha_sshfence_user", "hdfs")
+ $shared_edits_dir = extlookup("hadoop_ha_shared_edits_dir", "/hdfs_shared")
+ }
+
include common-hdfs
package { "hadoop-hdfs-datanode":
@@ -213,17 +229,79 @@ class hadoop {
user => "hdfs",
command => "/bin/bash -c 'hadoop fs -mkdir $title && hadoop fs -chmod $perm $title && hadoop fs -chown $user $title'",
unless => "/bin/bash -c 'hadoop fs -ls $name >/dev/null 2>&1'",
- require => [ Service["hadoop-hdfs-namenode"], Exec["namenode format"] ],
+ require => Service["hadoop-hdfs-namenode"],
}
+ Exec <| title == "activate nn1" |> -> Exec["HDFS init $title"]
}
- define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"]) {
+ define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"], $ha = 'disabled') {
+ $first_namenode = inline_template("<%= host.to_a[0] %>")
$hadoop_namenode_host = $host
$hadoop_namenode_port = $port
$hadoop_namenode_thrift_port = $thrift_port
$hadoop_security_authentication = $auth
+ if ($ha == 'enabled') {
+ $sshfence_user = extlookup("hadoop_ha_sshfence_user", "hdfs")
+ $sshfence_user_home = extlookup("hadoop_ha_sshfence_user_home", "/var/lib/hadoop-hdfs")
+ $sshfence_keydir = "$sshfence_user_home/.ssh"
+ $sshfence_keypath = "$sshfence_keydir/id_sshfence"
+ $sshfence_privkey = extlookup("hadoop_ha_sshfence_privkey", "$extlookup_datadir/hadoop/id_sshfence")
+ $sshfence_pubkey = extlookup("hadoop_ha_sshfence_pubkey", "$extlookup_datadir/hadoop/id_sshfence.pub")
+ $shared_edits_dir = extlookup("hadoop_ha_shared_edits_dir", "/hdfs_shared")
+ $nfs_server = extlookup("hadoop_ha_nfs_server", "")
+ $nfs_path = extlookup("hadoop_ha_nfs_path", "")
+
+ file { $sshfence_keydir:
+ ensure => directory,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0700',
+ require => Package["hadoop-hdfs"],
+ }
+
+ file { $sshfence_keypath:
+ source => $sshfence_privkey,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0600',
+ before => Service["hadoop-hdfs-namenode"],
+ require => File[$sshfence_keydir],
+ }
+
+ file { "$sshfence_keydir/authorized_keys":
+ source => $sshfence_pubkey,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0600',
+ before => Service["hadoop-hdfs-namenode"],
+ require => File[$sshfence_keydir],
+ }
+
+ file { $shared_edits_dir:
+ ensure => directory,
+ }
+
+ if ($nfs_server) {
+ if (!$nfs_path) {
+ fail("No nfs share specified for shared edits dir")
+ }
+
+ require nfs::client
+
+ mount { $shared_edits_dir:
+ ensure => "mounted",
+ atboot => true,
+ device => "${nfs_server}:${nfs_path}",
+ fstype => "nfs",
+ options => "tcp,soft,timeo=10,intr,rsize=32768,wsize=32768",
+ require => File[$shared_edits_dir],
+ before => Service["hadoop-hdfs-namenode"],
+ }
+ }
+ }
+
include common-hdfs
package { "hadoop-hdfs-namenode":
@@ -235,16 +313,35 @@ class hadoop {
ensure => running,
hasstatus => true,
subscribe => [Package["hadoop-hdfs-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [Package["hadoop-hdfs-namenode"], Exec["namenode format"]],
+ require => [Package["hadoop-hdfs-namenode"]],
}
+ Exec <| tag == "namenode-format" |> -> Service["hadoop-hdfs-namenode"]
Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-namenode"]
- exec { "namenode format":
- user => "hdfs",
- command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
- creates => "${namenode_data_dirs[0]}/current/VERSION",
- require => [ Package["hadoop-hdfs-namenode"], File[$dirs] ],
- }
+ if ($::fqdn == $first_namenode) {
+ exec { "namenode format":
+ user => "hdfs",
+ command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
+ creates => "${namenode_data_dirs[0]}/current/VERSION",
+ require => [ Package["hadoop-hdfs-namenode"], File[$dirs] ],
+ tag => "namenode-format",
+ }
+
+ if ($ha == "enabled") {
+ exec { "activate nn1":
+ command => "/usr/bin/hdfs haadmin -transitionToActive nn1",
+ user => "hdfs",
+ unless => "/usr/bin/test $(/usr/bin/hdfs haadmin -getServiceState nn1) = active",
+ require => Service["hadoop-hdfs-namenode"],
+ }
+ }
+ } elsif ($ha == "enabled") {
+ hadoop::namedir_copy { $namenode_data_dirs:
+ source => $first_namenode,
+ ssh_identity => $sshfence_keypath,
+ require => File[$sshfence_keypath],
+ }
+ }
file {
"/etc/default/hadoop-hdfs-namenode":
@@ -261,6 +358,15 @@ class hadoop {
}
}
+ define namedir_copy ($source, $ssh_identity) {
+ exec { "copy namedir $title from first namenode":
+ command => "/usr/bin/rsync -avz -e '/usr/bin/ssh -oStrictHostKeyChecking=no -i $ssh_identity' '${source}:$title/' '$title/'",
+ user => "hdfs",
+ tag => "namenode-format",
+ creates => "$title/current/VERSION",
+ }
+ }
+
define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple") {
$hadoop_secondarynamenode_port = $port
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml?rev=1306090&r1=1306089&r2=1306090&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml Wed Mar 28 01:02:06 2012
@@ -16,14 +16,20 @@
<!-- See the License for the specific language governing permissions and -->
<!-- limitations under the License. -->
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
<configuration>
<property>
<!-- URI of NN. Fully qualified. No IP.-->
- <name>fs.default.name</name>
- <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
+ <name>fs.defaultFS</name>
+<% if ha == "enabled" -%>
+ <value>hdfs://<%= nameservice_id %></value>
+<% else -%>
+ <value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>
+<% end -%>
</property>
+
<property>
<name>hadoop.security.authentication</name>
<value><%= hadoop_security_authentication %></value>
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1306090&r1=1306089&r2=1306090&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml Wed Mar 28 01:02:06 2012
@@ -16,9 +16,64 @@
<!-- See the License for the specific language governing permissions and -->
<!-- limitations under the License. -->
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
<configuration>
-<% if hadoop_security_authentication == "kerberos" %>
+<% if ha == "enabled" -%>
+ <!-- HA -->
+ <property>
+ <name>dfs.federation.nameservices</name>
+ <value><%= nameservice_id %></value>
+ </property>
+
+ <property>
+ <name>dfs.ha.namenodes.<%= nameservice_id %></name>
+ <value><%= (1..namenode_hosts.length).map { |n| "nn#{n}" }.join(",") %></value>
+ </property>
+
+<% namenode_hosts.each_with_index do |host,idx| -%>
+ <property>
+ <name>dfs.namenode.rpc-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
+ <value><%= host %>:<%= hadoop_namenode_port %></value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.http-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
+ <value><%= host %>:50070</value>
+ </property>
+
+<% end -%>
+<% if has_variable?("shared_edits_dir") -%>
+ <property>
+ <name>dfs.namenode.shared.edits.dir</name>
+ <value><%= shared_edits_dir %></value>
+ </property>
+
+<% end -%>
+ <property>
+ <name>dfs.client.failover.proxy.provider.<%= nameservice_id %></name>
+ <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+ </property>
+
+<% if has_variable?("sshfence_privkey") -%>
+ <property>
+ <name>dfs.ha.fencing.methods</name>
+ <value>sshfence(<%= sshfence_user %>)</value>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.ssh.private-key-files</name>
+ <value><%= sshfence_keypath %></value>
+ </property>
+
+<% else -%>
+ <property>
+ <name>dfs.ha.fencing.methods</name>
+ <value></value>
+ </property>
+
+<% end -%>
+<% elsif hadoop_security_authentication == "kerberos" -%>
<property>
<name>dfs.block.access.token.enable</name>
<value>true</value>
@@ -27,7 +82,7 @@
<!-- NameNode security config -->
<property>
<name>dfs.https.address</name>
- <value><%= hadoop_namenode_host %>:50475</value>
+ <value><%= namenode_hosts[0] %>:50475</value>
</property>
<property>
<name>dfs.https.port</name>
@@ -58,11 +113,11 @@
<!-- Secondary NameNode security config -->
<property>
<name>dfs.secondary.http.address</name>
- <value><%= hadoop_namenode_host %>:0</value>
+ <value><%= namenode_hosts[0] %>:0</value>
</property>
<property>
<name>dfs.secondary.https.address</name>
- <value><%= hadoop_namenode_host %>:50495</value>
+ <value><%= namenode_hosts[0] %>:50495</value>
</property>
<property>
<name>dfs.secondary.https.port</name>
@@ -106,16 +161,10 @@
<name>dfs.datanode.kerberos.https.principal</name>
<value>host/_HOST@<%= kerberos_realm %></value>
</property>
-<% end %>
+<% end -%>
<!-- name node -->
<property>
- <!-- URI of NN. Fully qualified. No IP.-->
- <name>fs.default.name</name>
- <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
- </property>
-
- <property>
<name>dfs.data.dir</name>
<value><%= hdfs_data_dirs.join(",") %></value>
</property>
@@ -132,29 +181,31 @@
</property>
<!-- Enable Hue plugins -->
-<% if has_variable?("hadoop_dfs_namenode_plugins") %>
+<% if has_variable?("hadoop_dfs_namenode_plugins") -%>
<property>
<name>dfs.namenode.plugins</name>
<value><%= hadoop_dfs_namenode_plugins %></value>
<description>Comma-separated list of namenode plug-ins to be activated.
</description>
</property>
-<% end %>
-<% if has_variable?("hadoop_dfs_datanode_plugins") %>
+
+<% end -%>
+<% if has_variable?("hadoop_dfs_datanode_plugins") -%>
<property>
<name>dfs.datanode.plugins</name>
<value><%= hadoop_dfs_datanode_plugins %></value>
<description>Comma-separated list of datanode plug-ins to be activated.
</description>
</property>
-<% end %>
-<% if has_variable?("hadoop_namenode_thrift_port") %>
+
+<% end -%>
+<% if has_variable?("hadoop_namenode_thrift_port") -%>
<property>
<name>dfs.thrift.address</name>
<value>0.0.0.0:<%= hadoop_namenode_thrift_port %></value>
</property>
-<% end %>
+<% end -%>
<!-- increase the number of datanode transceivers way above the default of 256
- this is for hbase -->
<property>
@@ -163,20 +214,20 @@
</property>
<!-- Configurations for large cluster -->
-<% if has_variable?("hadoop_config_dfs_block_size") %>
+<% if has_variable?("hadoop_config_dfs_block_size") -%>
<property>
<name>dfs.block.size</name>
<value><%= hadoop_config_dfs_block_size %></value>
</property>
-<% end %>
-<% if has_variable?("hadoop_config_namenode_handler_count") %>
+<% end -%>
+<% if has_variable?("hadoop_config_namenode_handler_count") -%>
<property>
<name>dfs.namenode.handler.count</name>
<value><%= hadoop_config_namenode_handler_count %></value>
</property>
-<% end %>
+<% end -%>
<property>
<name>dfs.webhdfs.enabled</name>
<value>true</value>
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml?rev=1306090&r1=1306089&r2=1306090&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml Wed Mar 28 01:02:06 2012
@@ -16,6 +16,7 @@
<!-- See the License for the specific language governing permissions and -->
<!-- limitations under the License. -->
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
<configuration>
<% if has_variable?("hadoop_core_proxyusers") -%>
@@ -63,12 +64,12 @@
</property>
<property>
<name>httpfs.hadoop.conf:dfs.namenode.kerberos.principal</name>
- <value>hdfs/<%= hadoop_namenode_host %>@<%= kerberos_realm %></value>
+ <value>hdfs/<%= namenode_hosts[0] %>@<%= kerberos_realm %></value>
</property>
<% end -%>
<property>
<name>httpfs.hadoop.conf:fs.default.name</name>
- <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
+ <value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>
</property>
</configuration>
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client.pp?rev=1306090&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client.pp (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client.pp Wed Mar 28 01:02:06 2012
@@ -0,0 +1,19 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class nfs::client {
+ include nfs::client::install
+ require nfs::client::service
+}
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp?rev=1306090&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp Wed Mar 28 01:02:06 2012
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class nfs::client::install inherits nfs::client::params {
+ package { $package_names:
+ ensure => present,
+ }
+}
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp?rev=1306090&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp Wed Mar 28 01:02:06 2012
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class nfs::client::params {
+ case "$::operatingsystem $::operatingsystemrelease" {
+ /(CentOS|RedHat) 6/: {
+ $package_names = [ "rpcbind", "nfs-utils" ]
+ $portmap_service = "rpcbind"
+ $idmapd_service = "rpcidmapd"
+ $nfs_version = 4
+ }
+
+ /(CentOS|RedHat) 5/: {
+ $package_names = [ "portmap", "nfs-utils" ]
+ $portmap_service = "portmap"
+ $nfs_version = 3
+ }
+
+ /SLES 11/: {
+ $package_names = [ "rpcbind", "nfs-client" ]
+ $portmap_service = "rpcbind"
+ $nfs_version = 3
+ }
+
+ /(Debian|Ubuntu)/: {
+ $package_names = [ "nfs-common", "portmap" ]
+ $portmap_service = "portmap"
+ $nfs_version = 3
+ }
+
+ default: {
+ fail("nfs::client::params not supported on $::operatingsystem $::operatingsystemerelease")
+ }
+ }
+}
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp?rev=1306090&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp Wed Mar 28 01:02:06 2012
@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class nfs::client::service inherits nfs::client::params {
+
+ include nfs::client::install
+
+ service { $portmap_service:
+ ensure => running,
+ enable => true,
+ require => Package[$package_names],
+ }
+
+ if ($nfs_version >= 4) {
+ service { $idmapd_service:
+ ensure => running,
+ enable => true,
+ require => [ Service[$portmap_service], Package[$package_names] ],
+ pattern => "rpc.idmapd",
+ }
+ }
+}
Added: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/init.pp?rev=1306090&view=auto
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/init.pp (added)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/nfs/manifests/init.pp Wed Mar 28 01:02:06 2012
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class nfs {
+ require nfs::client
+}