You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2012/04/24 01:16:49 UTC
svn commit: r1329506 - in /incubator/bigtop/trunk/bigtop-deploy/puppet:
manifests/cluster.pp modules/hadoop/manifests/init.pp
modules/hadoop/templates/core-site.xml modules/hadoop/templates/hdfs-site.xml
Author: rvs
Date: Mon Apr 23 23:16:49 2012
New Revision: 1329506
URL: http://svn.apache.org/viewvc?rev=1329506&view=rev
Log:
BIGTOP-539. update puppet code for BIGTOP-534
Modified:
incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp Mon Apr 23 23:16:49 2012
@@ -22,12 +22,12 @@ class hadoop_cluster_node {
$hadoop_ha = $standby_head_node ? {
"" => disabled,
- default => enabled,
+ default => extlookup("hadoop_ha", "manual"),
}
$hadoop_namenode_host = $hadoop_ha ? {
- "enabled" => [ $hadoop_head_node, $standby_head_node ],
- default => $hadoop_head_node,
+ "disabled" => $hadoop_head_node,
+ default => [ $hadoop_head_node, $standby_head_node ],
}
$hadoop_namenode_port = extlookup("hadoop_namenode_port", "17020")
$hadoop_namenode_thrift_port = extlookup("hadoop_namenode_thrift_port", "10090")
@@ -37,8 +37,8 @@ class hadoop_cluster_node {
# $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
$hadoop_ha_nameservice_id = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
$hadoop_namenode_uri = $hadoop_ha ? {
- "enabled" => "hdfs://${hadoop_ha_nameservice_id}:8020",
- default => "hdfs://$hadoop_namenode_host:$hadoop_namenode_port",
+ "disabled" => "hdfs://$hadoop_namenode_host:$hadoop_namenode_port",
+ default => "hdfs://${hadoop_ha_nameservice_id}:8020",
}
$hadoop_rm_host = $hadoop_head_node
@@ -56,6 +56,7 @@ class hadoop_cluster_node {
$hadoop_jobtracker_thrift_port = extlookup("hadoop_jobtracker_thrift_port", "9290")
$hadoop_mapred_jobtracker_plugins = extlookup("hadoop_mapred_jobtracker_plugins", "")
$hadoop_mapred_tasktracker_plugins = extlookup("hadoop_mapred_tasktracker_plugins", "")
+ $hadoop_ha_zookeeper_quorum = "${hadoop_head_node}:2181"
# $hadoop_mapred_jobtracker_plugins="org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin"
# $hadoop_mapred_tasktracker_plugins="org.apache.hadoop.mapred.TaskTrackerCmonInst"
@@ -134,9 +135,10 @@ class hadoop_head_node inherits hadoop_c
# thrift_port => $hadoop_namenode_thrift_port,
auth => $hadoop_security_authentication,
ha => $hadoop_ha,
+ zk => $hadoop_ha_zookeeper_quorum,
}
- if ($hadoop_ha != "enabled") {
+ if ($hadoop_ha == "disabled") {
hadoop::secondarynamenode { "secondary namenode":
namenode_host => $hadoop_namenode_host,
namenode_port => $hadoop_namenode_port,
@@ -198,7 +200,7 @@ class hadoop_head_node inherits hadoop_c
}
}
-class standby_head_node inherits hadoop_worker_node {
+class standby_head_node inherits hadoop_cluster_node {
hadoop::namenode { "namenode":
host => $hadoop_namenode_host,
port => $hadoop_namenode_port,
@@ -206,6 +208,7 @@ class standby_head_node inherits hadoop_
# thrift_port => $hadoop_namenode_thrift_port,
auth => $hadoop_security_authentication,
ha => $hadoop_ha,
+ zk => $hadoop_ha_zookeeper_quorum,
}
}
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Mon Apr 23 23:16:49 2012
@@ -75,11 +75,11 @@ class hadoop {
}
class common-hdfs inherits common {
- if ($auth == "kerberos" and $ha == "enabled") {
+ if ($auth == "kerberos" and $ha != "disabled") {
fail("High-availability secure clusters are not currently supported")
}
- if ($ha == 'enabled') {
+ if ($ha != 'disabled') {
$nameservice_id = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
}
@@ -126,7 +126,7 @@ class hadoop {
$hadoop_datanode_port = $port
$hadoop_security_authentication = $auth
- if ($ha == 'enabled') {
+ if ($ha != 'disabled') {
# Needed by hdfs-site.xml
$sshfence_keydir = "/usr/lib/hadoop/.ssh"
$sshfence_keypath = "$sshfence_keydir/id_sshfence"
@@ -234,7 +234,7 @@ class hadoop {
Exec <| title == "activate nn1" |> -> Exec["HDFS init $title"]
}
- define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"], $ha = 'disabled') {
+ define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"], $ha = 'disabled', $zk = '') {
$first_namenode = inline_template("<%= host.to_a[0] %>")
$hadoop_namenode_host = $host
@@ -242,7 +242,7 @@ class hadoop {
$hadoop_namenode_thrift_port = $thrift_port
$hadoop_security_authentication = $auth
- if ($ha == 'enabled') {
+ if ($ha != 'disabled') {
$sshfence_user = extlookup("hadoop_ha_sshfence_user", "hdfs")
$sshfence_user_home = extlookup("hadoop_ha_sshfence_user_home", "/var/lib/hadoop-hdfs")
$sshfence_keydir = "$sshfence_user_home/.ssh"
@@ -318,24 +318,50 @@ class hadoop {
Exec <| tag == "namenode-format" |> -> Service["hadoop-hdfs-namenode"]
Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-namenode"]
+ if ($ha == "auto") {
+ package { "hadoop-hdfs-zkfc":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ service { "hadoop-hdfs-zkfc":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-hdfs-zkfc"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [Package["hadoop-hdfs-zkfc"]],
+ }
+ Service <| title == "hadoop-hdfs-zkfc" |> -> Service <| title == "hadoop-hdfs-namenode" |>
+ }
+
if ($::fqdn == $first_namenode) {
exec { "namenode format":
user => "hdfs",
- command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
+ command => "/bin/bash -c 'yes Y | hdfs namenode -format >> /var/lib/hadoop-hdfs/nn.format.log 2>&1'",
creates => "${namenode_data_dirs[0]}/current/VERSION",
- require => [ Package["hadoop-hdfs-namenode"], File[$dirs] ],
+ require => [ Package["hadoop-hdfs-namenode"], File[$dirs], File["/etc/hadoop/conf/hdfs-site.xml"] ],
tag => "namenode-format",
}
-
- if ($ha == "enabled") {
- exec { "activate nn1":
- command => "/usr/bin/hdfs haadmin -transitionToActive nn1",
- user => "hdfs",
- unless => "/usr/bin/test $(/usr/bin/hdfs haadmin -getServiceState nn1) = active",
- require => Service["hadoop-hdfs-namenode"],
+
+ if ($ha != "disabled") {
+ if ($ha == "auto") {
+ exec { "namenode zk format":
+ user => "hdfs",
+ command => "/bin/bash -c 'yes N | hdfs zkfc -formatZK >> /var/lib/hadoop-hdfs/zk.format.log 2>&1 || :'",
+ require => [ Package["hadoop-hdfs-zkfc"], File["/etc/hadoop/conf/hdfs-site.xml"] ],
+ tag => "namenode-format",
+ }
+ Service <| title == "zookeeper-server" |> -> Exec <| title == "namenode zk format" |>
+ Exec <| title == "namenode zk format" |> -> Service <| title == "hadoop-hdfs-zkfc" |>
+ } else {
+ exec { "activate nn1":
+ command => "/usr/bin/hdfs haadmin -transitionToActive nn1",
+ user => "hdfs",
+ unless => "/usr/bin/test $(/usr/bin/hdfs haadmin -getServiceState nn1) = active",
+ require => Service["hadoop-hdfs-namenode"],
+ }
}
}
- } elsif ($ha == "enabled") {
+ } elsif ($ha != "disabled") {
hadoop::namedir_copy { $namenode_data_dirs:
source => $first_namenode,
ssh_identity => $sshfence_keypath,
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml Mon Apr 23 23:16:49 2012
@@ -22,7 +22,7 @@
<property>
<!-- URI of NN. Fully qualified. No IP.-->
<name>fs.defaultFS</name>
-<% if ha == "enabled" -%>
+<% if ha != "disabled" -%>
<value>hdfs://<%= nameservice_id %></value>
<% else -%>
<value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1329506&r1=1329505&r2=1329506&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml Mon Apr 23 23:16:49 2012
@@ -19,8 +19,22 @@
<% namenode_hosts = hadoop_namenode_host.to_a -%>
<configuration>
-<% if ha == "enabled" -%>
+<% if ha != "disabled" -%>
<!-- HA -->
+
+<% if ha == "auto" -%>
+ <property>
+ <name>dfs.ha.automatic-failover.enabled</name>
+ <value>true</value>
+ </property>
+<% if has_variable?("zk") -%>
+ <property>
+ <name>ha.zookeeper.quorum</name>
+ <value><%= zk %></value>
+ </property>
+<% end -%>
+<% end -%>
+
<property>
<name>dfs.federation.nameservices</name>
<value><%= nameservice_id %></value>