You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2012/02/11 04:41:45 UTC
svn commit: r1243001 - in
/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet:
manifests/cluster.pp modules/hadoop-hbase/manifests/init.pp
modules/hadoop-oozie/manifests/init.pp modules/hadoop/manifests/init.pp
modules/kerberos/manifests/init.pp
Author: rvs
Date: Sat Feb 11 03:41:45 2012
New Revision: 1243001
URL: http://svn.apache.org/viewvc?rev=1243001&view=rev
Log:
BIGTOP-396. Missing resource dependencies in puppet for secure clusters (Patrick Taylor Ramsey via rvs)
Modified:
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp?rev=1243001&r1=1243000&r2=1243001&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/manifests/cluster.pp Sat Feb 11 03:41:45 2012
@@ -66,14 +66,6 @@ class hadoop_cluster_node {
$kerberos_kdc_server = extlookup("hadoop_kerberos_kdc_server")
include kerberos::client
- kerberos::client::host_keytab { ["hdfs", "httpfs", "yarn", "mapred", "hbase", "oozie"]:
- princs_map => { hdfs => [ "host", "hdfs" ],
- httpfs => [ "httpfs" ],
- yarn => [ "yarn" ],
- mapred => [ "mapred" ],
- hbase => [ "hbase" ],
- oozie => [ "oozie" ], },
- }
}
}
@@ -113,7 +105,7 @@ class hadoop_worker_node inherits hadoop
class hadoop_head_node inherits hadoop_cluster_node {
if ($hadoop_security_authentication == "kerberos") {
- include kerberos::kdc, kerberos::kdc::admin_server
+ include kerberos::server
}
hadoop::namenode { "namenode":
@@ -169,6 +161,7 @@ class hadoop_head_node inherits hadoop_c
}
hadoop::create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user", "/hbase", "/benchmarks", "/user/jenkins", "/user/hive", "/user/root", "/user/history" ]:
+ auth => $hadoop_security_authentication,
hdfs_dirs_meta => { "/tmp" => { perm => "777", user => "hdfs" },
"/mapred" => { perm => "755", user => "mapred" },
"/system" => { perm => "755", user => "hdfs" },
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp?rev=1243001&r1=1243000&r2=1243001&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp Sat Feb 11 03:41:45 2012
@@ -22,6 +22,11 @@ class hadoop-hbase {
class common-server-config {
include client-package
+ if ($kerberos_realm) {
+ require kerberos::client
+ kerberos::host_keytab { "hbase":
+ }
+ }
file { "/etc/hbase/conf/hbase-site.xml":
content => template("hadoop-hbase/hbase-site.xml"),
@@ -51,6 +56,7 @@ class hadoop-hbase {
hasrestart => true,
hasstatus => true,
}
+ Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-regionserver"]
}
define master($rootdir, $zookeeper_quorum, $kerberos_realm = "", $heap_size="1024") {
@@ -67,5 +73,6 @@ class hadoop-hbase {
hasrestart => true,
hasstatus => true,
}
+ Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-master"]
}
}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp?rev=1243001&r1=1243000&r2=1243001&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp Sat Feb 11 03:41:45 2012
@@ -21,6 +21,12 @@ class hadoop-oozie {
}
define server($kerberos_realm = "") {
+ if ($kerberos_realm) {
+ require kerberos::client
+ kerberos::host_keytab { "oozie":
+ }
+ }
+
package { "oozie":
ensure => latest,
}
@@ -36,6 +42,7 @@ class hadoop-oozie {
hasrestart => true,
hasstatus => true,
}
+ Kerberos::Host_keytab <| title == "oozie" |> -> Service["oozie"]
}
}
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1243001&r1=1243000&r2=1243001&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Sat Feb 11 03:41:45 2012
@@ -19,7 +19,23 @@ class hadoop {
* Common definitions for hadoop nodes.
* They all need these files so we can access hdfs/jobs from any node
*/
+
+ class kerberos {
+ require kerberos::client
+
+ kerberos::host_keytab { "hdfs":
+ princs => [ "host", "hdfs" ],
+ }
+
+ kerberos::host_keytab { [ "yarn", "mapred" ]:
+ }
+ }
+
class common {
+ if ($auth == "kerberos") {
+ include hadoop::kerberos
+ }
+
file {
"/etc/hadoop/conf/hadoop-env.sh":
content => template('hadoop/hadoop-env.sh'),
@@ -126,13 +142,14 @@ class hadoop {
subscribe => [Package["hadoop-hdfs-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
require => [ Package["hadoop-hdfs-datanode"], File[$dirs] ],
}
+ Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-datanode"]
file { $dirs:
ensure => directory,
owner => hdfs,
group => hdfs,
mode => 755,
- require => [Package["hadoop-hdfs"]],
+ require => [ Package["hadoop-hdfs"] ],
}
}
@@ -169,12 +186,28 @@ class hadoop {
subscribe => [Package["hadoop-httpfs"], File["/etc/hadoop-httpfs/conf/httpfs-site.xml"], File["/etc/hadoop-httpfs/conf/httpfs-env.sh"], File["/etc/hadoop-httpfs/conf/httpfs-signature.secret"]],
require => [ Package["hadoop-httpfs"] ],
}
+ Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-httpfs"]
}
- define create_hdfs_dirs($hdfs_dirs_meta) {
+ class kinit {
+ include hadoop::kerberos
+
+ exec { "HDFS kinit":
+ command => "/usr/bin/kinit -kt /etc/hdfs.keytab hdfs/$fqdn && /usr/bin/kinit -R",
+ user => "hdfs",
+ require => Kerberos::Host_keytab["hdfs"],
+ }
+ }
+
+ define create_hdfs_dirs($hdfs_dirs_meta, $auth="simple") {
$user = $hdfs_dirs_meta[$title][user]
$perm = $hdfs_dirs_meta[$title][perm]
+ if ($auth == "kerberos") {
+ require hadoop::kinit
+ Exec["HDFS kinit"] -> Exec["HDFS init $title"]
+ }
+
exec { "HDFS init $title":
user => "hdfs",
command => "/bin/bash -c 'hadoop fs -mkdir $title && hadoop fs -chmod $perm $title && hadoop fs -chown $user $title'",
@@ -203,6 +236,7 @@ class hadoop {
subscribe => [Package["hadoop-hdfs-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
require => [Package["hadoop-hdfs-namenode"], Exec["namenode format"]],
}
+ Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-namenode"]
exec { "namenode format":
user => "hdfs",
@@ -238,6 +272,7 @@ class hadoop {
subscribe => [Package["hadoop-hdfs-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
require => [Package["hadoop-hdfs-secondarynamenode"]],
}
+ Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-secondarynamenode"]
}
@@ -258,9 +293,11 @@ class hadoop {
service { "hadoop-yarn-resourcemanager":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-yarn-resourcemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/hadoop/conf/yarn-site.xml"]],
- require => [ Package["hadoop-yarn-resourcemanager"] ]
+ subscribe => [Package["hadoop-yarn-resourcemanager"], File["/etc/hadoop/conf/hadoop-env.sh"],
+ File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+ require => [ Package["hadoop-yarn-resourcemanager"] ],
}
+ Kerberos::Host_keytab <| title == "yarn" |> -> Service["hadoop-yarn-resourcemanager"]
}
define historyserver ($host = $fqdn, $port = "10020", $webapp_port = "19888", $auth = "simple") {
@@ -279,9 +316,11 @@ class hadoop {
service { "hadoop-mapreduce-historyserver":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-mapreduce-historyserver"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/hadoop/conf/mapred-site.xml"]],
+ subscribe => [Package["hadoop-mapreduce-historyserver"], File["/etc/hadoop/conf/hadoop-env.sh"],
+ File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
require => [Package["hadoop-mapreduce-historyserver"]],
}
+ Kerberos::Host_keytab <| title == "yarn" |> -> Service["hadoop-mapreduce-historyserver"]
}
@@ -300,9 +339,11 @@ class hadoop {
service { "hadoop-yarn-nodemanager":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-yarn-nodemanager"], File["/etc/hadoop/conf/hadoop-env.sh"], File["/etc/hadoop/conf/yarn-site.xml"]],
+ subscribe => [Package["hadoop-yarn-nodemanager"], File["/etc/hadoop/conf/hadoop-env.sh"],
+ File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
require => [ Package["hadoop-yarn-nodemanager"], File[$dirs] ],
}
+ Kerberos::Host_keytab <| title == "yarn" |> -> Service["hadoop-yarn-nodemanager"]
file { $dirs:
ensure => directory,
Modified: incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp?rev=1243001&r1=1243000&r2=1243001&view=diff
==============================================================================
--- incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp (original)
+++ incubator/bigtop/branches/hadoop-0.23/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp Sat Feb 11 03:41:45 2012
@@ -61,15 +61,16 @@ class kerberos {
}
class kdc inherits kerberos::site {
- package { "$package_name_kdc":
+ package { $package_name_kdc:
ensure => installed,
}
- file { "$kdc_etc_path":
+ file { $kdc_etc_path:
ensure => directory,
owner => root,
group => root,
mode => "0700",
+ require => Package["$package_name_kdc"],
}
file { "${kdc_etc_path}/kdc.conf":
content => template('kerberos/kdc.conf'),
@@ -98,7 +99,7 @@ class kerberos {
require => [Package["$package_name_kdc"], File["${kdc_etc_path}/kdc.conf"], File["/etc/krb5.conf"]],
}
- service { "$service_name_kdc":
+ service { $service_name_kdc:
ensure => running,
require => [Package["$package_name_kdc"], File["${kdc_etc_path}/kdc.conf"], Exec["kdb5_util"]],
subscribe => File["${kdc_etc_path}/kdc.conf"],
@@ -125,32 +126,50 @@ class kerberos {
}
class client inherits kerberos::site {
- define create_princs {
- exec { "addprinc.$title":
- path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
- command => "kadmin -w secure -p kadmin/admin -q 'addprinc -randkey $title/$fqdn'",
- unless => "kadmin -w secure -p kadmin/admin -q listprincs | grep -q $title/$fqdn"
- }
+ package { $package_name_client:
+ ensure => installed,
}
+ }
- define host_keytab($fqdn = "$hostname.$domain", $princs_map) {
- $princs = $princs_map[$title]
- $keytab = "/etc/${title}.keytab"
- $exports = inline_template("<%= princs.join('/$fqdn ') + '/$fqdn ' %>")
+ class server {
+ include kerberos::client
- create_princs { $princs:
- }
+ class { "kerberos::kdc": }
+ ->
+ Class["kerberos::client"]
+
+ class { "kerberos::kdc::admin_server": }
+ ->
+ Class["kerberos::client"]
+ }
- exec { "xst.$title":
- path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
- command => "kadmin -w secure -p kadmin/admin -q 'xst -k $keytab $exports' ; chown $title $keytab",
- unless => "klist -kt $keytab 2>/dev/null | grep -q $title/$fqdn",
- require => [ Create_princs[$princs] ],
- }
+ define create_princs {
+ exec { "addprinc.$title":
+ path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
+ command => "kadmin -w secure -p kadmin/admin -q 'addprinc -randkey $title/$fqdn'",
+ unless => "kadmin -w secure -p kadmin/admin -q listprincs | grep -q $title/$fqdn",
+ require => Package[$kerberos::site::package_name_client],
}
+ }
- package { "$package_name_client":
- ensure => installed,
+ define host_keytab($fqdn = "$hostname.$domain", $princs = undef) {
+ $real_princs = $princs ? {
+ undef => [ $title ],
+ default => $princs,
+ }
+
+ $keytab = "/etc/${title}.keytab"
+ $exports = inline_template("<%= real_princs.join('/$fqdn ') + '/$fqdn ' %>")
+
+ create_princs { $real_princs:
+ }
+
+ exec { "xst.$title":
+ path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
+ command => "kadmin -w secure -p kadmin/admin -q 'xst -k $keytab $exports' ; chown $title $keytab",
+ unless => "klist -kt $keytab 2>/dev/null | grep -q $title/$fqdn",
+ require => [ Create_princs[$real_princs] ],
}
}
+
}