You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2012/01/12 03:06:40 UTC
svn commit: r1230369 - in /incubator/bigtop/trunk/bigtop-deploy/puppet: ./
config/ manifests/ modules/bigtop_util/ modules/bigtop_util/lib/
modules/bigtop_util/lib/puppet/ modules/bigtop_util/lib/puppet/parser/
modules/bigtop_util/lib/puppet/parser/fun...
Author: rvs
Date: Thu Jan 12 02:06:39 2012
New Revision: 1230369
URL: http://svn.apache.org/viewvc?rev=1230369&view=rev
Log:
BIGTOP-313. Move puppet configuration out of manifest and into extlookup() CSV (Patrick Taylor Ramsey via rvs)
Added:
incubator/bigtop/trunk/bigtop-deploy/puppet/README.md
incubator/bigtop/trunk/bigtop-deploy/puppet/config/
incubator/bigtop/trunk/bigtop-deploy/puppet/config/site.csv.example
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/append_each.rb
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/get_setting.rb
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_dirs.rb
Modified:
incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/README.md
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/README.md?rev=1230369&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/README.md (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/README.md Thu Jan 12 02:06:39 2012
@@ -0,0 +1,43 @@
+# Puppet classes for deploying Hadoop
+
+## Configuration
+
+manifests/init.pp expects configuration to live in CSV at $confdir/config/site.csv,
+which takes the form
+
+<pre>
+key,value[,value2,value3]
+</pre>
+
+An example is provided at config/site.csv.example. These values are loaded using
+puppet's extlookup() mechanism.
+
+Any options not defined there will revert to a default value defined in
+manifests/cluster.pp, with the following exceptions (which are required):
+
+* hadoop\_head\_node: must be set to the FQDN of the name node of your cluster (which will also
+ become its job tracker and gateway)
+* bigtop\_yumrepo\_uri: uri of a repository containing packages for hadoop as built by Bigtop.
+
+$confdir is the directory that puppet will look into for its configuration. On most systems,
+this will be either /etc/puppet/ or /etc/puppetlabs/puppet/. You may override this value by
+specifying --confdir=path/to/config/dir on the puppet command line.
+
+For other options that may be set here, look for calls to extlookup() in manifests/cluster.pp.
+Note that if hadoop\_storage\_dirs is left unset, puppet will attempt to guess which directories
+to use.
+
+## Usage
+
+Make sure that the bigtop-deploy directory is available on every node of your cluster, and then
+run the following on those nodes:
+
+<pre>
+# mkdir /etc/puppet/config
+# cat > /etc/puppet/config/site.csv <<EOF
+# hadoop_head_node,hadoopmaster.example.com
+# hadoop_storage_dirs,/data/1,/data/2
+# bigtop_yumrepo_uri,http://mirror.example.com/path/to/mirror/
+# EOF
+# puppet -d --modulepath=bigtop-deploy/puppet/modules bigtop-deploy/puppet/manifests/site.pp
+</pre>
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/config/site.csv.example
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/config/site.csv.example?rev=1230369&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/config/site.csv.example (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/config/site.csv.example Thu Jan 12 02:06:39 2012
@@ -0,0 +1,3 @@
+hadoop_head_node,hadoopmaster.example.com
+hadoop_storage_dirs,/data/1,/data/2,/data/3,/data/4
+bigtop_yumrepo_uri,http://mirror.example.com/path/to/mirror/
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp?rev=1230369&r1=1230368&r2=1230369&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp Thu Jan 12 02:06:39 2012
@@ -14,33 +14,43 @@
# limitations under the License.
class hadoop_cluster_node {
- $hadoop_namenode_host="$hadoop_head_node"
- $hadoop_namenode_port="17020"
- $hadoop_namenode_thrift_port="10090"
+ require bigtop_util
+
+ $hadoop_namenode_host = $hadoop_head_node
+ $hadoop_namenode_port = extlookup("hadoop_namenode_port", "17020")
+ $hadoop_namenode_thrift_port = extlookup("hadoop_namenode_thrift_port", "10090")
+ $hadoop_dfs_namenode_plugins = extlookup("hadoop_dfs_namenode_plugins", "")
+ $hadoop_dfs_datanode_plugins = extlookup("hadoop_dfs_datanode_plugins", "")
# $hadoop_dfs_namenode_plugins="org.apache.hadoop.thriftfs.NamenodePlugin"
# $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
- $hadoop_jobtracker_host="$hadoop_head_node"
- $hadoop_jobtracker_port="8021"
- $hadoop_jobtracker_thrift_port="9290"
+ $hadoop_jobtracker_host = $hadoop_head_node
+ $hadoop_jobtracker_port = extlookup("hadoop_jobtracker_port", "8021")
+ $hadoop_jobtracker_thrift_port = extlookup("hadoop_jobtracker_thrift_port", "9290")
+ $hadoop_mapred_jobtracker_plugins = extlookup("hadoop_mapred_jobtracker_plugins", "")
+ $hadoop_mapred_tasktracker_plugins = extlookup("hadoop_mapred_tasktracker_plugins", "")
# $hadoop_mapred_jobtracker_plugins="org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin"
# $hadoop_mapred_tasktracker_plugins="org.apache.hadoop.mapred.TaskTrackerCmonInst"
$hadoop_core_proxyusers = { oozie => { groups => 'root,hadoop,jenkins,oozie,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" } }
- $hadoop_hbase_rootdir = "hdfs://$hadoop_namenode_host:$hadoop_namenode_port/hbase"
- $hadoop_hbase_zookeeper_quorum = "$hadoop_head_node"
+ $hbase_relative_rootdir = extlookup("hadoop_hbase_rootdir", "/hbase")
+ $hadoop_hbase_rootdir = "hdfs://$hadoop_namenode_host:$hadoop_namenode_port/$hbase_relative_rootdir"
+ $hadoop_hbase_zookeeper_quorum = $hadoop_head_node
$hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"]
- $namenode_data_dirs = ["/mnt/namenode"]
- $hdfs_data_dirs = ["/mnt/hdfs"]
- $mapred_data_dirs = ["/mnt/scratch"]
+ # Set from facter if available
+ $roots = extlookup("hadoop_storage_dirs", split($hadoop_storage_dirs, ";"))
+ $namenode_data_dirs = extlookup("hadoop_namenode_data_dirs", append_each("/namenode", $roots))
+ $hdfs_data_dirs = extlookup("hadoop_hdfs_data_dirs", append_each("/hdfs", $roots))
+ $mapred_data_dirs = extlookup("hadoop_mapred_data_dirs", append_each("/mapred", $roots))
+ $hadoop_security_authentication = extlookup("hadoop_security", "simple")
if ($hadoop_security_authentication == "kerberos") {
- $kerberos_domain = "compute-1.internal"
- $kerberos_realm = "EXAMPLE.COM"
- $kerberos_kdc_server = "$hadoop_head_node"
+ $kerberos_domain = extlookup("hadoop_kerberos_domain")
+ $kerberos_realm = extlookup("hadoop_kerberos_realm")
+ $kerberos_kdc_server = extlookup("hadoop_kerberos_kdc_server")
include kerberos::client
kerberos::client::host_keytab { ["hdfs", "mapred", "hbase", "oozie"]:
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp?rev=1230369&r1=1230368&r2=1230369&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp Thu Jan 12 02:06:39 2012
@@ -13,10 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+require bigtop_util
+$puppet_confdir = get_setting("confdir")
+$default_yumrepo = "http://bigtop01.cloudera.org:8080/view/RCs/job/Bigtop-trunk-rc-zookeeper-3.4.0/label=centos5/lastSuccessfulBuild/artifact/output/"
+$extlookup_datadir="$puppet_confdir/config"
+$extlookup_precedence = ["site", "default"]
+
stage {"pre": before => Stage["main"]}
yumrepo { "Bigtop":
- baseurl => "http://bigtop01.cloudera.org:8080/view/RCs/job/Bigtop-trunk-rc-zookeeper-3.4.0/label=centos5/lastSuccessfulBuild/artifact/output/",
+ baseurl => extlookup("bigtop_yumrepo_uri", $default_yumrepo),
descr => "Bigtop packages",
enabled => 1,
gpgcheck => 0,
@@ -32,6 +38,11 @@ package { "jdk":
import "cluster.pp"
node default {
+ # Fails if hadoop_head_node is unset
+ if (!$::hadoop_head_node) {
+ $hadoop_head_node = extlookup("hadoop_head_node")
+ }
+
if ($hadoop_head_node == $fqdn) {
include hadoop_gateway_node
} else {
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/append_each.rb
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/append_each.rb?rev=1230369&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/append_each.rb (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/append_each.rb Thu Jan 12 02:06:39 2012
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Append a string to every element of an array
+
+Puppet::Parser::Functions::newfunction(:append_each, :type => :rvalue) do |args|
+ suffix = (args[0].is_a? Array) ? args[0].join("") : args[0]
+ inputs = (args[1].is_a? Array) ? args[1] : [ args[1] ]
+ inputs.map { |item| item + suffix }
+end
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/get_setting.rb
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/get_setting.rb?rev=1230369&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/get_setting.rb (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/lib/puppet/parser/functions/get_setting.rb Thu Jan 12 02:06:39 2012
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Retrieve a value from Puppet.settings (systemwide puppet configuration)
+Puppet::Parser::Functions::newfunction(:get_setting, :type => :rvalue) do |args|
+ ret = Puppet[args[0].to_sym]
+ ret.nil? ? :undef : ret
+end
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/manifests/init.pp?rev=1230369&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/bigtop_util/manifests/init.pp Thu Jan 12 02:06:39 2012
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class bigtop_util {
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_dirs.rb
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_dirs.rb?rev=1230369&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_dirs.rb (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_dirs.rb Thu Jan 12 02:06:39 2012
@@ -0,0 +1,14 @@
+Facter.add("hadoop_storage_dirs") do
+ setcode do
+ [ Facter.value("hadoop_storage_dir_pattern"),
+ "/data/[0-9]*",
+ "/mnt" ].reject(&:nil?).each do |pattern|
+
+ storage_dirs = Dir.glob(pattern) \
+ .select { |path| File.directory? path } \
+ .join(";")
+
+ break storage_dirs if storage_dirs.size > 0
+ end
+ end
+end
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1230369&r1=1230368&r2=1230369&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Thu Jan 12 02:06:39 2012
@@ -136,7 +136,7 @@ class hadoop {
exec { "namenode format":
user => "hdfs",
command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
- creates => inline_template("<%= hadoop_storage_locations.split(';')[0] %>/namenode/image"),
+ creates => "${namenode_data_dirs[0]}/image"),
require => [ Package["hadoop-namenode"], File[$dirs] ],
}
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1230369&r1=1230368&r2=1230369&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml Thu Jan 12 02:06:39 2012
@@ -108,12 +108,12 @@
<property>
<name>dfs.data.dir</name>
- <value><% hadoop_storage_locations.split(";").each do |storage_location| %><%= storage_location%>/hdfs,<% end %></value>
+ <value><%= hdfs_data_dirs.join(",") %></value>
</property>
<property>
<name>dfs.name.dir</name>
- <value><% hadoop_storage_locations.split(";").each do |storage_location| %><%= storage_location%>/namenode,<% end %></value>
+ <value><%= namenode_data_dirs.join(",") %></value>
</property>
<!-- Enable Hue plugins -->
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml?rev=1230369&r1=1230368&r2=1230369&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml Thu Jan 12 02:06:39 2012
@@ -101,7 +101,7 @@
<property>
<name>mapred.local.dir</name>
- <value><% hadoop_storage_locations.split(";").each do |storage_location| %><%= storage_location%>/scratch,<% end %></value>
+ <value><%= mapred_data_dirs.join(",") %></value>
<final>true</final>
</property>
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg?rev=1230369&r1=1230368&r2=1230369&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg Thu Jan 12 02:06:39 2012
@@ -1,4 +1,4 @@
-mapred.local.dir=<% hadoop_storage_locations.split(";").each do |storage_location| %><%= storage_location%>/scratch,<% end %>
+mapred.local.dir=<%= mapred_data_dirs.join(",") %>
hadoop.log.dir=/var/log/hadoop/
mapreduce.tasktracker.group=mapred
min.user.id=0