You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vg...@apache.org on 2011/12/14 03:50:11 UTC
svn commit: r1214038 - in /incubator/ambari/trunk: ./
agent/src/main/resources/puppet/ agent/src/main/resources/puppet/manifests/
agent/src/main/resources/puppet/modules/
agent/src/main/resources/puppet/modules/hadoop/
agent/src/main/resources/puppet/m...
Author: vgogate
Date: Wed Dec 14 02:50:11 2011
New Revision: 1214038
URL: http://svn.apache.org/viewvc?rev=1214038&view=rev
Log:
AMBARI-161. Add puppet module for Hadoop to agent resources
Added:
incubator/ambari/trunk/agent/src/main/resources/puppet/
incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/
incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/facter/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/dirs_between.rb
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_category_name.rb
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_files.rb
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_parent_dirs.rb
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_env.erb
incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_properties.erb
Modified:
incubator/ambari/trunk/CHANGES.txt
Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1214038&r1=1214037&r2=1214038&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Wed Dec 14 02:50:11 2011
@@ -2,6 +2,8 @@ Ambari Change log
Release 0.1.0 - unreleased
+ AMBARI-161. Add puppet module for Hadoop to agent resources (vgogate)
+
AMBARI-159. Temporarily disabled security (until Ambari upgrades to
Python2.7). (ddas)
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/manifests/site.pp Wed Dec 14 02:50:11 2011
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+stage {"pre": before => Stage["main"]}
+
+yumrepo { "Bigtop":
+ baseurl => "http://bigtop01.cloudera.org:8080/job/Bigtop-trunk-matrix/label=centos5/lastSuccessfulBuild/artifact/output/",
+ descr => "Bigtop packages",
+ enabled => 1,
+ gpgcheck => 0,
+}
+
+package { "jdk":
+ ensure => "installed",
+}
+
+node default {
+ notice($fqdn)
+
+ if !$ambari_stack_install_dir {
+ $ambari_stack_install_dir = "/var/ambari/"
+ }
+ notice ($ambari_stack_install_dir)
+
+ /* ensure new directories in the path are present */
+ $stack_path = "${ambari_stack_install_dir}/${ambari_cluster_name}"
+ $stack_path_intermediate_dirs = dirs_between ($stack_path)
+ file {$stack_path_intermediate_dirs:
+ ensure => directory
+ }
+
+ if ($fqdn in $role_to_nodes[namenode]) {
+ hadoop::namenode {"namenode":
+ ambari_role_prefix => "${stack_path}/namenode",
+ }
+ }
+
+ /* hadoop.security.authentication make global variable */
+ if ($fqdn in $role_to_nodes[datanode]) {
+ hadoop::datanode {"datanode":
+ ambari_role_prefix => "${stack_path}/datanode",
+ auth_type => "simple"
+ }
+ }
+}
+
+Yumrepo<||> -> Package<||>
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb Wed Dec 14 02:50:11 2011
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We make the assumption hadoop's data files will be located in /data/
+# Puppet needs to know where they are
+Facter.add("hadoop_storage_locations") do
+ setcode do
+
+ data_dir_path = "/data/"
+ storage_locations = ""
+
+ # We need to check /data/ exist
+ if File.directory?(data_dir_path)
+
+ # We assume all data directory will be a number
+ Dir.foreach(data_dir_path) { |directory|
+ storage_locations += (data_dir_path + directory + ';') if directory =~ /\d+/
+ }
+ end
+
+ # Return the list of storage locations for hadoop
+ if storage_locations == ""
+ storage_locations = "/mnt"
+ end
+ storage_locations
+ end
+end
+
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/dirs_between.rb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/dirs_between.rb?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/dirs_between.rb (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/dirs_between.rb Wed Dec 14 02:50:11 2011
@@ -0,0 +1,25 @@
+module Puppet::Parser::Functions
+ newfunction(:dirs_between, :type => :rvalue, :doc => "Generate a list of pathnames") do |args|
+ subbottom = args[0]
+ subdirs = []
+ while subbottom != "/"
+ subbottom, component = File.split(subbottom)
+ subdirs.unshift(component)
+ end
+ dir = '/'
+ paths = [ ]
+ newpaths = [ ]
+ while subdirs.length > 0
+ component = subdirs.shift()
+ dir = File.join(dir, component)
+ paths.push(dir)
+ end
+ paths.each do |d|
+ if !File.exists?(d)
+# Dir.mkdir(d)
+ newpaths.push(d)
+ end
+ end
+ return newpaths
+ end
+end
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_category_name.rb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_category_name.rb?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_category_name.rb (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_category_name.rb Wed Dec 14 02:50:11 2011
@@ -0,0 +1,5 @@
+module Puppet::Parser::Functions
+ newfunction(:get_category_name, :type => :rvalue) do |args|
+ return File.basename(args[0])
+ end
+end
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_files.rb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_files.rb?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_files.rb (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_files.rb Wed Dec 14 02:50:11 2011
@@ -0,0 +1,10 @@
+module Puppet::Parser::Functions
+ newfunction(:get_files, :type => :rvalue) do |args|
+ hadoop_conf_dir = args[0]
+ hadoop_stack_conf = args[1]
+ role_name = args[2]
+ files = Array.new
+ hadoop_stack_conf[role_name].keys.each {|fname| files << ""+hadoop_conf_dir+"/"+fname}
+ return files
+ end
+end
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_parent_dirs.rb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_parent_dirs.rb?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_parent_dirs.rb (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/lib/puppet/parser/functions/get_parent_dirs.rb Wed Dec 14 02:50:11 2011
@@ -0,0 +1,10 @@
+module Puppet::Parser::Functions
+ newfunction(:get_parent_dirs, :type => :rvalue) do |args|
+ dir = args[0]
+ dirs = Array.new
+ temp = "/"
+ # dir.split('/').each {|x| temp = temp + x + "/", dirs << temp}
+ dir.split('/').each {|x| temp = temp+x+"/", dirs << temp}
+ return dirs
+ end
+end
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/manifests/init.pp Wed Dec 14 02:50:11 2011
@@ -0,0 +1,132 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop {
+ /**
+ * Common definitions for hadoop nodes.
+ * They all need these files so we can access hdfs/jobs from any node
+ */
+ class common {
+ package { "hadoop":
+ ensure => latest,
+ }
+
+ package { "hadoop-native":
+ ensure => latest,
+ require => [Package["hadoop"]],
+ }
+ }
+
+ define datanode ($ambari_role_prefix, $auth_type = "simple") {
+
+ include common
+
+ $ambari_role_name = "datanode"
+ $hadoop_conf_dir = "${ambari_role_prefix}/etc/hadoop"
+ file {["${ambari_role_prefix}", "${ambari_role_prefix}/etc", "${ambari_role_prefix}/etc/hadoop"]:
+ ensure => directory
+ }
+ notice ($ambari_role_prefix)
+ notice ($ambari_role_name)
+ $files = get_files ($hadoop_conf_dir, $::hadoop_stack_conf, $ambari_role_name)
+ notice($files)
+
+ /* Create config files for each category */
+ create_config_file {$files:
+ conf_map => $::hadoop_stack_conf[$title],
+ require => [Package["hadoop"]]
+ }
+
+ package { "hadoop-datanode":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ if ($auth_type == "kerberos") {
+ package { "hadoop-sbin":
+ ensure => latest,
+ require => [Package["hadoop"]],
+ }
+ }
+ }
+
+ define namenode ($ambari_role_prefix) {
+
+ include common
+
+ $ambari_role_name = "namenode"
+ $hadoop_conf_dir = "${ambari_role_prefix}/etc/hadoop"
+ file {["${ambari_role_prefix}", "${ambari_role_prefix}/etc", "${ambari_role_prefix}/etc/hadoop"]:
+ ensure => directory
+ }
+ notice ($ambari_role_prefix)
+ notice ($ambari_role_name)
+ $files = get_files ($hadoop_conf_dir, $::hadoop_stack_conf, $ambari_role_name)
+ notice($files)
+
+ /* Create config files for each category */
+ create_config_file {$files:
+ conf_map => $::hadoop_stack_conf[$title],
+ require => [Package["hadoop"]]
+ }
+
+ package { "hadoop-namenode":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+ }
+
+ define client ($ambari_role_prefix) {
+
+ include common
+
+ $ambari_role_name = "client"
+ $hadoop_conf_dir = "${ambari_role_prefix}/etc/hadoop"
+ file {["${ambari_role_prefix}", "${ambari_role_prefix}/etc", "${ambari_role_prefix}/etc/hadoop"]:
+ ensure => directory
+ }
+ notice ($ambari_role_prefix)
+ $files = get_files ($hadoop_conf_dir, $::hadoop_stack_conf, $ambari_role_name)
+ notice($files)
+
+ /* Create config files for each category */
+ create_config_file {$files:
+ conf_map => $::hadoop_stack_conf[$title],
+ require => [Package["hadoop"]]
+ }
+
+ package { ["hadoop-doc", "hadoop-source", "hadoop-debuginfo",
+ "hadoop-fuse", "hadoop-libhdfs", "hadoop-pipes"]:
+ ensure => latest,
+ require => [Package["jdk"], Package["hadoop"]],
+ }
+ }
+
+ define create_config_file ($conf_map) {
+ $category = get_category_name ($title)
+ $conf_category_map = $conf_map[$category]
+ if $category == 'hadoop-env.sh' {
+ file {"$title":
+ ensure => present,
+ content => template('hadoop/config_env.erb'),
+ }
+ } else {
+ file {"$title":
+ ensure => present,
+ content => template('hadoop/config_properties.erb'),
+ }
+ }
+ }
+}
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_env.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_env.erb?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_env.erb (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_env.erb Wed Dec 14 02:50:11 2011
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more -->
+<!-- contributor license agreements. See the NOTICE file distributed with -->
+<!-- this work for additional information regarding copyright ownership. -->
+<!-- The ASF licenses this file to You under the Apache License, Version 2.0 -->
+<!-- (the "License"); you may not use this file except in compliance with -->
+<!-- the License. You may obtain a copy of the License at -->
+<!-- -->
+<!-- http://www.apache.org/licenses/LICENSE-2.0 -->
+<!-- -->
+<!-- Unless required by applicable law or agreed to in writing, software -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS, -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
+<!-- See the License for the specific language governing permissions and -->
+<!-- limitations under the License. -->
+
+<% require 'erubis' -%>
+<% conf_category_map.each do |key,value| -%>
+export <%= key %>="<%= Erubis::Eruby.new(value).result(binding) %>"
+<% end -%>
Added: incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_properties.erb
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_properties.erb?rev=1214038&view=auto
==============================================================================
--- incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_properties.erb (added)
+++ incubator/ambari/trunk/agent/src/main/resources/puppet/modules/hadoop/templates/config_properties.erb Wed Dec 14 02:50:11 2011
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more -->
+<!-- contributor license agreements. See the NOTICE file distributed with -->
+<!-- this work for additional information regarding copyright ownership. -->
+<!-- The ASF licenses this file to You under the Apache License, Version 2.0 -->
+<!-- (the "License"); you may not use this file except in compliance with -->
+<!-- the License. You may obtain a copy of the License at -->
+<!-- -->
+<!-- http://www.apache.org/licenses/LICENSE-2.0 -->
+<!-- -->
+<!-- Unless required by applicable law or agreed to in writing, software -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS, -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
+<!-- See the License for the specific language governing permissions and -->
+<!-- limitations under the License. -->
+
+<configuration>
+<% require 'erubis' -%>
+<% conf_category_map.each do |key,value| -%>
+ <property>
+ <name><%= key %></name>
+ <value><%= Erubis::Eruby.new(value).result(binding) %></value>
+ </property>
+<% end -%>
+</configuration>