You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2011/10/15 01:33:46 UTC
svn commit: r1183561 [1/2] - in /incubator/bigtop/trunk/bigtop-deploy: ./
puppet/ puppet/manifests/ puppet/modules/ puppet/modules/hadoop-flume/
puppet/modules/hadoop-flume/manifests/ puppet/modules/hadoop-flume/tests/
puppet/modules/hadoop-hbase/ pupp...
Author: rvs
Date: Fri Oct 14 23:33:44 2011
New Revision: 1183561
URL: http://svn.apache.org/viewvc?rev=1183561&view=rev
Log:
BIGTOP-95. Puppet-based deployment infrastructure needs to be added to bigtop
Added:
incubator/bigtop/trunk/bigtop-deploy/
incubator/bigtop/trunk/bigtop-deploy/puppet/
incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/templates/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/templates/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/templates/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/taskcontroller.cfg
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/lib/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/lib/facter/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/lib/facter/kadm_keytab.rb
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/templates/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/templates/kadm5.acl
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/templates/kdc.conf
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/templates/krb5.conf
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/tests/
incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/tests/init.pp
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/cluster.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,180 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+node hadoop_cluster_node {
+ $hadoop_head_node="ip-10-114-221-125.ec2.internal"
+
+ $hadoop_namenode_host="$hadoop_head_node"
+ $hadoop_namenode_port="17020"
+ $hadoop_namenode_thrift_port="10090"
+ # $hadoop_dfs_namenode_plugins="org.apache.hadoop.thriftfs.NamenodePlugin"
+ # $hadoop_dfs_datanode_plugins="org.apache.hadoop.thriftfs.DatanodePlugin"
+
+ $hadoop_jobtracker_host="$hadoop_head_node"
+ $hadoop_jobtracker_port="8021"
+ $hadoop_jobtracker_thrift_port="9290"
+ # $hadoop_mapred_jobtracker_plugins="org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin"
+ # $hadoop_mapred_tasktracker_plugins="org.apache.hadoop.mapred.TaskTrackerCmonInst"
+
+ $hadoop_core_proxyusers = { oozie => { groups => '*', hosts => '*' } }
+
+ $hadoop_hbase_rootdir = "hdfs://$hadoop_namenode_host:$hadoop_namenode_port/hbase"
+ $hadoop_hbase_zookeeper_quorum = "$hadoop_head_node"
+
+ $hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"]
+
+ # $hadoop_security_authentication="kerberos"
+
+ # $kerberos_domain = "example.com"
+ # $kerberos_realm = "EXAMPLE.COM"
+ # $kerberos_kdc_server = "c0405"
+
+ # include kerberos::client
+ # kerberos::client::host_keytab { ["hdfs", "mapred", "hbase", "oozie"]:
+ # princs_map => { hdfs => [ "host", "hdfs" ],
+ # mapred => [ "mapred" ],
+ # hbase => [ "hbase" ],
+ # oozie => [ "oozie" ], },
+ # }
+}
+
+node hadoop_worker_node inherits hadoop_cluster_node {
+ hadoop::datanode { "datanode":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ # auth => $hadoop_security_authentication,
+ }
+
+ hadoop::tasktracker { "tasktracker":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ jobtracker_host => $hadoop_jobtracker_host,
+ jobtracker_port => $hadoop_jobtracker_port,
+ # auth => $hadoop_security_authentication,
+ }
+
+ hadoop-hbase::server { "hbase region server":
+ rootdir => $hadoop_hbase_rootdir,
+ zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
+ # kerberos_realm => $kerberos_realm,
+ }
+
+ $hdfs_data_dir = ["/mnt/data"]
+ $mapred_data_dir = [ "/mnt/scratch" ]
+
+ file {
+ $mapred_data_dir:
+ ensure => directory,
+ owner => mapred,
+ group => mapred,
+ mode => 755,
+ }
+
+ file {
+ $hdfs_data_dir:
+ ensure => directory,
+ owner => hdfs,
+ group => hdfs,
+ mode => 700,
+ }
+}
+
+
+node hadoop_head_node inherits hadoop_cluster_node {
+
+ # include kerberos::kdc, kerberos::kdc::admin_server
+
+ hadoop::namenode { "namenode":
+ port => $hadoop_namenode_port,
+ jobtracker_host => $hadoop_jobtracker_host,
+ jobtracker_port => $hadoop_jobtracker_port,
+ # thrift_port => $hadoop_namenode_thrift_port,
+ # auth => $hadoop_security_authentication,
+ }
+
+ hadoop::secondarynamenode { "secondary namenode":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ # auth => $hadoop_security_authentication,
+ }
+
+ hadoop::jobtracker { "jobtracker":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ host => $hadoop_jobtracker_host,
+ port => $hadoop_jobtracker_port,
+ # thrift_port => $hadoop_jobtracker_thrift_port,
+ # auth => $hadoop_security_authentication,
+ }
+
+ hadoop-hbase::master { "hbase master":
+ rootdir => $hadoop_hbase_rootdir,
+ zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
+ # kerberos_realm => $kerberos_realm,
+ }
+
+ hadoop-oozie::server { "oozie server":
+ # kerberos_realm => $kerberos_realm,
+ }
+
+ hadoop-zookeeper::server { "zookeeper":
+ myid => "0",
+ ensemble => $hadoop_zookeeper_ensemble,
+ }
+
+ $namenode_data_dir = ["/mnt/namenode"]
+
+ file { $namenode_data_dir:
+ ensure => directory,
+ owner => hdfs,
+ group => hdfs,
+ mode => 700,
+ }
+
+ $mapred_data_dir = ["/mnt/scratch"]
+
+ file { $mapred_data_dir:
+ ensure => directory,
+ owner => mapred,
+ group => mapred,
+ mode => 755,
+ }
+}
+
+node hadoop_gateway_node inherits hadoop_head_node {
+ # hadoop::client { "gateway":
+ # namenode_host => $hadoop_namenode_host,
+ # namenode_port => $hadoop_namenode_port,
+ # jobtracker_host => $hadoop_jobtracker_host,
+ # jobtracker_port => $hadoop_jobtracker_port,
+ # # auth => $hadoop_security_authentication,
+ # }
+
+ hadoop-pig::client { "pig client":
+ }
+ hadoop-hive::client { "hive client":
+ hbase_zookeeper_quorum => $hadoop_hbase_zookeeper_quorum,
+ }
+ hadoop-sqoop::client { "sqoop client":
+ }
+ hadoop-oozie::client { "oozie client":
+ }
+ hadoop-hbase::client { "hbase client":
+ }
+ hadoop-zookeeper::client { "zookeeper client":
+ }
+ hadoop-flume::client { "flume client":
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/manifests/site.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+stage {"pre": before => Stage["main"]}
+
+yumrepo { "Bigtop":
+ baseurl => "http://bigtop01.cloudera.org:8080/job/Bigtop-trunk-centos5/lastSuccessfulBuild/artifact/output/",
+ descr => "Bigtop packages",
+ enabled => 1,
+ gpgcheck => 0
+}
+
+package { "jdk":
+ ensure => "installed",
+}
+
+import "cluster.pp"
+
+node default inherits hadoop_worker_node {
+}
+
+node 'ip-10-114-221-125.ec2.internal' inherits hadoop_head_node {
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-flume {
+ define client {
+ package { "flume":
+ ensure => latest,
+ }
+ }
+
+ # It really is flume node, but node is a reserved keyword for puppet
+ define agent {
+ package { "flume-node":
+ ensure => latest,
+ }
+
+ service { "flume-node":
+ ensure => running,
+ require => Package["flume-node"],
+ # FIXME: this need to be fixed in upstream flume
+ hasstatus => false,
+ hasrestart => true,
+ }
+ }
+
+ define master {
+ package { "flume-master":
+ ensure => latest,
+ }
+
+ service { "flume-master":
+ ensure => running,
+ require => Package["flume-node"],
+ # FIXME: this need to be fixed in upstream flume
+ hasstatus => true,
+ hasrestart => true,
+ }
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-flume/tests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop-flume::client { "test-flume": }
+hadoop-flume::agent { "test-flume-node": }
+hadoop-flume::master { "test-flume-master": }
+
+
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,71 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-hbase {
+ class client-package {
+ package { "hadoop-hbase":
+ ensure => latest,
+ }
+ }
+
+ class common-server-config {
+ include client-package
+
+ file { "/etc/hbase/conf/hbase-site.xml":
+ content => template("hadoop-hbase/hbase-site.xml"),
+ require => Package["hadoop-hbase"],
+ }
+ file { "/etc/hbase/conf/hbase-env.sh":
+ content => template("hadoop-hbase/hbase-env.sh"),
+ require => Package["hadoop-hbase"],
+ }
+ }
+
+ define client {
+ include client-package
+ }
+
+ define server($rootdir, $zookeeper_quorum, $kerberos_realm = "") {
+ include common-server-config
+
+ package { "hadoop-hbase-regionserver":
+ ensure => latest,
+ }
+
+ service { "hadoop-hbase-regionserver":
+ ensure => running,
+ require => Package["hadoop-hbase-regionserver"],
+ subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+
+ define master($rootdir, $zookeeper_quorum, $kerberos_realm = "") {
+ include common-server-config
+
+ package { "hadoop-hbase-master":
+ ensure => latest,
+ }
+
+ service { "hadoop-hbase-master":
+ ensure => running,
+ require => Package["hadoop-hbase-master"],
+ subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-env.sh Fri Oct 14 23:33:44 2011
@@ -0,0 +1,76 @@
+#
+#/**
+# * Copyright 2007 The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+# Set environment variables here.
+
+# The java implementation to use. Java 1.6 required.
+# export JAVA_HOME=/usr/java/jdk1.6.0/
+
+# Extra Java CLASSPATH elements. Optional.
+export HBASE_CLASSPATH=/etc/hadoop-0.20/conf
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+# export HBASE_HEAPSIZE=1000
+
+# Extra Java runtime options.
+# Below are what we set by default. May only work with SUN JVM.
+# For more on why as well as other possible settings,
+# see http://wiki.apache.org/hadoop/PerformanceTuning
+export HBASE_OPTS="$HBASE_OPTS -ea -XX:+UseConcMarkSweepGC -XX:+CMSIncrementalMode"
+
+# Uncomment below to enable java garbage collection logging.
+# export HBASE_OPTS="$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log"
+
+# Uncomment and adjust to enable JMX exporting
+# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.
+# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html
+#
+# export HBASE_JMX_BASE="-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false"
+# export HBASE_MASTER_OPTS="$HBASE_MASTER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10101"
+# export HBASE_REGIONSERVER_OPTS="$HBASE_REGIONSERVER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10102"
+# export HBASE_THRIFT_OPTS="$HBASE_THRIFT_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103"
+# export HBASE_ZOOKEEPER_OPTS="$HBASE_ZOOKEEPER_OPTS $HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104"
+
+# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.
+# export HBASE_REGIONSERVERS=${HBASE_HOME}/conf/regionservers
+
+# Extra ssh options. Empty by default.
+# export HBASE_SSH_OPTS="-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR"
+
+# Where log files are stored. $HBASE_HOME/logs by default.
+# export HBASE_LOG_DIR=${HBASE_HOME}/logs
+
+# A string representing this instance of hbase. $USER by default.
+# export HBASE_IDENT_STRING=$USER
+
+# The scheduling priority for daemon processes. See 'man nice'.
+# export HBASE_NICENESS=10
+
+# The directory where pid files are stored. /tmp by default.
+# export HBASE_PID_DIR=/var/hadoop/pids
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+# export HBASE_SLAVE_SLEEP=0.1
+
+# Tell HBase whether it should manage it's own instance of Zookeeper or not.
+# export HBASE_MANAGES_ZK=true
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/templates/hbase-site.xml Fri Oct 14 23:33:44 2011
@@ -0,0 +1,58 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.cluster.distributed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hbase.zookeeper.quorum</name>
+ <value><%= zookeeper_quorum %></value>
+ </property>
+
+ <property>
+ <name>hbase.rootdir</name>
+ <value><%= rootdir %></value>
+ </property>
+
+<% if kerberos_realm != "" %>
+ <property>
+ <name>hbase.regionserver.kerberos.principal</name>
+ <value>hbase/_HOST@<%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>hbase.regionserver.keytab.file</name>
+ <value>/etc/hbase.keytab</value>
+ </property>
+ <property>
+ <name>hbase.master.kerberos.principal</name>
+ <value>hbase/_HOST@<%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>hbase.master.keytab.file</name>
+ <value>/etc/hbase.keytab</value>
+ </property>
+<% end %>
+</configuration>
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop-hbase::client { "hbase shell": }
+hadoop-hbase::master { "test-hbase-master":
+ rootdir => "hdfs://localhost:17020/hbase",
+ zookeeper_quorum => "localhost",
+ kerberos_realm => "KRB.YOU.ORG",
+}
+hadoop-hbase::server { "test-hbase-server":
+ rootdir => "hdfs://localhost:17020/hbase",
+ zookeeper_quorum => "localhost",
+ kerberos_realm => "KRB.YOU.ORG",
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,27 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-hive {
+ define client($hbase_master = "", $hbase_zookeeper_quorum = "") {
+ package { "hadoop-hive":
+ ensure => latest,
+ }
+
+ file { "/etc/hive/conf/hive-site.xml":
+ content => template('hadoop-hive/hive-site.xml'),
+ require => Package["hadoop-hive"],
+ }
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml Fri Oct 14 23:33:44 2011
@@ -0,0 +1,63 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more -->
+<!-- contributor license agreements. See the NOTICE file distributed with -->
+<!-- this work for additional information regarding copyright ownership. -->
+<!-- The ASF licenses this file to You under the Apache License, Version 2.0 -->
+<!-- (the "License"); you may not use this file except in compliance with -->
+<!-- the License. You may obtain a copy of the License at -->
+<!-- -->
+<!-- http://www.apache.org/licenses/LICENSE-2.0 -->
+<!-- -->
+<!-- Unless required by applicable law or agreed to in writing, software -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS, -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
+<!-- See the License for the specific language governing permissions and -->
+<!-- limitations under the License. -->
+
+<configuration>
+
+<!-- Hive Configuration can either be stored in this file or in the hadoop configuration files -->
+<!-- that are implied by Hadoop setup variables. -->
+<!-- Aside from Hadoop setup variables - this file is provided as a convenience so that Hive -->
+<!-- users do not have to edit hadoop configuration files (that may be managed as a centralized -->
+<!-- resource). -->
+
+<!-- Hive Execution Parameters -->
+
+<% if hbase_master != "" %>
+<property>
+ <name>hbase.master</name>
+ <value><%= hbase_master %></value>
+ <description>http://wiki.apache.org/hadoop/Hive/HBaseIntegration</description>
+</property>
+<% end %>
+
+<% if hbase_zookeeper_quorum != "" %>
+<property>
+ <name>hbase.zookeeper.quorum</name>
+ <value><%= hbase_zookeeper_quorum %></value>
+ <description>http://wiki.apache.org/hadoop/Hive/HBaseIntegration</description>
+</property>
+<% end %>
+
+<property>
+ <name>javax.jdo.option.ConnectionURL</name>
+ <value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value>
+ <description>JDBC connect string for a JDBC metastore</description>
+</property>
+
+<property>
+ <name>javax.jdo.option.ConnectionDriverName</name>
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+ <description>Driver class name for a JDBC metastore</description>
+</property>
+
+<property>
+ <name>hive.hwi.war.file</name>
+ <value>/usr/lib/hive/lib/hive-hwi-0.7.0-cdh3u1-SNAPSHOT.war</value>
+ <description>This is the WAR file with the jsp content for Hive Web Interface</description>
+</property>
+
+</configuration>
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-hive/tests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop-hive::client { "test-hive": }
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-oozie {
+ define client($kerberos_realm = "") {
+ package { "oozie-client":
+ ensure => latest,
+ }
+ }
+
+ define server($kerberos_realm = "") {
+ package { "oozie":
+ ensure => latest,
+ }
+
+ file { "/etc/oozie/oozie-site.xml":
+ content => template("hadoop-oozie/oozie-site.xml"),
+ require => Package["oozie"],
+ }
+
+ service { "oozie":
+ ensure => running,
+ require => Package["oozie"],
+ hasrestart => true,
+ hasstatus => true,
+ }
+
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/templates/oozie-site.xml Fri Oct 14 23:33:44 2011
@@ -0,0 +1,311 @@
+<?xml version="1.0"?>
+<!--
+ Copyright (c) 2010 Yahoo! Inc. All rights reserved.
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+
+<!--
+ Refer to the oozie-default.xml file for the complete list of
+ Oozie configuration properties and their default values.
+-->
+
+ <property>
+ <name>oozie.service.ActionService.executor.ext.classes</name>
+ <value>
+ org.apache.oozie.action.hadoop.HiveActionExecutor,
+ org.apache.oozie.action.hadoop.SqoopActionExecutor
+ </value>
+ </property>
+
+ <property>
+ <name>oozie.service.SchemaService.wf.ext.schemas</name>
+ <value>hive-action-0.2.xsd,sqoop-action-0.2.xsd</value>
+ </property>
+
+ <property>
+ <name>oozie.system.id</name>
+ <value>oozie-${user.name}</value>
+ <description>
+ The Oozie system ID.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.systemmode</name>
+ <value>NORMAL</value>
+ <description>
+ System mode for Oozie at startup.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.AuthorizationService.security.enabled</name>
+ <value>false</value>
+ <description>
+ Specifies whether security (user name/admin role) is enabled or not.
+ If disabled any user can manage Oozie system and manage any job.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.PurgeService.older.than</name>
+ <value>30</value>
+ <description>
+ Jobs older than this value, in days, will be purged by the PurgeService.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.PurgeService.purge.interval</name>
+ <value>3600</value>
+ <description>
+ Interval at which the purge service will run, in seconds.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.CallableQueueService.queue.size</name>
+ <value>10000</value>
+ <description>Max callable queue size</description>
+ </property>
+
+ <property>
+ <name>oozie.service.CallableQueueService.threads</name>
+ <value>10</value>
+ <description>Number of threads used for executing callables</description>
+ </property>
+
+ <property>
+ <name>oozie.service.CallableQueueService.callable.concurrency</name>
+ <value>3</value>
+ <description>
+ Maximum concurrency for a given callable type.
+ Each command is a callable type (submit, start, run, signal, job, jobs, suspend,resume, etc).
+ Each action type is a callable type (Map-Reduce, Pig, SSH, FS, sub-workflow, etc).
+ All commands that use action executors (action-start, action-end, action-kill and action-check) use
+ the action type as the callable type.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.coord.normal.default.timeout
+ </name>
+ <value>120</value>
+ <description>Default timeout for a coordinator action input check (in minutes) for normal job.
+ -1 means infinite timeout</description>
+ </property>
+
+ <property>
+ <name>oozie.db.schema.name</name>
+ <value>oozie</value>
+ <description>
+ Oozie DataBase Name
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.StoreService.create.db.schema</name>
+ <value>true</value>
+ <description>
+ Creates Oozie DB.
+
+ If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP.
+ If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.StoreService.jdbc.driver</name>
+ <value>org.apache.derby.jdbc.EmbeddedDriver</value>
+ <description>
+ JDBC driver class.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.StoreService.jdbc.url</name>
+ <value>jdbc:derby:${oozie.data.dir}/${oozie.db.schema.name}-db;create=true</value>
+ <description>
+ JDBC URL.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.StoreService.jdbc.username</name>
+ <value>sa</value>
+ <description>
+ DB user name.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.StoreService.jdbc.password</name>
+ <value> </value>
+ <description>
+ DB user password.
+
+ IMPORTANT: if password is emtpy leave a 1 space string, the service trims the value,
+ if empty Configuration assumes it is NULL.
+
+ IMPORTANT: if the StoreServicePasswordService is active, it will reset this value with the value given in
+ the console.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.StoreService.pool.max.active.conn</name>
+ <value>10</value>
+ <description>
+ Max number of connections.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.HadoopAccessorService.kerberos.enabled</name>
+ <value><%= if (kerberos_realm != "") ; "true" else "false" end %></value>
+ <description>
+ Indicates if Oozie is configured to use Kerberos.
+ </description>
+ </property>
+
+ <property>
+ <name>local.realm</name>
+ <value><%= kerberos_realm %></value>
+ <description>
+ Kerberos Realm used by Oozie and Hadoop. Using 'local.realm' to be aligned with Hadoop configuration
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.HadoopAccessorService.keytab.file</name>
+ <value>/etc/oozie.keytab</value>
+ <description>
+ Location of the Oozie user keytab file.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.HadoopAccessorService.kerberos.principal</name>
+ <value>${user.name}/<%= fqdn %>@${local.realm}</value>
+ <description>
+ Kerberos principal for Oozie service.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.HadoopAccessorService.jobTracker.whitelist</name>
+ <value> </value>
+ <description>
+ Whitelisted job tracker for Oozie service.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.HadoopAccessorService.nameNode.whitelist</name>
+ <value> </value>
+ <description>
+ Whitelisted job tracker for Oozie service.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.service.WorkflowAppService.system.libpath</name>
+ <value>/user/${user.name}/share/lib</value>
+ <description>
+ System library path to use for workflow applications.
+ This path is added to workflow application if their job properties sets
+ the property 'oozie.use.system.libpath' to true.
+ </description>
+ </property>
+
+ <property>
+ <name>use.system.libpath.for.mapreduce.and.pig.jobs</name>
+ <value>false</value>
+ <description>
+ If set to true, submissions of MapReduce and Pig jobs will include
+ automatically the system library path, thus not requiring users to
+ specify where the Pig JAR files are. Instead, the ones from the system
+ library path are used.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.authentication.type</name>
+ <value>simple</value>
+ <description>
+ Defines authentication used for Oozie HTTP endpoint.
+ Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME#
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.authentication.token.validity</name>
+ <value>36000</value>
+ <description>
+ Indicates how long (in seconds) an authentication token is valid before it has
+ to be renewed.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.authentication.signature.secret</name>
+ <value>oozie</value>
+ <description>
+ The signature secret for signing the authentication tokens.
+ If not set a random secret is generated at startup time.
+ In order to authentiation to work correctly across multiple hosts
+ the secret must be the same across al the hosts.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.authentication.cookie.domain</name>
+ <value></value>
+ <description>
+ The domain to use for the HTTP cookie that stores the authentication token.
+ In order to authentiation to work correctly across multiple hosts
+ the domain must be correctly set.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description>
+ Indicates if anonymous requests are allowed.
+ This setting is meaningful only when using 'simple' authentication.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.authentication.kerberos.principal</name>
+ <value>HTTP/<%= fqdn %>@${local.realm}</value>
+ <description>
+ Indicates the Kerberos principal to be used for HTTP endpoint.
+ The principal MUST start with 'HTTP/' as per Kerberos HTTP SPNEGO specification.
+ </description>
+ </property>
+
+ <property>
+ <name>oozie.authentication.kerberos.keytab</name>
+ <value>${oozie.service.HadoopAccessorService.keytab.file}</value>
+ <description>
+ Location of the keytab file with the credentials for the principal.
+ Referring to the same keytab file Oozie uses for its Kerberos credentials for Hadoop.
+ </description>
+ </property>
+
+</configuration>
+
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-oozie/tests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop-oozie::server { "test-oozie-server": }
+hadoop-oozie::client { "test-oozie": }
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-pig {
+ define client {
+ package { "hadoop-pig":
+ ensure => latest,
+ require => Package["hadoop-0.20"],
+ }
+
+ file { "/etc/pig/conf/pig.properties":
+ content => template('hadoop-pig/pig.properties'),
+ require => Package["hadoop-pig"],
+ owner => "root", /* FIXME: I'm really no sure about these -- we might end */
+ mode => "755", /* up deploying/testing a different thing compared */
+ /* to a straight rpm/deb deployment */
+ }
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/templates/pig.properties Fri Oct 14 23:33:44 2011
@@ -0,0 +1,70 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Pig configuration file. All values can be overwritten by command line arguments.
+# see bin/pig -help
+
+# log4jconf log4j configuration file
+# log4jconf=./conf/log4j.properties
+
+# brief logging (no timestamps)
+brief=false
+
+# clustername, name of the hadoop jobtracker. If no port is defined port 50020 will be used.
+#cluster
+
+#debug level, INFO is default
+debug=INFO
+
+# a file that contains pig script
+#file=
+
+# load jarfile, colon separated
+#jar=
+
+#verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+#exectype local|mapreduce, mapreduce is default
+#exectype=mapreduce
+# hod realted properties
+#ssh.gateway
+#hod.expect.root
+#hod.expect.uselatest
+#hod.command
+#hod.config.dir
+#hod.param
+
+
+#Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+#EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+#This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+
+######################
+# Everything below this line is Yahoo specific. Note that I've made
+# (almost) no changes to the lines above to make merging in from Apache
+# easier. Any values I don't want from above I override below.
+#
+# This file is configured for use with HOD on the production clusters. If you
+# want to run pig with a static cluster you will need to remove everything
+# below this line and set the cluster value (above) to the
+# hostname and port of your job tracker.
+
+exectype=mapreduce
+log.file=
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-pig/tests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include pig
+pig::client { "test-pig": }
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-sqoop {
+ define client {
+ package { "sqoop":
+ ensure => latest,
+ }
+
+ # FIXME: this is NOT supposed to be needed
+ # but for now that's the easiest way to add
+ # extra jars to sqoop classpath
+ file { "/usr/lib/sqoop/lib/mysql-connector-java-5.1.12.jar":
+ ensure => "/usr/share/java/mysql-connector-java-5.1.12.jar",
+ }
+ }
+
+ define metastore {
+ package { "sqoop-metastore":
+ ensure => latest,
+ }
+
+ service { "sqoop-metastore":
+ ensure => running,
+ require => Package["sqoop-metastore"],
+ hasstatus => true,
+ hasrestart => true,
+ }
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-sqoop/tests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop-sqoop::client { "test-sqoop": }
+hadoop-sqoop::metastore { "test-sqoop-metastore": }
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop-zookeeper {
+ define client {
+ package { "hadoop-zookeeper":
+ ensure => latest,
+ }
+ }
+
+ define server($myid, $ensemble = ["localhost:2888:3888"]) {
+ package { "hadoop-zookeeper-server":
+ ensure => latest,
+ }
+
+ service { "hadoop-zookeeper-server":
+ ensure => running,
+ require => Package["hadoop-zookeeper-server"],
+ subscribe => File["/etc/zookeeper/zoo.cfg"],
+ hasrestart => true,
+ hasstatus => true,
+ }
+
+ file { "/etc/zookeeper/zoo.cfg":
+ content => template("hadoop-zookeeper/zoo.cfg"),
+ require => Package["hadoop-zookeeper-server"],
+ }
+
+ file { "/var/zookeeper/myid":
+ content => inline_template("<%= myid %>"),
+ require => Package["hadoop-zookeeper-server"],
+ }
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/templates/zoo.cfg Fri Oct 14 23:33:44 2011
@@ -0,0 +1,31 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+maxClientCnxns=50
+# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=10
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=5
+# the directory where the snapshot is stored.
+dataDir=/var/zookeeper
+# the port at which the clients will connect
+clientPort=2181
+<% ensemble.each_with_index do |server,idx| %>
+server.<%= idx %>=<%= server %>
+<% end %>
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop-zookeeper/tests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+hadoop-zookeeper::client { "zoo visitor": }
+hadoop-zookeeper::server { "test-oozie-server":
+ myid => "0",
+ ensemble => ["foo:2888:3888", "bar:2888:3888", "baz:2888:3888"],
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/lib/facter/hadoop_storage_locations.rb Fri Oct 14 23:33:44 2011
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We make the assumption hadoop's data files will be located in /data/
+# Puppet needs to know where they are
+Facter.add("hadoop_storage_locations") do
+ setcode do
+
+ data_dir_path = "/data/"
+ storage_locations = ""
+
+ # We need to check /data/ exist
+ if File.directory?(data_dir_path)
+
+ # We assume all data directory will be a number
+ Dir.foreach(data_dir_path) { |directory|
+ storage_locations += (data_dir_path + directory + ';') if directory =~ /\d+/
+ }
+ end
+
+ # Return the list of storage locations for hadoop
+ if storage_locations == ""
+ storage_locations = "/mnt"
+ end
+ storage_locations
+ end
+end
+
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Fri Oct 14 23:33:44 2011
@@ -0,0 +1,223 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class hadoop {
+
+ /**
+ * Common definitions for hadoop nodes.
+ * They all need these files so we can access hdfs/jobs from any node
+ */
+ class common {
+ file {
+ "/etc/hadoop/conf/core-site.xml":
+ content => template('hadoop/core-site.xml'),
+ }
+
+ file {
+ "/etc/hadoop/conf/mapred-site.xml":
+ content => template('hadoop/mapred-site.xml'),
+ }
+
+ file {
+ "/etc/hadoop/conf/hdfs-site.xml":
+ content => template('hadoop/hdfs-site.xml'),
+ }
+
+ file {
+ "/etc/hadoop/conf/hadoop-env.sh":
+ content => template('hadoop/hadoop-env.sh'),
+ }
+
+ package { "hadoop":
+ ensure => latest,
+ }
+
+ package { ["hadoop-native"]: # KRB: "hadoop-sbin"
+ ensure => latest,
+ require => [Package["hadoop"], Yumrepo["Bigtop"]],
+ }
+
+ }
+
+
+ define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple") {
+
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_datanode_port = $port
+ $hadoop_security_authentication = $auth
+
+ include common
+
+ package { "hadoop-datanode":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ service { "hadoop-datanode":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [ Package["hadoop-datanode"] ],
+ }
+
+
+ }
+
+ define create_hdfs_dirs() {
+ $hdfs_dirs_meta = { "/tmp" => { perm => "777", user => "hdfs" },
+ "/mapred" => { perm => "755", user => "mapred" },
+ "/system" => { perm => "755", user => "hdfs" },
+ "/user" => { perm => "755", user => "hdfs" },
+ "/hbase" => { perm => "755", user => "hbase" },
+ "/benchmarks" => { perm => "777", user => "hdfs" },
+ "/user/hudson" => { perm => "777", user => "hudson"},
+ "/user/hive" => { perm => "777", user => "hive" } }
+
+
+ $user = $hdfs_dirs_meta[$title][user]
+ $perm = $hdfs_dirs_meta[$title][perm]
+
+ exec { "HDFS init $title":
+ user => "hdfs",
+ command => "/bin/bash -c 'hadoop fs -mkdir $title && hadoop fs -chown $user $name && hadoop fs -chmod $perm $name'",
+ unless => "/bin/bash -c 'hadoop fs -ls $name >/dev/null 2>&1'",
+ require => Service["hadoop-namenode"],
+ }
+ }
+
+ define namenode ($jobtracker_host, $jobtracker_port, $host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple") {
+
+ $hadoop_namenode_host = $host
+ $hadoop_namenode_port = $port
+ $hadoop_namenode_thrift_port = $thrift_port
+ $hadoop_jobtracker_host = $jobtracker_host
+ $hadoop_jobtracker_port = $jobtracker_port
+ $hadoop_security_authentication = $auth
+
+ include common
+
+ package { "hadoop-namenode":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ service { "hadoop-namenode":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [Package["hadoop-namenode"], Exec["namenode format"]],
+ }
+
+ exec { "namenode format":
+ user => "hdfs",
+ command => "/bin/bash -c 'yes Y | hadoop namenode -format'",
+ creates => inline_template("<%= hadoop_storage_locations.split(';')[0] %>/namenode/image"),
+ require => [Package["hadoop-namenode"]],
+ }
+
+ create_hdfs_dirs { [ "/mapred", "/tmp", "/system", "/user", "/hbase", "/benchmarks", "/user/hudson", "/user/hive" ]:
+ }
+ }
+
+
+ define jobtracker ($namenode_host, $namenode_port, $host = $fqdn, $port = "8021", $thrift_port = "9290", $auth = "simple") {
+
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_jobtracker_thrift_port = $thrift_port
+ $hadoop_jobtracker_host = $host
+ $hadoop_jobtracker_port = $port
+ $hadoop_security_authentication = $auth
+
+ include common
+
+ package { "hadoop-jobtracker":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ service { "hadoop-jobtracker":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-jobtracker"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [ Package["hadoop-jobtracker"] ]
+ }
+ }
+
+
+ define tasktracker ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple"){
+
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_jobtracker_host = $jobtracker_host
+ $hadoop_jobtracker_port = $jobtracker_port
+ $hadoop_security_authentication = $auth
+
+ include common
+
+ package { "hadoop-tasktracker":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ file { "/etc/hadoop/conf/taskcontroller.cfg":
+ content => template('hadoop/taskcontroller.cfg'),
+ }
+
+ service { "hadoop-tasktracker":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-tasktracker"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [ Package["hadoop-tasktracker"], File["/etc/hadoop/conf/taskcontroller.cfg"] ],
+ }
+ }
+
+
+ define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple") {
+
+ $hadoop_secondarynamenode_port = $port
+ $hadoop_security_authentication = $auth
+ include common
+
+ package { "hadoop-secondarynamenode":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ service { "hadoop-secondarynamenode":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [Package["hadoop-secondarynamenode"]],
+ }
+ }
+
+ define client ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple") {
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_jobtracker_host = $jobtracker_host
+ $hadoop_jobtracker_port = $jobtracker_port
+ $hadoop_security_authentication = $auth
+
+ include common
+
+ package { ["hadoop-doc", "hadoop-source", "hadoop-debuginfo",
+ "hadoop-fuse", "hadoop-libhdfs", "hadoop-pipes"]:
+ ensure => latest,
+ require => [Package["jdk"], Package["hadoop"]],
+ }
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml Fri Oct 14 23:33:44 2011
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more -->
+<!-- contributor license agreements. See the NOTICE file distributed with -->
+<!-- this work for additional information regarding copyright ownership. -->
+<!-- The ASF licenses this file to You under the Apache License, Version 2.0 -->
+<!-- (the "License"); you may not use this file except in compliance with -->
+<!-- the License. You may obtain a copy of the License at -->
+<!-- -->
+<!-- http://www.apache.org/licenses/LICENSE-2.0 -->
+<!-- -->
+<!-- Unless required by applicable law or agreed to in writing, software -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS, -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
+<!-- See the License for the specific language governing permissions and -->
+<!-- limitations under the License. -->
+
+<configuration>
+
+ <property>
+ <!-- URI of NN. Fully qualified. No IP.-->
+ <name>fs.default.name</name>
+ <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
+ </property>
+
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value><%= hadoop_security_authentication %></value>
+ </property>
+
+<% if hadoop_security_authentication == "kerberos" %>
+ <property>
+ <name>hadoop.security.authorization</name>
+ <value>true</value>
+ </property>
+<% end %>
+
+<% if has_variable?("hadoop_security_group_mapping") %>
+ <property>
+ <name>hadoop.security.group.mapping</name>
+ <value><%= hadoop_security_group_mapping %></value>
+ </property>
+<% end %>
+
+<% if has_variable?("hadoop_core_proxyusers") %>
+<% hadoop_core_proxyusers.each do |superuser, data| %>
+ <property>
+ <name>hadoop.proxyuser.<%= superuser %>.hosts</name>
+ <value><%= data['hosts'] %></value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.<%= superuser %>.groups</name>
+ <value><%= data['groups'] %></value>
+ </property>
+<% end %>
+
+<% end %>
+ <property>
+ <name>dfs.permissions</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>io.compression.codecs</name>
+ <value>org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.BZip2Codec<% if has_variable?("hadoop_snappy_codec") %>,org.apache.hadoop.io.compress.SnappyCodec<% end %></value>
+ </property>
+ <property>
+ <name>io.compression.codec.lzo.class</name>
+ <value>com.hadoop.compression.lzo.LzoCodec</value>
+ </property>
+
+<% if has_variable?("hadoop_config_fs_inmemory_size_mb") %>
+ <property>
+ <name>fs.inmemory.size.mb</name>
+ <value><%= hadoop_config_fs_inmemory_size_mb %></value>
+ </property>
+<% end %>
+
+<% if has_variable?("hadoop_config_io_file_buffer_size") %>
+ <property>
+ <name>io.file.buffer.size</name>
+ <value><%= hadoop_config_io_file_buffer_size %></value>
+ </property>
+<% end %>
+
+</configuration>
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh?rev=1183561&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh Fri Oct 14 23:33:44 2011
@@ -0,0 +1,90 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+<% def shell_config(shell_var, *puppet_var)
+ puppet_var = puppet_var[0] || shell_var.downcase
+ if has_variable? puppet_var
+ return "export #{shell_var}=#{scope.lookupvar(puppet_var)}"
+ else
+ return "#export #{shell_var}="
+ end
+ end %>
+# WARNING: Heavy puppet machinery is involved managing this file,
+# your edits stand no chance
+#
+# Set Hadoop-specific environment variables here.
+
+# The only required environment variable is JAVA_HOME. All others are
+# optional. When running a distributed configuration it is best to
+# set JAVA_HOME in this file, so that it is correctly defined on
+# remote nodes.
+
+# The java implementation to use. Required.
+<%= shell_config("JAVA_HOME", "hadoop_java_home") %>
+
+# Extra Java CLASSPATH elements. Optional.
+<%= shell_config("HADOOP_CLASSPATH") %>
+
+# The maximum amount of heap to use, in MB. Default is 1000.
+<%= shell_config("HADOOP_HEAPSIZE") %>
+
+# Extra Java runtime options. Empty by default.
+<%= shell_config("HADOOP_OPTS") %>
+
+# Command specific options appended to HADOOP_OPTS when specified
+<%= shell_config("HADOOP_NAMENODE_OPTS") %>
+<%= shell_config("HADOOP_SECONDARYNAMENODE_OPTS") %>
+<%= shell_config("HADOOP_DATANODE_OPTS") %>
+<%= shell_config("HADOOP_BALANCER_OPTS") %>
+<%= shell_config("HADOOP_JOBTRACKER_OPTS") %>
+<%= shell_config("HADOOP_TASKTRACKER_OPTS") %>
+
+# The following applies to multiple commands (fs, dfs, fsck, distcp etc)
+<%= shell_config("HADOOP_CLIENT_OPTS") %>
+
+# Extra ssh options. Empty by default.
+<%= shell_config("HADOOP_SSH_OPTS") %>
+
+# Where log files are stored. $HADOOP_HOME/logs by default.
+<%= shell_config("HADOOP_LOG_DIR") %>
+
+# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.
+<%= shell_config("HADOOP_SLAVES") %>
+
+# host:path where hadoop code should be rsync'd from. Unset by default.
+<%= shell_config("HADOOP_MASTER") %>
+
+# Seconds to sleep between slave commands. Unset by default. This
+# can be useful in large clusters, where, e.g., slave rsyncs can
+# otherwise arrive faster than the master can service them.
+<%= shell_config("HADOOP_SLAVE_SLEEP") %>
+
+# The directory where pid files are stored. /tmp by default.
+<%= shell_config("HADOOP_PID_DIR") %>
+
+# A string representing this instance of hadoop. $USER by default.
+<%= shell_config("HADOOP_IDENT_STRING") %>
+
+# The scheduling priority for daemon processes. See 'man nice'.
+<%= shell_config("HADOOP_NICENESS") %>
+
+### WARNING: the following is NOT really optional. It is a shame that stock Hadoop
+### hadoop_env.sh doesn't make it clear -- you can NOT turn com.sun.management.jmxremote off
+### and have a working Hadoop cluster.
+export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_NAMENODE_OPTS"
+export HADOOP_SECONDARYNAMENODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_SECONDARYNAMENODE_OPTS"
+export HADOOP_DATANODE_OPTS="-Dcom.sun.management.jmxremote $HADOOP_DATANODE_OPTS"
+export HADOOP_BALANCER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_BALANCER_OPTS"
+export HADOOP_JOBTRACKER_OPTS="-Dcom.sun.management.jmxremote $HADOOP_JOBTRACKER_OPTS"