You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2012/04/16 18:10:32 UTC
svn commit: r1326670 [2/47] - in /incubator/bigtop/trunk: ./
bigtop-deploy/puppet/manifests/
bigtop-deploy/puppet/modules/hadoop-hbase/manifests/
bigtop-deploy/puppet/modules/hadoop-hbase/templates/
bigtop-deploy/puppet/modules/hadoop-oozie/manifests/ ...
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp Mon Apr 16 16:10:22 2012
@@ -22,12 +22,15 @@ class hadoop {
class kerberos {
require kerberos::client
-
+
kerberos::host_keytab { "hdfs":
princs => [ "host", "hdfs" ],
+ spnego => true,
}
kerberos::host_keytab { [ "yarn", "mapred" ]:
+ tag => "mapreduce",
+ spnego => true,
}
}
@@ -37,84 +40,172 @@ class hadoop {
}
file {
- "/etc/hadoop/conf/core-site.xml":
- content => template('hadoop/core-site.xml'),
+ "/etc/hadoop/conf/hadoop-env.sh":
+ content => template('hadoop/hadoop-env.sh'),
require => [Package["hadoop"]],
}
- file {
- "/etc/hadoop/conf/mapred-site.xml":
- content => template('hadoop/mapred-site.xml'),
- require => [Package["hadoop"]],
+ package { "hadoop":
+ ensure => latest,
+ require => Package["jdk"],
}
+ #FIXME: package { "hadoop-native":
+ # ensure => latest,
+ # require => [Package["hadoop"]],
+ #}
+ }
+
+ class common-yarn inherits common {
+ package { "hadoop-yarn":
+ ensure => latest,
+ require => [Package["jdk"], Package["hadoop"]],
+ }
+
file {
- "/etc/hadoop/conf/hdfs-site.xml":
- content => template('hadoop/hdfs-site.xml'),
+ "/etc/hadoop/conf/yarn-site.xml":
+ content => template('hadoop/yarn-site.xml'),
require => [Package["hadoop"]],
}
+ file { "/etc/hadoop/conf/container-executor.cfg":
+ content => template('hadoop/container-executor.cfg'),
+ require => [Package["hadoop"]],
+ }
+ }
+
+ class common-hdfs inherits common {
+ if ($auth == "kerberos" and $ha == "enabled") {
+ fail("High-availability secure clusters are not currently supported")
+ }
+
+ if ($ha == 'enabled') {
+ $nameservice_id = extlookup("hadoop_ha_nameservice_id", "ha-nn-uri")
+ }
+
+ package { "hadoop-hdfs":
+ ensure => latest,
+ require => [Package["jdk"], Package["hadoop"]],
+ }
+
file {
- "/etc/hadoop/conf/hadoop-env.sh":
- content => template('hadoop/hadoop-env.sh'),
+ "/etc/hadoop/conf/core-site.xml":
+ content => template('hadoop/core-site.xml'),
require => [Package["hadoop"]],
}
file {
- "/etc/default/hadoop":
- content => template('hadoop/hadoop'),
+ "/etc/hadoop/conf/hdfs-site.xml":
+ content => template('hadoop/hdfs-site.xml'),
require => [Package["hadoop"]],
}
+ }
- package { "hadoop":
+ class common-mapred-app inherits common-hdfs {
+ package { "hadoop-mapreduce":
ensure => latest,
+ require => [Package["jdk"], Package["hadoop"]],
}
- package { "hadoop-native":
- ensure => latest,
+ file {
+ "/etc/hadoop/conf/mapred-site.xml":
+ content => template('hadoop/mapred-site.xml'),
+ require => [Package["hadoop"]],
+ }
+
+ file { "/etc/hadoop/conf/taskcontroller.cfg":
+ content => template('hadoop/taskcontroller.cfg'),
require => [Package["hadoop"]],
}
}
+ define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple", $dirs = ["/tmp/data"], $ha = 'disabled') {
- define datanode ($namenode_host, $namenode_port, $port = "50075", $auth = "simple", $dirs = ["/tmp/data"]) {
-
- $hadoop_namenode_host = $namenode_host
- $hadoop_namenode_port = $namenode_port
- $hadoop_datanode_port = $port
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_datanode_port = $port
$hadoop_security_authentication = $auth
- include common
+ if ($ha == 'enabled') {
+ # Needed by hdfs-site.xml
+ $sshfence_keydir = "/usr/lib/hadoop/.ssh"
+ $sshfence_keypath = "$sshfence_keydir/id_sshfence"
+ $sshfence_user = extlookup("hadoop_ha_sshfence_user", "hdfs")
+ $shared_edits_dir = extlookup("hadoop_ha_shared_edits_dir", "/hdfs_shared")
+ }
+
+ include common-hdfs
- package { "hadoop-datanode":
+ package { "hadoop-hdfs-datanode":
ensure => latest,
require => Package["jdk"],
}
- if ($hadoop_security_authentication == "kerberos") {
- package { "hadoop-sbin":
- ensure => latest,
- require => [Package["hadoop"]],
- }
+ file {
+ "/etc/default/hadoop-hdfs-datanode":
+ content => template('hadoop/hadoop-hdfs'),
+ require => [Package["hadoop-hdfs-datanode"]],
}
- service { "hadoop-datanode":
+ service { "hadoop-hdfs-datanode":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [ Package["hadoop-datanode"], File[$dirs] ],
+ subscribe => [Package["hadoop-hdfs-datanode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [ Package["hadoop-hdfs-datanode"], File[$dirs] ],
}
- Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-datanode"]
+ Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-datanode"]
file { $dirs:
ensure => directory,
owner => hdfs,
group => hdfs,
mode => 755,
- require => [Package["hadoop"]],
+ require => [ Package["hadoop-hdfs"] ],
}
}
+ define httpfs ($namenode_host, $namenode_port, $port = "14000", $auth = "simple", $secret = "hadoop httpfs secret") {
+
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_httpfs_port = $port
+ $hadoop_security_authentication = $auth
+
+ if ($auth == "kerberos") {
+ kerberos::host_keytab { "httpfs":
+ spnego => true,
+ }
+ }
+
+ package { "hadoop-httpfs":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ file { "/etc/hadoop-httpfs/conf/httpfs-site.xml":
+ content => template('hadoop/httpfs-site.xml'),
+ require => [Package["hadoop-httpfs"]],
+ }
+
+ file { "/etc/hadoop-httpfs/conf/httpfs-env.sh":
+ content => template('hadoop/httpfs-env.sh'),
+ require => [Package["hadoop-httpfs"]],
+ }
+
+ file { "/etc/hadoop-httpfs/conf/httpfs-signature.secret":
+ content => inline_template("<%= secret %>"),
+ require => [Package["hadoop-httpfs"]],
+ }
+
+ service { "hadoop-httpfs":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-httpfs"], File["/etc/hadoop-httpfs/conf/httpfs-site.xml"], File["/etc/hadoop-httpfs/conf/httpfs-env.sh"], File["/etc/hadoop-httpfs/conf/httpfs-signature.secret"]],
+ require => [ Package["hadoop-httpfs"] ],
+ }
+ Kerberos::Host_keytab <| title == "httpfs" |> -> Service["hadoop-httpfs"]
+ }
+
class kinit {
include hadoop::kerberos
@@ -138,140 +229,293 @@ class hadoop {
user => "hdfs",
command => "/bin/bash -c 'hadoop fs -mkdir $title && hadoop fs -chmod $perm $title && hadoop fs -chown $user $title'",
unless => "/bin/bash -c 'hadoop fs -ls $name >/dev/null 2>&1'",
- require => [ Service["hadoop-namenode"], Exec["namenode format"] ],
+ require => Service["hadoop-hdfs-namenode"],
}
+ Exec <| title == "activate nn1" |> -> Exec["HDFS init $title"]
}
- define namenode ($jobtracker_host, $jobtracker_port, $host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"]) {
+ define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"], $ha = 'disabled') {
+ $first_namenode = inline_template("<%= host.to_a[0] %>")
$hadoop_namenode_host = $host
$hadoop_namenode_port = $port
$hadoop_namenode_thrift_port = $thrift_port
- $hadoop_jobtracker_host = $jobtracker_host
- $hadoop_jobtracker_port = $jobtracker_port
$hadoop_security_authentication = $auth
- include common
+ if ($ha == 'enabled') {
+ $sshfence_user = extlookup("hadoop_ha_sshfence_user", "hdfs")
+ $sshfence_user_home = extlookup("hadoop_ha_sshfence_user_home", "/var/lib/hadoop-hdfs")
+ $sshfence_keydir = "$sshfence_user_home/.ssh"
+ $sshfence_keypath = "$sshfence_keydir/id_sshfence"
+ $sshfence_privkey = extlookup("hadoop_ha_sshfence_privkey", "$extlookup_datadir/hadoop/id_sshfence")
+ $sshfence_pubkey = extlookup("hadoop_ha_sshfence_pubkey", "$extlookup_datadir/hadoop/id_sshfence.pub")
+ $shared_edits_dir = extlookup("hadoop_ha_shared_edits_dir", "/hdfs_shared")
+ $nfs_server = extlookup("hadoop_ha_nfs_server", "")
+ $nfs_path = extlookup("hadoop_ha_nfs_path", "")
+
+ file { $sshfence_keydir:
+ ensure => directory,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0700',
+ require => Package["hadoop-hdfs"],
+ }
+
+ file { $sshfence_keypath:
+ source => $sshfence_privkey,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0600',
+ before => Service["hadoop-hdfs-namenode"],
+ require => File[$sshfence_keydir],
+ }
+
+ file { "$sshfence_keydir/authorized_keys":
+ source => $sshfence_pubkey,
+ owner => 'hdfs',
+ group => 'hdfs',
+ mode => '0600',
+ before => Service["hadoop-hdfs-namenode"],
+ require => File[$sshfence_keydir],
+ }
+
+ file { $shared_edits_dir:
+ ensure => directory,
+ }
+
+ if ($nfs_server) {
+ if (!$nfs_path) {
+ fail("No nfs share specified for shared edits dir")
+ }
+
+ require nfs::client
+
+ mount { $shared_edits_dir:
+ ensure => "mounted",
+ atboot => true,
+ device => "${nfs_server}:${nfs_path}",
+ fstype => "nfs",
+ options => "tcp,soft,timeo=10,intr,rsize=32768,wsize=32768",
+ require => File[$shared_edits_dir],
+ before => Service["hadoop-hdfs-namenode"],
+ }
+ }
+ }
+
+ include common-hdfs
- package { "hadoop-namenode":
+ package { "hadoop-hdfs-namenode":
ensure => latest,
require => Package["jdk"],
}
- service { "hadoop-namenode":
+ service { "hadoop-hdfs-namenode":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [Package["hadoop-namenode"], Exec["namenode format"]],
+ subscribe => [Package["hadoop-hdfs-namenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [Package["hadoop-hdfs-namenode"]],
}
- Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-namenode"]
+ Exec <| tag == "namenode-format" |> -> Service["hadoop-hdfs-namenode"]
+ Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-namenode"]
- exec { "namenode format":
- user => "hdfs",
- command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
- creates => "${namenode_data_dirs[0]}/image",
- require => [ Package["hadoop-namenode"], File[$dirs] ],
- }
+ if ($::fqdn == $first_namenode) {
+ exec { "namenode format":
+ user => "hdfs",
+ command => "/bin/bash -c 'yes Y | hadoop namenode -format >> /tmp/nn.format.log 2>&1'",
+ creates => "${namenode_data_dirs[0]}/current/VERSION",
+ require => [ Package["hadoop-hdfs-namenode"], File[$dirs] ],
+ tag => "namenode-format",
+ }
+
+ if ($ha == "enabled") {
+ exec { "activate nn1":
+ command => "/usr/bin/hdfs haadmin -transitionToActive nn1",
+ user => "hdfs",
+ unless => "/usr/bin/test $(/usr/bin/hdfs haadmin -getServiceState nn1) = active",
+ require => Service["hadoop-hdfs-namenode"],
+ }
+ }
+ } elsif ($ha == "enabled") {
+ hadoop::namedir_copy { $namenode_data_dirs:
+ source => $first_namenode,
+ ssh_identity => $sshfence_keypath,
+ require => File[$sshfence_keypath],
+ }
+ }
+
+ file {
+ "/etc/default/hadoop-hdfs-namenode":
+ content => template('hadoop/hadoop-hdfs'),
+ require => [Package["hadoop-hdfs-namenode"]],
+ }
file { $dirs:
ensure => directory,
owner => hdfs,
group => hdfs,
mode => 700,
- require => [Package["hadoop"]],
+ require => [Package["hadoop-hdfs"]],
}
}
+ define namedir_copy ($source, $ssh_identity) {
+ exec { "copy namedir $title from first namenode":
+ command => "/usr/bin/rsync -avz -e '/usr/bin/ssh -oStrictHostKeyChecking=no -i $ssh_identity' '${source}:$title/' '$title/'",
+ user => "hdfs",
+ tag => "namenode-format",
+ creates => "$title/current/VERSION",
+ }
+ }
+
+ define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple") {
- define jobtracker ($namenode_host, $namenode_port, $host = $fqdn, $port = "8021", $thrift_port = "9290", $auth = "simple", $dirs = ["/tmp/mr"]) {
+ $hadoop_secondarynamenode_port = $port
+ $hadoop_security_authentication = $auth
- $hadoop_namenode_host = $namenode_host
- $hadoop_namenode_port = $namenode_port
- $hadoop_jobtracker_thrift_port = $thrift_port
- $hadoop_jobtracker_host = $host
- $hadoop_jobtracker_port = $port
- $hadoop_security_authentication = $auth
+ include common-hdfs
- include common
+ package { "hadoop-hdfs-secondarynamenode":
+ ensure => latest,
+ require => Package["jdk"],
+ }
- package { "hadoop-jobtracker":
- ensure => latest,
- require => Package["jdk"],
- }
+ file {
+ "/etc/default/hadoop-hdfs-secondarynamenode":
+ content => template('hadoop/hadoop-hdfs'),
+ require => [Package["hadoop-hdfs-secondarynamenode"]],
+ }
- service { "hadoop-jobtracker":
+ service { "hadoop-hdfs-secondarynamenode":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-jobtracker"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [ Package["hadoop-jobtracker"], File[$dirs] ],
+ subscribe => [Package["hadoop-hdfs-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hdfs-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
+ require => [Package["hadoop-hdfs-secondarynamenode"]],
}
-
- file { $dirs:
- ensure => directory,
- owner => mapred,
- group => mapred,
- mode => 755,
- require => [Package["hadoop"]],
+ Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-hdfs-secondarynamenode"]
+ }
+
+
+ define resourcemanager ($host = $fqdn, $port = "8040", $rt_port = "8025", $sc_port = "8030", $thrift_port = "9290", $auth = "simple") {
+ $hadoop_rm_host = $host
+ $hadoop_rm_port = $port
+ $hadoop_rt_port = $rt_port
+ $hadoop_sc_port = $sc_port
+ $hadoop_security_authentication = $auth
+
+ include common-yarn
+
+ package { "hadoop-yarn-resourcemanager":
+ ensure => latest,
+ require => Package["jdk"],
+ }
+
+ service { "hadoop-yarn-resourcemanager":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-yarn-resourcemanager"], File["/etc/hadoop/conf/hadoop-env.sh"],
+ File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+ require => [ Package["hadoop-yarn-resourcemanager"] ],
}
- Kerberos::Host_keytab <| title == "mapred" |> -> Service["hadoop-jobtracker"]
+ Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-yarn-resourcemanager"]
}
+ define proxyserver ($host = $fqdn, $port = "8088", $auth = "simple") {
+ $hadoop_ps_host = $host
+ $hadoop_ps_port = $port
+ $hadoop_security_authentication = $auth
- define tasktracker ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple", $dirs = ["/tmp/mr"]){
+ include common-yarn
- $hadoop_namenode_host = $namenode_host
- $hadoop_namenode_port = $namenode_port
- $hadoop_jobtracker_host = $jobtracker_host
- $hadoop_jobtracker_port = $jobtracker_port
- $hadoop_security_authentication = $auth
+ package { "hadoop-yarn-proxyserver":
+ ensure => latest,
+ require => Package["jdk"],
+ }
- include common
+ service { "hadoop-yarn-proxyserver":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-yarn-proxyserver"], File["/etc/hadoop/conf/hadoop-env.sh"],
+ File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+ require => [ Package["hadoop-yarn-proxyserver"] ],
+ }
+ Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-yarn-proxyserver"]
+ }
- package { "hadoop-tasktracker":
- ensure => latest,
- require => Package["jdk"],
- }
-
- file { "/etc/hadoop/conf/taskcontroller.cfg":
- content => template('hadoop/taskcontroller.cfg'),
+ define historyserver ($host = $fqdn, $port = "10020", $webapp_port = "19888", $auth = "simple") {
+ $hadoop_hs_host = $host
+ $hadoop_hs_port = $port
+ $hadoop_hs_webapp_port = $app_port
+ $hadoop_security_authentication = $auth
+
+ include common-mapred-app
+
+ package { "hadoop-mapreduce-historyserver":
+ ensure => latest,
+ require => Package["jdk"],
}
- service { "hadoop-tasktracker":
+ service { "hadoop-mapreduce-historyserver":
ensure => running,
hasstatus => true,
- subscribe => [Package["hadoop-tasktracker"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/mapred-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [ Package["hadoop-tasktracker"], File["/etc/hadoop/conf/taskcontroller.cfg"], File[$dirs] ],
+ subscribe => [Package["hadoop-mapreduce-historyserver"], File["/etc/hadoop/conf/hadoop-env.sh"],
+ File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+ require => [Package["hadoop-mapreduce-historyserver"]],
+ }
+ Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-mapreduce-historyserver"]
+ }
+
+
+ define nodemanager ($rm_host, $rm_port, $rt_port, $auth = "simple", $dirs = ["/tmp/yarn"]){
+ $hadoop_rm_host = $rm_host
+ $hadoop_rm_port = $rm_port
+ $hadoop_rt_port = $rt_port
+
+ include common-yarn
+
+ package { "hadoop-yarn-nodemanager":
+ ensure => latest,
+ require => Package["jdk"],
}
+ service { "hadoop-yarn-nodemanager":
+ ensure => running,
+ hasstatus => true,
+ subscribe => [Package["hadoop-yarn-nodemanager"], File["/etc/hadoop/conf/hadoop-env.sh"],
+ File["/etc/hadoop/conf/yarn-site.xml"], File["/etc/hadoop/conf/core-site.xml"]],
+ require => [ Package["hadoop-yarn-nodemanager"], File[$dirs] ],
+ }
+ Kerberos::Host_keytab <| tag == "mapreduce" |> -> Service["hadoop-yarn-nodemanager"]
+
file { $dirs:
ensure => directory,
- owner => mapred,
- group => mapred,
+ owner => yarn,
+ group => yarn,
mode => 755,
- require => [Package["hadoop"]],
+ require => [Package["hadoop-yarn"]],
}
- Kerberos::Host_keytab <| title == "mapred" |> -> Service["hadoop-tasktracker"]
}
-
- define secondarynamenode ($namenode_host, $namenode_port, $port = "50090", $auth = "simple") {
-
- $hadoop_secondarynamenode_port = $port
+ define mapred-app ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple", $jobhistory_host = "", $jobhistory_port="10020", $dirs = ["/tmp/mr"]){
+ $hadoop_namenode_host = $namenode_host
+ $hadoop_namenode_port = $namenode_port
+ $hadoop_jobtracker_host = $jobtracker_host
+ $hadoop_jobtracker_port = $jobtracker_port
$hadoop_security_authentication = $auth
- include common
- package { "hadoop-secondarynamenode":
- ensure => latest,
- require => Package["jdk"],
- }
+ include common-mapred-app
- service { "hadoop-secondarynamenode":
- ensure => running,
- hasstatus => true,
- subscribe => [Package["hadoop-secondarynamenode"], File["/etc/hadoop/conf/core-site.xml"], File["/etc/hadoop/conf/hadoop-env.sh"]],
- require => [Package["hadoop-secondarynamenode"]],
+ if ($jobhistory_host != "") {
+ $hadoop_hs_host = $jobhistory_host
+ $hadoop_hs_port = $jobhistory_port
+ }
+
+ file { $dirs:
+ ensure => directory,
+ owner => yarn,
+ group => yarn,
+ mode => 755,
+ require => [Package["hadoop-mapreduce"]],
}
- Kerberos::Host_keytab <| title == "hdfs" |> -> Service["hadoop-secondarynamenode"]
}
define client ($namenode_host, $namenode_port, $jobtracker_host, $jobtracker_port, $auth = "simple") {
@@ -281,12 +525,12 @@ class hadoop {
$hadoop_jobtracker_port = $jobtracker_port
$hadoop_security_authentication = $auth
- include common
+ include common-mapred-app
- package { ["hadoop-doc", "hadoop-source", "hadoop-debuginfo",
- "hadoop-fuse", "hadoop-libhdfs", "hadoop-pipes"]:
+ # FIXME: "hadoop-source", "hadoop-fuse", "hadoop-pipes"
+ package { ["hadoop-doc", "hadoop-debuginfo", "hadoop-libhdfs"]:
ensure => latest,
- require => [Package["jdk"], Package["hadoop"]],
+ require => [Package["jdk"], Package["hadoop"], Package["hadoop-hdfs"], Package["hadoop-mapreduce"]],
}
}
}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg Mon Apr 16 16:10:22 2012
@@ -0,0 +1,3 @@
+yarn.nodemanager.linux-container-executor.group=yarn
+#banned.users=foo,bar
+#min.user.id=1000
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml Mon Apr 16 16:10:22 2012
@@ -16,14 +16,20 @@
<!-- See the License for the specific language governing permissions and -->
<!-- limitations under the License. -->
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
<configuration>
<property>
<!-- URI of NN. Fully qualified. No IP.-->
- <name>fs.default.name</name>
- <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
+ <name>fs.defaultFS</name>
+<% if ha == "enabled" -%>
+ <value>hdfs://<%= nameservice_id %></value>
+<% else -%>
+ <value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>
+<% end -%>
</property>
+
<property>
<name>hadoop.security.authentication</name>
<value><%= hadoop_security_authentication %></value>
@@ -56,11 +62,6 @@
<% end %>
<% end %>
- <property>
- <name>dfs.permissions</name>
- <value>false</value>
- </property>
-
<% if has_variable?("hadoop_snappy_codec") %>
<property>
<name>io.compression.codecs</name>
Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs (from r1325587, incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs&p1=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-hdfs Mon Apr 16 16:10:22 2012
@@ -11,18 +11,16 @@
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
-export HADOOP_HOME_WARN_SUPPRESS=true
-export HADOOP_HOME=/usr/lib/hadoop
-export HADOOP_PID_DIR=/var/run/hadoop
-export HADOOP_LOG_DIR=/var/log/hadoop
+# limitations under the License.
+export HADOOP_PID_DIR=/var/run/hadoop-hdfs
+export HADOOP_LOG_DIR=/var/log/hadoop-hdfs
export HADOOP_NAMENODE_USER=hdfs
export HADOOP_SECONDARYNAMENODE_USER=hdfs
export HADOOP_DATANODE_USER=hdfs
+export HADOOP_IDENT_STRING=hdfs
+
<% if hadoop_security_authentication == "kerberos" %>
export HADOOP_SECURE_DN_USER=hdfs
export HADOOP_SECURE_DN_PID_DIR=$HADOOP_PID_DIR
export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_LOG_DIR
<% end %>
-export HADOOP_JOBTRACKER_USER=mapred
-export HADOOP_TASKTRACKER_USER=mapred
-export HADOOP_IDENT_STRING=hadoop
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml Mon Apr 16 16:10:22 2012
@@ -16,9 +16,64 @@
<!-- See the License for the specific language governing permissions and -->
<!-- limitations under the License. -->
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
<configuration>
-<% if hadoop_security_authentication == "kerberos" %>
+<% if ha == "enabled" -%>
+ <!-- HA -->
+ <property>
+ <name>dfs.federation.nameservices</name>
+ <value><%= nameservice_id %></value>
+ </property>
+
+ <property>
+ <name>dfs.ha.namenodes.<%= nameservice_id %></name>
+ <value><%= (1..namenode_hosts.length).map { |n| "nn#{n}" }.join(",") %></value>
+ </property>
+
+<% namenode_hosts.each_with_index do |host,idx| -%>
+ <property>
+ <name>dfs.namenode.rpc-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
+ <value><%= host %>:<%= hadoop_namenode_port %></value>
+ </property>
+
+ <property>
+ <name>dfs.namenode.http-address.<%= nameservice_id %>.nn<%= idx+1 %></name>
+ <value><%= host %>:50070</value>
+ </property>
+
+<% end -%>
+<% if has_variable?("shared_edits_dir") -%>
+ <property>
+ <name>dfs.namenode.shared.edits.dir</name>
+ <value><%= shared_edits_dir %></value>
+ </property>
+
+<% end -%>
+ <property>
+ <name>dfs.client.failover.proxy.provider.<%= nameservice_id %></name>
+ <value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
+ </property>
+
+<% if has_variable?("sshfence_privkey") -%>
+ <property>
+ <name>dfs.ha.fencing.methods</name>
+ <value>sshfence(<%= sshfence_user %>)</value>
+ </property>
+
+ <property>
+ <name>dfs.ha.fencing.ssh.private-key-files</name>
+ <value><%= sshfence_keypath %></value>
+ </property>
+
+<% else -%>
+ <property>
+ <name>dfs.ha.fencing.methods</name>
+ <value></value>
+ </property>
+
+<% end -%>
+<% elsif hadoop_security_authentication == "kerberos" -%>
<property>
<name>dfs.block.access.token.enable</name>
<value>true</value>
@@ -27,7 +82,7 @@
<!-- NameNode security config -->
<property>
<name>dfs.https.address</name>
- <value><%= hadoop_namenode_host %>:50475</value>
+ <value><%= namenode_hosts[0] %>:50475</value>
</property>
<property>
<name>dfs.https.port</name>
@@ -45,15 +100,24 @@
<name>dfs.namenode.kerberos.https.principal</name>
<value>host/_HOST@<%= kerberos_realm %></value>
</property>
+ <property>
+ <name>dfs.web.authentication.kerberos.keytab</name>
+ <value>/etc/hdfs.keytab</value> <!-- path to the HDFS keytab -->
+ </property>
+ <property>
+ <name>dfs.web.authentication.kerberos.principal</name>
+ <value>HTTP/_HOST@<%= kerberos_realm %></value>
+ </property>
+
<!-- Secondary NameNode security config -->
<property>
<name>dfs.secondary.http.address</name>
- <value><%= hadoop_namenode_host %>:0</value>
+ <value><%= namenode_hosts[0] %>:0</value>
</property>
<property>
<name>dfs.secondary.https.address</name>
- <value><%= hadoop_namenode_host %>:50495</value>
+ <value><%= namenode_hosts[0] %>:50495</value>
</property>
<property>
<name>dfs.secondary.https.port</name>
@@ -97,16 +161,10 @@
<name>dfs.datanode.kerberos.https.principal</name>
<value>host/_HOST@<%= kerberos_realm %></value>
</property>
-<% end %>
+<% end -%>
<!-- name node -->
<property>
- <!-- URI of NN. Fully qualified. No IP.-->
- <name>fs.default.name</name>
- <value>hdfs://<%= hadoop_namenode_host %>:<%= hadoop_namenode_port%></value>
- </property>
-
- <property>
<name>dfs.data.dir</name>
<value><%= hdfs_data_dirs.join(",") %></value>
</property>
@@ -116,30 +174,38 @@
<value><%= namenode_data_dirs.join(",") %></value>
</property>
+ <property>
+ <name>dfs.permissions.superusergroup</name>
+ <value>hadoop</value>
+ <description>The name of the group of super-users.</description>
+ </property>
+
<!-- Enable Hue plugins -->
-<% if has_variable?("hadoop_dfs_namenode_plugins") %>
+<% if has_variable?("hadoop_dfs_namenode_plugins") -%>
<property>
<name>dfs.namenode.plugins</name>
<value><%= hadoop_dfs_namenode_plugins %></value>
<description>Comma-separated list of namenode plug-ins to be activated.
</description>
</property>
-<% end %>
-<% if has_variable?("hadoop_dfs_datanode_plugins") %>
+
+<% end -%>
+<% if has_variable?("hadoop_dfs_datanode_plugins") -%>
<property>
<name>dfs.datanode.plugins</name>
<value><%= hadoop_dfs_datanode_plugins %></value>
<description>Comma-separated list of datanode plug-ins to be activated.
</description>
</property>
-<% end %>
-<% if has_variable?("hadoop_namenode_thrift_port") %>
+
+<% end -%>
+<% if has_variable?("hadoop_namenode_thrift_port") -%>
<property>
<name>dfs.thrift.address</name>
<value>0.0.0.0:<%= hadoop_namenode_thrift_port %></value>
</property>
-<% end %>
+<% end -%>
<!-- increase the number of datanode transceivers way above the default of 256
- this is for hbase -->
<property>
@@ -148,18 +214,23 @@
</property>
<!-- Configurations for large cluster -->
-<% if has_variable?("hadoop_config_dfs_block_size") %>
+<% if has_variable?("hadoop_config_dfs_block_size") -%>
<property>
<name>dfs.block.size</name>
<value><%= hadoop_config_dfs_block_size %></value>
</property>
-<% end %>
-<% if has_variable?("hadoop_config_namenode_handler_count") %>
+<% end -%>
+<% if has_variable?("hadoop_config_namenode_handler_count") -%>
<property>
<name>dfs.namenode.handler.count</name>
<value><%= hadoop_config_namenode_handler_count %></value>
</property>
-<% end %>
+
+<% end -%>
+ <property>
+ <name>dfs.webhdfs.enabled</name>
+ <value>true</value>
+ </property>
</configuration>
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-env.sh Mon Apr 16 16:10:22 2012
@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License. See accompanying LICENSE file.
+#
+
+# Set httpfs specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs HttpFS
+# Java System properties for HttpFS should be specified in this variable
+#
+# export CATALINA_OPTS=
+
+# HttpFS logs directory
+#
+# export HTTPFS_LOG=${HTTPFS_HOME}/logs
+
+# HttpFS temporary directory
+#
+# export HTTPFS_TEMP=${HTTPFS_HOME}/temp
+
+# The HTTP port used by HttpFS
+#
+export HTTPFS_HTTP_PORT=<%= hadoop_httpfs_port %>
+
+# The Admin port used by HttpFS
+#
+# export HTTPFS_ADMIN_PORT=`expr ${HTTPFS_HTTP_PORT} + 1`
+
+# The hostname HttpFS server runs on
+#
+# export HTTPFS_HTTP_HOSTNAME=`hostname -f`
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/httpfs-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,75 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!-- Licensed to the Apache Software Foundation (ASF) under one or more -->
+<!-- contributor license agreements. See the NOTICE file distributed with -->
+<!-- this work for additional information regarding copyright ownership. -->
+<!-- The ASF licenses this file to You under the Apache License, Version 2.0 -->
+<!-- (the "License"); you may not use this file except in compliance with -->
+<!-- the License. You may obtain a copy of the License at -->
+<!-- -->
+<!-- http://www.apache.org/licenses/LICENSE-2.0 -->
+<!-- -->
+<!-- Unless required by applicable law or agreed to in writing, software -->
+<!-- distributed under the License is distributed on an "AS IS" BASIS, -->
+<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
+<!-- See the License for the specific language governing permissions and -->
+<!-- limitations under the License. -->
+
+<% namenode_hosts = hadoop_namenode_host.to_a -%>
+<configuration>
+
+<% if has_variable?("hadoop_core_proxyusers") -%>
+<% hadoop_core_proxyusers.sort.each do |superuser, data| -%>
+ <property>
+ <name>httpfs.proxyuser.<%= superuser %>.hosts</name>
+ <value><%= data['hosts'] %></value>
+ </property>
+ <property>
+ <name>httpfs.proxyuser.<%= superuser %>.groups</name>
+ <value><%= data['groups'] %></value>
+ </property>
+<% end -%>
+<% end -%>
+
+<% if hadoop_security_authentication == "kerberos" -%>
+ <property>
+ <name>httpfs.authentication.type</name>
+ <value>kerberos</value>
+ </property>
+ <property>
+ <name>kerberos.realm</name>
+ <value><%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>httpfs.authentication.kerberos.principal</name>
+ <value>HTTP/<%= fqdn %></value>
+ </property>
+ <property>
+ <name>httpfs.authentication.kerberos.keytab</name>
+ <value>/etc/httpfs.keytab</value>
+ </property>
+
+ <property>
+ <name>httpfs.hadoop.authentication.type</name>
+ <value>kerberos</value>
+ </property>
+ <property>
+ <name>httpfs.hadoop.authentication.kerberos.principal</name>
+ <value>httpfs/<%= fqdn %></value>
+ </property>
+ <property>
+ <name>httpfs.hadoop.authentication.kerberos.keytab</name>
+ <value>${httpfs.authentication.kerberos.keytab}</value>
+ </property>
+ <property>
+ <name>httpfs.hadoop.conf:dfs.namenode.kerberos.principal</name>
+ <value>hdfs/<%= namenode_hosts[0] %>@<%= kerberos_realm %></value>
+ </property>
+<% end -%>
+
+ <property>
+ <name>httpfs.hadoop.conf:fs.default.name</name>
+ <value>hdfs://<%= namenode_hosts[0] %>:<%= hadoop_namenode_port%></value>
+ </property>
+</configuration>
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml Mon Apr 16 16:10:22 2012
@@ -19,6 +19,16 @@
<configuration>
<% if hadoop_security_authentication == "kerberos" %>
+ <!-- HistoryServer security configs -->
+ <property>
+ <name>mapreduce.jobhistory.principal</name>
+ <value>mapred/_HOST@<%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.keytab</name>
+ <value>/etc/mapred.keytab</value> <!-- path to the MapReduce keytab -->
+ </property>
+
<!-- JobTracker security configs -->
<property>
<name>mapreduce.jobtracker.kerberos.principal</name>
@@ -49,7 +59,7 @@
<!-- TaskController settings -->
<property>
- <name>mapred.task.tracker.task-controller</name>
+ <name>mapreduce.tasktracker.taskcontroller</name>
<value>org.apache.hadoop.mapred.LinuxTaskController</value>
</property>
<property>
@@ -60,7 +70,7 @@
<% if has_variable?("mapred_acls_enabled") %>
<property>
- <name>mapred.acls.enabled</name>
+ <name>mapreduce.cluster.acls.enabled</name>
<value><%= mapred_acls_enabled %></value>
</property>
<% end %>
@@ -68,7 +78,7 @@
<!-- specify JobTracker TaskScheduler -->
<% if has_variable?("hadoop_jobtracker_taskscheduler") %>
<property>
- <name>mapred.jobtracker.taskScheduler</name>
+ <name>mapreduce.jobtracker.taskscheduler</name>
<value><%= hadoop_jobtracker_taskscheduler %></value>
</property>
<% end %>
@@ -94,20 +104,46 @@
</property>
<% end %>
+<% if has_variable?("hadoop_hs_host") %>
+ <property>
+ <name>mapreduce.jobhistory.address</name>
+ <value><%= hadoop_hs_host %>:<%= hadoop_hs_port %></value>
+ </property>
+ <property>
+ <name>mapreduce.jobhistory.webapp.address</name>
+ <value><%= hadoop_hs_host %>:<%= hadoop_hs_webapp_port %></value>
+ </property>
+<% end %>
+
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+
<property>
- <name>mapred.job.tracker</name>
+ <name>mapreduce.admin.user.env</name>
+ <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+ </property>
+
+ <property>
+ <name>mapreduce.jobtracker.address</name>
<value><%= hadoop_jobtracker_host %>:<%= hadoop_jobtracker_port%></value>
</property>
<property>
- <name>mapred.local.dir</name>
+ <name>mapreduce.cluster.local.dir</name>
<value><%= mapred_data_dirs.join(",") %></value>
<final>true</final>
</property>
+ <!-- property>
+ <name>mapreduce.cluster.temp.dir</name>
+ <value>/mapred/system</value>
+ </property -->
+
<property>
- <name>mapred.child.java.opts</name>
- <value>-Xmx1024m</value>
+ <name>mapreduce.jobtracker.system.dir</name>
+ <value>/mapred/system</value>
</property>
<property>
@@ -116,8 +152,13 @@
</property>
<property>
- <name>mapred.system.dir</name>
- <value>/mapred/system</value>
+ <name>yarn.app.mapreduce.am.staging-dir</name>
+ <value>/user</value>
+ </property>
+
+ <property>
+ <name>mapred.child.java.opts</name>
+ <value>-Xmx1024m</value>
</property>
<% if has_variable?("hadoop_config_mapred_child_ulimit") %>
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,136 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+<% if hadoop_security_authentication == "kerberos" %>
+ <!-- JobTracker security configs -->
+ <property>
+ <name>yarn.resourcemanager.principal</name>
+ <value>yarn/_HOST@<%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>yarn.resourcemanager.keytab</name>
+ <value>/etc/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.web-proxy.principal</name>
+ <value>yarn/_HOST@<%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>yarn.web-proxy.keytab</name>
+ <value>/etc/yarn.keytab</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.principal</name>
+ <value>yarn/_HOST@<%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.keytab</name>
+ <value>/etc/yarn.keytab</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.container-executor.class</name>
+ <value>org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor</value>
+ </property>
+ <property>
+ <name>yarn.nodemanager.linux-container-executor.group</name>
+ <value>hadoop</value>
+ </property>
+
+ <property>
+ <name>yarn.web-proxy.principal</name>
+ <value>yarn/_HOST@<%= kerberos_realm %></value>
+ </property>
+ <property>
+ <name>yarn.web-proxy.keytab</name>
+ <value>/etc/yarn.keytab</value>
+ </property>
+<% end %>
+
+<%
+=begin FIXME: MAPREDUCE-3916 -%>
+ <property>
+ <name>yarn.web-proxy.address</name>
+ <value><%= hadoop_ps_host %>:<%= hadoop_ps_port %></value>
+ </property>
+<%
+=end -%>
+
+ <property>
+ <name>yarn.resourcemanager.resource-tracker.address</name>
+ <value><%= hadoop_rm_host %>:<%= hadoop_rt_port %></value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.address</name>
+ <value><%= hadoop_rm_host %>:<%= hadoop_rm_port %></value>
+ </property>
+
+ <property>
+ <name>yarn.resourcemanager.scheduler.address</name>
+ <value><%= hadoop_rm_host %>:<%= hadoop_sc_port %></value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce.shuffle</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value><%= yarn_data_dirs.join(",") %></value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <description>Where to store container logs.</description>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>/var/log/hadoop-yarn/containers</value>
+ </property>
+
+ <property>
+ <description>Where to aggregate logs to.</description>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/var/log/hadoop-yarn/apps</value>
+ </property>
+
+ <property>
+ <description>Classpath for typical applications.</description>
+ <name>yarn.application.classpath</name>
+ <value>
+ $HADOOP_CONF_DIR,
+ $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
+ $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
+ $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
+ $YARN_HOME/*,$YARN_HOME/lib/*
+ </value>
+ </property>
+</configuration>
Modified: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp Mon Apr 16 16:10:22 2012
@@ -126,6 +126,11 @@ class kerberos {
}
class client inherits kerberos::site {
+ # Required for SPNEGO
+ @principal { "HTTP":
+
+ }
+
package { $package_name_client:
ensure => installed,
}
@@ -143,7 +148,7 @@ class kerberos {
Class["kerberos::client"]
}
- define create_princs {
+ define principal {
exec { "addprinc.$title":
path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
command => "kadmin -w secure -p kadmin/admin -q 'addprinc -randkey $title/$fqdn'",
@@ -152,23 +157,35 @@ class kerberos {
}
}
- define host_keytab($fqdn = "$hostname.$domain", $princs = undef) {
- $real_princs = $princs ? {
+ define host_keytab($fqdn = "$hostname.$domain", $princs = undef, $spnego = disabled) {
+
+ require "kerberos::client"
+
+ $needed_princs = $princs ? {
undef => [ $title ],
default => $princs,
}
$keytab = "/etc/${title}.keytab"
- $exports = inline_template("<%= real_princs.join('/$fqdn ') + '/$fqdn ' %>")
+ $exports = inline_template("<%= needed_princs.map { |x| x+'/$fqdn' }.join(' ') %>")
+ $spnego_export = $spnego ? {
+ /(true|enabled)/ => "HTTP/$fqdn",
+ default => "",
+ }
+
+ principal { $needed_princs:
- create_princs { $real_princs:
}
exec { "xst.$title":
path => $kerberos::site::exec_path, # BUG: I really shouldn't need to do a FQVN here
- command => "kadmin -w secure -p kadmin/admin -q 'xst -k $keytab $exports' ; chown $title $keytab",
+ command => "kadmin -w secure -p kadmin/admin -q 'xst -k $keytab $exports $spnego_export' ; chown $title $keytab",
unless => "klist -kt $keytab 2>/dev/null | grep -q $title/$fqdn",
- require => [ Create_princs[$real_princs] ],
+ require => [ Kerberos::Principal[$needed_princs] ],
+ }
+
+ if ($spnego =~ /(true|enabled)/) {
+ Kerberos::Principal <| title == "HTTP" |> -> Exec["xst.$title"]
}
}
Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client.pp Mon Apr 16 16:10:22 2012
@@ -1,12 +1,11 @@
-#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,6 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -ex
-
-mvn clean install -DskipTests "$@"
+class nfs::client {
+ include nfs::client::install
+ require nfs::client::service
+}
Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/install.pp Mon Apr 16 16:10:22 2012
@@ -1,12 +1,11 @@
-#!/bin/sh
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -14,7 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-set -ex
-
-mvn clean install -Dmahout.skip.distribution=false -DskipTests "$@"
-
+class nfs::client::install inherits nfs::client::params {
+ package { $package_names:
+ ensure => present,
+ }
+}
Added: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp (added)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/params.pp Mon Apr 16 16:10:22 2012
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class nfs::client::params {
+ case "$::operatingsystem $::operatingsystemrelease" {
+ /(CentOS|RedHat) 6/: {
+ $package_names = [ "rpcbind", "nfs-utils" ]
+ $portmap_service = "rpcbind"
+ $idmapd_service = "rpcidmapd"
+ $nfs_version = 4
+ }
+
+ /(CentOS|RedHat) 5/: {
+ $package_names = [ "portmap", "nfs-utils" ]
+ $portmap_service = "portmap"
+ $nfs_version = 3
+ }
+
+ /SLES 11/: {
+ $package_names = [ "rpcbind", "nfs-client" ]
+ $portmap_service = "rpcbind"
+ $nfs_version = 3
+ }
+
+ /(Debian|Ubuntu)/: {
+ $package_names = [ "nfs-common", "portmap" ]
+ $portmap_service = "portmap"
+ $nfs_version = 3
+ }
+
+ default: {
+ fail("nfs::client::params not supported on $::operatingsystem $::operatingsystemerelease")
+ }
+ }
+}
Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/oozie/oozie-env.sh)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/oozie/oozie-env.sh&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/oozie/oozie-env.sh (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp Mon Apr 16 16:10:22 2012
@@ -1,13 +1,11 @@
-#! /bin/bash
-
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,10 +13,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-export OOZIE_CONFIG=/etc/oozie/conf
-export OOZIE_DATA=/var/lib/oozie
-export OOZIE_LOG=/var/log/oozie
-export CATALINA_BASE=${OOZIE_DATA}/oozie-server
-export CATALINA_TMPDIR=/var/tmp/oozie
-export CATALINA_PID=/var/run/oozie/oozie.pid
+class nfs::client::service inherits nfs::client::params {
+
+ include nfs::client::install
+
+ service { $portmap_service:
+ ensure => running,
+ enable => true,
+ require => Package[$package_names],
+ }
+ if ($nfs_version >= 4) {
+ service { $idmapd_service:
+ ensure => running,
+ enable => true,
+ require => [ Service[$portmap_service], Package[$package_names] ],
+ pattern => "rpc.idmapd",
+ }
+ }
+}
Propchange: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/client/service.pp
------------------------------------------------------------------------------
svn:eol-style = native
Copied: incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp?p2=incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/mapred.conf (original)
+++ incubator/bigtop/trunk/bigtop-deploy/puppet/modules/nfs/manifests/init.pp Mon Apr 16 16:10:22 2012
@@ -1,11 +1,11 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
+# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,5 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-mapred - nofile 32768
-mapred - nproc 65536
+class nfs {
+ require nfs::client
+}
Copied: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build?p2=incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build&p1=incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/mahout/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,5 +16,10 @@
set -ex
-mvn clean install -Dmahout.skip.distribution=false -DskipTests "$@"
+rm -f jsvc
+pushd unix
+ ./configure --prefix=/usr
+ make
+ cp -f jsvc ../jsvc
+popd
Added: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh Mon Apr 16 16:10:22 2012
@@ -0,0 +1,90 @@
+#!/bin/bash -x
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -ex
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --build-dir=DIR path to hive/build/dist
+ --prefix=PREFIX path to install into
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'build-dir:' \
+ -l 'bin-dir:' \
+ -l 'doc-dir:' \
+ -l 'man-dir:' \
+ -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --bin-dir)
+ BIN_DIR=$2 ; shift 2
+ ;;
+ --man-dir)
+ MAN_DIR=$2 ; shift 2
+ ;;
+ --doc-dir)
+ DOC_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX BUILD_DIR; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing param: $var
+ usage
+ fi
+done
+
+BIN_DIR=${BIN_DIR:-$PREFIX/usr/lib/bigtop-utils}
+DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/bigtop-jsvc}
+MAN_DIR=${MAN_DIR:-$PREFIX/usr/man}
+
+#libexec
+install -d -m 0755 ${BIN_DIR}
+cp ${BUILD_DIR}/jsvc ${BIN_DIR}
+
+# docs
+install -d -m 0755 ${DOC_DIR}
+cp ${BUILD_DIR}/*.txt ${DOC_DIR}/
+cp ${BUILD_DIR}/README ${DOC_DIR}/
Propchange: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-jsvc/install_jsvc.sh
------------------------------------------------------------------------------
svn:executable = *
Copied: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/pig/do-component-build)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build?p2=incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build&p1=incubator/bigtop/trunk/bigtop-packages/src/common/pig/do-component-build&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/pig/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,6 +16,10 @@
set -ex
-ant -Djavac.version=1.6 -Djava5.home=${JAVA5_HOME} -Dforrest.home=${FORREST_HOME} -Dversion=${PIG_BASE_VERSION} package "$@"
-
+mkdir dl
+ant -Dbase.path=dl clean download
+ant -Dbase.path=dl
+ant -Dbase.path=dl -Dskip.installer=true -f dist.xml dist-static package-tgz
+mkdir build
+tar -C build --strip-components=1 -xzf output/release/v*/bin/apache-tomcat-*[0-9].tar.gz
Added: incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/bigtop-tomcat/install_tomcat.sh Mon Apr 16 16:10:22 2012
@@ -0,0 +1,91 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+usage() {
+ echo "
+usage: $0 <options>
+ Required not-so-options:
+ --build-dir=DIR path to Whirr dist.dir
+ --prefix=PREFIX path to install into
+
+ Optional options:
+ --doc-dir=DIR path to install docs into [/usr/share/doc/whirr]
+ --lib-dir=DIR path to install Whirr home [/usr/lib/whirr]
+ "
+ exit 1
+}
+
+OPTS=$(getopt \
+ -n $0 \
+ -o '' \
+ -l 'prefix:' \
+ -l 'doc-dir:' \
+ -l 'lib-dir:' \
+ -l 'build-dir:' -- "$@")
+
+if [ $? != 0 ] ; then
+ usage
+fi
+
+eval set -- "$OPTS"
+while true ; do
+ case "$1" in
+ --prefix)
+ PREFIX=$2 ; shift 2
+ ;;
+ --build-dir)
+ BUILD_DIR=$2 ; shift 2
+ ;;
+ --doc-dir)
+ DOC_DIR=$2 ; shift 2
+ ;;
+ --lib-dir)
+ LIB_DIR=$2 ; shift 2
+ ;;
+ --)
+ shift ; break
+ ;;
+ *)
+ echo "Unknown option: $1"
+ usage
+ exit 1
+ ;;
+ esac
+done
+
+for var in PREFIX BUILD_DIR ; do
+ if [ -z "$(eval "echo \$$var")" ]; then
+ echo Missing param: $var
+ usage
+ fi
+done
+
+MAN_DIR=$PREFIX/usr/share/man/man1
+DOC_DIR=${DOC_DIR:-$PREFIX/usr/share/doc/bigtop-tomcat}
+LIB_DIR=${LIB_DIR:-$PREFIX/usr/lib/bigtop-tomcat}
+
+# First we'll move everything into lib
+install -d -m 0755 $LIB_DIR
+(cd $BUILD_DIR && tar -cf - .) | (cd $LIB_DIR && tar -xf -)
+
+rm -rf $LIB_DIR/webapps
+mv -f $LIB_DIR/conf $LIB_DIR/conf.template
+
+install -d -m 0755 $DOC_DIR
+mv $LIB_DIR/{RUNNING.txt,RELEASE-NOTES,NOTICE,LICENSE} $DOC_DIR
Modified: incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/flume/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,4 +16,4 @@
set -ex
-mvn clean install -DskipTests "$@"
+mvn clean install -DskipTests -Dhadoop.profile=23 "$@"
Modified: incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/flume/flume-node.init Mon Apr 16 16:10:22 2012
@@ -15,10 +15,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-# Starts a Flume node
+# Starts a Flume NG node
#
# chkconfig: 345 90 10
-# description: Flume node
+# description: Flume NG node
#
### BEGIN INIT INFO
# Provides: flume-node
@@ -28,7 +28,7 @@
# Should-Stop:
# Default-Start: 3 4 5
# Default-Stop: 0 1 2 6
-# Short-Description: Flume node
+# Short-Description: Flume NG node
### END INIT INFO
. /lib/lsb/init-functions
@@ -53,7 +53,7 @@ FLUME_RUN=/var/run/flume
FLUME_HOME=/usr/lib/flume
LOCKFILE="/var/lock/subsys/flume-node"
-desc="Flume node daemon"
+desc="Flume NG node daemon"
FLUME_CONF_FILE=${FLUME_CONF_FILE:-${FLUME_CONF_DIR}/flume.conf}
EXEC_PATH=/usr/bin/flume-ng
@@ -110,19 +110,19 @@ checkstatus(){
case "$status" in
$STATUS_RUNNING)
- log_success_msg "Flume node is running"
+ log_success_msg "Flume NG node is running"
;;
$STATUS_DEAD)
- log_failure_msg "Flume node is dead and pid file exists"
+ log_failure_msg "Flume NG node is dead and pid file exists"
;;
$STATUS_DEAD_AND_LOCK)
- log_failure_msg "Flume node is dead and lock file exists"
+ log_failure_msg "Flume NG node is dead and lock file exists"
;;
$STATUS_NOT_RUNNING)
- log_failure_msg "Flume node is not running"
+ log_failure_msg "Flume NG node is not running"
;;
*)
- log_failure_msg "Flume node status is unknown"
+ log_failure_msg "Flume NG node status is unknown"
;;
esac
return $status
Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/README Mon Apr 16 16:10:22 2012
@@ -0,0 +1,4 @@
+A minimum amount of configuration to make Hadoop daemons
+self-contained and useful. It would be really nice if the
+empty configuration of Hadoop had the same effect by
+default (along the lines of hbase-default.xml).
Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/core-site.xml Mon Apr 16 16:10:22 2012
@@ -18,8 +18,4 @@
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
- <property>
- <name>mapred.job.tracker</name>
- <value>localhost:8021</value>
- </property>
</configuration>
Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/hdfs-site.xml Mon Apr 16 16:10:22 2012
@@ -19,7 +19,7 @@
<configuration>
<property>
- <name>mapred.job.tracker</name>
- <value>localhost:8021</value>
+ <name>dfs.name.dir</name>
+ <value>/var/lib/hadoop-hdfs/cache/hdfs/dfs/name</value>
</property>
</configuration>
Copied: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml (from r1325587, incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml)
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml?p2=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml&p1=incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml&r1=1325587&r2=1326670&rev=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/mapred-site.xml Mon Apr 16 16:10:22 2012
@@ -22,4 +22,10 @@
<name>mapred.job.tracker</name>
<value>localhost:8021</value>
</property>
+
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+ </property>
+
</configuration>
Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce.shuffle</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>List of directories to store localized files in.</description>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>/var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir</value>
+ </property>
+
+ <property>
+ <description>Where to store container logs.</description>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>/var/log/hadoop-yarn/containers</value>
+ </property>
+
+ <property>
+ <description>Where to aggregate logs to.</description>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/var/log/hadoop-yarn/apps</value>
+ </property>
+
+ <property>
+ <description>Classpath for typical applications.</description>
+ <name>yarn.application.classpath</name>
+ <value>
+ $HADOOP_CONF_DIR,
+ $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
+ $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
+ $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
+ $YARN_HOME/*,$YARN_HOME/lib/*
+ </value>
+ </property>
+</configuration>
Propchange: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.empty/yarn-site.xml
------------------------------------------------------------------------------
svn:executable = *
Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/core-site.xml Mon Apr 16 16:10:22 2012
@@ -23,11 +23,6 @@
<value>hdfs://localhost:8020</value>
</property>
- <property>
- <name>hadoop.tmp.dir</name>
- <value>/var/lib/hadoop/cache/${user.name}</value>
- </property>
-
<!-- OOZIE proxy user setting -->
<property>
<name>hadoop.proxyuser.oozie.hosts</name>
@@ -38,4 +33,14 @@
<value>*</value>
</property>
+ <!-- HTTPFS proxy user setting -->
+ <property>
+ <name>hadoop.proxyuser.httpfs.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.httpfs.groups</name>
+ <value>*</value>
+ </property>
+
</configuration>
Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/hdfs-site.xml Mon Apr 16 16:10:22 2012
@@ -22,10 +22,6 @@
<name>dfs.replication</name>
<value>1</value>
</property>
- <property>
- <name>dfs.permissions</name>
- <value>false</value>
- </property>
<!-- Immediately exit safemode as soon as one DataNode checks in.
On a multi-node cluster, these configurations must be removed. -->
<property>
@@ -37,8 +33,19 @@
<value>1</value>
</property>
<property>
- <!-- specify this so that running 'hadoop namenode -format' formats the right dir -->
- <name>dfs.name.dir</name>
- <value>/var/lib/hadoop/cache/hadoop/dfs/name</value>
+ <name>hadoop.tmp.dir</name>
+ <value>/var/lib/hadoop-hdfs/cache/${user.name}</value>
+ </property>
+ <property>
+ <name>dfs.namenode.name.dir</name>
+ <value>/var/lib/hadoop-hdfs/cache/${user.name}/dfs/name</value>
+ </property>
+ <property>
+ <name>dfs.namenode.checkpoint.dir</name>
+ <value>/var/lib/hadoop-hdfs/cache/${user.name}/dfs/namesecondary</value>
+ </property>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/var/lib/hadoop-hdfs/cache/${user.name}/dfs/data</value>
</property>
</configuration>
Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/mapred-site.xml Mon Apr 16 16:10:22 2012
@@ -22,4 +22,21 @@
<name>mapred.job.tracker</name>
<value>localhost:8021</value>
</property>
+
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+
+ <property>
+ <name>mapreduce.admin.user.env</name>
+ <value>CLASSPATH=/etc/hadoop/conf/*:/usr/lib/hadoop/*:/usr/lib/hadoop/lib/*:/usr/lib/hadoop/lib/native/*</value>
+ </property>
+
+ <property>
+ <description>To set the value of tmp directory for map and reduce tasks.</description>
+ <name>mapreduce.task.tmp.dir</name>
+ <value>/var/lib/hadoop-mapreduce/cache/${user.name}/tasks</value>
+ </property>
+
</configuration>
Added: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml?rev=1326670&view=auto
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml (added)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/conf.pseudo/yarn-site.xml Mon Apr 16 16:10:22 2012
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce.shuffle</value>
+ </property>
+
+ <property>
+ <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
+ <value>org.apache.hadoop.mapred.ShuffleHandler</value>
+ </property>
+
+ <property>
+ <name>yarn.log-aggregation-enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>List of directories to store localized files in.</description>
+ <name>yarn.nodemanager.local-dirs</name>
+ <value>/var/lib/hadoop-yarn/cache/${user.name}/nm-local-dir</value>
+ </property>
+
+ <property>
+ <description>Where to store container logs.</description>
+ <name>yarn.nodemanager.log-dirs</name>
+ <value>/var/log/hadoop-yarn/containers</value>
+ </property>
+
+ <property>
+ <description>Where to aggregate logs to.</description>
+ <name>yarn.nodemanager.remote-app-log-dir</name>
+ <value>/var/log/hadoop-yarn/apps</value>
+ </property>
+
+ <property>
+ <description>Classpath for typical applications.</description>
+ <name>yarn.application.classpath</name>
+ <value>
+ $HADOOP_CONF_DIR,
+ $HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,
+ $HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,
+ $HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,
+ $YARN_HOME/*,$YARN_HOME/lib/*
+ </value>
+ </property>
+</configuration>
Modified: incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build
URL: http://svn.apache.org/viewvc/incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build?rev=1326670&r1=1326669&r2=1326670&view=diff
==============================================================================
--- incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build (original)
+++ incubator/bigtop/trunk/bigtop-packages/src/common/hadoop/do-component-build Mon Apr 16 16:10:22 2012
@@ -16,47 +16,28 @@
set -ex
-# FIXME: BIGTOP-137
-chmod 777 src/c++/task-controller/configure || :
+if [ -z "$BUNDLE_SNAPPY" ] ; then
+ [ -f /usr/lib/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib"
+ [ -f /usr/lib64/libsnappy.so ] && BUNDLE_SNAPPY="-Dsnappy.prefix=x -Dbundle.snappy=true -Dsnappy.lib=/usr/lib64"
+fi
+mkdir build
+mkdir build/src
-# This is a poor man version comparison of autoconf.
-# Current autotools files target autotools 2.61 or sup.
-# There will be some build issues if we try to regenerate them with a lesser version of autotools
-# So we will only regenerate the autotools files if they are installed and their version is greater than
-# 2.60
-DO_AUTORECONF=""
-if [ -n `which autoconf` ]; then
-
- AUTOCONF_VERSION=`autoconf --version | head -n 1 | sed -e 's/.* \([.0-9]\+\)$/\1/'`
- echo $AUTOCONF_VERSION
-
- if [[ $AUTOCONF_VERSION > 2.60 ]]; then
- DO_AUTORECONF="create-c++-configure"
- fi
-fi
+# Build artifacts
+MAVEN_SKIP_TESTS="-DskipTests -DskipTest -DskipITs"
+mvn $BUNDLE_SNAPPY -Dcontainer-executor.conf.dir=/etc/hadoop/conf -Pdist -Pnative -Pfuse -Psrc -Dtar ${MAVEN_SKIP_TESTS} install "$@"
+mvn site site:stage ${MAVEN_SKIP_TESTS} $@
+
+(cd build ; tar --strip-components=1 -xzvf ../hadoop-dist/target/hadoop-${HADOOP_VERSION}.tar.gz)
+(cd build/src ; tar --strip-components=1 -xzvf ../../hadoop-dist/target/hadoop-${HADOOP_VERSION}-src.tar.gz)
+
+# Copy generated docs
+mkdir -p build/share/doc/
+cp -r target/site/* build/share/doc/
-pushd src/native
- autoreconf -if
-popd
-
-LIBJVMS=`find ${JAVA_HOME}/jre/lib/ -name "libjvm.so" -exec dirname {} \;`
-LIBRARIES="-lcrypto -lm "
-for jvm in $LIBJVMS;
-do
- LIBRARIES="${LIBRARIES} -L${jvm}"
-done
-
-# FIXME: the LDFLAGS="-lcrypto" is a workaround for MAPREDUCE-2127
-# FIXME: the FUSE_HOME="..." and LDFLAGS="-lm" is a workaround for HADOOP-3344
-LDFLAGS="$LIBRARIES" FUSE_HOME=`pwd`/build/c++/${HADOOP_ARCH} ant \
- -Djava5.home=$JAVA5_HOME \
- -Dforrest.home=$FORREST_HOME \
- -Dversion=${HADOOP_VERSION} \
- -Dcompile.native=true \
- -Dhadoop.conf.dir=/etc/hadoop/conf \
- -Dlibhdfs=1 -Dfusedfs=true -Dcompile.c++=true -Djdiff.build.dir=build/docs/jdiff \
- $DO_AUTORECONF task-controller api-report bin-package compile-contrib package \
- "$@"
+# Create a manifest for hadoop client package
+(cd hadoop-client/target/hadoop-client-*/share/hadoop/client/lib ; ls) > build/hadoop-client.list
-
+# Copy fuse output to the build directory
+cp hadoop-hdfs-project/hadoop-hdfs/src/contrib/fuse-dfs/target/fuse-dfs/src/{fuse_dfs,fuse_dfs_wrapper.sh} build/bin/