You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2015/01/19 16:43:47 UTC
bigtop git commit: BIGTOP-1602. Update and clean up puppet config
file templates.
Repository: bigtop
Updated Branches:
refs/heads/master 2d7d7ca6d -> ff8a22d72
BIGTOP-1602. Update and clean up puppet config file templates.
Move option ha.zookeeper.quorum from hdfs-site.xml to core-site.xml. Switch
from dfs.{name,data}.dir to dfs.{name,data}node.{name,data}.dir with file://
scheme. Remove unused, disabled and obsolete thrift port options from puppet
modules.
Signed-off-by: Evans Ye <ev...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ff8a22d7
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ff8a22d7
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ff8a22d7
Branch: refs/heads/master
Commit: ff8a22d72651f38121818d0dce7ed7e6329cbd0b
Parents: 2d7d7ca
Author: Michael Weiser <m....@science-computing.de>
Authored: Mon Jan 19 10:35:31 2015 +0100
Committer: Evans Ye <ev...@apache.org>
Committed: Mon Jan 19 15:38:33 2015 +0000
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 6 -----
.../puppet/modules/hadoop/manifests/init.pp | 5 ++--
.../modules/hadoop/templates/core-site.xml | 6 +++++
.../modules/hadoop/templates/hdfs-site.xml | 24 ++++----------------
.../modules/hadoop/templates/mapred-site.xml | 7 ------
5 files changed, 13 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ff8a22d7/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index d83b536..9abd305 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -31,7 +31,6 @@ class hadoop_cluster_node {
default => [ $hadoop_head_node, $standby_head_node ],
}
$hadoop_namenode_port = extlookup("hadoop_namenode_port", "17020")
- $hadoop_namenode_thrift_port = extlookup("hadoop_namenode_thrift_port", "10090")
$hadoop_dfs_namenode_plugins = extlookup("hadoop_dfs_namenode_plugins", "")
$hadoop_dfs_datanode_plugins = extlookup("hadoop_dfs_datanode_plugins", "")
# $hadoop_dfs_namenode_plugins="org.apache.hadoop.thriftfs.NamenodePlugin"
@@ -46,7 +45,6 @@ class hadoop_cluster_node {
$hadoop_rt_port = extlookup("hadoop_rt_port", "8025")
$hadoop_rm_port = extlookup("hadoop_rm_port", "8032")
$hadoop_sc_port = extlookup("hadoop_sc_port", "8030")
- $hadoop_rt_thrift_port = extlookup("hadoop_rt_thrift_port", "9290")
$hadoop_hs_host = $hadoop_head_node
$hadoop_hs_port = extlookup("hadoop_hs_port", "10020")
@@ -57,7 +55,6 @@ class hadoop_cluster_node {
$hadoop_jobtracker_host = $hadoop_head_node
$hadoop_jobtracker_port = extlookup("hadoop_jobtracker_port", "8021")
- $hadoop_jobtracker_thrift_port = extlookup("hadoop_jobtracker_thrift_port", "9290")
$hadoop_mapred_jobtracker_plugins = extlookup("hadoop_mapred_jobtracker_plugins", "")
$hadoop_mapred_tasktracker_plugins = extlookup("hadoop_mapred_tasktracker_plugins", "")
@@ -238,7 +235,6 @@ if ($hadoop_security_authentication == "kerberos") {
host => $hadoop_namenode_host,
port => $hadoop_namenode_port,
dirs => $namenode_data_dirs,
- # thrift_port => $hadoop_namenode_thrift_port,
auth => $hadoop_security_authentication,
ha => $hadoop_ha,
zk => $hadoop_ha_zookeeper_quorum,
@@ -258,7 +254,6 @@ if ($hadoop_security_authentication == "kerberos") {
port => $hadoop_rm_port,
rt_port => $hadoop_rt_port,
sc_port => $hadoop_sc_port,
- # thrift_port => $hadoop_jobtracker_thrift_port,
auth => $hadoop_security_authentication,
}
@@ -336,7 +331,6 @@ class standby_head_node inherits hadoop_cluster_node {
host => $hadoop_namenode_host,
port => $hadoop_namenode_port,
dirs => $namenode_data_dirs,
- # thrift_port => $hadoop_namenode_thrift_port,
auth => $hadoop_security_authentication,
ha => $hadoop_ha,
zk => $hadoop_ha_zookeeper_quorum,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ff8a22d7/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index ef4ffa8..32eebe2 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -288,12 +288,11 @@ class hadoop {
Exec <| title == "activate nn1" |> -> Exec["HDFS rsync $title"]
}
- define namenode ($host = $fqdn , $port = "8020", $thrift_port= "10090", $auth = "simple", $dirs = ["/tmp/nn"], $ha = 'disabled', $zk = '') {
+ define namenode ($host = $fqdn , $port = "8020", $auth = "simple", $dirs = ["/tmp/nn"], $ha = 'disabled', $zk = '') {
$first_namenode = inline_template("<%= Array(@host)[0] %>")
$hadoop_namenode_host = $host
$hadoop_namenode_port = $port
- $hadoop_namenode_thrift_port = $thrift_port
$hadoop_security_authentication = $auth
if ($ha != 'disabled') {
@@ -474,7 +473,7 @@ class hadoop {
}
- define resourcemanager ($host = $fqdn, $port = "8032", $rt_port = "8025", $sc_port = "8030", $thrift_port = "9290", $auth = "simple") {
+ define resourcemanager ($host = $fqdn, $port = "8032", $rt_port = "8025", $sc_port = "8030", $auth = "simple") {
$hadoop_rm_host = $host
$hadoop_rm_port = $port
$hadoop_rt_port = $rt_port
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ff8a22d7/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
index e1bb799..0b8717b 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/core-site.xml
@@ -29,6 +29,12 @@
<% end -%>
</property>
+<% if @ha == "auto" and @zk -%>
+ <property>
+ <name>ha.zookeeper.quorum</name>
+ <value><%= @zk %></value>
+ </property>
+<% end -%>
<property>
<name>hadoop.security.authentication</name>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ff8a22d7/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
index 88c7277..351508d 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
@@ -22,17 +22,11 @@
<% if @ha != "disabled" -%>
<!-- HA -->
-<% if @ha == "auto" -%>
+<% if @ha == "auto" -%>
<property>
<name>dfs.ha.automatic-failover.enabled</name>
<value>true</value>
</property>
-<% if @zk -%>
- <property>
- <name>ha.zookeeper.quorum</name>
- <value><%= @zk %></value>
- </property>
-<% end -%>
<% end -%>
<property>
@@ -177,10 +171,9 @@
</property>
<% end -%>
- <!-- name node -->
<property>
- <name>dfs.data.dir</name>
- <value><%= @hdfs_data_dirs.join(",") %></value>
+ <name>dfs.datanode.data.dir</name>
+ <value><%= @hdfs_data_dirs.map { |dir| "file://#{dir}" }.join(",") %></value>
</property>
<% if @hdfs_support_append %>
@@ -203,8 +196,8 @@
<% end %>
<property>
- <name>dfs.name.dir</name>
- <value><%= @namenode_data_dirs.join(",") %></value>
+ <name>dfs.namenode.name.dir</name>
+ <value><%= @namenode_data_dirs.map { |dir| "file://#{dir}" }.join(",") %></value>
</property>
<property>
@@ -232,13 +225,6 @@
</property>
<% end -%>
-<% if @hadoop_namenode_thrift_port -%>
- <property>
- <name>dfs.thrift.address</name>
- <value>0.0.0.0:<%= @hadoop_namenode_thrift_port %></value>
- </property>
-
-<% end -%>
<!-- increase the number of datanode transceivers way above the default of 256
- this is for hbase -->
<property>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ff8a22d7/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
index a2517e5..5bf9777 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
@@ -302,13 +302,6 @@
</property>
<% end %>
-<% if @hadoop_jobtracker_thrift_port %>
- <!-- Enable Hue plugins -->
- <property>
- <name>jobtracker.thrift.address</name>
- <value>0.0.0.0:<%= @hadoop_jobtracker_thrift_port %></value>
- </property>
-<% end %>
<% if @hadoop_mapred_jobtracker_plugins %>
<property>
<name>mapred.jobtracker.plugins</name>