You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by rv...@apache.org on 2013/10/04 22:33:26 UTC
[2/2] git commit: BIGTOP-1106. update our puppet deployment topology
BIGTOP-1106. update our puppet deployment topology
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/5de186bc
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/5de186bc
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/5de186bc
Branch: refs/heads/master
Commit: 5de186bc956b0f19ec9a000b9a343baaa249b915
Parents: 822a6ea
Author: Roman Shaposhnik <rv...@cloudera.com>
Authored: Thu Oct 3 17:50:05 2013 -0700
Committer: Roman Shaposhnik <rv...@cloudera.com>
Committed: Fri Oct 4 13:21:20 2013 -0700
----------------------------------------------------------------------
bigtop-deploy/puppet/manifests/cluster.pp | 108 +++++++++++--------
.../modules/hadoop-hbase/manifests/init.pp | 25 ++++-
.../puppet/modules/hadoop-hbase/tests/init.pp | 5 +-
3 files changed, 86 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5de186bc/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 303cc74..cec9cec 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -59,36 +59,44 @@ class hadoop_cluster_node {
$hadoop_jobtracker_thrift_port = extlookup("hadoop_jobtracker_thrift_port", "9290")
$hadoop_mapred_jobtracker_plugins = extlookup("hadoop_mapred_jobtracker_plugins", "")
$hadoop_mapred_tasktracker_plugins = extlookup("hadoop_mapred_tasktracker_plugins", "")
- $hadoop_ha_zookeeper_quorum = "${hadoop_head_node}:2181"
- # $hadoop_mapred_jobtracker_plugins="org.apache.hadoop.thriftfs.ThriftJobTrackerPlugin"
- # $hadoop_mapred_tasktracker_plugins="org.apache.hadoop.mapred.TaskTrackerCmonInst"
+
+ $hadoop_zookeeper_port = extlookup("hadoop_zookeeper_port", "2181")
+ $solrcloud_port = extlookup("solrcloud_port", "1978")
+ $solrcloud_admin_port = extlookup("solrcloud_admin_port", "1979")
+ $hadoop_oozie_port = extlookup("hadoop_oozie_port", "11000")
+ $hadoop_httpfs_port = extlookup("hadoop_httpfs_port", "14000")
+ $hadoop_rm_http_port = extlookup("hadoop_rm_http_port", "8088")
+ $hadoop_rm_proxy_port = extlookup("hadoop_rm_proxy_port", "8088")
+ $hadoop_history_server_port = extlookup("hadoop_history_server_port", "19888")
+ $hbase_thrift_port = extlookup("hbase_thrift_port", "9090")
+
+ $hadoop_ha_zookeeper_quorum = "${hadoop_head_node}:${hadoop_zookeeper_port}"
+ $solrcloud_zk = "${hadoop_head_node}:${hadoop_zookeeper_port}"
+ $hbase_thrift_address = "${hadoop_head_node}:${hbase_thrift_port}"
+ $hadoop_oozie_url = "http://${hadoop_head_node}:${hadoop_oozie_port}/oozie"
+ $hadoop_httpfs_url = "http://${hadoop_head_node}:${hadoop_httpfs_port}/webhdfs/v1"
+ $sqoop_server_url = "http://${hadoop_head_node}:${sqoop_server_port}/sqoop"
+ $solrcloud_url = "http://${hadoop_head_node}:${solrcloud_port}/solr/"
+ $hadoop_rm_url = "http://${hadoop_head_node}:${hadoop_rm_http_port}"
+ $hadoop_rm_proxy_url = "http://${hadoop_head_node}:${hadoop_rm_proxy_port}"
+ $hadoop_history_server_url = "http://${hadoop_head_node}:${hadoop_history_server_port}"
$bigtop_real_users = [ 'jenkins', 'testuser', 'hudson' ]
- $hadoop_core_proxyusers = { oozie => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" },
- hue => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" },
- httpfs => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "${hadoop_head_node},localhost,127.0.0.1" } }
+ $hadoop_core_proxyusers = { oozie => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "*" },
+ hue => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "*" },
+ httpfs => { groups => 'hudson,testuser,root,hadoop,jenkins,oozie,httpfs,hue,users', hosts => "*" } }
$hbase_relative_rootdir = extlookup("hadoop_hbase_rootdir", "/hbase")
$hadoop_hbase_rootdir = "$hadoop_namenode_uri$hbase_relative_rootdir"
$hadoop_hbase_zookeeper_quorum = $hadoop_head_node
$hbase_heap_size = extlookup("hbase_heap_size", "1024")
+ $hbase_thrift_server = $hadoop_head_node
$giraph_zookeeper_quorum = $hadoop_head_node
$hadoop_zookeeper_ensemble = ["$hadoop_head_node:2888:3888"]
- $hadoop_oozie_url = "http://${hadoop_head_node}:11000/oozie"
- $hadoop_httpfs_url = "http://${hadoop_head_node}:14000/webhdfs/v1"
- $hadoop_rm_url = "http://${hadoop_head_node}:8088"
- $hadoop_rm_proxy_url = "http://${hadoop_head_node}:8088"
- $hadoop_history_server_url = "http://${hadoop_head_node}:19888"
-
- $solrcloud_collections = ["collection1"]
- $solrcloud_port = "1978"
- $solrcloud_port_admin = "1979"
- $solrcloud_zk = "${hadoop_head_node}:2181"
-
# Set from facter if available
$roots = extlookup("hadoop_storage_dirs", split($hadoop_storage_dirs, ";"))
$namenode_data_dirs = extlookup("hadoop_namenode_data_dirs", append_each("/namenode", $roots))
@@ -157,10 +165,11 @@ class hadoop_worker_node inherits hadoop_cluster_node {
}
solr::server { "solrcloud server":
- collections => $solrcloud_collections,
port => $solrcloud_port,
- port_admin => $solrcloud_port_admin,
+ port_admin => $solrcloud_admin_port,
zk => $solrcloud_zk,
+ root_url => $hadoop_namenode_uri,
+ kerberos_realm => $kerberos_realm,
}
}
@@ -212,12 +221,6 @@ class hadoop_head_node inherits hadoop_worker_node {
auth => $hadoop_security_authentication,
}
- hadoop::httpfs { "httpfs":
- namenode_host => $hadoop_namenode_host,
- namenode_port => $hadoop_namenode_port,
- auth => $hadoop_security_authentication,
- }
-
hadoop-hbase::master { "hbase master":
rootdir => $hadoop_hbase_rootdir,
heap_size => $hbase_heap_size,
@@ -229,11 +232,6 @@ class hadoop_head_node inherits hadoop_worker_node {
kerberos_realm => $kerberos_realm,
}
- hadoop-sqoop::server { "sqoop server":
- }
- hadoop-sqoop::client { "sqoop client":
- }
-
hcatalog::server { "hcatalog server":
kerberos_realm => $kerberos_realm,
}
@@ -241,20 +239,6 @@ class hadoop_head_node inherits hadoop_worker_node {
kerberos_realm => $kerberos_realm,
}
- hue::server { "hue server":
- rm_url => $hadoop_rm_url,
- rm_proxy_url => $hadoop_rm_proxy_url,
- history_server_url => $hadoop_history_server_url,
- webhdfs_url => $hadoop_httpfs_url,
- rm_host => $hadoop_rm_host,
- rm_port => $hadoop_rm_port,
- oozie_url => $hadoop_oozie_url,
- default_fs => $hadoop_namenode_uri,
- kerberos_realm => $kerberos_realm,
- }
- Hadoop::Httpfs<||> -> Hue::Server<||>
- Hadoop-sqoop::Client<||> -> Hue::Server<||>
-
hadoop-zookeeper::server { "zookeeper":
myid => "0",
ensemble => $hadoop_zookeeper_ensemble,
@@ -270,7 +254,6 @@ class hadoop_head_node inherits hadoop_worker_node {
Exec<| title == "init hdfs" |> -> Hadoop-hbase::Master<||>
Exec<| title == "init hdfs" |> -> Hadoop::Resourcemanager<||>
Exec<| title == "init hdfs" |> -> Hadoop::Historyserver<||>
- Exec<| title == "init hdfs" |> -> Hadoop::Httpfs<||>
Exec<| title == "init hdfs" |> -> Hadoop::Rsync_hdfs<||>
Exec<| title == "init hdfs" |> -> Hadoop-oozie::Server<||>
@@ -290,6 +273,37 @@ class standby_head_node inherits hadoop_cluster_node {
}
class hadoop_gateway_node inherits hadoop_cluster_node {
+ $hbase_thrift_address = "${fqdn}:${hbase_thrift_port}"
+ $hadoop_httpfs_url = "http://${fqdn}:${hadoop_httpfs_port}/webhdfs/v1"
+ $sqoop_server_url = "http://${fqdn}:${sqoop_server_port}/sqoop"
+ $solrcloud_url = "http://${fqdn}:${solrcloud_port}/solr/"
+
+ hadoop-sqoop::server { "sqoop server":
+ }
+
+ hadoop::httpfs { "httpfs":
+ namenode_host => $hadoop_namenode_host,
+ namenode_port => $hadoop_namenode_port,
+ auth => $hadoop_security_authentication,
+ }
+
+ hue::server { "hue server":
+ rm_url => $hadoop_rm_url,
+ rm_proxy_url => $hadoop_rm_proxy_url,
+ history_server_url => $hadoop_history_server_url,
+ webhdfs_url => $hadoop_httpfs_url,
+ sqoop_url => $sqoop_server_url,
+ solr_url => $solrcloud_url,
+ hbase_thrift_url => $hbase_thrift_address,
+ rm_host => $hadoop_rm_host,
+ rm_port => $hadoop_rm_port,
+ oozie_url => $hadoop_oozie_url,
+ default_fs => $hadoop_namenode_uri,
+ kerberos_realm => $kerberos_realm,
+ }
+ Hadoop::Httpfs<||> -> Hue::Server<||>
+ Hadoop-hbase::Client<||> -> Hue::Server<||>
+
hadoop::client { "hadoop client":
namenode_host => $hadoop_namenode_host,
namenode_port => $hadoop_namenode_port,
@@ -313,7 +327,9 @@ class hadoop_gateway_node inherits hadoop_cluster_node {
}
hadoop-oozie::client { "oozie client":
}
- hadoop-hbase::client { "hbase client":
+ hadoop-hbase::client { "hbase thrift client":
+ thrift => true,
+ kerberos_realm => $kerberos_realm,
}
hadoop-zookeeper::client { "zookeeper client":
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5de186bc/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
index e102986..3bbaa8a 100644
--- a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
@@ -20,7 +20,7 @@ class hadoop-hbase {
}
}
- class common-server-config {
+ class common-config {
include client-package
if ($kerberos_realm) {
require kerberos::client
@@ -45,12 +45,27 @@ class hadoop-hbase {
}
}
- define client {
- include client-package
+ define client($thrift = false, $kerberos_realm = "") {
+ include common-config
+
+ if ($thrift) {
+ package { "hbase-thrift":
+ ensure => latest,
+ }
+
+ service { "hbase-thrift":
+ ensure => running,
+ require => Package["hbase-thrift"],
+ subscribe => File["/etc/hbase/conf/hbase-site.xml", "/etc/hbase/conf/hbase-env.sh"],
+ hasrestart => true,
+ hasstatus => true,
+ }
+ Kerberos::Host_keytab <| title == "hbase" |> -> Service["hbase-thrift"]
+ }
}
define server($rootdir, $zookeeper_quorum, $kerberos_realm = "", $heap_size="1024") {
- include common-server-config
+ include common-config
package { "hbase-regionserver":
ensure => latest,
@@ -67,7 +82,7 @@ class hadoop-hbase {
}
define master($rootdir, $zookeeper_quorum, $kerberos_realm = "", $heap_size="1024") {
- include common-server-config
+ include common-config
package { "hbase-master":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/5de186bc/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp
index f25c08a..c6fc7c0 100644
--- a/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-hbase/tests/init.pp
@@ -13,7 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-hadoop-hbase::client { "hbase shell": }
+hadoop-hbase::client { "hbase thrift client":
+ thrift => true,
+ kerberos_realm => "KRB.YOU.ORG",
+}
hadoop-hbase::master { "test-hbase-master":
rootdir => "hdfs://localhost:17020/hbase",
zookeeper_quorum => "localhost",