You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2015/09/12 18:37:35 UTC
bigtop git commit: BIGTOP-1746: Introduce roles in bigtop puppet
deployment
Repository: bigtop
Updated Branches:
refs/heads/master 5aab80ca6 -> 6616117e3
BIGTOP-1746: Introduce roles in bigtop puppet deployment
Signed-off-by: Evans Ye <ev...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6616117e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6616117e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6616117e
Branch: refs/heads/master
Commit: 6616117e303971419a5059b54e2fed5838a567d6
Parents: 5aab80c
Author: Gajendran <vi...@amazon.com>
Authored: Sun Aug 30 13:33:21 2015 -0700
Committer: Evans Ye <ev...@apache.org>
Committed: Sat Sep 12 16:36:03 2015 +0000
----------------------------------------------------------------------
.../puppet/hieradata/bigtop/cluster.yaml | 28 ++
bigtop-deploy/puppet/hieradata/site.yaml | 2 +
bigtop-deploy/puppet/manifests/cluster.pp | 336 ++++++++++---------
bigtop-deploy/puppet/manifests/site.pp | 28 +-
.../puppet/modules/bigtop-util/Gemfile | 8 +
.../puppet/modules/bigtop-util/Gemfile.lock | 51 +++
.../puppet/modules/bigtop-util/README.md | 23 ++
.../puppet/modules/bigtop-util/Rakefile | 2 +
.../lib/puppet/parser/functions/get_roles.rb | 61 ++++
.../spec/functions/get_roles_spec.rb | 60 ++++
.../puppet/modules/crunch/manifests/init.pp | 7 +
.../puppet/modules/giraph/manifests/init.pp | 7 +
.../modules/hadoop-flume/manifests/init.pp | 7 +
.../modules/hadoop-hbase/manifests/init.pp | 18 +
.../modules/hadoop-hive/manifests/init.pp | 12 +-
.../modules/hadoop-hive/templates/hive-site.xml | 4 +-
.../modules/hadoop-oozie/manifests/init.pp | 16 +
.../puppet/modules/hadoop-pig/manifests/init.pp | 7 +
.../modules/hadoop-zookeeper/manifests/init.pp | 11 +
.../puppet/modules/hadoop/manifests/init.pp | 64 +++-
.../modules/hadoop/templates/hadoop-env.sh | 4 +-
.../modules/hadoop/templates/mapred-site.xml | 15 -
.../puppet/modules/hcatalog/manifests/init.pp | 11 +
.../puppet/modules/hue/manifests/init.pp | 12 +
.../modules/ignite-hadoop/manifests/init.pp | 6 +
.../puppet/modules/kerberos/manifests/init.pp | 9 +
.../puppet/modules/mahout/manifests/init.pp | 7 +
.../puppet/modules/solr/manifests/init.pp | 7 +
.../puppet/modules/spark/manifests/init.pp | 11 +
.../puppet/modules/sqoop2/manifests/init.pp | 11 +
.../puppet/modules/tachyon/manifests/init.pp | 11 +
.../puppet/modules/tez/manifests/init.pp | 8 +
.../puppet/modules/ycsb/manifests/init.pp | 7 +
33 files changed, 662 insertions(+), 209 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 85583d1..41d477b 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -24,6 +24,27 @@
# - hadoop
# - yarn
+# A list of roles can be assigned to each node in a cluster via "bigtop::roles"
+# hiera conf. This is useful if you want to run specific daemons/processes of a component
+# (lets say hadoop) on specific nodes. To enable roles, you need to explicitly set
+# "bigtop::roles_enabled" conf to true. By default, it is not enabled. If not enabled,
+# bigtop will honour the hadoop_cluster_node::cluster_components parameter and assume
+# respective roles to each node based on hadoop_head_node, hadoop_gateway_node
+# and other nodes (worker nodes) classification. For full list of roles
+# supported by each component, refer to $roles_map datastructure in cluster.pp file
+# for example:
+
+#bigtop::roles
+# - namenode
+# - resourcemanager
+
+# By specifying above roles, you instruct a node to run only namenode and resourcemanager.
+# When roles are disabled, all worker daemons for each component assume hadoop_head_node as
+# their master. With roles enabled, this assumption may no longer be valid and its upto the user
+# to specify the FQDN for each master daemons to all nodes so that worker daemons can communicate
+# with master. FQDN and port for master daemons of each component can be specified using below
+# configurations.
+
# Storage directories (will be created if doesn't exist)
#hadoop::hadoop_storage_dirs:
# - /data/1
@@ -141,3 +162,10 @@ hadoop-hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_conf
# tez
hadoop::common::tez_conf_dir: "/etc/tez/conf"
hadoop::common::tez_jars: "/usr/lib/tez"
+
+# to enable tez in hadoop, uncomment the lines below
+# hadoop::common::use_tez: true
+# hadoop::common_mapred_app::mapreduce_framework_name: "yarn-tez"
+
+# to enable tez in hive, uncomment the lines below
+# hadoop-hive::client::hive_execution_engine: "tez"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/hieradata/site.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index f8c1b7b..70c0529 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -35,5 +35,7 @@ hadoop::hadoop_storage_dirs:
# - zookeeper
# - ycsb
+#bigtop::roles_enabled: false
+
#bigtop::jdk_package_name: "openjdk-7-jre-headless"
#bigtop::bigtop_repo_uri: "http://mirror.example.com/path/to/mirror/"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 19cfc65..15a247b 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -13,37 +13,103 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+$roles_map = {
+ hdfs-non-ha => {
+ master => ["namenode"],
+ worker => ["datanode"],
+ standby => ["secondarynamenode"],
+ },
+ hdfs-ha => {
+ master => ["namenode"],
+ worker => ["datanode"],
+ standby => ["standby-namenode"],
+ },
+ yarn => {
+ master => ["resourcemanager"],
+ worker => ["nodemanager"],
+ client => ["hadoop-client"],
+ # mapred is the default app which runs on yarn.
+ library => ["mapred-app"],
+ },
+ mapred => {
+ library => ["mapred-app"],
+ },
+ hbase => {
+ master => ["hbase-master"],
+ worker => ["hbase-server"],
+ client => ["hbase-client"],
+ },
+ ignite-hadoop => {
+ worker => ["ignite-server"],
+ },
+ solrcloud => {
+ worker => ["solr-server"],
+ },
+ spark => {
+ master => ["spark-master"],
+ worker => ["spark-worker"],
+ },
+ tachyon => {
+ master => ["tachyon-master"],
+ worker => ["tachyon-worker"],
+ },
+ flume => {
+ worker => ["flume-agent"],
+ },
+ kerberos => {
+ master => ["kerberos-server"],
+ },
+ oozie => {
+ master => ["oozie-server"],
+ client => ["oozie-client"],
+ },
+ hcat => {
+ master => ["hcatalog-server"],
+ gateway_server => ["webhcat-server"],
+ },
+ sqoop => {
+ gateway_server => ["sqoop-server"],
+ client => ["sqoop-client"],
+ },
+ httpfs => {
+ gateway_server => ["httpfs-server"],
+ },
+ hue => {
+ gateway_server => ["hue-server"],
+ },
+ mahout => {
+ client => ["mahout-client"],
+ },
+ giraph => {
+ client => ["giraph-client"],
+ },
+ crunch => {
+ client => ["crunch-client"],
+ },
+ pig => {
+ client => ["pig-client"],
+ },
+ hive => {
+ client => ["hive-client"],
+ },
+ tez => {
+ client => ["tez-client"],
+ },
+ zookeeper => {
+ worker => ["zookeeper-server"],
+ client => ["zookeeper-client"],
+ },
+ ycsb => {
+ client => ["ycsb-client"],
+ },
+}
+
class hadoop_cluster_node (
$hadoop_security_authentication = hiera("hadoop::hadoop_security_authentication", "simple"),
-
- # Lookup component array or comma separated components (i.e.
- # hadoop,spark,hbase ) as a default via facter.
- $cluster_components = "$::components"
+ $bigtop_real_users = [ 'jenkins', 'testuser', 'hudson' ],
+ $cluster_components = ["all"]
) {
- # Ensure (even if a single value) that the type is an array.
- if is_array($cluster_components) {
- $components = $cluster_components
- } else {
- $components = any2array($cluster_components, ",")
- }
-
- $all = ($components[0] == undef)
-
- if ($hadoop_security_authentication == "kerberos") {
- include kerberos::client
- }
-
- # Flume agent is the only component that goes on EVERY node in the cluster
- if ($all or "flume" in $components) {
- include hadoop-flume::agent
- }
-}
-
-
-class hadoop_worker_node (
- $bigtop_real_users = [ 'jenkins', 'testuser', 'hudson' ]
- ) inherits hadoop_cluster_node {
user { $bigtop_real_users:
ensure => present,
system => false,
@@ -53,168 +119,106 @@ class hadoop_worker_node (
if ($hadoop_security_authentication == "kerberos") {
kerberos::host_keytab { $bigtop_real_users: }
User<||> -> Kerberos::Host_keytab<||>
+ include kerberos::client
}
- include hadoop::datanode
- if ($all or "yarn" in $components) {
- include hadoop::nodemanager
- }
- if ($all or "hbase" in $components) {
- include hadoop-hbase::server
- }
-
- if ($all or "ignite-hadoop" in $components) {
- ignite-hadoop::server { "ignite-hadoop-node": }
- }
-
- ### If mapred is not installed, yarn can fail.
- ### So, when we install yarn, we also need mapred for now.
- ### This dependency should be cleaned up eventually.
- if ($all or "mapred-app" or "yarn" in $components) {
- include hadoop::mapred-app
- }
-
- if ($all or "solrcloud" in $components) {
- include solr::server
- }
+ $hadoop_head_node = hiera("bigtop::hadoop_head_node")
+ $standby_head_node = hiera("bigtop::standby_head_node", "")
+ $hadoop_gateway_node = hiera("bigtop::hadoop_gateway_node", $hadoop_head_node)
- if ($all or "spark" in $components) {
- include spark::worker
+ $ha_enabled = $standby_head_node ? {
+ "" => false,
+ default => true,
}
- if ($all or "tachyon" in $components) {
- include tachyon::worker
+ # look into alternate hiera datasources configured using this path in
+ # hiera.yaml
+ $hadoop_hiera_ha_path = $ha_enabled ? {
+ false => "noha",
+ true => "ha",
}
-
}
-class hadoop_head_node inherits hadoop_worker_node {
- exec { "init hdfs":
- path => ['/bin','/sbin','/usr/bin','/usr/sbin'],
- command => 'bash -x /usr/lib/hadoop/libexec/init-hdfs.sh',
- require => Package['hadoop-hdfs'],
- timeout => 0
- }
- Class['Hadoop::Namenode'] -> Class['Hadoop::Datanode'] -> Exec<| title == "init hdfs" |>
-
-if ($hadoop_security_authentication == "kerberos") {
- include kerberos::server
- include kerberos::kdc
- include kerberos::kdc::admin_server
- }
-
- include hadoop::namenode
-
- if ($hadoop::common_hdfs::ha == "disabled") {
- include hadoop::secondarynamenode
- }
-
- if ($all or "yarn" in $components) {
- include hadoop::resourcemanager
- include hadoop::historyserver
- include hadoop::proxyserver
- Exec<| title == "init hdfs" |> -> Class['Hadoop::Resourcemanager'] -> Class['Hadoop::Nodemanager']
- Exec<| title == "init hdfs" |> -> Class['Hadoop::Historyserver']
- }
-
- if ($all or "hbase" in $components) {
- include hadoop-hbase::master
- Exec<| title == "init hdfs" |> -> Class['Hadoop-hbase::Master']
- }
-
- if ($all or "oozie" in $components) {
- include hadoop-oozie::server
- if ($all or "mapred-app" in $components) {
- Class['Hadoop::Mapred-app'] -> Class['Hadoop-oozie::Server']
+class node_with_roles ($roles = hiera("bigtop::roles")) inherits hadoop_cluster_node {
+ define deploy_module($roles) {
+ class { "${name}::deploy":
+ roles => $roles,
}
- Exec<| title == "init hdfs" |> -> Class['Hadoop-oozie::Server']
}
- if ($all or "hcat" in $components) {
- include hcatalog::server
- include hcatalog::webhcat::server
+ $modules = [
+ "crunch",
+ "giraph",
+ "hadoop",
+ "hadoop-hbase",
+ "ignite-hadoop",
+ "hadoop-flume",
+ "hadoop-hive",
+ "hadoop-oozie",
+ "hadoop-pig",
+ "sqoop2",
+ "hadoop-zookeeper",
+ "hcatalog",
+ "hue",
+ "mahout",
+ "solr",
+ "spark",
+ "tachyon",
+ "tez",
+ "ycsb",
+ "kerberos"
+ ]
+
+ deploy_module { $modules:
+ roles => $roles,
}
+}
- if ($all or "spark" in $components) {
- include spark::master
- }
+class node_with_components inherits hadoop_cluster_node {
- if ($all or "tachyon" in $components) {
- include tachyon::master
+ # Ensure (even if a single value) that the type is an array.
+ if (is_array($cluster_components)) {
+ $components_array = $cluster_components
+ } else {
+ if ($cluster_components == undef) {
+ $components_array = ["all"]
+ } else {
+ $components_array = [$cluster_components]
+ }
}
- if ($all or "hbase" in $components) {
- include hadoop-zookeeper::server
+ $given_components = $components_array[0] ? {
+ "all" => delete(keys($roles_map), ["hdfs-non-ha", "hdfs-ha"]),
+ default => $components_array,
}
-
- if ($all or "tez" in $components) {
- include tez::client
- Class['tez::client'] -> Exec<| title == "init hdfs" |>
+ $ha_dependent_components = $ha_enabled ? {
+ true => ["hdfs-ha"],
+ default => ["hdfs-non-ha"],
}
+ $components = concat($given_components, $ha_dependent_components)
- # class hadoop::rsync_hdfs isn't used anywhere
- #Exec<| title == "init hdfs" |> -> Class['Hadoop::Rsync_hdfs']
-}
-
-class standby_head_node inherits hadoop_cluster_node {
- include hadoop::namenode
-}
-
-class hadoop_gateway_node inherits hadoop_cluster_node {
- if ($all or "sqoop2" in $components) {
- include sqoop2::server
- }
+ $master_role_types = ["master", "worker", "library"]
+ $standby_role_types = ["standby", "library"]
+ $worker_role_types = ["worker", "library"]
+ $gateway_role_types = ["client", "gateway_server"]
- if ($all or "httpfs" in $components) {
- include hadoop::httpfs
- if ($all or "hue" in $components) {
- Class['Hadoop::Httpfs'] -> Class['Hue::Server']
- }
- }
-
- if ($all or "hue" in $components) {
- include hue::server
- if ($all or "hbase" in $components) {
- Class['Hadoop-hbase::Client'] -> Class['Hue::Server']
+ if ($::fqdn == $hadoop_head_node or $::fqdn == $hadoop_gateway_node) {
+ if ($hadoop_gateway_node == $hadoop_head_node) {
+ $role_types = concat($master_role_types, $gateway_role_types)
+ } elsif ($::fqdn == $hadoop_head_node) {
+ $role_types = $master_role_types
+ } else {
+ $role_types = $gateway_role_types
}
+ } elsif ($::fqdn == $standby_head_node) {
+ $role_types = $standby_role_types
+ } else {
+ $role_types = $worker_role_types
}
- include hadoop::client
+ $roles = get_roles($components, $role_types, $roles_map)
- if ($all or "mahout" in $components) {
- include mahout::client
- }
- if ($all or "giraph" in $components) {
- include giraph::client
- }
- if ($all or "crunch" in $components) {
- include crunch::client
- }
- if ($all or "pig" in $components) {
- include hadoop-pig::client
- }
- if ($all or "hive" in $components) {
- include hadoop-hive::client
- }
- if ($all or "sqoop2" in $components) {
- include sqoop2::client
- }
- if ($all or "sqoop" in $components) {
- include sqoop::client
- }
- if ($all or "oozie" in $components) {
- include hadoop-oozie::client
- }
- if ($all or "hbase" in $components) {
- include hadoop-hbase::client
- }
- if ($all or "zookeeper" in $components) {
- include hadoop-zookeeper::client
- }
- if ($all or "tez" in $components) {
- include tez::client
- }
- if ($all or "ycsb" in $components) {
- include ycsb::client
+ class { 'node_with_roles':
+ roles => $roles,
}
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/manifests/site.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/site.pp b/bigtop-deploy/puppet/manifests/site.pp
index 8e8276c..ae71627 100644
--- a/bigtop-deploy/puppet/manifests/site.pp
+++ b/bigtop-deploy/puppet/manifests/site.pp
@@ -56,31 +56,17 @@ package { $jdk_package_name:
import "cluster.pp"
node default {
- $hadoop_head_node = hiera("bigtop::hadoop_head_node")
- $standby_head_node = hiera("bigtop::standby_head_node", "")
- $hadoop_gateway_node = hiera("bigtop::hadoop_gateway_node", $hadoop_head_node)
- # look into alternate hiera datasources configured using this path in
- # hiera.yaml
- $hadoop_hiera_ha_path = $standby_head_node ? {
- "" => "noha",
- default => "ha",
- }
+ $roles_enabled = hiera("bigtop::roles_enabled", false)
- case $::fqdn {
- $hadoop_head_node: {
- include hadoop_head_node
- }
- $standby_head_node: {
- include standby_head_node
- }
- default: {
- include hadoop_worker_node
- }
+ if (!is_bool($roles_enabled)) {
+ fail("bigtop::roles hiera conf is not of type boolean. It should be set to either true or false")
}
- if ($hadoop_gateway_node == $::fqdn) {
- include hadoop_gateway_node
+ if ($roles_enabled) {
+ include node_with_roles
+ } else {
+ include node_with_components
}
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/Gemfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/bigtop-util/Gemfile b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile
new file mode 100644
index 0000000..4abd416
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile
@@ -0,0 +1,8 @@
+source 'https://rubygems.org'
+
+group :test do
+ gem 'rake'
+ gem 'puppet', ENV['PUPPET_VERSION'] || '~> 3.7.0'
+ gem 'rspec-puppet', git: 'https://github.com/rodjek/rspec-puppet.git'
+ gem 'puppetlabs_spec_helper'
+end
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock
new file mode 100644
index 0000000..a24c0b3
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/bigtop-util/Gemfile.lock
@@ -0,0 +1,51 @@
+GIT
+ remote: https://github.com/rodjek/rspec-puppet.git
+ revision: 8459e14807977244c00bdbcf190062c529b63474
+ specs:
+ rspec-puppet (2.0.0)
+ rspec (~> 2.0)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ CFPropertyList (2.2.8)
+ diff-lcs (1.2.5)
+ facter (2.4.1)
+ CFPropertyList (~> 2.2.6)
+ hiera (1.3.4)
+ json_pure
+ json_pure (1.8.2)
+ metaclass (0.0.4)
+ mocha (1.1.0)
+ metaclass (~> 0.0.1)
+ puppet (3.7.4)
+ facter (> 1.6, < 3)
+ hiera (~> 1.0)
+ json_pure
+ puppet-lint (1.1.0)
+ puppet-syntax (2.0.0)
+ rake
+ puppetlabs_spec_helper (0.9.1)
+ mocha
+ puppet-lint
+ puppet-syntax
+ rake
+ rspec-puppet
+ rake (10.4.2)
+ rspec (2.99.0)
+ rspec-core (~> 2.99.0)
+ rspec-expectations (~> 2.99.0)
+ rspec-mocks (~> 2.99.0)
+ rspec-core (2.99.2)
+ rspec-expectations (2.99.2)
+ diff-lcs (>= 1.1.3, < 2.0)
+ rspec-mocks (2.99.3)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ puppet (~> 3.7.0)
+ puppetlabs_spec_helper
+ rake
+ rspec-puppet!
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/README.md
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/bigtop-util/README.md b/bigtop-deploy/puppet/modules/bigtop-util/README.md
new file mode 100644
index 0000000..287504c
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/bigtop-util/README.md
@@ -0,0 +1,23 @@
+# Bigtop Util Puppet Module
+
+This module provides common utility functions used by other Bigtop puppet modules.
+
+## Testing
+
+Test dependencies are managed by Bundler:
+
+```bash
+gem install bundler
+```
+
+To install the required gems for testing, `cd` into `bigtop-util` and use:
+
+```bash
+bundle install --path vendor/bundle
+```
+
+Now unit tests can be run using:
+
+```bash
+bundle exec rake spec
+```
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/Rakefile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/bigtop-util/Rakefile b/bigtop-deploy/puppet/modules/bigtop-util/Rakefile
new file mode 100644
index 0000000..14f1c24
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/bigtop-util/Rakefile
@@ -0,0 +1,2 @@
+require 'rubygems'
+require 'puppetlabs_spec_helper/rake_tasks'
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb b/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb
new file mode 100644
index 0000000..3f78d31
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/bigtop-util/lib/puppet/parser/functions/get_roles.rb
@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Function to return an array of roles given the roles mapping, array of components and role types to lookup.
+# Example map:
+# $map = {
+# component1 => {
+# role_type1 => ["xxx"],
+# role_type2 => ["yyy"],
+# role_type3 => ["zzz"],
+# }
+# }
+# calling get_roles(["component1"], ["role_type1", "role_type2"], $map) will return ["xxx", "yyy"]
+
+Puppet::Parser::Functions.newfunction(:get_roles, :type => :rvalue) do |arguments|
+ if arguments.size != 3
+ fail Puppet::ParseError, "get_roles() method: Incorrect number of arguments. arguments given #{arguments.size} for 3"
+ end
+
+ components = arguments[0]
+ role_types = arguments[1]
+ roles_map = arguments[2]
+
+ unless components.is_a? Array
+ fail Puppet::ParseError, "get_roles(): Requires first argument to be array"
+ end
+
+ unless role_types.is_a? Array
+ fail Puppet::ParseError, "get_roles(): Requires second argument to be array"
+ end
+
+ unless roles_map.is_a? Hash
+ fail Puppet::ParseError, "get_roles(): Requires third argument to be hash"
+ end
+
+ roles = Array.new
+ components.each do |component|
+ role_types.each do |role_type|
+ if roles_map.key?(component)
+ component_map = roles_map[component]
+ if component_map.key?(role_type)
+ temp_roles = component_map[role_type]
+ roles.concat(temp_roles)
+ end
+ end
+ end
+ end
+ roles
+end
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb b/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb
new file mode 100644
index 0000000..5a680e5
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/bigtop-util/spec/functions/get_roles_spec.rb
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'puppetlabs_spec_helper/module_spec_helper'
+
+describe 'get_roles' do
+ let(:scope) { PuppetlabsSpec::PuppetInternals.scope }
+ subject { scope.function_get_roles([components, role_types, roles_map]) }
+
+ context 'simple test' do
+ let(:components) { ["c1", "c2"] }
+
+ let(:role_types) { ["rt1", "rt3"] }
+
+ let(:roles_map) do
+ {
+ "c1" => {
+ "rt1" => ["r1"],
+ "rt2" => ["r2"]
+ },
+ "c2" => {
+ "rt3" => ["r3"],
+ "rt4" => ["r4"]
+ },
+ }
+ end
+
+ it { is_expected.to match_array(["r1", "r3"]) }
+ end
+
+ context 'missing roles_type/component test' do
+ let(:components) { ["c1", "c2"] }
+
+ let(:role_types) { ["rt1", "rt3"] }
+
+ let(:roles_map) do
+ {
+ "c1" => {
+ "rt1" => ["r10", "r11"],
+ "rt2" => ["r2"]
+ },
+ }
+ end
+
+ it { is_expected.to match_array(["r10", "r11"]) }
+ end
+end
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/crunch/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/crunch/manifests/init.pp b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp
index b31edf6..fa190eb 100644
--- a/bigtop-deploy/puppet/modules/crunch/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/crunch/manifests/init.pp
@@ -14,6 +14,13 @@
# limitations under the License.
class crunch {
+
+ class deploy ($roles) {
+ if ("crunch-client" in $roles) {
+ include crunch::client
+ }
+ }
+
class client {
package { ["crunch", "crunch-doc"]:
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/giraph/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/giraph/manifests/init.pp b/bigtop-deploy/puppet/modules/giraph/manifests/init.pp
index 1dc0d9b..a52a133 100644
--- a/bigtop-deploy/puppet/modules/giraph/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/giraph/manifests/init.pp
@@ -14,6 +14,13 @@
# limitations under the License.
class giraph {
+
+ class deploy ($roles) {
+ if ("giraph-client" in $roles) {
+ include giraph::client
+ }
+ }
+
class client($zookeeper_quorum = 'localhost') {
package { "giraph":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp
index daf352a..e6012cf 100644
--- a/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-flume/manifests/init.pp
@@ -14,6 +14,13 @@
# limitations under the License.
class hadoop-flume {
+
+ class deploy ($roles) {
+ if ("flume-agent" in $roles) {
+ include hadoop-flume::agent
+ }
+ }
+
class agent($sources = [], $sinks = [], $channels = []) {
package { "flume-agent":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
index 454e978..f61a23b 100644
--- a/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-hbase/manifests/init.pp
@@ -14,6 +14,24 @@
# limitations under the License.
class hadoop-hbase {
+
+ class deploy ($roles) {
+ if ("hbase-server" in $roles) {
+ include hadoop-hbase::server
+ }
+
+ if ("hbase-master" in $roles) {
+ include hadoop::init_hdfs
+ include hadoop-hbase::master
+ include hadoop-zookeeper::server
+ Class['Hadoop::Init_hdfs'] -> Class['Hadoop-hbase::Master']
+ }
+
+ if ("hbase-client" in $roles) {
+ include hadoop-hbase::client
+ }
+ }
+
class client-package {
package { "hbase":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp
index f9dede4..b0eee0a 100644
--- a/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-hive/manifests/init.pp
@@ -14,7 +14,17 @@
# limitations under the License.
class hadoop-hive {
- class client($hbase_master = "", $hbase_zookeeper_quorum = "") {
+
+ class deploy ($roles) {
+ if ("hive-client" in $roles) {
+ include hadoop-hive::client
+ }
+ }
+
+ class client($hbase_master = "",
+ $hbase_zookeeper_quorum = "",
+ $hive_execution_engine = "mr") {
+
package { "hive":
ensure => latest,
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
index 4e33907..2aca237 100644
--- a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
@@ -42,12 +42,10 @@
</property>
<% end %>
-<% if @all or @components.include? "tez" -%>
<property>
<name>hive.execution.engine</name>
- <value>tez</value>
+ <value><%= @hive_execution_engine %></value>
</property>
-<% end %>
<property>
<name>javax.jdo.option.ConnectionURL</name>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
index f1177e9..0829360 100644
--- a/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-oozie/manifests/init.pp
@@ -14,6 +14,22 @@
# limitations under the License.
class hadoop-oozie {
+
+ class deploy ($roles) {
+ if ("oozie-client" in $roles) {
+ include hadoop-oozie::client
+ }
+
+ if ("oozie-server" in $roles) {
+ include hadoop::init_hdfs
+ include hadoop-oozie::server
+ Class['Hadoop::Init_hdfs'] -> Class['Hadoop-oozie::Server']
+ if ("mapred-app" in $roles) {
+ Class['Hadoop::Mapred-app'] -> Class['Hadoop-oozie::Server']
+ }
+ }
+ }
+
class client {
package { "oozie-client":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp
index 37bfde0..3ffaa70 100644
--- a/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-pig/manifests/init.pp
@@ -14,6 +14,13 @@
# limitations under the License.
class hadoop-pig {
+
+ class deploy ($roles) {
+ if ("pig-client" in $roles) {
+ include hadoop-pig::client
+ }
+ }
+
class client {
package { "pig":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp
index d8bccfe..a44834c 100644
--- a/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop-zookeeper/manifests/init.pp
@@ -14,6 +14,17 @@
# limitations under the License.
class hadoop-zookeeper {
+
+ class deploy ($roles) {
+ if ("zookeeper-client" in $roles) {
+ include hadoop-zookeeper::client
+ }
+
+ if ("zookeeper-server" in $roles) {
+ include hadoop-zookeeper::server
+ }
+ }
+
class client {
package { "zookeeper":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 57c739c..777c26e 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -27,6 +27,67 @@ class hadoop ($hadoop_security_authentication = "simple",
include stdlib
+ class deploy ($roles) {
+ if ("datanode" in $roles) {
+ include hadoop::datanode
+ }
+
+ if ("namenode" in $roles) {
+ include hadoop::init_hdfs
+ include hadoop::namenode
+
+ if ("datanode" in $roles) {
+ Class['Hadoop::Namenode'] -> Class['Hadoop::Datanode'] -> Class['Hadoop::Init_hdfs']
+ } else {
+ Class['Hadoop::Namenode'] -> Class['Hadoop::Init_hdfs']
+ }
+ }
+
+ if ($hadoop::common_hdfs::ha != "disabled" and "standby-namenode" in $roles) {
+ include hadoop::namenode
+ }
+
+ if ("mapred-app" in $roles) {
+ include hadoop::mapred-app
+ }
+
+ if ("nodemanager" in $roles) {
+ include hadoop::nodemanager
+ }
+
+ if ("resourcemanager" in $roles) {
+ include hadoop::resourcemanager
+ include hadoop::historyserver
+ include hadoop::proxyserver
+
+ Class['Hadoop::Init_hdfs'] -> Class['Hadoop::Resourcemanager']
+ if ("nodemanager" in $roles) {
+ Class['Hadoop::Resourcemanager'] -> Class['Hadoop::Nodemanager']
+ }
+ Class['Hadoop::Init_hdfs'] -> Class['Hadoop::Historyserver']
+ }
+
+ if ($hadoop::common_hdfs::ha == "disabled" and "secondarynamenode" in $roles) {
+ include hadoop::secondarynamenode
+ }
+
+ if ("httpfs-server" in $roles) {
+ include hadoop::httpfs
+ }
+
+ if ("hadoop-client" in $roles) {
+ include hadoop::client
+ }
+ }
+
+ class init_hdfs {
+ exec { "init hdfs":
+ path => ['/bin','/sbin','/usr/bin','/usr/sbin'],
+ command => 'bash -x /usr/lib/hadoop/libexec/init-hdfs.sh',
+ require => Package['hadoop-hdfs']
+ }
+ }
+
class common ($hadoop_java_home = undef,
$hadoop_classpath = undef,
$hadoop_heapsize = undef,
@@ -46,6 +107,7 @@ class hadoop ($hadoop_security_authentication = "simple",
$hadoop_pid_dir = undef,
$hadoop_ident_string = undef,
$hadoop_niceness = undef,
+ $use_tez = false,
$tez_conf_dir = undef,
$tez_jars = undef,
) inherits hadoop {
@@ -298,7 +360,7 @@ class hadoop ($hadoop_security_authentication = "simple",
$mapreduce_jobhistory_host = undef,
$mapreduce_jobhistory_port = "10020",
$mapreduce_jobhistory_webapp_port = "19888",
- $mapreduce_framework_name = undef,
+ $mapreduce_framework_name = "yarn",
$mapred_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/mapred"),
$mapreduce_cluster_temp_dir = "/mapred/system",
$yarn_app_mapreduce_am_staging_dir = "/user",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
index f959f5b..a1397c2 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
@@ -80,14 +80,14 @@
# The scheduling priority for daemon processes. See 'man nice'.
<%= shell_config("HADOOP_NICENESS") %>
+<% if (@use_tez) -%>
# tez environment, needed to enable tez
-<% if (@all or @components.include? "tez") -%>
<%= shell_config("TEZ_CONF_DIR") %>
<%= shell_config("TEZ_JARS") %>
# Add tez into HADOOP_CLASSPATH
export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*
-<% end -%>
+<% end -%>
### WARNING: the following is NOT really optional. It is a shame that stock Hadoop
### hadoop_env.sh doesn't make it clear -- you can NOT turn com.sun.management.jmxremote off
### and have a working Hadoop cluster.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
index 2723e82..ea49926 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
@@ -92,21 +92,6 @@
<value><%= @mapreduce_framework_name %></value>
</property>
-<!-- If not specified, we automatically choose the best framework for users -->
-<% else -%>
-<% if @all or @components.include? "tez" -%>
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn-tez</value>
- </property>
-
-<% else -%>
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
-
-<% end -%>
<% end -%>
<property>
<name>mapreduce.cluster.local.dir</name>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
index 6585dd3..d78956d 100644
--- a/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hcatalog/manifests/init.pp
@@ -14,6 +14,17 @@
# limitations under the License.
class hcatalog {
+
+ class deploy ($roles) {
+ if ("hcatalog-server" in $roles) {
+ include hcatalog::server
+ }
+
+ if ("webhcat-server" in $roles) {
+ include hcatalog::webhcat::server
+ }
+ }
+
class server($port = "9083", $kerberos_realm = "") {
package { "hcatalog-server":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/hue/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hue/manifests/init.pp b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
index 687a055..72d08c0 100644
--- a/bigtop-deploy/puppet/modules/hue/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hue/manifests/init.pp
@@ -14,6 +14,18 @@
# limitations under the License.
class hue {
+ class deploy ($roles) {
+ if ("hue-server" in $roles) {
+ include hue::server
+ if ("httpfs-server" in $roles) {
+ Class['Hadoop::Httpfs'] -> Class['Hue::Server']
+ }
+ if ("hbase-client" in $roles) {
+ Class['Hadoop-hbase::Client'] -> Class['Hue::Server']
+ }
+ }
+ }
+
class server($sqoop2_url = "http://localhost:12000/sqoop", $solr_url = "http://localhost:8983/solr/", $hbase_thrift_url = "",
$webhdfs_url, $rm_host, $rm_port, $oozie_url, $rm_proxy_url, $history_server_url,
$hive_host = "", $hive_port = "10000",
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
index 6a967c6..de658a5 100644
--- a/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/ignite-hadoop/manifests/init.pp
@@ -14,6 +14,12 @@
# limitations under the License.
class ignite-hadoop {
+ class deploy ($roles) {
+ if ("ignite-server" in $roles) {
+ ignite-hadoop::server { "ignite-hadoop-node": }
+ }
+ }
+
define server() {
package { "ignite-hadoop":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp b/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
index 4a726a0..946f425 100644
--- a/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/kerberos/manifests/init.pp
@@ -14,6 +14,15 @@
# limitations under the License.
class kerberos {
+
+ class deploy ($roles) {
+ if ("kerberos-server" in $roles) {
+ include kerberos::server
+ include kerberos::kdc
+ include kerberos::kdc::admin_server
+ }
+ }
+
class site ($domain = inline_template('<%= domain %>'),
$realm = inline_template('<%= domain.upcase %>'),
$kdc_server = 'localhost',
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/mahout/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/mahout/manifests/init.pp b/bigtop-deploy/puppet/modules/mahout/manifests/init.pp
index 0d9bd8c..0c55a9b 100644
--- a/bigtop-deploy/puppet/modules/mahout/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/mahout/manifests/init.pp
@@ -14,6 +14,13 @@
# limitations under the License.
class mahout {
+
+ class deploy ($roles) {
+ if ("mahout-client" in $roles) {
+ include mahout::client
+ }
+ }
+
class client {
package { "mahout":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/solr/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/solr/manifests/init.pp b/bigtop-deploy/puppet/modules/solr/manifests/init.pp
index 119fbd1..c90e0e7 100644
--- a/bigtop-deploy/puppet/modules/solr/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/solr/manifests/init.pp
@@ -14,6 +14,13 @@
# limitations under the License.
class solr {
+
+ class deploy ($roles) {
+ if ("solr-server" in $roles) {
+ include solr::server
+ }
+ }
+
class server($port = "1978", $port_admin = "1979", $zk = "localhost:2181", $root_url = "hdfs://localhost:8020/solr", $kerberos_realm = "") {
package { "solr-server":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/spark/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/spark/manifests/init.pp b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
index 2f87ead..9b33bb9 100644
--- a/bigtop-deploy/puppet/modules/spark/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/spark/manifests/init.pp
@@ -14,6 +14,17 @@
# limitations under the License.
class spark {
+
+ class deploy ($roles) {
+ if ("spark-master" in $roles) {
+ include spark::master
+ }
+
+ if ("spark-worker" in $roles) {
+ include spark::worker
+ }
+ }
+
class common ($master_host = $fqdn, $master_port = "7077", $master_ui_port = "18080") {
package { "spark-core":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp b/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp
index 9648483..60b6939 100644
--- a/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/sqoop2/manifests/init.pp
@@ -14,6 +14,17 @@
# limitations under the License.
class sqoop2 {
+
+ class deploy ($roles) {
+ if ("sqoop-client" in $roles) {
+ include sqoop2::client
+ }
+
+ if ("sqoop-server" in $roles) {
+ include sqoop2::server
+ }
+ }
+
class client {
package { "sqoop2-client":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
index 8d53b25..ef7e5df 100644
--- a/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/tachyon/manifests/init.pp
@@ -10,6 +10,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
class tachyon {
+
+ class deploy ($roles) {
+ if ("tachyon-master" in $roles) {
+ include tachyon::master
+ }
+
+ if ("tachyon-worker" in $roles) {
+ include tachyon::worker
+ }
+ }
+
class common ($master_host){
package { "tachyon-tfs":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/tez/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/tez/manifests/init.pp b/bigtop-deploy/puppet/modules/tez/manifests/init.pp
index 6db2220..4e307aa 100644
--- a/bigtop-deploy/puppet/modules/tez/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/tez/manifests/init.pp
@@ -14,6 +14,14 @@
# limitations under the License.
class tez {
+ class deploy ($roles) {
+ if ("tez-client" in $roles) {
+ include hadoop::init_hdfs
+ include tez::client
+ Class['Tez::Client'] -> Class['Hadoop::Init_hdfs']
+ }
+ }
+
class client {
package { "tez":
ensure => latest,
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6616117e/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp b/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp
index 8e7e7d5..ed5c86a 100644
--- a/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/ycsb/manifests/init.pp
@@ -14,6 +14,13 @@
# limitations under the License.
class ycsb {
+
+ class deploy ($roles) {
+ if ("ycsb-client" in $roles) {
+ include ycsb::client
+ }
+ }
+
class client {
package { ["ycsb"]:
ensure => latest,