You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2015/02/08 17:44:04 UTC
bigtop git commit: BIGTOP-1655. Update VM and Docker provisioner to
work with hiera converted puppet recipes
Repository: bigtop
Updated Branches:
refs/heads/master 1472a0e11 -> b00e9da8f
BIGTOP-1655. Update VM and Docker provisioner to work with hiera converted puppet recipes
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b00e9da8
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b00e9da8
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b00e9da8
Branch: refs/heads/master
Commit: b00e9da8f2b98ffbd0bc7233a1a5452b83a9271b
Parents: 1472a0e
Author: Evans Ye <ev...@apache.org>
Authored: Sat Feb 7 17:18:12 2015 +0000
Committer: Evans Ye <ev...@apache.org>
Committed: Sun Feb 8 14:09:20 2015 +0000
----------------------------------------------------------------------
.../vm/vagrant-puppet-docker/Vagrantfile | 2 +-
.../vm/vagrant-puppet-docker/docker-hadoop.sh | 29 ++++++++++++--------
.../vm/vagrant-puppet-docker/provision.sh | 16 ++++++-----
.../vm/vagrant-puppet-docker/vagrantconfig.yaml | 2 +-
.../vagrantconfig_debian.yaml | 2 +-
bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 18 ++++++------
.../vm/vagrant-puppet-vm/vagrantconfig.yaml | 2 +-
7 files changed, 40 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile
index f800df7..f082142 100755
--- a/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/Vagrantfile
@@ -20,7 +20,7 @@ require 'fileutils'
# number of instances
$num_instances = 3
-#$vagrantyamlconf = "vagrantconfig.yaml"
+$vagrantyamlconf = "vagrantconfig.yaml"
# load configuration from config.rb
CONFIG = File.join(File.dirname(__FILE__), "config.rb")
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
index 3b44c4f..6459e85 100755
--- a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
@@ -32,24 +32,29 @@ create() {
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`)
repo=$(get-yaml-config repo)
- components=$(get-yaml-config components)
+ components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
distro=$(get-yaml-config distro)
- echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh" |vagrant ssh ${nodes[0]}
- echo "/vagrant/provision.sh $hadoop_head_node $repo $components" |vagrant ssh ${nodes[0]}
- bigtop-puppet ${nodes[0]}
- for ((i=1 ; i<${#nodes[*]} ; i++)); do
+
+ # setup environment before running bigtop puppet deployment
+ for node in ${nodes[*]}; do
(
- echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh" |vagrant ssh ${nodes[$i]}
- echo "/vagrant/provision.sh $hadoop_head_node $repo $components" |vagrant ssh ${nodes[$i]}
- bigtop-puppet ${nodes[$i]}
+ echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh" |vagrant ssh $node
+ echo "/vagrant/provision.sh $hadoop_head_node $repo \"$components\"" |vagrant ssh $node
) &
done
wait
+
+ # run bigtop puppet (master node need to be provisioned before slave nodes)
+ bigtop-puppet ${nodes[0]}
+ for ((i=1 ; i<${#nodes[*]} ; i++)); do
+ bigtop-puppet ${nodes[$i]} &
+ done
+ wait
}
provision() {
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
- for node in $nodes; do
+ for node in ${nodes[*]}; do
bigtop-puppet $node &
done
wait
@@ -62,16 +67,16 @@ smoke-tests() {
destroy() {
- rm -rf ./hosts ./config ./config.rb
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
- for node in $nodes; do
+ rm -rvf ./hosts ./config.rb
+ for node in ${nodes[*]}; do
vagrant destroy -f $node
done
wait
}
bigtop-puppet() {
- echo "puppet apply -d --confdir=/vagrant --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules:/usr/share/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp" |vagrant ssh $1
+ echo "puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules:/usr/share/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp" |vagrant ssh $1
}
get-yaml-config() {
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh b/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
index 8d1fa90..b9f1cfb 100755
--- a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
@@ -32,11 +32,13 @@ else
jdk="java-1.7.0-openjdk-devel.x86_64"
fi
-mkdir /vagrant/config
-cat > /vagrant/config/site.csv << EOF
-hadoop_head_node,$1
-hadoop_storage_dirs,/data/1,/data/2
-bigtop_yumrepo_uri,$2
-jdk_package_name,$jdk
-components,$3
+mkdir -p /etc/puppet/hieradata
+cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet
+cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/
+cat > /etc/puppet/hieradata/site.yaml << EOF
+bigtop::hadoop_head_node: $1
+hadoop::hadoop_storage_dirs: [/data/1, /data/2]
+Bigtop::bigtop_yumrepo_uri: $2
+bigtop::jdk_package_name: $jdk
+hadoop_cluster_node::cluster_components: $3
EOF
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
index bb540e2..c5b75b4 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
@@ -8,7 +8,7 @@ boot2docker:
repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"
distro: centos
-components: "hadoop,yarn"
+components: [hadoop, yarn]
namenode_ui_port: "50070"
yarn_ui_port: "8088"
hbase_ui_port: "60010"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
index bd22455..5a8e772 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
@@ -8,7 +8,7 @@ boot2docker:
repo: "http://to.do/notyet"
distro: debian
-components: "hadoop,yarn"
+components: [hadoop, yarn]
namenode_ui_port: "50070"
yarn_ui_port: "8088"
hbase_ui_port: "60010"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
index 2f49b4c..8b49f09 100755
--- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
@@ -47,13 +47,15 @@ chkconfig iptables off
cat /dev/null > /etc/hosts
echo "Bigtop yum repo = #{repo}"
# Prepare puppet configuration file
-mkdir /vagrant/config
-cat > /vagrant/config/site.csv << EOF
-hadoop_head_node,#{bigtop_master}
-hadoop_storage_dirs,/data/1,/data/2
-bigtop_yumrepo_uri,#{repo}
-jdk_package_name,java-1.7.0-openjdk-devel.x86_64
-components,#{components}
+mkdir -p /etc/puppet/hieradata
+cp /bigtop-home/bigtop-deploy/puppet/hiera.yaml /etc/puppet
+cp -r /bigtop-home/bigtop-deploy/puppet/hieradata/bigtop/ /etc/puppet/hieradata/
+cat > /etc/puppet/hieradata/site.yaml << EOF
+bigtop::hadoop_head_node: #{bigtop_master}
+hadoop::hadoop_storage_dirs: [/data/1, /data/2]
+Bigtop::bigtop_yumrepo_uri: #{repo}
+bigtop::jdk_package_name: java-1.7.0-openjdk-devel.x86_64
+hadoop_cluster_node::cluster_components: #{components}
EOF
SCRIPT
@@ -108,7 +110,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
puppet.module_path = "../../puppet/modules/"
puppet.manifests_path = "../../puppet/manifests/"
puppet.manifest_file = "site.pp"
- puppet.options = "--debug --confdir=/vagrant"
+ puppet.options = "--debug"
end
if run_smoke_tests then
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b00e9da8/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
index baa4cb4..390a3b1 100644
--- a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
@@ -3,7 +3,7 @@ number_cpus: 1
box: "puppetlabs/centos-6.5-64-nocm"
repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"
num_instances: 1
-components: "hadoop,yarn"
+components: [hadoop, yarn]
enable_local_yum: false
run_smoke_tests: false
distro: centos