You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ja...@apache.org on 2014/08/26 14:05:49 UTC
git commit: BIGTOP-1416. Refactor vagrant-puppet's Vagrantfile to
make instance number configurable
Repository: bigtop
Updated Branches:
refs/heads/master da7ce0f3e -> da224ef25
BIGTOP-1416. Refactor vagrant-puppet's Vagrantfile to make instance number configurable
Signed-off-by: jay@apache.org <jayunit100>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/da224ef2
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/da224ef2
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/da224ef2
Branch: refs/heads/master
Commit: da224ef2507a229716e1fd97d310cc66a68bad71
Parents: da7ce0f
Author: evans_ye <in...@gmail.com>
Authored: Mon Aug 25 14:28:40 2014 +0800
Committer: jay@apache.org <jayunit100>
Committed: Tue Aug 26 08:05:25 2014 -0400
----------------------------------------------------------------------
bigtop-deploy/vm/vagrant-puppet/README.md | 28 ++--
bigtop-deploy/vm/vagrant-puppet/Vagrantfile | 155 +++++++----------------
bigtop-deploy/vm/vagrant-puppet/startup.sh | 40 ------
3 files changed, 59 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/da224ef2/bigtop-deploy/vm/vagrant-puppet/README.md
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/README.md b/bigtop-deploy/vm/vagrant-puppet/README.md
index 70afb36..91acadd 100644
--- a/bigtop-deploy/vm/vagrant-puppet/README.md
+++ b/bigtop-deploy/vm/vagrant-puppet/README.md
@@ -19,8 +19,8 @@
## Overview
-The startup.sh script runs one of 3 vagrant templates, and creates a bigtop virtual hadoop cluster for you, by
-pulling from existing publishing bigtop repositories. This cluster can be used:
+The Vagrantfile definition creates a bigtop virtual hadoop cluster for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
- to test bigtop smoke tests
- to test bigtop puppet recipes
@@ -45,21 +45,13 @@ vagrant plugin install vagrant-cachier
3) To provision a 3 node Apache Hadoop cluster on top of vagrant boxes
```
-./startup.sh --cluster
+vagrant up
```
-4) See options with -h specified
+4) You can specify number of nodes you'd like to provision by modifying `num_instances` in Vagrantfile
```
-$ ./startup.sh -h
-
- usage: startup.sh [options]
-
- -s, --standalone deploy a standalone hadoop vm
-
- -c, --cluster deploy a 3 node hadoop cluster
-
- -h, --help
+num_instances = 5
```
##Example:
@@ -67,11 +59,15 @@ $ ./startup.sh -h
5) Run hbase-test.sh to evaluate the deployment.
##Configure Apache Hadoop ecosystem components
-* Choose the ecosystem you want to be deployed by modify components in provision.sh.
+* Choose the ecosystem you want to be deployed by modifying components in provision.sh.
```
- components,hadoop,hbase,...
+ components,hadoop,hbase,yarn,mapred-app,...
```
-By default, Apache Hadoop and Apache HBase will be installed.
+By default, Apache Hadoop, YARN, and Apache HBase will be installed.
See `bigtop-deploy/puppet/config/site.csv.example` for more details.
+
+##Note:
+
+For bigtop 0.7.0 code base, you must change the value of the [yarn-site.xml](https://github.com/apache/bigtop/blob/master/bigtop-deploy/puppet/modules/hadoop/templates/yarn-site.xml) yarn.nodemanager.aux.services from "mapreduce_shuffle" to "mapreduce.shuffle" before `vagrant up`
http://git-wip-us.apache.org/repos/asf/bigtop/blob/da224ef2/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
index 15a3dec..c6825ad 100755
--- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
@@ -20,127 +20,66 @@ VAGRANTFILE_API_VERSION = "2"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
vmbox = "puppetlab-centos-64-nocm"
- # Virtual Box implementation
+ # Virtual box implementation
vmboxurl = "http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box"
- # Network environment settings
- bigtop1_hostname = "bigtop1.vagrant"
- bigtop1_ip = "10.10.10.12"
- bigtop2_hostname = "bigtop2.vagrant"
- bigtop2_ip = "10.10.10.13"
- bigtop3_hostname = "bigtop3.vagrant"
- bigtop3_ip = "10.10.10.14"
+ # number of instances
+ num_instances = 3
- # The bigtop puppet recipes
+ # the bigtop puppet recipes
bigtop_puppet = "../../puppet"
# manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager)
- config.hostmanager.enabled = true
+ # use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
+ if Vagrant.has_plugin?("vagrant-hostmanager")
+ config.hostmanager.enabled = true
+ end
# use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
if Vagrant.has_plugin?("vagrant-cachier")
config.cache.scope = :box
end
- # First node in the cluster
- config.vm.define :bigtop1 do |bigtop1|
-
- bigtop1.vm.box = vmbox
- bigtop1.vm.box_url = vmboxurl
-
- bigtop1.vm.provider :virtualbox do |vb|
- vb.customize ["modifyvm", :id, "--memory", "1024"]
- end
-
- bigtop1.vm.network :private_network, ip: bigtop1_ip
- bigtop1.vm.hostname = bigtop1_hostname
-
- bigtop1.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
-
- # Shell provisioner to setup basic vm environment
- bigtop1.vm.provision :shell do |shell|
- shell.path = "provision.sh"
- shell.args = bigtop1_hostname + " " + bigtop1_ip + " " + bigtop1_hostname + " " + bigtop1_ip
- end
-
- bigtop1.vm.provision :hostmanager
-
- # Deploy Apache Hadoop and it's ecosystem
- bigtop1.vm.provision :puppet do |puppet|
- puppet.module_path = bigtop_puppet + "/modules"
- puppet.manifests_path = bigtop_puppet + "/manifests"
- puppet.manifest_file = "site.pp"
- puppet.options = "--debug --confdir=/bigtop-puppet"
- end
-
- end
-
-
- # Second node in the cluster
- config.vm.define :bigtop2 do |bigtop2|
-
- bigtop2.vm.box = vmbox
- bigtop2.vm.box_url = vmboxurl
-
- bigtop2.vm.provider :virtualbox do |vb|
- vb.customize ["modifyvm", :id, "--memory", "1024"]
- end
-
- bigtop2.vm.network :private_network, ip: bigtop2_ip
- bigtop2.vm.hostname = bigtop2_hostname
-
- bigtop2.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
-
- # Shell provisioner to setup basic vm environment
- bigtop2.vm.provision :shell do |shell|
- shell.path = "provision.sh"
- shell.args = bigtop1_hostname + " " + bigtop1_ip + " " + bigtop2_hostname + " " + bigtop2_ip
- end
-
- bigtop2.vm.provision :hostmanager
-
- # Deploy Apache Hadoop and it's ecosystem
- bigtop2.vm.provision :puppet do |puppet|
- puppet.module_path = bigtop_puppet + "/modules"
- puppet.manifests_path = bigtop_puppet + "/manifests"
- puppet.manifest_file = "site.pp"
- puppet.options = "--debug --confdir=/bigtop-puppet"
- end
-
- end
-
-
- # Third node in the cluster
- config.vm.define :bigtop3 do |bigtop3|
-
- bigtop3.vm.box = vmbox
- bigtop3.vm.box_url = vmboxurl
-
- bigtop3.vm.provider :virtualbox do |vb|
- vb.customize ["modifyvm", :id, "--memory", "1024"]
- end
-
- bigtop3.vm.network :private_network, ip: bigtop3_ip
- bigtop3.vm.hostname = bigtop3_hostname
-
- bigtop3.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
-
- # Shell provisioner to setup basic vm environment
- bigtop3.vm.provision :shell do |shell|
- shell.path = "provision.sh"
- shell.args = bigtop1_hostname + " " + bigtop1_ip + " " + bigtop3_hostname + " " + bigtop3_ip
+ # declare the variable first, will be override in following loop
+ bigtop_master = ""
+
+ # nodes definition
+ (1..num_instances).each do |i|
+ config.vm.define "bigtop#{i}" do |bigtop|
+
+ bigtop.vm.box = vmbox
+ bigtop.vm.box_url = vmboxurl
+ bigtop_hostname="bigtop#{i}.vagrant"
+ bigtop_ip="10.10.10.1#{i}"
+ if "#{i}" == "1"
+ bigtop_master = bigtop_hostname
+ end
+
+ bigtop.vm.provider :virtualbox do |vb|
+ vb.customize ["modifyvm", :id, "--memory", "1024"]
+ end
+
+ bigtop.vm.network :private_network, ip: bigtop_ip
+ bigtop.vm.hostname = bigtop_hostname
+
+ bigtop.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
+
+ # shell provisioner to setup basic vm environment
+ bigtop.vm.provision :shell do |shell|
+ shell.path = "provision.sh"
+ shell.args = bigtop_master
+ end
+
+ bigtop.vm.provision :hostmanager
+
+ # deploy Apache Hadoop and it's ecosystem
+ bigtop.vm.provision :puppet do |puppet|
+ puppet.module_path = bigtop_puppet + "/modules"
+ puppet.manifests_path = bigtop_puppet + "/manifests"
+ puppet.manifest_file = "site.pp"
+ puppet.options = "--debug --confdir=/bigtop-puppet"
+ end
end
-
- bigtop3.vm.provision :hostmanager
-
- # Deploy Apache Hadoop and it's ecosystem
- bigtop3.vm.provision :puppet do |puppet|
- puppet.module_path = bigtop_puppet + "/modules"
- puppet.manifests_path = bigtop_puppet + "/manifests"
- puppet.manifest_file = "site.pp"
- puppet.options = "--debug --confdir=/bigtop-puppet"
- end
-
end
end
http://git-wip-us.apache.org/repos/asf/bigtop/blob/da224ef2/bigtop-deploy/vm/vagrant-puppet/startup.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/startup.sh b/bigtop-deploy/vm/vagrant-puppet/startup.sh
deleted file mode 100755
index 34b52ad..0000000
--- a/bigtop-deploy/vm/vagrant-puppet/startup.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-usage() {
- echo "usage: `basename $0` [options]"
- echo " -s, --standalone deploy a standalone hadoop VM"
- echo " -c, --cluster deploy a 3 node hadoop cluster"
- echo " -h, --help"
- exit 1
-}
-
-case "$1" in
--s|--standalone)
- vagrant up bigtop1
- shift;;
--c|--cluster)
- vagrant up --provision-with shell && vagrant provision --provision-with hostmanager,puppet
- shift;;
--h|--help)
- usage
- shift;;
-"")
- usage
- shift
- break;;
-esac