You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ja...@apache.org on 2014/10/21 04:19:45 UTC
git commit: BIGTOP-1417. Dockerize the puppetized vagrant deployer
Repository: bigtop
Updated Branches:
refs/heads/master 3b4ae8aba -> 7d76e1c12
BIGTOP-1417. Dockerize the puppetized vagrant deployer
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/7d76e1c1
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/7d76e1c1
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/7d76e1c1
Branch: refs/heads/master
Commit: 7d76e1c12d96760944e495c7777acd4e8efb4142
Parents: 3b4ae8a
Author: evans_ye <in...@gmail.com>
Authored: Tue Oct 21 03:34:26 2014 +0800
Committer: jay@apache.org <jayunit100>
Committed: Mon Oct 20 22:18:56 2014 -0400
----------------------------------------------------------------------
bigtop-deploy/vm/docker-puppet/Dockerfile | 33 +++++
bigtop-deploy/vm/docker-puppet/README.md | 123 +++++++++++++++++++
bigtop-deploy/vm/docker-puppet/Vagrantfile | 69 +++++++++++
.../vm/docker-puppet/boot2docker/Vagrantfile | 18 +++
bigtop-deploy/vm/docker-puppet/docker-hadoop.sh | 88 +++++++++++++
bigtop-deploy/vm/docker-puppet/provision.sh | 45 +++++++
6 files changed, 376 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/Dockerfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/docker-puppet/Dockerfile b/bigtop-deploy/vm/docker-puppet/Dockerfile
new file mode 100644
index 0000000..8e65126
--- /dev/null
+++ b/bigtop-deploy/vm/docker-puppet/Dockerfile
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM bigtop/seed:centos-6.4
+
+# enable ssh
+RUN yum -y install openssh-server openssh-clients sudo
+RUN sed -i.bak s/UsePAM\ yes/UsePAM\ no/ /etc/ssh/sshd_config
+RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key
+RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
+
+# requiretty off
+RUN sed -i.bak 's/requiretty/!requiretty/' /etc/sudoers
+
+# setup vagrant account
+RUN mkdir /root/.ssh/
+RUN chmod 0755 /root/.ssh
+RUN wget http://github.com/mitchellh/vagrant/raw/master/keys/vagrant.pub --no-check-certificate -O /root/.ssh/authorized_keys
+RUN chmod 0644 /root/.ssh/authorized_keys
+
+CMD /usr/sbin/sshd -D
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/README.md
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/docker-puppet/README.md b/bigtop-deploy/vm/docker-puppet/README.md
new file mode 100644
index 0000000..3b54bea
--- /dev/null
+++ b/bigtop-deploy/vm/docker-puppet/README.md
@@ -0,0 +1,123 @@
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+------------------------------------------------------------------------------------------------------------------------------------------------------
+
+#BigTop docker provisioner
+
+## Overview
+
+The Vagrantfile definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
+
+- to test bigtop smoke tests
+- to test bigtop puppet recipes
+
+## Prerequisites
+
+### OS X and Windows
+
+* Install [VirtualBox](https://www.virtualbox.org/wiki/Downloads)
+
+* Install [Vagrant](http://www.vagrantup.com/downloads.html)
+
+### Linux
+
+* [Kernel Requirements](http://docker.readthedocs.org/en/v0.5.3/installation/kernel/)
+
+* Install [Docker](https://docs.docker.com/installation/)
+
+* Install [Vagrant](http://www.vagrantup.com/downloads.html)
+
+## Getting Started
+
+* Create a 3 node Bigtop Hadoop cluster from scratch
+
+```
+cd bigtop/bigtop-deploy/vm/docker-puppet
+./docker-hadoop.sh --build-image --create 3
+```
+
+## USAGE
+
+1) Build up the base Docker image that supports Vagrant.
+
+```
+./docker-hadoop.sh --build-image
+```
+
+2) Create a Bigtop Hadoop cluster by given # of node. (will place a file called config.rb)
+
+```
+./docker-hadoop.sh --create 3
+```
+
+3) Destroy the cluster.
+
+```
+./docker-hadoop.sh --destroy
+```
+
+4) Update your cluster after doing configuration changes. (re-run puppet apply)
+
+```
+./docker-hadoop.sh --provision
+```
+
+5) Chain your operations with-in one command.
+
+```
+./docker-hadoop.sh --build-image --create 5 --destroy
+```
+
+Commands will be executed by following order:
+
+```
+build-image => create 5 node cluster => destroy the cluster
+```
+
+6) Run hbase-test.sh to evaluate the deployment.
+
+```
+../vagrant-puppet/hbase-test.sh
+```
+
+7) See helper message:
+
+```
+./docker-hadoop.sh -h
+usage: docker-hadoop.sh [options]
+ -b, --build-image Build base Docker image for Bigtop Hadoop
+ (must be exectued at least once before creating cluster)
+ -c NUM_INSTANCES, --create NUM_INSTANCES Create a docker based Bigtop Hadoop cluster
+ -p, --provision Deploy configuration changes
+ -d, --destroy Destroy the cluster
+ -h, --help
+```
+
+##Configure Apache Hadoop ecosystem components
+
+* Choose the ecosystem you want to be deployed by modifying components in provision.sh.
+
+```
+ components,hadoop,hbase,yarn,mapred-app,...
+```
+
+By default, Apache Hadoop, YARN, and Apache HBase will be installed.
+See `bigtop-deploy/puppet/config/site.csv.example` for more details.
+
+##Notes
+
+* Users currently using vagrant 1.6+ is strongly recommanded to upgrade to 1.6.4+, otherwise you will encounter the [issue](https://github.com/mitchellh/vagrant/issues/3769) when installing plguins
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/docker-puppet/Vagrantfile b/bigtop-deploy/vm/docker-puppet/Vagrantfile
new file mode 100755
index 0000000..ea7153a
--- /dev/null
+++ b/bigtop-deploy/vm/docker-puppet/Vagrantfile
@@ -0,0 +1,69 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fileutils'
+
+# number of instances
+$num_instances = 3
+# the bigtop puppet recipes
+$bigtop_puppet = "../../puppet"
+
+# load configuration from config.rb
+CONFIG = File.join(File.dirname(__FILE__), "config.rb")
+if File.exist?(CONFIG)
+ require CONFIG
+end
+
+ENV['VAGRANT_DEFAULT_PROVIDER'] ||= 'docker'
+
+VAGRANTFILE_API_VERSION = "2"
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+ # image definition
+ config.vm.define "image", autostart: false do |image|
+ image.vm.provider "docker" do |d|
+ d.build_dir = "."
+ d.build_args = ["-t","bigtop/ssh:centos-6.4"]
+ d.vagrant_machine = "boot2docker"
+ d.vagrant_vagrantfile = "./boot2docker/Vagrantfile"
+ end
+ image.ssh.username = "root"
+ image.ssh.port = 22
+ end
+
+ # nodes definition
+ (1..$num_instances).each do |i|
+ config.vm.define "bigtop#{i}" do |bigtop|
+ # docker container settings
+ bigtop.vm.provider "docker" do |d|
+ d.image = "bigtop/ssh:centos-6.4"
+ d.create_args = ["--privileged=true"]
+ if "#{i}" == "1"
+ d.ports = ["50070:50070","8088:8088","60010:60010"]
+ end
+ d.vagrant_machine = "boot2docker"
+ d.vagrant_vagrantfile = "./boot2docker/Vagrantfile"
+ end
+ bigtop.ssh.username = "root"
+ bigtop.ssh.port = 22
+ bigtop.vm.hostname = "bigtop#{i}.docker"
+ bigtop.vm.synced_folder $bigtop_puppet, "/bigtop-puppet"
+ end
+ end
+
+end
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile b/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile
new file mode 100644
index 0000000..8fa04f8
--- /dev/null
+++ b/bigtop-deploy/vm/docker-puppet/boot2docker/Vagrantfile
@@ -0,0 +1,18 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+
+ config.vm.define "boot2docker" do |node|
+ node.vm.box = "yungsang/boot2docker"
+ node.vm.provider "virtualbox" do |vb|
+ vb.customize ["modifyvm", :id, "--memory", "4196"]
+ end
+ node.vm.network "forwarded_port", host: 50070, guest: 50070
+ node.vm.network "forwarded_port", host: 8088, guest: 8088
+ node.vm.network "forwarded_port", host: 60010, guest: 60010
+ end
+
+end
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh
new file mode 100755
index 0000000..449b5fb
--- /dev/null
+++ b/bigtop-deploy/vm/docker-puppet/docker-hadoop.sh
@@ -0,0 +1,88 @@
+#!/bin/bash
+
+usage() {
+ echo "usage: $PROG [options]"
+ echo " -b, --build-image Build base Docker image for Bigtop Hadoop"
+ echo " (must be exectued at least once before creating cluster)"
+ echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a docker based Bigtop Hadoop cluster"
+ echo " -p, --provision Deploy configuration changes"
+ echo " -d, --destroy Destroy the cluster"
+ echo " -h, --help"
+ exit 1
+}
+
+build-image() {
+ vagrant up image --provider docker
+ {
+ echo "echo -e '\nBUILD IMAGE SUCCESS.\n'" |vagrant ssh image
+ } || {
+ >&2 echo -e "\nBUILD IMAGE FAILED!\n"
+ exit 2
+ }
+ vagrant destroy image -f
+}
+
+create() {
+ echo "\$num_instances = $1" > config.rb
+ vagrant up --no-parallel
+ nodes=(`vagrant status |grep running |awk '{print $1}'`)
+ hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`)
+ echo "/vagrant/provision.sh $hadoop_head_node" |vagrant ssh ${nodes[0]}
+ bigtop-puppet ${nodes[0]}
+ for ((i=1 ; i<${#nodes[*]} ; i++)); do
+ (
+ echo "/vagrant/provision.sh $hadoop_head_node" |vagrant ssh ${nodes[$i]}
+ bigtop-puppet ${nodes[$i]}
+ ) &
+ done
+ wait
+}
+
+provision() {
+ nodes=(`vagrant status |grep running |awk '{print $1}'`)
+ for node in $nodes; do
+ bigtop-puppet $node &
+ done
+ wait
+}
+
+destroy() {
+ vagrant destroy -f
+ rm -f ./hosts
+}
+
+bigtop-puppet() {
+ echo "puppet apply -d --confdir=/bigtop-puppet --modulepath=/bigtop-puppet/modules /bigtop-puppet/manifests/site.pp" |vagrant ssh $1
+}
+
+PROG=`basename $0`
+ARGS=`getopt -o "bc:pdh" -l "build-image,create:,provision,destroy,help" -n $PROG -- "$@"`
+
+if [ $? -ne 0 ]; then
+ usage
+fi
+
+eval set -- "$ARGS"
+
+while true; do
+ case "$1" in
+ -b|--build-image)
+ build-image
+ shift;;
+ -c|--create)
+ create $2
+ shift 2;;
+ -p|--provision)
+ provision
+ shift;;
+ -d|--destroy)
+ destroy
+ shift;;
+ -h|--help)
+ usage
+ shift;;
+ --)
+ shift
+ break;;
+ esac
+done
http://git-wip-us.apache.org/repos/asf/bigtop/blob/7d76e1c1/bigtop-deploy/vm/docker-puppet/provision.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/docker-puppet/provision.sh b/bigtop-deploy/vm/docker-puppet/provision.sh
new file mode 100755
index 0000000..929999d
--- /dev/null
+++ b/bigtop-deploy/vm/docker-puppet/provision.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install puppet agent
+yum -y install http://yum.puppetlabs.com/puppetlabs-release-el-6.noarch.rpm
+yum -y install puppet-2.7.24-1.el6.noarch
+
+sysctl kernel.hostname=`hostname -f`
+
+# Unmount device /etc/hosts and replace it by a shared hosts file
+echo -e "`hostname -i`\t`hostname -f`" >> /vagrant/hosts
+umount /etc/hosts
+mv /etc/hosts /etc/hosts.bak
+ln -s /vagrant/hosts /etc/hosts
+
+# Prepare puppet configuration file
+cat > /bigtop-puppet/config/site.csv << EOF
+hadoop_head_node,$1
+hadoop_storage_dirs,/data/1,/data/2
+bigtop_yumrepo_uri,http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/
+jdk_package_name,java-1.7.0-openjdk-devel.x86_64
+components,hadoop,hbase,yarn,mapred-app
+EOF
+
+mkdir -p /data/{1,2}
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+yum -y install rng-tools
+sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
+service rngd start