You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2016/08/04 19:47:49 UTC
[01/10] bigtop git commit: BIGTOP-2507. Make buildstamp configurable
by a bom file [Forced Update!]
Repository: bigtop
Updated Branches:
refs/heads/BIGTOP-2253 2fe4c1f5d -> 27efdf525 (forced update)
BIGTOP-2507. Make buildstamp configurable by a bom file
Signed-off-by: Konstantin Boudnik <ko...@epam.com>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/b3526b92
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/b3526b92
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/b3526b92
Branch: refs/heads/BIGTOP-2253
Commit: b3526b9271619fb641e8f5e48cd8356364b3dd51
Parents: 4e97c40
Author: zenmyo <tz...@gmail.com>
Authored: Fri Jul 29 17:25:51 2016 +0900
Committer: Konstantin Boudnik <ko...@epam.com>
Committed: Wed Aug 3 14:30:22 2016 -0700
----------------------------------------------------------------------
packages.gradle | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/b3526b92/packages.gradle
----------------------------------------------------------------------
diff --git a/packages.gradle b/packages.gradle
index 40c04e3..e0e3d7b 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -45,7 +45,7 @@ config.bigtop.builddir = projectDir.absolutePath + "/build"
config.bigtop.outputdir = projectDir.absolutePath + "/output"
config.bigtop.distdir = projectDir.absolutePath + "/dist"
config.bigtop.dldir = projectDir.absolutePath + "/dl"
-config.bigtop.buildstamp = 1
+if(!config.bigtop.buildstamp) config.bigtop.buildstamp = 1
def bomVersions = []
[07/10] bigtop git commit: BIGTOP-2279. Add provisioner into
release-assembly.xml
Posted by of...@apache.org.
BIGTOP-2279. Add provisioner into release-assembly.xml
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6bfb16fd
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6bfb16fd
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6bfb16fd
Branch: refs/heads/BIGTOP-2253
Commit: 6bfb16fd9875c9aa4be56ba83d38d8295c85448b
Parents: 67f08c7
Author: Evans Ye <ev...@apache.org>
Authored: Wed Apr 13 02:56:54 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:45:20 2016 +0200
----------------------------------------------------------------------
src/assembly/release-assembly.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6bfb16fd/src/assembly/release-assembly.xml
----------------------------------------------------------------------
diff --git a/src/assembly/release-assembly.xml b/src/assembly/release-assembly.xml
index 20d90db..0868001 100644
--- a/src/assembly/release-assembly.xml
+++ b/src/assembly/release-assembly.xml
@@ -38,6 +38,7 @@
<include>gradle*</include>
<include>docs/**</include>
<include>docker/**</include>
+ <include>provisioner/**</include>
<include>src/**</include>
<include>test/**</include>
<include>*.bom</include>
[04/10] bigtop git commit: BIGTOP-2273. Add exec command into
docker-hadoop.sh
Posted by of...@apache.org.
BIGTOP-2273. Add exec command into docker-hadoop.sh
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4c89e663
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4c89e663
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4c89e663
Branch: refs/heads/BIGTOP-2253
Commit: 4c89e66390f0b602a7ed7177aa730cd5dd1ce960
Parents: c1b07de
Author: Evans Ye <ev...@apache.org>
Authored: Sun Jan 24 13:49:36 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:45:20 2016 +0200
----------------------------------------------------------------------
provisioner/docker/README.md | 25 +++++++++++++++++++-----
provisioner/docker/docker-hadoop.sh | 33 ++++++++++++++++++++++++++++----
2 files changed, 49 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4c89e663/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
index 51bac85..fbc82b4 100644
--- a/provisioner/docker/README.md
+++ b/provisioner/docker/README.md
@@ -60,19 +60,31 @@ service docker start
./docker-hadoop.sh --destroy
```
-3) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
+3) Get into the first container (the master)
+
+```
+./docker-hadoop.sh --exec 1 bash
+```
+
+4) Execute a command on the second container
+
+```
+./docker-hadoop.sh --exec 2 hadoop fs -ls /
+```
+
+5) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
```
./docker-hadoop.sh --provision
```
-4) Run Bigtop smoke tests
+6) Run Bigtop smoke tests
```
./docker-hadoop.sh --smoke-tests
```
-5) Chain your operations with-in one command.
+7) Chain your operations with-in one command.
```
./docker-hadoop.sh --create 5 --smoke-tests --destroy
@@ -84,7 +96,7 @@ Commands will be executed by following order:
create 5 node cluster => run smoke tests => destroy the cluster
```
-6) See helper message:
+8) See helper message:
```
./docker-hadoop.sh -h
@@ -92,9 +104,12 @@ usage: docker-hadoop.sh [-C file ] args
-C file Use alternate file for config.yaml
commands:
-c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster
+ -d, --destroy Destroy the cluster
+ -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number.
+ For example: docker-hadoop.sh --exec 1 bash
+ docker-hadoop.sh --exec docker_bigtop_1 bash
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
- -d, --destroy Destroy the cluster
-h, --help
```
http://git-wip-us.apache.org/repos/asf/bigtop/blob/4c89e663/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index e8bd767..d8392ba 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -22,9 +22,12 @@ usage() {
echo " -C file Use alternate file for config.yaml"
echo " commands:"
echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
+ echo " -d, --destroy Destroy the cluster"
+ echo " -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number."
+ echo " For example: $PROG --exec 1 bash"
+ echo " $PROG --exec docker_bigtop_1 bash"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
- echo " -d, --destroy Destroy the cluster"
echo " -h, --help"
exit 1
}
@@ -137,6 +140,20 @@ get-yaml-config() {
cat ${yamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r'
}
+execute() {
+ re='^[0-9]+$'
+ if [[ $1 =~ $re ]] ; then
+ no=$1
+ shift
+ nodes=(`docker-compose ps -q`)
+ docker exec -ti ${nodes[$((no-1))]} $@
+ else
+ name=$1
+ shift
+ docker exec -ti $name $@
+ fi
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -160,15 +177,23 @@ while [ $# -gt 0 ]; do
fi
yamlconf=$2
shift 2;;
+ -d|--destroy)
+ destroy
+ shift;;
+ -e|--exec)
+ if [ $# -lt 3 ]; then
+ echo "exec command takes 2 parameters: 1) instance no 2) command to be executed" 1>&2
+ usage
+ fi
+ shift
+ execute $@
+ shift $#;;
-p|--provision)
provision
shift;;
-s|--smoke-tests)
smoke-tests
shift;;
- -d|--destroy)
- destroy
- shift;;
-h|--help)
usage
shift;;
[06/10] bigtop git commit: BIGTOP-2387. New Docker provisioner
destroy will delete all docker containers on the host
Posted by of...@apache.org.
BIGTOP-2387. New Docker provisioner destroy will delete all docker containers on the host
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/17129e62
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/17129e62
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/17129e62
Branch: refs/heads/BIGTOP-2253
Commit: 17129e625957a290100662a9092a00aac618c40c
Parents: 07157d7
Author: Evans Ye <ev...@apache.org>
Authored: Sun Apr 24 23:50:57 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:45:20 2016 +0200
----------------------------------------------------------------------
provisioner/docker/README.md | 1 +
provisioner/docker/docker-hadoop.sh | 60 +++++++++++++++++++++-----------
2 files changed, 40 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/17129e62/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
index c95b218..d1c0f2e 100644
--- a/provisioner/docker/README.md
+++ b/provisioner/docker/README.md
@@ -109,6 +109,7 @@ usage: docker-hadoop.sh [-C file ] args
For example: docker-hadoop.sh --exec 1 bash
docker-hadoop.sh --exec docker_bigtop_1 bash
-E, --env-check Check whether required tools has been installed
+ -l, --list List out container status for the cluster
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
-h, --help
http://git-wip-us.apache.org/repos/asf/bigtop/blob/17129e62/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 93c8c9c..89ac6bb 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -15,8 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet
-
usage() {
echo "usage: $PROG [-C file ] args"
echo " -C file Use alternate file for config.yaml"
@@ -27,6 +25,7 @@ usage() {
echo " For example: $PROG --exec 1 bash"
echo " $PROG --exec docker_bigtop_1 bash"
echo " -E, --env-check Check whether required tools has been installed"
+ echo " -l, --list List out container status for the cluster"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
echo " -h, --help"
@@ -34,6 +33,12 @@ usage() {
}
create() {
+ if [ -e .provision_id ]; then
+ echo "Cluster already exist! Run ./$PROG -d to destroy the cluster or delete .provision_id file and containers manually."
+ exit 1;
+ fi
+ echo "`date +'%Y%m%d_%H%M%S'`_R$RANDOM" > .provision_id
+ PROVISION_ID=`cat .provision_id`
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
mkdir -p config/hieradata 2> /dev/null
cat /dev/null > ./config/hiera.yaml
@@ -41,15 +46,15 @@ create() {
export DOCKER_IMAGE=$(get-yaml-config docker image)
# Startup instances
- docker-compose scale bigtop=$1
+ docker-compose -p $PROVISION_ID scale bigtop=$1
if [ $? -ne 0 ]; then
echo "Docker container(s) startup failed!";
exit 1;
fi
# Get the headnode FQDN
- nodes=(`docker-compose ps -q`)
- hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${nodes[0]}`
+ NODES=(`docker-compose -p $PROVISION_ID ps -q`)
+ hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${NODES[0]}`
# Fetch configurations form specificed yaml config file
repo=$(get-yaml-config repo)
@@ -66,10 +71,9 @@ create() {
}
generate-hosts() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
entry=`docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node`
- docker exec ${nodes[0]} bash -c "echo $entry >> /etc/hosts"
+ docker exec ${NODES[0]} bash -c "echo $entry >> /etc/hosts"
done
wait
@@ -89,40 +93,39 @@ EOF
}
copy-to-instances() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
docker cp $1 $node:$2 &
done
wait
}
bootstrap() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
+ docker cp $1 $node:$2 &
docker exec $node bash -c "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$1.sh $2" &
done
wait
}
provision() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
bigtop-puppet $node &
done
wait
}
smoke-tests() {
- nodes=(`docker-compose ps -q`)
- hadoop_head_node=${nodes:0:12}
+ hadoop_head_node=${NODES:0:12}
smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`"
docker exec $hadoop_head_node bash -c "bash -x /bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh $smoke_test_components"
}
destroy() {
- docker-compose stop
- docker-compose rm -f
- rm -rvf ./config
+ if [ -n "$PROVISION_ID" ]; then
+ docker-compose -p $PROVISION_ID stop
+ docker-compose -p $PROVISION_ID rm -f
+ fi
+ rm -rvf ./config .provision_id
}
bigtop-puppet() {
@@ -147,8 +150,7 @@ execute() {
if [[ $1 =~ $re ]] ; then
no=$1
shift
- nodes=(`docker-compose ps -q`)
- docker exec -ti ${nodes[$((no-1))]} $@
+ docker exec -ti ${NODES[$((no-1))]} $@
else
name=$1
shift
@@ -166,6 +168,10 @@ env-check() {
ruby -v || exit 1
}
+list() {
+ docker-compose -p $PROVISION_ID ps
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -173,6 +179,15 @@ if [ $# -eq 0 ]; then
fi
yamlconf="config.yaml"
+
+BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet
+if [ -e .provision_id ]; then
+ PROVISION_ID=`cat .provision_id`
+fi
+if [ -n "$PROVISION_ID" ]; then
+ NODES=`docker-compose -p $PROVISION_ID ps -q`
+fi
+
while [ $# -gt 0 ]; do
case "$1" in
-c|--create)
@@ -204,6 +219,9 @@ while [ $# -gt 0 ]; do
-E|--env-check)
env-check
shift;;
+ -l|--list)
+ list
+ shift;;
-p|--provision)
provision
shift;;
[09/10] bigtop git commit: BIGTOP-2505: Support systemd containers,
fix exec (-e) flag
Posted by of...@apache.org.
BIGTOP-2505: Support systemd containers, fix exec (-e) flag
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/af42c2f1
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/af42c2f1
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/af42c2f1
Branch: refs/heads/BIGTOP-2253
Commit: af42c2f13c94952fa2e51032a3a189f53421aef9
Parents: 17129e6
Author: Olaf Flebbe <of...@oflebbe.de>
Authored: Sat Jul 23 12:34:12 2016 +0200
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:45:38 2016 +0200
----------------------------------------------------------------------
provisioner/docker/config_centos7.yaml | 32 +++++++++++++++++
provisioner/docker/docker-compose.yml | 2 +-
provisioner/docker/docker-hadoop.sh | 10 +++---
provisioner/utils/setup-env-centos.sh | 36 -------------------
provisioner/utils/setup-env-debian.sh | 34 ------------------
provisioner/utils/smoke-tests.sh | 56 -----------------------------
6 files changed, 38 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/af42c2f1/provisioner/docker/config_centos7.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml
new file mode 100644
index 0000000..3610a8f
--- /dev/null
+++ b/provisioner/docker/config_centos7.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:centos-7"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64"
+distro: centos
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "java-1.7.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/af42c2f1/provisioner/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml
index 95831d1..d71b247 100644
--- a/provisioner/docker/docker-compose.yml
+++ b/provisioner/docker/docker-compose.yml
@@ -1,6 +1,6 @@
bigtop:
image: ${DOCKER_IMAGE}
- command: sleep infinity
+ command: /sbin/init
domainname: bigtop.apache.org
privileged: true
volumes:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/af42c2f1/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 89ac6bb..4b1c0df 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -41,8 +41,8 @@ create() {
PROVISION_ID=`cat .provision_id`
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
mkdir -p config/hieradata 2> /dev/null
- cat /dev/null > ./config/hiera.yaml
- cat /dev/null > ./config/hosts
+ echo > ./config/hiera.yaml
+ echo > ./config/hosts
export DOCKER_IMAGE=$(get-yaml-config docker image)
# Startup instances
@@ -81,8 +81,8 @@ generate-hosts() {
generate-config() {
echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata"
- cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml
- yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/
+ cat $BIGTOP_PUPPET_DIR/hiera.yaml >> ./config/hiera.yaml
+ cp -vfr $BIGTOP_PUPPET_DIR/hieradata ./config/
cat > ./config/hieradata/site.yaml << EOF
bigtop::hadoop_head_node: $1
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
@@ -185,7 +185,7 @@ if [ -e .provision_id ]; then
PROVISION_ID=`cat .provision_id`
fi
if [ -n "$PROVISION_ID" ]; then
- NODES=`docker-compose -p $PROVISION_ID ps -q`
+ NODES=(`docker-compose -p $PROVISION_ID ps -q`)
fi
while [ $# -gt 0 ]; do
http://git-wip-us.apache.org/repos/asf/bigtop/blob/af42c2f1/provisioner/utils/setup-env-centos.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh
deleted file mode 100755
index bb25d28..0000000
--- a/provisioner/utils/setup-env-centos.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-enable_local_repo=${1:-false}
-
-bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
-
-# Setup rng-tools to improve virtual machine entropy performance.
-# The poor entropy performance will cause kerberos provisioning failed.
-yum -y install rng-tools
-sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
-service rngd start
-
-if [ $enable_local_repo == "true" ]; then
- echo "Enabling local yum."
- yum -y install yum-utils
- sudo echo "gpgcheck=0" >> /etc/yum.conf
- sudo yum-config-manager --add-repo file:///bigtop-home/output
-else
- echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
-fi
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/af42c2f1/provisioner/utils/setup-env-debian.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh
deleted file mode 100755
index c5768d9..0000000
--- a/provisioner/utils/setup-env-debian.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-enable_local_repo=${1:-false}
-
-bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
-
-# Setup rng-tools to improve virtual machine entropy performance.
-# The poor entropy performance will cause kerberos provisioning failed.
-apt-get -y install rng-tools
-sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
-service rng-tools start
-
-if [ $enable_local_repo == "true" ]; then
- echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
- apt-get update
-else
- echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
-fi
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/af42c2f1/provisioner/utils/smoke-tests.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
deleted file mode 100755
index 8dac31c..0000000
--- a/provisioner/utils/smoke-tests.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HCFS_USER="hdfs"
-SMOKE_TESTS=${1:-mapreduce,pig}
-
-# Autodetect JAVA_HOME
-if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-else
- >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
- exit 2
-fi
-
-echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
-
-export HADOOP_CONF_DIR=/etc/hadoop/conf/
-export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
-export HIVE_HOME=/usr/lib/hive/
-export PIG_HOME=/usr/lib/pig/
-export FLUME_HOME=/usr/lib/flume/
-export SQOOP_HOME=/usr/lib/sqoop/
-export HIVE_CONF_DIR=/etc/hive/conf/
-export MAHOUT_HOME="/usr/lib/mahout"
-
-su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root'
-su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
-su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
-
-if [ -f /etc/debian_version ] ; then
- apt-get -y install pig hive flume mahout sqoop
-else
- yum install -y pig hive flume mahout sqoop
-fi
-ALL_SMOKE_TASKS=""
-for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
- ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
-done
-cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
-# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
-# permissions as used for smoke-tests execution
-rm -rf buildSrc/build/test-results/binary
-
[02/10] bigtop git commit: BIGTOP-2467 Zookeeper Puppet script does
not setup Zookeeper node ids correctly
Posted by of...@apache.org.
BIGTOP-2467 Zookeeper Puppet script does not setup Zookeeper node ids correctly
Zookeeper wants its peer ids to be consistent across nodes, but the
current puppet script auto-generates ids for nodes, based on their index
in a list. This breaks Zookeeper quorum.
We fix the issue by requiring that puppet take a list comprising
explicit ids and ip addresses, rather than just a list of ip addresses.
Added backwards compatibilty for old way of doing ensemble.
Should prevent us from breaking existing puppet scripts.
Signed-off-by: Olaf Flebbe <of...@oflebbe.de>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/87e7d01d
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/87e7d01d
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/87e7d01d
Branch: refs/heads/BIGTOP-2253
Commit: 87e7d01da2558347f9ea64983506d9d8d944a4b2
Parents: b3526b9
Author: Pete Vander Giessen <pe...@gmail.com>
Authored: Wed Jun 8 16:02:15 2016 -0400
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 20:41:26 2016 +0200
----------------------------------------------------------------------
bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml | 2 +-
.../puppet/modules/hadoop_zookeeper/manifests/init.pp | 2 +-
.../puppet/modules/hadoop_zookeeper/templates/zoo.cfg | 8 +++++++-
bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp | 2 +-
4 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/87e7d01d/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 55f5bcd..60c59b1 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -148,7 +148,7 @@ qfs::common::chunkserver_client_port: "22000"
hadoop_zookeeper::server::myid: "0"
hadoop_zookeeper::server::ensemble:
- - "%{hiera('bigtop::hadoop_head_node')}:2888:3888"
+ - ["0", "%{hiera('bigtop::hadoop_head_node')}:2888:3888"]
hadoop_zookeeper::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
# those are only here because they were present as extlookup keys previously
http://git-wip-us.apache.org/repos/asf/bigtop/blob/87e7d01d/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp
index 6965090..31a609e 100644
--- a/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/manifests/init.pp
@@ -61,7 +61,7 @@ class hadoop_zookeeper (
class server($myid,
$port = "2181",
$datadir = "/var/lib/zookeeper",
- $ensemble = ["localhost:2888:3888"],
+ $ensemble = [$myid, "localhost:2888:3888"],
$kerberos_realm = $hadoop_zookeeper::kerberos_realm,
) inherits hadoop_zookeeper {
include common
http://git-wip-us.apache.org/repos/asf/bigtop/blob/87e7d01d/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg b/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg
index 69d98c4..e255a0a 100644
--- a/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg
+++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/templates/zoo.cfg
@@ -26,8 +26,14 @@ syncLimit=5
dataDir=<%= @datadir %>
# the port at which the clients will connect
clientPort=<%= @port %>
-<% @ensemble.each_with_index do |server,idx| %>
+<% if !@ensemble[0].nil? && @ensemble[0].length == 2 %>
+ <% @ensemble.each do |idx, ip| %>
+server.<%= idx %>=<%= ip %>
+ <% end %>
+<% else %>
+ <% @ensemble.each_with_index do |server,idx| %>
server.<%= idx %>=<%= server %>
+ <% end %>
<% end %>
# purge snapshots every day
autopurge.purgeInterval=24
http://git-wip-us.apache.org/repos/asf/bigtop/blob/87e7d01d/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp b/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp
index 0340f10..7158714 100644
--- a/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop_zookeeper/tests/init.pp
@@ -16,5 +16,5 @@
hadoop_zookeeper::client { "zoo visitor": }
hadoop_zookeeper::server { "test-oozie-server":
myid => "0",
- ensemble => ["foo:2888:3888", "bar:2888:3888", "baz:2888:3888"],
+ ensemble => [[0, "foo:2888:3888"], [1, "bar:2888:3888"], [2, "baz:2888:3888"]],
}
[03/10] bigtop git commit: BIGTOP-2386. Got permission denied when
running new Docker provisioner as non-root
Posted by of...@apache.org.
BIGTOP-2386. Got permission denied when running new Docker provisioner as non-root
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/67f08c7e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/67f08c7e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/67f08c7e
Branch: refs/heads/BIGTOP-2253
Commit: 67f08c7e13311833cfe578cff8b527113b317c66
Parents: 4c89e66
Author: Evans Ye <ev...@apache.org>
Authored: Tue Apr 12 18:53:01 2016 +0000
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:45:20 2016 +0200
----------------------------------------------------------------------
provisioner/docker/docker-hadoop.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/67f08c7e/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index d8392ba..2323456 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -34,7 +34,7 @@ usage() {
create() {
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
- mkdir config 2> /dev/null
+ mkdir -p config/hieradata 2> /dev/null
cat /dev/null > ./config/hiera.yaml
cat /dev/null > ./config/hosts
export DOCKER_IMAGE=$(get-yaml-config docker image)
@@ -67,7 +67,8 @@ create() {
generate-hosts() {
nodes=(`docker-compose ps -q`)
for node in ${nodes[*]}; do
- echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts
+ entry=`docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node`
+ docker exec ${nodes[0]} bash -c "echo $entry >> /etc/hosts"
done
wait
[08/10] bigtop git commit: BIGTOP-2312. Add environment check
Posted by of...@apache.org.
BIGTOP-2312. Add environment check
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/07157d71
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/07157d71
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/07157d71
Branch: refs/heads/BIGTOP-2253
Commit: 07157d713e970ef39cda882d6dd2cd4d9b3b5a9d
Parents: 6bfb16f
Author: Evans Ye <ev...@apache.org>
Authored: Sat Apr 23 08:14:39 2016 +0000
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:45:20 2016 +0200
----------------------------------------------------------------------
provisioner/docker/README.md | 1 +
provisioner/docker/docker-hadoop.sh | 15 +++++++++++++++
2 files changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/07157d71/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
index fbc82b4..c95b218 100644
--- a/provisioner/docker/README.md
+++ b/provisioner/docker/README.md
@@ -108,6 +108,7 @@ usage: docker-hadoop.sh [-C file ] args
-e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number.
For example: docker-hadoop.sh --exec 1 bash
docker-hadoop.sh --exec docker_bigtop_1 bash
+ -E, --env-check Check whether required tools has been installed
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
-h, --help
http://git-wip-us.apache.org/repos/asf/bigtop/blob/07157d71/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 2323456..93c8c9c 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -26,6 +26,7 @@ usage() {
echo " -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number."
echo " For example: $PROG --exec 1 bash"
echo " $PROG --exec docker_bigtop_1 bash"
+ echo " -E, --env-check Check whether required tools has been installed"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
echo " -h, --help"
@@ -155,6 +156,16 @@ execute() {
fi
}
+env-check() {
+ echo "Environment check..."
+ echo "Check docker:"
+ docker -v || exit 1
+ echo "Check docker-compose:"
+ docker-compose -v || exit 1
+ echo "Check ruby:"
+ ruby -v || exit 1
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -169,6 +180,7 @@ while [ $# -gt 0 ]; do
echo "Create requires a number" 1>&2
usage
fi
+ env-check
create $2
shift 2;;
-C|--conf)
@@ -189,6 +201,9 @@ while [ $# -gt 0 ]; do
shift
execute $@
shift $#;;
+ -E|--env-check)
+ env-check
+ shift;;
-p|--provision)
provision
shift;;
[05/10] bigtop git commit: BIGTOP-2254. Replace the Docker
orchestration tool from Vagrant to Docker Compose
Posted by of...@apache.org.
BIGTOP-2254. Replace the Docker orchestration tool from Vagrant to Docker Compose
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c1b07de7
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c1b07de7
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c1b07de7
Branch: refs/heads/BIGTOP-2253
Commit: c1b07de70d6012f9efe9c5477da85628d89cbf05
Parents: 87e7d01
Author: Evans Ye <ev...@apache.org>
Authored: Sat Jan 23 10:33:03 2016 +0000
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:45:20 2016 +0200
----------------------------------------------------------------------
provisioner/docker/.gitignore | 1 +
provisioner/docker/README.md | 131 +++++++++++++++++++++
provisioner/docker/config.yaml | 32 ++++++
provisioner/docker/config_debian.yaml | 32 ++++++
provisioner/docker/config_ubuntu.yaml | 32 ++++++
provisioner/docker/docker-compose.yml | 10 ++
provisioner/docker/docker-hadoop.sh | 179 +++++++++++++++++++++++++++++
provisioner/utils/setup-env-centos.sh | 36 ++++++
provisioner/utils/setup-env-debian.sh | 34 ++++++
provisioner/utils/smoke-tests.sh | 56 +++++++++
10 files changed, 543 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/docker/.gitignore
----------------------------------------------------------------------
diff --git a/provisioner/docker/.gitignore b/provisioner/docker/.gitignore
new file mode 100644
index 0000000..f733c4b
--- /dev/null
+++ b/provisioner/docker/.gitignore
@@ -0,0 +1 @@
+config/
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
new file mode 100644
index 0000000..51bac85
--- /dev/null
+++ b/provisioner/docker/README.md
@@ -0,0 +1,131 @@
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+------------------------------------------------------------------------------------------------------------------------------------------------------
+
+#BigTop Docker provisioner
+
+## Overview
+
+The Docker Compose definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
+
+- to test bigtop smoke tests
+- to test bigtop puppet recipes
+- to run integration test with your application
+
+This has been verified on Docker Engine 1.9.1, with api version 1.15, and Docker Compose 1.5.2 on Amazon Linux 2015.09 release.
+
+## Prerequisites
+
+### OS X and Windows
+
+* Install [Docker Toolbox](https://www.docker.com/docker-toolbox)
+
+### Linux
+
+* Install [Docker](https://docs.docker.com/installation/)
+
+* Install [Docker Compose](https://docs.docker.com/compose/install/)
+
+* Start the Docker daemon
+
+```
+service docker start
+```
+
+## USAGE
+
+1) Create a Bigtop Hadoop cluster by given # of node.
+
+```
+./docker-hadoop.sh --create 3
+```
+
+2) Destroy the cluster.
+
+```
+./docker-hadoop.sh --destroy
+```
+
+3) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
+
+```
+./docker-hadoop.sh --provision
+```
+
+4) Run Bigtop smoke tests
+
+```
+./docker-hadoop.sh --smoke-tests
+```
+
+5) Chain your operations with-in one command.
+
+```
+./docker-hadoop.sh --create 5 --smoke-tests --destroy
+```
+
+Commands will be executed by following order:
+
+```
+create 5 node cluster => run smoke tests => destroy the cluster
+```
+
+6) See helper message:
+
+```
+./docker-hadoop.sh -h
+usage: docker-hadoop.sh [-C file ] args
+ -C file Use alternate file for config.yaml
+ commands:
+ -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster
+ -p, --provision Deploy configuration changes
+ -s, --smoke-tests Run Bigtop smoke tests
+ -d, --destroy Destroy the cluster
+ -h, --help
+```
+
+##Configurations
+
+* There are several parameters can be configured in config.yaml:
+
+1) Modify memory limit for Docker containers
+
+```
+docker:
+ memory_size: "2048"
+
+```
+
+2) Use different host ports mapping for web UIs
+
+```
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+
+```
+Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed
+
+
+##Configure Apache Hadoop ecosystem components
+* Choose the ecosystem you want to be deployed by modifying components in config.yaml
+
+```
+components: "hadoop, hbase, yarn,..."
+```
+
+By default, Apache Hadoop and YARN will be installed.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/docker/config.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml
new file mode 100644
index 0000000..86f35e7
--- /dev/null
+++ b/provisioner/docker/config.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:centos-6"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/6/x86_64"
+distro: centos
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "java-1.7.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/docker/config_debian.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_debian.yaml b/provisioner/docker/config_debian.yaml
new file mode 100644
index 0000000..f6b11ae
--- /dev/null
+++ b/provisioner/docker/config_debian.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:debian-8"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/debian/8/x86_64"
+distro: debian
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "openjdk-7-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/docker/config_ubuntu.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_ubuntu.yaml b/provisioner/docker/config_ubuntu.yaml
new file mode 100644
index 0000000..e4ea6f3
--- /dev/null
+++ b/provisioner/docker/config_ubuntu.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:ubuntu-14.04"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/ubuntu/trusty/x86_64"
+distro: debian
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "openjdk-7-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml
new file mode 100644
index 0000000..95831d1
--- /dev/null
+++ b/provisioner/docker/docker-compose.yml
@@ -0,0 +1,10 @@
+bigtop:
+ image: ${DOCKER_IMAGE}
+ command: sleep infinity
+ domainname: bigtop.apache.org
+ privileged: true
+ volumes:
+ - ../../:/bigtop-home
+ - ./config/hiera.yaml:/etc/puppet/hiera.yaml
+ - ./config/hieradata:/etc/puppet/hieradata
+ - ./config/hosts:/etc/hosts
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
new file mode 100755
index 0000000..e8bd767
--- /dev/null
+++ b/provisioner/docker/docker-hadoop.sh
@@ -0,0 +1,179 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet
+
+usage() {
+ echo "usage: $PROG [-C file ] args"
+ echo " -C file Use alternate file for config.yaml"
+ echo " commands:"
+ echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
+ echo " -p, --provision Deploy configuration changes"
+ echo " -s, --smoke-tests Run Bigtop smoke tests"
+ echo " -d, --destroy Destroy the cluster"
+ echo " -h, --help"
+ exit 1
+}
+
+create() {
+ # Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
+ mkdir config 2> /dev/null
+ cat /dev/null > ./config/hiera.yaml
+ cat /dev/null > ./config/hosts
+ export DOCKER_IMAGE=$(get-yaml-config docker image)
+
+ # Startup instances
+ docker-compose scale bigtop=$1
+ if [ $? -ne 0 ]; then
+ echo "Docker container(s) startup failed!";
+ exit 1;
+ fi
+
+ # Get the headnode FQDN
+ nodes=(`docker-compose ps -q`)
+ hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${nodes[0]}`
+
+ # Fetch configurations form specificed yaml config file
+ repo=$(get-yaml-config repo)
+ components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
+ jdk=$(get-yaml-config jdk)
+ distro=$(get-yaml-config distro)
+ enable_local_repo=$(get-yaml-config enable_local_repo)
+ generate-config "$hadoop_head_node" "$repo" "$components" "$jdk"
+
+ # Start provisioning
+ generate-hosts
+ bootstrap $distro $enable_local_repo
+ provision
+}
+
+generate-hosts() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts
+ done
+ wait
+
+}
+
+generate-config() {
+ echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata"
+ cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml
+ yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/
+ cat > ./config/hieradata/site.yaml << EOF
+bigtop::hadoop_head_node: $1
+hadoop::hadoop_storage_dirs: [/data/1, /data/2]
+bigtop::bigtop_repo_uri: $2
+hadoop_cluster_node::cluster_components: $3
+bigtop::jdk_package_name: $4
+EOF
+}
+
+copy-to-instances() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ docker cp $1 $node:$2 &
+ done
+ wait
+}
+
+bootstrap() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ docker exec $node bash -c "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$1.sh $2" &
+ done
+ wait
+}
+
+provision() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ bigtop-puppet $node &
+ done
+ wait
+}
+
+smoke-tests() {
+ nodes=(`docker-compose ps -q`)
+ hadoop_head_node=${nodes:0:12}
+ smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`"
+ docker exec $hadoop_head_node bash -c "bash -x /bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh $smoke_test_components"
+}
+
+destroy() {
+ docker-compose stop
+ docker-compose rm -f
+ rm -rvf ./config
+}
+
+bigtop-puppet() {
+ docker exec $1 bash -c 'puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp'
+}
+
+get-yaml-config() {
+ RUBY_EXE=ruby
+ if [ $# -eq 1 ]; then
+ RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1'];"
+ elif [ $# -eq 2 ]; then
+ RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1']['$2'];"
+ else
+ echo "The yaml config retrieval function can only take 1 or 2 parameters.";
+ exit 1;
+ fi
+ cat ${yamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r'
+}
+
+PROG=`basename $0`
+
+if [ $# -eq 0 ]; then
+ usage
+fi
+
+yamlconf="config.yaml"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -c|--create)
+ if [ $# -lt 2 ]; then
+ echo "Create requires a number" 1>&2
+ usage
+ fi
+ create $2
+ shift 2;;
+ -C|--conf)
+ if [ $# -lt 2 ]; then
+ echo "Alternative config file for config.yaml" 1>&2
+ usage
+ fi
+ yamlconf=$2
+ shift 2;;
+ -p|--provision)
+ provision
+ shift;;
+ -s|--smoke-tests)
+ smoke-tests
+ shift;;
+ -d|--destroy)
+ destroy
+ shift;;
+ -h|--help)
+ usage
+ shift;;
+ *)
+ echo "Unknown argument: '$1'" 1>&2
+ usage;;
+ esac
+done
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/utils/setup-env-centos.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh
new file mode 100755
index 0000000..bb25d28
--- /dev/null
+++ b/provisioner/utils/setup-env-centos.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+yum -y install rng-tools
+sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
+service rngd start
+
+if [ $enable_local_repo == "true" ]; then
+ echo "Enabling local yum."
+ yum -y install yum-utils
+ sudo echo "gpgcheck=0" >> /etc/yum.conf
+ sudo yum-config-manager --add-repo file:///bigtop-home/output
+else
+ echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
+fi
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/utils/setup-env-debian.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh
new file mode 100755
index 0000000..c5768d9
--- /dev/null
+++ b/provisioner/utils/setup-env-debian.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+apt-get -y install rng-tools
+sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
+service rng-tools start
+
+if [ $enable_local_repo == "true" ]; then
+ echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
+ apt-get update
+else
+ echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
+fi
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c1b07de7/provisioner/utils/smoke-tests.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
new file mode 100755
index 0000000..8dac31c
--- /dev/null
+++ b/provisioner/utils/smoke-tests.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HCFS_USER="hdfs"
+SMOKE_TESTS=${1:-mapreduce,pig}
+
+# Autodetect JAVA_HOME
+if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+else
+ >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
+ exit 2
+fi
+
+echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
+
+export HADOOP_CONF_DIR=/etc/hadoop/conf/
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
+export HIVE_HOME=/usr/lib/hive/
+export PIG_HOME=/usr/lib/pig/
+export FLUME_HOME=/usr/lib/flume/
+export SQOOP_HOME=/usr/lib/sqoop/
+export HIVE_CONF_DIR=/etc/hive/conf/
+export MAHOUT_HOME="/usr/lib/mahout"
+
+su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
+
+if [ -f /etc/debian_version ] ; then
+ apt-get -y install pig hive flume mahout sqoop
+else
+ yum install -y pig hive flume mahout sqoop
+fi
+ALL_SMOKE_TASKS=""
+for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
+ ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
+done
+cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
+# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
+# permissions as used for smoke-tests execution
+rm -rf buildSrc/build/test-results/binary
+
[10/10] bigtop git commit: BIGTOP-2508. Make Docker Provisioner to
work with Docker on Mac
Posted by of...@apache.org.
BIGTOP-2508. Make Docker Provisioner to work with Docker on Mac
Signed-off-by: Olaf Flebbe <of...@oflebbe.de>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/27efdf52
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/27efdf52
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/27efdf52
Branch: refs/heads/BIGTOP-2253
Commit: 27efdf525551803a8c640b8ea9ebd28d1d8ab70f
Parents: af42c2f
Author: Evans Ye <ev...@apache.org>
Authored: Sun Jul 10 15:33:22 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Thu Aug 4 21:47:17 2016 +0200
----------------------------------------------------------------------
provisioner/docker/config/hiera.yaml | 1 +
provisioner/docker/config/hieradata/PLACEHOLDER | 0
provisioner/docker/config/hosts | 1 +
provisioner/docker/docker-hadoop.sh | 4 +++-
4 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/27efdf52/provisioner/docker/config/hiera.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config/hiera.yaml b/provisioner/docker/config/hiera.yaml
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/provisioner/docker/config/hiera.yaml
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/27efdf52/provisioner/docker/config/hieradata/PLACEHOLDER
----------------------------------------------------------------------
diff --git a/provisioner/docker/config/hieradata/PLACEHOLDER b/provisioner/docker/config/hieradata/PLACEHOLDER
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/27efdf52/provisioner/docker/config/hosts
----------------------------------------------------------------------
diff --git a/provisioner/docker/config/hosts b/provisioner/docker/config/hosts
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/provisioner/docker/config/hosts
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/27efdf52/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 4b1c0df..9c7794f 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -125,7 +125,9 @@ destroy() {
docker-compose -p $PROVISION_ID stop
docker-compose -p $PROVISION_ID rm -f
fi
- rm -rvf ./config .provision_id
+ echo > ./config/hiera.yaml
+ echo > ./config/hosts
+ rm -rvf ./config/hieradata/bigtop ./config/hieradata/site.yaml .provision_id
}
bigtop-puppet() {