You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by of...@apache.org on 2016/08/09 18:27:04 UTC
[01/11] bigtop git commit: BIGTOP-2513: Some RAT exclude rules
defined in build.gradle are not defined in pom.xml [Forced Update!]
Repository: bigtop
Updated Branches:
refs/heads/BIGTOP-2253 27efdf525 -> 47d1f6970 (forced update)
BIGTOP-2513: Some RAT exclude rules defined in build.gradle are not defined in pom.xml
Signed-off-by: Konstantin Boudnik <ko...@epam.com>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/f13db7a2
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/f13db7a2
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/f13db7a2
Branch: refs/heads/BIGTOP-2253
Commit: f13db7a260f157394085ff4f2e07f225268962a7
Parents: 87e7d01
Author: Kengo Seki <se...@apache.org>
Authored: Tue Aug 9 01:18:27 2016 +0900
Committer: Konstantin Boudnik <ko...@epam.com>
Committed: Mon Aug 8 11:39:02 2016 -0700
----------------------------------------------------------------------
pom.xml | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/f13db7a2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 21fe0b5..fb038f5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -305,6 +305,7 @@
<exclude>bigtop-tests/smoke-tests/phoenix/*.csv</exclude>
<exclude>bigtop-tests/smoke-tests/phoenix/*.sql</exclude>
<exclude>bigtop-tests/smoke-tests/ignite-hadoop/*.data</exclude>
+ <exclude>bigtop-tests/smoke-tests/tajo/table1/*.csv</exclude>
<exclude>**/target/**</exclude>
<exclude>**/build/**</exclude>
<exclude>**/.gradle/**</exclude>
@@ -355,7 +356,9 @@
bigtop-deploy/puppet/modules/hadoop/templates/container-executor.cfg
</exclude>
<exclude>bigtop-deploy/puppet/README.md</exclude>
+ <exclude>bigtop-deploy/puppet/modules/qfs/README.md</exclude>
<exclude>bigtop-deploy/puppet/config/site.csv.example</exclude>
+ <exclude>dl/**</exclude>
</excludes>
</configuration>
</plugin>
[06/11] bigtop git commit: BIGTOP-2254. Replace the Docker
orchestration tool from Vagrant to Docker Compose
Posted by of...@apache.org.
BIGTOP-2254. Replace the Docker orchestration tool from Vagrant to Docker Compose
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/d5415743
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/d5415743
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/d5415743
Branch: refs/heads/BIGTOP-2253
Commit: d5415743973e6b4db707dbc5ed9d4be458eaf40e
Parents: fc61d93
Author: Evans Ye <ev...@apache.org>
Authored: Sat Jan 23 10:33:03 2016 +0000
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/.gitignore | 1 +
provisioner/docker/README.md | 131 +++++++++++++++++++++
provisioner/docker/config.yaml | 32 ++++++
provisioner/docker/config_debian.yaml | 32 ++++++
provisioner/docker/config_ubuntu.yaml | 32 ++++++
provisioner/docker/docker-compose.yml | 10 ++
provisioner/docker/docker-hadoop.sh | 179 +++++++++++++++++++++++++++++
provisioner/utils/setup-env-centos.sh | 36 ++++++
provisioner/utils/setup-env-debian.sh | 34 ++++++
provisioner/utils/smoke-tests.sh | 56 +++++++++
10 files changed, 543 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/docker/.gitignore
----------------------------------------------------------------------
diff --git a/provisioner/docker/.gitignore b/provisioner/docker/.gitignore
new file mode 100644
index 0000000..f733c4b
--- /dev/null
+++ b/provisioner/docker/.gitignore
@@ -0,0 +1 @@
+config/
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
new file mode 100644
index 0000000..51bac85
--- /dev/null
+++ b/provisioner/docker/README.md
@@ -0,0 +1,131 @@
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+------------------------------------------------------------------------------------------------------------------------------------------------------
+
+#BigTop Docker provisioner
+
+## Overview
+
+The Docker Compose definition and wrapper script that creates Bigtop virtual Hadoop cluster on top of Docker containers for you, by pulling from existing publishing bigtop repositories.
+This cluster can be used:
+
+- to test bigtop smoke tests
+- to test bigtop puppet recipes
+- to run integration test with your application
+
+This has been verified on Docker Engine 1.9.1, with api version 1.15, and Docker Compose 1.5.2 on Amazon Linux 2015.09 release.
+
+## Prerequisites
+
+### OS X and Windows
+
+* Install [Docker Toolbox](https://www.docker.com/docker-toolbox)
+
+### Linux
+
+* Install [Docker](https://docs.docker.com/installation/)
+
+* Install [Docker Compose](https://docs.docker.com/compose/install/)
+
+* Start the Docker daemon
+
+```
+service docker start
+```
+
+## USAGE
+
+1) Create a Bigtop Hadoop cluster by given # of node.
+
+```
+./docker-hadoop.sh --create 3
+```
+
+2) Destroy the cluster.
+
+```
+./docker-hadoop.sh --destroy
+```
+
+3) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
+
+```
+./docker-hadoop.sh --provision
+```
+
+4) Run Bigtop smoke tests
+
+```
+./docker-hadoop.sh --smoke-tests
+```
+
+5) Chain your operations with-in one command.
+
+```
+./docker-hadoop.sh --create 5 --smoke-tests --destroy
+```
+
+Commands will be executed by following order:
+
+```
+create 5 node cluster => run smoke tests => destroy the cluster
+```
+
+6) See helper message:
+
+```
+./docker-hadoop.sh -h
+usage: docker-hadoop.sh [-C file ] args
+ -C file Use alternate file for config.yaml
+ commands:
+ -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster
+ -p, --provision Deploy configuration changes
+ -s, --smoke-tests Run Bigtop smoke tests
+ -d, --destroy Destroy the cluster
+ -h, --help
+```
+
+##Configurations
+
+* There are several parameters can be configured in config.yaml:
+
+1) Modify memory limit for Docker containers
+
+```
+docker:
+ memory_size: "2048"
+
+```
+
+2) Use different host ports mapping for web UIs
+
+```
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+
+```
+Note: If running on OS X or Windows, the boot2docker VM should be reloaded after ports changed
+
+
+##Configure Apache Hadoop ecosystem components
+* Choose the ecosystem you want to be deployed by modifying components in config.yaml
+
+```
+components: "hadoop, hbase, yarn,..."
+```
+
+By default, Apache Hadoop and YARN will be installed.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/docker/config.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config.yaml b/provisioner/docker/config.yaml
new file mode 100644
index 0000000..86f35e7
--- /dev/null
+++ b/provisioner/docker/config.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:centos-6"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/6/x86_64"
+distro: centos
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "java-1.7.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/docker/config_debian.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_debian.yaml b/provisioner/docker/config_debian.yaml
new file mode 100644
index 0000000..f6b11ae
--- /dev/null
+++ b/provisioner/docker/config_debian.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:debian-8"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/debian/8/x86_64"
+distro: debian
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "openjdk-7-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/docker/config_ubuntu.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_ubuntu.yaml b/provisioner/docker/config_ubuntu.yaml
new file mode 100644
index 0000000..e4ea6f3
--- /dev/null
+++ b/provisioner/docker/config_ubuntu.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:ubuntu-14.04"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/ubuntu/trusty/x86_64"
+distro: debian
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "openjdk-7-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml
new file mode 100644
index 0000000..95831d1
--- /dev/null
+++ b/provisioner/docker/docker-compose.yml
@@ -0,0 +1,10 @@
+bigtop:
+ image: ${DOCKER_IMAGE}
+ command: sleep infinity
+ domainname: bigtop.apache.org
+ privileged: true
+ volumes:
+ - ../../:/bigtop-home
+ - ./config/hiera.yaml:/etc/puppet/hiera.yaml
+ - ./config/hieradata:/etc/puppet/hieradata
+ - ./config/hosts:/etc/hosts
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
new file mode 100755
index 0000000..e8bd767
--- /dev/null
+++ b/provisioner/docker/docker-hadoop.sh
@@ -0,0 +1,179 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet
+
+usage() {
+ echo "usage: $PROG [-C file ] args"
+ echo " -C file Use alternate file for config.yaml"
+ echo " commands:"
+ echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
+ echo " -p, --provision Deploy configuration changes"
+ echo " -s, --smoke-tests Run Bigtop smoke tests"
+ echo " -d, --destroy Destroy the cluster"
+ echo " -h, --help"
+ exit 1
+}
+
+create() {
+ # Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
+ mkdir config 2> /dev/null
+ cat /dev/null > ./config/hiera.yaml
+ cat /dev/null > ./config/hosts
+ export DOCKER_IMAGE=$(get-yaml-config docker image)
+
+ # Startup instances
+ docker-compose scale bigtop=$1
+ if [ $? -ne 0 ]; then
+ echo "Docker container(s) startup failed!";
+ exit 1;
+ fi
+
+ # Get the headnode FQDN
+ nodes=(`docker-compose ps -q`)
+ hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${nodes[0]}`
+
+ # Fetch configurations form specificed yaml config file
+ repo=$(get-yaml-config repo)
+ components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
+ jdk=$(get-yaml-config jdk)
+ distro=$(get-yaml-config distro)
+ enable_local_repo=$(get-yaml-config enable_local_repo)
+ generate-config "$hadoop_head_node" "$repo" "$components" "$jdk"
+
+ # Start provisioning
+ generate-hosts
+ bootstrap $distro $enable_local_repo
+ provision
+}
+
+generate-hosts() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts
+ done
+ wait
+
+}
+
+generate-config() {
+ echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata"
+ cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml
+ yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/
+ cat > ./config/hieradata/site.yaml << EOF
+bigtop::hadoop_head_node: $1
+hadoop::hadoop_storage_dirs: [/data/1, /data/2]
+bigtop::bigtop_repo_uri: $2
+hadoop_cluster_node::cluster_components: $3
+bigtop::jdk_package_name: $4
+EOF
+}
+
+copy-to-instances() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ docker cp $1 $node:$2 &
+ done
+ wait
+}
+
+bootstrap() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ docker exec $node bash -c "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$1.sh $2" &
+ done
+ wait
+}
+
+provision() {
+ nodes=(`docker-compose ps -q`)
+ for node in ${nodes[*]}; do
+ bigtop-puppet $node &
+ done
+ wait
+}
+
+smoke-tests() {
+ nodes=(`docker-compose ps -q`)
+ hadoop_head_node=${nodes:0:12}
+ smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`"
+ docker exec $hadoop_head_node bash -c "bash -x /bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh $smoke_test_components"
+}
+
+destroy() {
+ docker-compose stop
+ docker-compose rm -f
+ rm -rvf ./config
+}
+
+bigtop-puppet() {
+ docker exec $1 bash -c 'puppet apply -d --modulepath=/bigtop-home/bigtop-deploy/puppet/modules:/etc/puppet/modules /bigtop-home/bigtop-deploy/puppet/manifests/site.pp'
+}
+
+get-yaml-config() {
+ RUBY_EXE=ruby
+ if [ $# -eq 1 ]; then
+ RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1'];"
+ elif [ $# -eq 2 ]; then
+ RUBY_SCRIPT="data = YAML::load(STDIN.read); puts data['$1']['$2'];"
+ else
+ echo "The yaml config retrieval function can only take 1 or 2 parameters.";
+ exit 1;
+ fi
+ cat ${yamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r'
+}
+
+PROG=`basename $0`
+
+if [ $# -eq 0 ]; then
+ usage
+fi
+
+yamlconf="config.yaml"
+while [ $# -gt 0 ]; do
+ case "$1" in
+ -c|--create)
+ if [ $# -lt 2 ]; then
+ echo "Create requires a number" 1>&2
+ usage
+ fi
+ create $2
+ shift 2;;
+ -C|--conf)
+ if [ $# -lt 2 ]; then
+ echo "Alternative config file for config.yaml" 1>&2
+ usage
+ fi
+ yamlconf=$2
+ shift 2;;
+ -p|--provision)
+ provision
+ shift;;
+ -s|--smoke-tests)
+ smoke-tests
+ shift;;
+ -d|--destroy)
+ destroy
+ shift;;
+ -h|--help)
+ usage
+ shift;;
+ *)
+ echo "Unknown argument: '$1'" 1>&2
+ usage;;
+ esac
+done
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/utils/setup-env-centos.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh
new file mode 100755
index 0000000..bb25d28
--- /dev/null
+++ b/provisioner/utils/setup-env-centos.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+yum -y install rng-tools
+sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
+service rngd start
+
+if [ $enable_local_repo == "true" ]; then
+ echo "Enabling local yum."
+ yum -y install yum-utils
+ sudo echo "gpgcheck=0" >> /etc/yum.conf
+ sudo yum-config-manager --add-repo file:///bigtop-home/output
+else
+ echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
+fi
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/utils/setup-env-debian.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh
new file mode 100755
index 0000000..c5768d9
--- /dev/null
+++ b/provisioner/utils/setup-env-debian.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+enable_local_repo=${1:-false}
+
+bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
+
+# Setup rng-tools to improve virtual machine entropy performance.
+# The poor entropy performance will cause kerberos provisioning failed.
+apt-get -y install rng-tools
+sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
+service rng-tools start
+
+if [ $enable_local_repo == "true" ]; then
+ echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
+ apt-get update
+else
+ echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
+fi
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/d5415743/provisioner/utils/smoke-tests.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
new file mode 100755
index 0000000..8dac31c
--- /dev/null
+++ b/provisioner/utils/smoke-tests.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+HCFS_USER="hdfs"
+SMOKE_TESTS=${1:-mapreduce,pig}
+
+# Autodetect JAVA_HOME
+if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+else
+ >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
+ exit 2
+fi
+
+echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
+
+export HADOOP_CONF_DIR=/etc/hadoop/conf/
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
+export HIVE_HOME=/usr/lib/hive/
+export PIG_HOME=/usr/lib/pig/
+export FLUME_HOME=/usr/lib/flume/
+export SQOOP_HOME=/usr/lib/sqoop/
+export HIVE_CONF_DIR=/etc/hive/conf/
+export MAHOUT_HOME="/usr/lib/mahout"
+
+su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
+
+if [ -f /etc/debian_version ] ; then
+ apt-get -y install pig hive flume mahout sqoop
+else
+ yum install -y pig hive flume mahout sqoop
+fi
+ALL_SMOKE_TASKS=""
+for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
+ ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
+done
+cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
+# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
+# permissions as used for smoke-tests execution
+rm -rf buildSrc/build/test-results/binary
+
[04/11] bigtop git commit: BIGTOP-2386. Got permission denied when
running new Docker provisioner as non-root
Posted by of...@apache.org.
BIGTOP-2386. Got permission denied when running new Docker provisioner as non-root
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/51b94e84
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/51b94e84
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/51b94e84
Branch: refs/heads/BIGTOP-2253
Commit: 51b94e846b8afe395f24c13e62b538a64b1b7af6
Parents: fb08b15
Author: Evans Ye <ev...@apache.org>
Authored: Tue Apr 12 18:53:01 2016 +0000
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/docker-hadoop.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/51b94e84/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index d8392ba..2323456 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -34,7 +34,7 @@ usage() {
create() {
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
- mkdir config 2> /dev/null
+ mkdir -p config/hieradata 2> /dev/null
cat /dev/null > ./config/hiera.yaml
cat /dev/null > ./config/hosts
export DOCKER_IMAGE=$(get-yaml-config docker image)
@@ -67,7 +67,8 @@ create() {
generate-hosts() {
nodes=(`docker-compose ps -q`)
for node in ${nodes[*]}; do
- echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts
+ entry=`docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node`
+ docker exec ${nodes[0]} bash -c "echo $entry >> /etc/hosts"
done
wait
[03/11] bigtop git commit: BIGTOP-2505: Support systemd containers,
fix exec (-e) flag
Posted by of...@apache.org.
BIGTOP-2505: Support systemd containers, fix exec (-e) flag
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/40a52870
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/40a52870
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/40a52870
Branch: refs/heads/BIGTOP-2253
Commit: 40a52870b5a4694ae3af9cf59734a8b2d35675ec
Parents: 6591e4b
Author: Olaf Flebbe <of...@oflebbe.de>
Authored: Sat Jul 23 12:34:12 2016 +0200
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/config_centos7.yaml | 32 +++++++++++++++++
provisioner/docker/docker-compose.yml | 2 +-
provisioner/docker/docker-hadoop.sh | 10 +++---
provisioner/utils/setup-env-centos.sh | 36 -------------------
provisioner/utils/setup-env-debian.sh | 34 ------------------
provisioner/utils/smoke-tests.sh | 56 -----------------------------
6 files changed, 38 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/docker/config_centos7.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config_centos7.yaml b/provisioner/docker/config_centos7.yaml
new file mode 100644
index 0000000..3610a8f
--- /dev/null
+++ b/provisioner/docker/config_centos7.yaml
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+docker:
+ memory_size: "4096"
+ image: "bigtop/deploy:centos-7"
+
+boot2docker:
+ memory_size: "4096"
+ number_cpus: "1"
+
+repo: "http://bigtop-repos.s3.amazonaws.com/releases/1.0.0/centos/7/x86_64"
+distro: centos
+components: [hadoop, yarn]
+namenode_ui_port: "50070"
+yarn_ui_port: "8088"
+hbase_ui_port: "60010"
+enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
+jdk: "java-1.7.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-compose.yml b/provisioner/docker/docker-compose.yml
index 95831d1..d71b247 100644
--- a/provisioner/docker/docker-compose.yml
+++ b/provisioner/docker/docker-compose.yml
@@ -1,6 +1,6 @@
bigtop:
image: ${DOCKER_IMAGE}
- command: sleep infinity
+ command: /sbin/init
domainname: bigtop.apache.org
privileged: true
volumes:
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 89ac6bb..4b1c0df 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -41,8 +41,8 @@ create() {
PROVISION_ID=`cat .provision_id`
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
mkdir -p config/hieradata 2> /dev/null
- cat /dev/null > ./config/hiera.yaml
- cat /dev/null > ./config/hosts
+ echo > ./config/hiera.yaml
+ echo > ./config/hosts
export DOCKER_IMAGE=$(get-yaml-config docker image)
# Startup instances
@@ -81,8 +81,8 @@ generate-hosts() {
generate-config() {
echo "Bigtop Puppet configurations are shared between instances, and can be modified under config/hieradata"
- cat $BIGTOP_PUPPET_DIR/hiera.yaml > ./config/hiera.yaml
- yes | cp -vr $BIGTOP_PUPPET_DIR/hieradata ./config/
+ cat $BIGTOP_PUPPET_DIR/hiera.yaml >> ./config/hiera.yaml
+ cp -vfr $BIGTOP_PUPPET_DIR/hieradata ./config/
cat > ./config/hieradata/site.yaml << EOF
bigtop::hadoop_head_node: $1
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
@@ -185,7 +185,7 @@ if [ -e .provision_id ]; then
PROVISION_ID=`cat .provision_id`
fi
if [ -n "$PROVISION_ID" ]; then
- NODES=`docker-compose -p $PROVISION_ID ps -q`
+ NODES=(`docker-compose -p $PROVISION_ID ps -q`)
fi
while [ $# -gt 0 ]; do
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/utils/setup-env-centos.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-centos.sh b/provisioner/utils/setup-env-centos.sh
deleted file mode 100755
index bb25d28..0000000
--- a/provisioner/utils/setup-env-centos.sh
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-enable_local_repo=${1:-false}
-
-bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
-
-# Setup rng-tools to improve virtual machine entropy performance.
-# The poor entropy performance will cause kerberos provisioning failed.
-yum -y install rng-tools
-sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
-service rngd start
-
-if [ $enable_local_repo == "true" ]; then
- echo "Enabling local yum."
- yum -y install yum-utils
- sudo echo "gpgcheck=0" >> /etc/yum.conf
- sudo yum-config-manager --add-repo file:///bigtop-home/output
-else
- echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
-fi
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/utils/setup-env-debian.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/setup-env-debian.sh b/provisioner/utils/setup-env-debian.sh
deleted file mode 100755
index c5768d9..0000000
--- a/provisioner/utils/setup-env-debian.sh
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-enable_local_repo=${1:-false}
-
-bash /bigtop-home/bigtop_toolchain/bin/puppetize.sh
-
-# Setup rng-tools to improve virtual machine entropy performance.
-# The poor entropy performance will cause kerberos provisioning failed.
-apt-get -y install rng-tools
-sed -i.bak 's@#HRNGDEVICE=/dev/null@HRNGDEVICE=/dev/urandom@' /etc/default/rng-tools
-service rng-tools start
-
-if [ $enable_local_repo == "true" ]; then
- echo "deb file:///bigtop-home/output/apt bigtop contrib" > /etc/apt/sources.list.d/bigtop-home_output.list
- apt-get update
-else
- echo "local yum = $enable_local_repo ; NOT Enabling local yum. Packages will be pulled from remote..."
-fi
-
http://git-wip-us.apache.org/repos/asf/bigtop/blob/40a52870/provisioner/utils/smoke-tests.sh
----------------------------------------------------------------------
diff --git a/provisioner/utils/smoke-tests.sh b/provisioner/utils/smoke-tests.sh
deleted file mode 100755
index 8dac31c..0000000
--- a/provisioner/utils/smoke-tests.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-HCFS_USER="hdfs"
-SMOKE_TESTS=${1:-mapreduce,pig}
-
-# Autodetect JAVA_HOME
-if [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-else
- >&2 echo -e "\nUNABLE TO DETECT JAVAHOME SINCE bigtop-utils NEEDS TO BE INSTALLED!\n"
- exit 2
-fi
-
-echo -e "\n===== START TO RUN SMOKE TESTS: $SMOKE_TESTS =====\n"
-
-export HADOOP_CONF_DIR=/etc/hadoop/conf/
-export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
-export HIVE_HOME=/usr/lib/hive/
-export PIG_HOME=/usr/lib/pig/
-export FLUME_HOME=/usr/lib/flume/
-export SQOOP_HOME=/usr/lib/sqoop/
-export HIVE_CONF_DIR=/etc/hive/conf/
-export MAHOUT_HOME="/usr/lib/mahout"
-
-su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root'
-su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
-su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
-
-if [ -f /etc/debian_version ] ; then
- apt-get -y install pig hive flume mahout sqoop
-else
- yum install -y pig hive flume mahout sqoop
-fi
-ALL_SMOKE_TASKS=""
-for s in `echo $SMOKE_TESTS | sed -e 's#,# #g'`; do
- ALL_SMOKE_TASKS="$ALL_SMOKE_TASKS bigtop-tests:smoke-tests:$s:test"
-done
-cd /bigtop-home && ./gradlew clean $ALL_SMOKE_TASKS -Psmoke.tests --info
-# BIGTOP-2244 workaround: clean the top level buildSrc/build with the same
-# permissions as used for smoke-tests execution
-rm -rf buildSrc/build/test-results/binary
-
[07/11] bigtop git commit: BIGTOP-2279. Add provisioner into
release-assembly.xml
Posted by of...@apache.org.
BIGTOP-2279. Add provisioner into release-assembly.xml
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a8bc09e8
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a8bc09e8
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a8bc09e8
Branch: refs/heads/BIGTOP-2253
Commit: a8bc09e89051019e8a0aaafddc13705365523258
Parents: 51b94e8
Author: Evans Ye <ev...@apache.org>
Authored: Wed Apr 13 02:56:54 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
src/assembly/release-assembly.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a8bc09e8/src/assembly/release-assembly.xml
----------------------------------------------------------------------
diff --git a/src/assembly/release-assembly.xml b/src/assembly/release-assembly.xml
index 20d90db..0868001 100644
--- a/src/assembly/release-assembly.xml
+++ b/src/assembly/release-assembly.xml
@@ -38,6 +38,7 @@
<include>gradle*</include>
<include>docs/**</include>
<include>docker/**</include>
+ <include>provisioner/**</include>
<include>src/**</include>
<include>test/**</include>
<include>*.bom</include>
[02/11] bigtop git commit: BIGTOP-2512: Wrong help message in
docker-hadoop.sh
Posted by of...@apache.org.
BIGTOP-2512: Wrong help message in docker-hadoop.sh
Signed-off-by: Konstantin Boudnik <ko...@epam.com>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/fc61d934
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/fc61d934
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/fc61d934
Branch: refs/heads/BIGTOP-2253
Commit: fc61d934dfe972339b7bf4e4cb6360979023efcd
Parents: f13db7a
Author: Kengo Seki <se...@apache.org>
Authored: Tue Aug 9 00:28:04 2016 +0900
Committer: Konstantin Boudnik <ko...@epam.com>
Committed: Mon Aug 8 11:42:27 2016 -0700
----------------------------------------------------------------------
bigtop-deploy/vm/vagrant-puppet-docker/README.md | 2 +-
bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fc61d934/bigtop-deploy/vm/vagrant-puppet-docker/README.md
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/README.md b/bigtop-deploy/vm/vagrant-puppet-docker/README.md
index 602b10b..fb9121c 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/README.md
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/README.md
@@ -106,7 +106,7 @@ create 5 node cluster => run smoke tests => destroy the cluster
```
./docker-hadoop.sh -h
usage: docker-hadoop.sh [options]
- -c NUM_INSTANCES, --create=NUM_INSTANCES Create a docker based Bigtop Hadoop cluster
+ -c NUM_INSTANCES, --create NUM_INSTANCES Create a docker based Bigtop Hadoop cluster
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
-d, --destroy Destroy the cluster
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fc61d934/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
index 57a6f6e..8c3628b 100755
--- a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
@@ -19,7 +19,7 @@ usage() {
echo "usage: $PROG [-C file ] args"
echo " -C file Use alternate file for vagrantconfig.yaml"
echo " commands:"
- echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
+ echo " -c NUM_INSTANCES, --create NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
echo " -d, --destroy Destroy the cluster"
[08/11] bigtop git commit: BIGTOP-2273. Add exec command into
docker-hadoop.sh
Posted by of...@apache.org.
BIGTOP-2273. Add exec command into docker-hadoop.sh
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/fb08b15a
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/fb08b15a
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/fb08b15a
Branch: refs/heads/BIGTOP-2253
Commit: fb08b15ae1c37c4e8cba93eff21393de96ff3d02
Parents: d541574
Author: Evans Ye <ev...@apache.org>
Authored: Sun Jan 24 13:49:36 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/README.md | 25 +++++++++++++++++++-----
provisioner/docker/docker-hadoop.sh | 33 ++++++++++++++++++++++++++++----
2 files changed, 49 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb08b15a/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
index 51bac85..fbc82b4 100644
--- a/provisioner/docker/README.md
+++ b/provisioner/docker/README.md
@@ -60,19 +60,31 @@ service docker start
./docker-hadoop.sh --destroy
```
-3) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
+3) Get into the first container (the master)
+
+```
+./docker-hadoop.sh --exec 1 bash
+```
+
+4) Execute a command on the second container
+
+```
+./docker-hadoop.sh --exec 2 hadoop fs -ls /
+```
+
+5) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
```
./docker-hadoop.sh --provision
```
-4) Run Bigtop smoke tests
+6) Run Bigtop smoke tests
```
./docker-hadoop.sh --smoke-tests
```
-5) Chain your operations with-in one command.
+7) Chain your operations with-in one command.
```
./docker-hadoop.sh --create 5 --smoke-tests --destroy
@@ -84,7 +96,7 @@ Commands will be executed by following order:
create 5 node cluster => run smoke tests => destroy the cluster
```
-6) See helper message:
+8) See helper message:
```
./docker-hadoop.sh -h
@@ -92,9 +104,12 @@ usage: docker-hadoop.sh [-C file ] args
-C file Use alternate file for config.yaml
commands:
-c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster
+ -d, --destroy Destroy the cluster
+ -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number.
+ For example: docker-hadoop.sh --exec 1 bash
+ docker-hadoop.sh --exec docker_bigtop_1 bash
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
- -d, --destroy Destroy the cluster
-h, --help
```
http://git-wip-us.apache.org/repos/asf/bigtop/blob/fb08b15a/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index e8bd767..d8392ba 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -22,9 +22,12 @@ usage() {
echo " -C file Use alternate file for config.yaml"
echo " commands:"
echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
+ echo " -d, --destroy Destroy the cluster"
+ echo " -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number."
+ echo " For example: $PROG --exec 1 bash"
+ echo " $PROG --exec docker_bigtop_1 bash"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
- echo " -d, --destroy Destroy the cluster"
echo " -h, --help"
exit 1
}
@@ -137,6 +140,20 @@ get-yaml-config() {
cat ${yamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r'
}
+execute() {
+ re='^[0-9]+$'
+ if [[ $1 =~ $re ]] ; then
+ no=$1
+ shift
+ nodes=(`docker-compose ps -q`)
+ docker exec -ti ${nodes[$((no-1))]} $@
+ else
+ name=$1
+ shift
+ docker exec -ti $name $@
+ fi
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -160,15 +177,23 @@ while [ $# -gt 0 ]; do
fi
yamlconf=$2
shift 2;;
+ -d|--destroy)
+ destroy
+ shift;;
+ -e|--exec)
+ if [ $# -lt 3 ]; then
+ echo "exec command takes 2 parameters: 1) instance no 2) command to be executed" 1>&2
+ usage
+ fi
+ shift
+ execute $@
+ shift $#;;
-p|--provision)
provision
shift;;
-s|--smoke-tests)
smoke-tests
shift;;
- -d|--destroy)
- destroy
- shift;;
-h|--help)
usage
shift;;
[05/11] bigtop git commit: BIGTOP-2508. Make Docker Provisioner to
work with Docker on Mac
Posted by of...@apache.org.
BIGTOP-2508. Make Docker Provisioner to work with Docker on Mac
Signed-off-by: Olaf Flebbe <of...@oflebbe.de>
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/71b0ee3e
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/71b0ee3e
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/71b0ee3e
Branch: refs/heads/BIGTOP-2253
Commit: 71b0ee3e7fc0c79ae689b3ae82ac3d5c0ba3641b
Parents: 40a5287
Author: Evans Ye <ev...@apache.org>
Authored: Sun Jul 10 15:33:22 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/config/hiera.yaml | 1 +
provisioner/docker/config/hieradata/PLACEHOLDER | 0
provisioner/docker/config/hosts | 1 +
provisioner/docker/docker-hadoop.sh | 4 +++-
4 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71b0ee3e/provisioner/docker/config/hiera.yaml
----------------------------------------------------------------------
diff --git a/provisioner/docker/config/hiera.yaml b/provisioner/docker/config/hiera.yaml
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/provisioner/docker/config/hiera.yaml
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71b0ee3e/provisioner/docker/config/hieradata/PLACEHOLDER
----------------------------------------------------------------------
diff --git a/provisioner/docker/config/hieradata/PLACEHOLDER b/provisioner/docker/config/hieradata/PLACEHOLDER
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71b0ee3e/provisioner/docker/config/hosts
----------------------------------------------------------------------
diff --git a/provisioner/docker/config/hosts b/provisioner/docker/config/hosts
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/provisioner/docker/config/hosts
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/bigtop/blob/71b0ee3e/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 4b1c0df..9c7794f 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -125,7 +125,9 @@ destroy() {
docker-compose -p $PROVISION_ID stop
docker-compose -p $PROVISION_ID rm -f
fi
- rm -rvf ./config .provision_id
+ echo > ./config/hiera.yaml
+ echo > ./config/hosts
+ rm -rvf ./config/hieradata/bigtop ./config/hieradata/site.yaml .provision_id
}
bigtop-puppet() {
[10/11] bigtop git commit: BIGTOP-2458: Add option to disable IP
hostname checking for DataNode registration
Posted by of...@apache.org.
BIGTOP-2458: Add option to disable IP hostname checking for DataNode registration
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/47d1f697
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/47d1f697
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/47d1f697
Branch: refs/heads/BIGTOP-2253
Commit: 47d1f6970644fbd1d2209d028b1f4f386f77e73b
Parents: 71b0ee3
Author: Cory Johns <jo...@gmail.com>
Authored: Thu May 26 15:34:49 2016 -0400
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
bigtop-deploy/puppet/modules/hadoop/manifests/init.pp | 1 +
bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml | 6 ++++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/47d1f697/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 90c9505..430e341 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -228,6 +228,7 @@ class hadoop ($hadoop_security_authentication = "simple",
$hadoop_http_authentication_signature_secret_file = "/etc/hadoop/conf/hadoop-http-authentication-signature-secret",
$hadoop_http_authentication_cookie_domain = regsubst($fqdn, "^[^\\.]+\\.", ""),
$generate_secrets = $hadoop::generate_secrets,
+ $namenode_datanode_registration_ip_hostname_check = true,
) inherits hadoop {
$sshfence_keydir = "$hadoop_ha_sshfence_user_home/.ssh"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/47d1f697/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
index 328cf6f..8216eb4 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hdfs-site.xml
@@ -297,4 +297,10 @@
<% end -%>
<% end -%>
+
+ <property>
+ <name>dfs.namenode.datanode.registration.ip-hostname-check</name>
+ <value><%= @namenode_datanode_registration_ip_hostname_check %></value>
+ </property>
+
</configuration>
[11/11] bigtop git commit: BIGTOP-2387. New Docker provisioner
destroy will delete all docker containers on the host
Posted by of...@apache.org.
BIGTOP-2387. New Docker provisioner destroy will delete all docker containers on the host
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6591e4bd
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6591e4bd
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6591e4bd
Branch: refs/heads/BIGTOP-2253
Commit: 6591e4bdf1afc7b856ce40a177d225fe03e01579
Parents: cea230c
Author: Evans Ye <ev...@apache.org>
Authored: Sun Apr 24 23:50:57 2016 +0800
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/README.md | 1 +
provisioner/docker/docker-hadoop.sh | 60 +++++++++++++++++++++-----------
2 files changed, 40 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6591e4bd/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
index c95b218..d1c0f2e 100644
--- a/provisioner/docker/README.md
+++ b/provisioner/docker/README.md
@@ -109,6 +109,7 @@ usage: docker-hadoop.sh [-C file ] args
For example: docker-hadoop.sh --exec 1 bash
docker-hadoop.sh --exec docker_bigtop_1 bash
-E, --env-check Check whether required tools has been installed
+ -l, --list List out container status for the cluster
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
-h, --help
http://git-wip-us.apache.org/repos/asf/bigtop/blob/6591e4bd/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 93c8c9c..89ac6bb 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -15,8 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet
-
usage() {
echo "usage: $PROG [-C file ] args"
echo " -C file Use alternate file for config.yaml"
@@ -27,6 +25,7 @@ usage() {
echo " For example: $PROG --exec 1 bash"
echo " $PROG --exec docker_bigtop_1 bash"
echo " -E, --env-check Check whether required tools has been installed"
+ echo " -l, --list List out container status for the cluster"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
echo " -h, --help"
@@ -34,6 +33,12 @@ usage() {
}
create() {
+ if [ -e .provision_id ]; then
+ echo "Cluster already exist! Run ./$PROG -d to destroy the cluster or delete .provision_id file and containers manually."
+ exit 1;
+ fi
+ echo "`date +'%Y%m%d_%H%M%S'`_R$RANDOM" > .provision_id
+ PROVISION_ID=`cat .provision_id`
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
mkdir -p config/hieradata 2> /dev/null
cat /dev/null > ./config/hiera.yaml
@@ -41,15 +46,15 @@ create() {
export DOCKER_IMAGE=$(get-yaml-config docker image)
# Startup instances
- docker-compose scale bigtop=$1
+ docker-compose -p $PROVISION_ID scale bigtop=$1
if [ $? -ne 0 ]; then
echo "Docker container(s) startup failed!";
exit 1;
fi
# Get the headnode FQDN
- nodes=(`docker-compose ps -q`)
- hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${nodes[0]}`
+ NODES=(`docker-compose -p $PROVISION_ID ps -q`)
+ hadoop_head_node=`docker inspect --format {{.Config.Hostname}}.{{.Config.Domainname}} ${NODES[0]}`
# Fetch configurations form specificed yaml config file
repo=$(get-yaml-config repo)
@@ -66,10 +71,9 @@ create() {
}
generate-hosts() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
entry=`docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node`
- docker exec ${nodes[0]} bash -c "echo $entry >> /etc/hosts"
+ docker exec ${NODES[0]} bash -c "echo $entry >> /etc/hosts"
done
wait
@@ -89,40 +93,39 @@ EOF
}
copy-to-instances() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
docker cp $1 $node:$2 &
done
wait
}
bootstrap() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
+ docker cp $1 $node:$2 &
docker exec $node bash -c "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$1.sh $2" &
done
wait
}
provision() {
- nodes=(`docker-compose ps -q`)
- for node in ${nodes[*]}; do
+ for node in ${NODES[*]}; do
bigtop-puppet $node &
done
wait
}
smoke-tests() {
- nodes=(`docker-compose ps -q`)
- hadoop_head_node=${nodes:0:12}
+ hadoop_head_node=${NODES:0:12}
smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`"
docker exec $hadoop_head_node bash -c "bash -x /bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh $smoke_test_components"
}
destroy() {
- docker-compose stop
- docker-compose rm -f
- rm -rvf ./config
+ if [ -n "$PROVISION_ID" ]; then
+ docker-compose -p $PROVISION_ID stop
+ docker-compose -p $PROVISION_ID rm -f
+ fi
+ rm -rvf ./config .provision_id
}
bigtop-puppet() {
@@ -147,8 +150,7 @@ execute() {
if [[ $1 =~ $re ]] ; then
no=$1
shift
- nodes=(`docker-compose ps -q`)
- docker exec -ti ${nodes[$((no-1))]} $@
+ docker exec -ti ${NODES[$((no-1))]} $@
else
name=$1
shift
@@ -166,6 +168,10 @@ env-check() {
ruby -v || exit 1
}
+list() {
+ docker-compose -p $PROVISION_ID ps
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -173,6 +179,15 @@ if [ $# -eq 0 ]; then
fi
yamlconf="config.yaml"
+
+BIGTOP_PUPPET_DIR=../../bigtop-deploy/puppet
+if [ -e .provision_id ]; then
+ PROVISION_ID=`cat .provision_id`
+fi
+if [ -n "$PROVISION_ID" ]; then
+ NODES=`docker-compose -p $PROVISION_ID ps -q`
+fi
+
while [ $# -gt 0 ]; do
case "$1" in
-c|--create)
@@ -204,6 +219,9 @@ while [ $# -gt 0 ]; do
-E|--env-check)
env-check
shift;;
+ -l|--list)
+ list
+ shift;;
-p|--provision)
provision
shift;;
[09/11] bigtop git commit: BIGTOP-2312. Add environment check
Posted by of...@apache.org.
BIGTOP-2312. Add environment check
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/cea230c2
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/cea230c2
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/cea230c2
Branch: refs/heads/BIGTOP-2253
Commit: cea230c26867e220a41e3f189840765a5984dfcd
Parents: a8bc09e
Author: Evans Ye <ev...@apache.org>
Authored: Sat Apr 23 08:14:39 2016 +0000
Committer: Olaf Flebbe <of...@oflebbe.de>
Committed: Tue Aug 9 20:26:21 2016 +0200
----------------------------------------------------------------------
provisioner/docker/README.md | 1 +
provisioner/docker/docker-hadoop.sh | 15 +++++++++++++++
2 files changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/cea230c2/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
index fbc82b4..c95b218 100644
--- a/provisioner/docker/README.md
+++ b/provisioner/docker/README.md
@@ -108,6 +108,7 @@ usage: docker-hadoop.sh [-C file ] args
-e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number.
For example: docker-hadoop.sh --exec 1 bash
docker-hadoop.sh --exec docker_bigtop_1 bash
+ -E, --env-check Check whether required tools has been installed
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
-h, --help
http://git-wip-us.apache.org/repos/asf/bigtop/blob/cea230c2/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 2323456..93c8c9c 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -26,6 +26,7 @@ usage() {
echo " -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number."
echo " For example: $PROG --exec 1 bash"
echo " $PROG --exec docker_bigtop_1 bash"
+ echo " -E, --env-check Check whether required tools has been installed"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
echo " -h, --help"
@@ -155,6 +156,16 @@ execute() {
fi
}
+env-check() {
+ echo "Environment check..."
+ echo "Check docker:"
+ docker -v || exit 1
+ echo "Check docker-compose:"
+ docker-compose -v || exit 1
+ echo "Check ruby:"
+ ruby -v || exit 1
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -169,6 +180,7 @@ while [ $# -gt 0 ]; do
echo "Create requires a number" 1>&2
usage
fi
+ env-check
create $2
shift 2;;
-C|--conf)
@@ -189,6 +201,9 @@ while [ $# -gt 0 ]; do
shift
execute $@
shift $#;;
+ -E|--env-check)
+ env-check
+ shift;;
-p|--provision)
provision
shift;;