You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2019/03/14 17:13:10 UTC
[bigtop] branch master updated: BIGTOP-3189. Allow to specify more
properties/options at command line for docker provisioner (#491)
This is an automated email from the ASF dual-hosted git repository.
evansye pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 21297aa BIGTOP-3189. Allow to specify more properties/options at command line for docker provisioner (#491)
21297aa is described below
commit 21297aa7b5afffc8b0d2c44e0fb7e44b2a32ff69
Author: Evans Ye <ev...@apache.org>
AuthorDate: Fri Mar 15 01:13:06 2019 +0800
BIGTOP-3189. Allow to specify more properties/options at command line for docker provisioner (#491)
---
build.gradle | 64 ++++++++++++++++++++++++++++++-----
packages.gradle | 2 +-
provisioner/docker/docker-hadoop.sh | 67 +++++++++++++++++++++++++------------
3 files changed, 101 insertions(+), 32 deletions(-)
diff --git a/build.gradle b/build.gradle
index d7e1045..dafa5c5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -257,10 +257,36 @@ task "toolchain-devtools"(type:Exec,
}
task "docker-provisioner"(type:Exec,
- description: 'Provision a Bigtop cluster on Docker container(s). Default to CentOS and 1 node.\nCreate a 3 node cluster:\n $ ./gradlew -Pnum_instances=3 docker-provisioner\nRun smoke tests on 3 node cluster:\n $ ./gradlew -Pnum_instances=3 -Prun_smoke_tests=true docker-provisioner\nSpecify configuration file (located in provisioner/docker/):\n $ ./gradlew -Pconfig=config_debian8.yaml docker-provisioner',
+ description: '''
+Provision a Bigtop stack cluster on Docker container(s). Default to CentOS and 1 node.
+Properties:
+ -Pconfig=[CONFIG_FILE] (located under provisioner/docker/)
+ -Penable_local_repo
+ -Pimage=[DOCKER_IMAGE] (overwrites -POS and -Pprefix)
+ -Pnum_instances=[NUM_INSTANCES]
+ -Pnexus=[NEXUS_URL] (NEXUS_URL is optional)
+ -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3]
+ -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...]
+ -Prun_smoke_tests (run test components defined in config file)
+ -Psmoke_tests=[COMPONENTS]
+ -Pstack=[COMPONENTS]
+
+For detailed explaination, please refer to the helper message:
+ $ provisioner/docker/docker-hadoop.sh --help
+
+Examples:
+Create a 3 node cluster:
+ $ ./gradlew -Pnum_instances=3 docker-provisioner
+Run smoke tests on 3 node cluster:
+ $ ./gradlew -Pnum_instances=3 -Prun_smoke_tests docker-provisioner
+Specify custom configuration file:
+ $ ./gradlew -Pconfig=config_debian-9.yaml docker-provisioner
+''',
group: DEPLOY_GROUP) {
def _config = project.hasProperty("config") ? config : "config.yaml"
def _num_instances = project.hasProperty("num_instances") ? num_instances : 1
+ def _OS = project.hasProperty("OS") ? OS : 'centos-7'
+ def _prefix = project.hasProperty("prefix") ? prefix : 'trunk'
def command = [
'./docker-hadoop.sh',
'-C', _config,
@@ -279,6 +305,18 @@ task "docker-provisioner"(type:Exec,
}
if ( project.hasProperty("nexus") ) {
command.add('--nexus')
+ command.add("${nexus}")
+ }
+ if ( project.hasProperty("enable_local_repo") ) {
+ command.add('--enable-local-repo')
+ }
+ if (project.hasProperty("OS") || project.hasProperty("prefix")) {
+ command.add('--image')
+ command.add("bigtop/puppet:${_prefix}-${_OS}")
+ }
+ if (project.hasProperty("image")) {
+ command.add('--image')
+ command.add("${image}")
}
workingDir 'provisioner/docker'
commandLine command
@@ -450,10 +488,14 @@ allprojects {
}
task "bigtop-puppet"(type:Exec,
- description: 'Build bigtop/puppet images.\n' +
- 'Usage:\n $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=STRING_TO_PREFIX bigtop-puppet\n' +
- 'Example:\n $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-puppet\n' +
- 'The built image name: bigtop/puppet:1.0.0-debian-9',
+ description: '''
+Build bigtop/puppet images
+Usage:
+ $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] bigtop-puppet
+Example:
+ $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-puppet
+ The built image name: bigtop/puppet:1.0.0-debian-9
+ ''',
group: DOCKERBUILD_GROUP) {
def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
def _OS = project.hasProperty("OS") ? OS : "centos-7"
@@ -465,10 +507,14 @@ task "bigtop-puppet"(type:Exec,
}
task "bigtop-slaves"(type:Exec,
- description: 'Build bigtop/slaves images.\n' +
- 'Usage:\n $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=STRING_TO_PREFIX bigtop-slaves\n' +
- 'Example:\n $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-slaves\n' +
- 'The built image name: bigtop/slaves:1.0.0-debian-9',
+ description: '''
+Build bigtop/slaves images
+Usage:
+ $ ./gradlew -POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] -Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] bigtop-slaves
+Example:
+ $ ./gradlew -POS=debian-9 -Pprefix=1.0.0 bigtop-slaves
+ The built image name: bigtop/slaves:1.0.0-debian-9
+ ''',
group: DOCKERBUILD_GROUP) {
def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
def _OS = project.hasProperty("OS") ? OS : "centos-7"
diff --git a/packages.gradle b/packages.gradle
index 202aa9d..2e069c2 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -629,7 +629,7 @@ def genTasks = { target ->
task "$target-pkg-ind" (
description: "Invoking a native binary packaging for $target in Docker. Usage: \$ ./gradlew " +
"-POS=[centos-7|fedora-26|debian-9|ubuntu-16.04|opensuse-42.3] " +
- "-Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] $target-pkg-ind" +
+ "-Pprefix=[trunk|1.2.1|1.2.0|1.1.0|...] $target-pkg-ind " +
"-Pnexus=[true|false]",
group: PACKAGES_GROUP) doLast {
def _prefix = project.hasProperty("prefix") ? prefix : "trunk"
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 2bc6fd9..bf8b53d 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -17,20 +17,29 @@
usage() {
echo "usage: $PROG [-C file] args"
- echo " -C file Use alternate file for config.yaml"
+ echo " -C file - Use alternate file for config.yaml"
echo " commands:"
- echo " -c NUM_INSTANCES, --create NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
- echo " -d, --destroy Destroy the cluster"
- echo " -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number."
- echo " For example: $PROG --exec 1 bash"
- echo " $PROG --exec docker_bigtop_1 bash"
- echo " -E, --env-check Check whether required tools has been installed"
- echo " -l, --list List out container status for the cluster"
- echo " -p, --provision Deploy configuration changes"
- echo " -n, --nexus [NEXUS_URL] Configure Nexus proxy to speed up test execution"
- echo " If no NEXUS_URL specified, default to http://NEXUS_IP:8081/nexus,"
- echo " where NEXUS_IP is the ip of the container named nexus"
- echo " -s, --smoke-tests Run Bigtop smoke tests"
+ echo " -c NUM_INSTANCES, --create NUM_INSTANCES - Create a Docker based Bigtop Hadoop cluster"
+ echo " -d, --destroy - Destroy the cluster"
+ echo " -e, --exec INSTANCE_NO|INSTANCE_NAME - Execute command on a specific instance. Instance can be specified by name or number"
+ echo " For example: $PROG --exec 1 bash"
+ echo " $PROG --exec docker_bigtop_1 bash"
+ echo " -E, --env-check - Check whether required tools has been installed"
+ echo " -k, --stack COMPONENTS - Overwrite the components to deploy defined in config file"
+ echo " COMPONENTS is a comma separated string"
+ echo " For example: $PROG -c 3 --stack hdfs"
+ echo " $PROG -c 3 --stack 'hdfs, yarn, spark'"
+ echo " -l, --list - List out container status for the cluster"
+ echo " -n, --nexus NEXUS_URL - Configure Nexus proxy to speed up test execution"
+ echo " NEXUS_URL is optional. If not specified, default to http://NEXUS_IP:8081/nexus"
+ echo " Where NEXUS_IP is the ip of container named nexus"
+ echo " -p, --provision - Deploy configuration changes"
+ echo " -r, --enable-local-repo - Whether to use repo created at local file system. You can get one by $ ./gradlew repo"
+ echo " -s, --smoke-tests COMPONENTS - Run Bigtop smoke tests"
+ echo " COMPONENTS is optional. If not specified, default to smoke_test_components in config file"
+ echo " COMPONENTS is a comma separated string"
+ echo " For example: $PROG -c 3 --smoke-tests hdfs"
+ echo " $PROG -c 3 --smoke-tests 'hdfs, yarn, mapreduce'"
echo " -h, --help"
exit 1
}
@@ -47,7 +56,9 @@ create() {
echo > ./config/hiera.yaml
echo > ./config/hosts
# set correct image name based on running architecture
- image_name=$(get-yaml-config docker image)
+ if [ -z ${image_name+x} ]; then
+ image_name=$(get-yaml-config docker image)
+ fi
running_arch=$(uname -m)
if [ "x86_64" == ${running_arch} ]; then
image_name=${image_name}
@@ -74,7 +85,9 @@ create() {
components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
fi
distro=$(get-yaml-config distro)
- enable_local_repo=$(get-yaml-config enable_local_repo)
+ if [ -z ${enable_local_repo+x} ]; then
+ enable_local_repo=$(get-yaml-config enable_local_repo)
+ fi
generate-config "$hadoop_head_node" "$repo" "$components"
# Start provisioning
@@ -268,6 +281,20 @@ while [ $# -gt 0 ]; do
-E|--env-check)
env-check
shift;;
+ -k|--stack)
+ if [ $# -lt 2 ]; then
+ log "No stack specified"
+ usage
+ fi
+ components="[$2]"
+ shift 2;;
+ -i|--image)
+ if [ $# -lt 2 ]; then
+ log "No image specified"
+ usage
+ fi
+ image_name=$2
+ shift 2;;
-l|--list)
list
shift;;
@@ -285,16 +312,12 @@ while [ $# -gt 0 ]; do
shift 2
fi
;;
- -k|--stack)
- if [ $# -lt 2 ]; then
- log "No stack specified"
- usage
- fi
- components="[$2]"
- shift 2;;
-p|--provision)
provision
shift;;
+ -r|--enable-local-repo)
+ enable_local_repo=true
+ shift;;
-s|--smoke-tests)
if [ $# -lt 2 ] || [[ $2 == -* ]]; then
shift