You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2016/04/23 10:24:36 UTC
[1/4] bigtop git commit: BIGTOP-2273. Add exec command into
docker-hadoop.sh [Forced Update!]
Repository: bigtop
Updated Branches:
refs/heads/BIGTOP-2253 52d5e65e9 -> 24aa75744 (forced update)
BIGTOP-2273. Add exec command into docker-hadoop.sh
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/43acb843
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/43acb843
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/43acb843
Branch: refs/heads/BIGTOP-2253
Commit: 43acb8434eb15cc11b1f9c7e923c9be0b2c8a318
Parents: d19b5c7
Author: Evans Ye <ev...@apache.org>
Authored: Sun Jan 24 13:49:36 2016 +0800
Committer: Evans Ye <ev...@apache.org>
Committed: Sat Apr 23 08:23:05 2016 +0000
----------------------------------------------------------------------
provisioner/docker/README.md | 23 +++++++++++++++++-----
provisioner/docker/docker-hadoop.sh | 33 ++++++++++++++++++++++++++++----
2 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/43acb843/provisioner/docker/README.md
----------------------------------------------------------------------
diff --git a/provisioner/docker/README.md b/provisioner/docker/README.md
index 51bac85..3866ee8 100644
--- a/provisioner/docker/README.md
+++ b/provisioner/docker/README.md
@@ -60,19 +60,31 @@ service docker start
./docker-hadoop.sh --destroy
```
-3) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
+3) Get into the first container (the master)
+
+```
+./docker-hadoop.sh --exec 1 bash
+```
+
+4) Execute a command on the second container
+
+```
+./docker-hadoop.sh --exec 2 hadoop fs -ls /
+```
+
+5) Update your cluster after doing configuration changes on ./config. (re-run puppet apply)
```
./docker-hadoop.sh --provision
```
-4) Run Bigtop smoke tests
+6) Run Bigtop smoke tests
```
./docker-hadoop.sh --smoke-tests
```
-5) Chain your operations with-in one command.
+7) Chain your operations with-in one command.
```
./docker-hadoop.sh --create 5 --smoke-tests --destroy
@@ -84,7 +96,7 @@ Commands will be executed by following order:
create 5 node cluster => run smoke tests => destroy the cluster
```
-6) See helper message:
+8) See helper message:
```
./docker-hadoop.sh -h
@@ -92,9 +104,10 @@ usage: docker-hadoop.sh [-C file ] args
-C file Use alternate file for config.yaml
commands:
-c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster
+ -d, --destroy Destroy the cluster
+ -e, --exec INSTANCE_NO COMMAND Execute command on a specific instance. For example: docker-hadoop.sh --exec 1 bash
-p, --provision Deploy configuration changes
-s, --smoke-tests Run Bigtop smoke tests
- -d, --destroy Destroy the cluster
-h, --help
```
http://git-wip-us.apache.org/repos/asf/bigtop/blob/43acb843/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index e8bd767..d8392ba 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -22,9 +22,12 @@ usage() {
echo " -C file Use alternate file for config.yaml"
echo " commands:"
echo " -c NUM_INSTANCES, --create=NUM_INSTANCES Create a Docker based Bigtop Hadoop cluster"
+ echo " -d, --destroy Destroy the cluster"
+ echo " -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number."
+ echo " For example: $PROG --exec 1 bash"
+ echo " $PROG --exec docker_bigtop_1 bash"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
- echo " -d, --destroy Destroy the cluster"
echo " -h, --help"
exit 1
}
@@ -137,6 +140,20 @@ get-yaml-config() {
cat ${yamlconf} | $RUBY_EXE -ryaml -e "$RUBY_SCRIPT" | tr -d '\r'
}
+execute() {
+ re='^[0-9]+$'
+ if [[ $1 =~ $re ]] ; then
+ no=$1
+ shift
+ nodes=(`docker-compose ps -q`)
+ docker exec -ti ${nodes[$((no-1))]} $@
+ else
+ name=$1
+ shift
+ docker exec -ti $name $@
+ fi
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -160,15 +177,23 @@ while [ $# -gt 0 ]; do
fi
yamlconf=$2
shift 2;;
+ -d|--destroy)
+ destroy
+ shift;;
+ -e|--exec)
+ if [ $# -lt 3 ]; then
+ echo "exec command takes 2 parameters: 1) instance no 2) command to be executed" 1>&2
+ usage
+ fi
+ shift
+ execute $@
+ shift $#;;
-p|--provision)
provision
shift;;
-s|--smoke-tests)
smoke-tests
shift;;
- -d|--destroy)
- destroy
- shift;;
-h|--help)
usage
shift;;
[2/4] bigtop git commit: BIGTOP-2386. Got permission denied when
running new Docker provisioner as non-root
Posted by ev...@apache.org.
BIGTOP-2386. Got permission denied when running new Docker provisioner as non-root
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/c3844399
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/c3844399
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/c3844399
Branch: refs/heads/BIGTOP-2253
Commit: c3844399abf144918ec8160c7e246fdfad241f5f
Parents: 43acb84
Author: Evans Ye <ev...@apache.org>
Authored: Tue Apr 12 18:53:01 2016 +0000
Committer: Evans Ye <ev...@apache.org>
Committed: Sat Apr 23 08:23:09 2016 +0000
----------------------------------------------------------------------
provisioner/docker/docker-hadoop.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/c3844399/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index d8392ba..2323456 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -34,7 +34,7 @@ usage() {
create() {
# Create a shared /etc/hosts and hiera.yaml that will be both mounted to each container soon
- mkdir config 2> /dev/null
+ mkdir -p config/hieradata 2> /dev/null
cat /dev/null > ./config/hiera.yaml
cat /dev/null > ./config/hosts
export DOCKER_IMAGE=$(get-yaml-config docker image)
@@ -67,7 +67,8 @@ create() {
generate-hosts() {
nodes=(`docker-compose ps -q`)
for node in ${nodes[*]}; do
- echo `docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node` >> ./config/hosts
+ entry=`docker inspect --format "{{.NetworkSettings.IPAddress}} {{.Config.Hostname}}.{{.Config.Domainname}}" $node`
+ docker exec ${nodes[0]} bash -c "echo $entry >> /etc/hosts"
done
wait
[4/4] bigtop git commit: BIGTOP-2312. Add environment check
Posted by ev...@apache.org.
BIGTOP-2312. Add environment check
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/24aa7574
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/24aa7574
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/24aa7574
Branch: refs/heads/BIGTOP-2253
Commit: 24aa757442d4e144f28398cf4bc697e601eaeeff
Parents: 9d9f086
Author: Evans Ye <ev...@apache.org>
Authored: Sat Apr 23 08:14:39 2016 +0000
Committer: Evans Ye <ev...@apache.org>
Committed: Sat Apr 23 08:23:54 2016 +0000
----------------------------------------------------------------------
provisioner/docker/docker-hadoop.sh | 15 +++++++++++++++
1 file changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/24aa7574/provisioner/docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/provisioner/docker/docker-hadoop.sh b/provisioner/docker/docker-hadoop.sh
index 2323456..93c8c9c 100755
--- a/provisioner/docker/docker-hadoop.sh
+++ b/provisioner/docker/docker-hadoop.sh
@@ -26,6 +26,7 @@ usage() {
echo " -e, --exec INSTANCE_NO|INSTANCE_NAME Execute command on a specific instance. Instance can be specified by name or number."
echo " For example: $PROG --exec 1 bash"
echo " $PROG --exec docker_bigtop_1 bash"
+ echo " -E, --env-check Check whether required tools has been installed"
echo " -p, --provision Deploy configuration changes"
echo " -s, --smoke-tests Run Bigtop smoke tests"
echo " -h, --help"
@@ -155,6 +156,16 @@ execute() {
fi
}
+env-check() {
+ echo "Environment check..."
+ echo "Check docker:"
+ docker -v || exit 1
+ echo "Check docker-compose:"
+ docker-compose -v || exit 1
+ echo "Check ruby:"
+ ruby -v || exit 1
+}
+
PROG=`basename $0`
if [ $# -eq 0 ]; then
@@ -169,6 +180,7 @@ while [ $# -gt 0 ]; do
echo "Create requires a number" 1>&2
usage
fi
+ env-check
create $2
shift 2;;
-C|--conf)
@@ -189,6 +201,9 @@ while [ $# -gt 0 ]; do
shift
execute $@
shift $#;;
+ -E|--env-check)
+ env-check
+ shift;;
-p|--provision)
provision
shift;;
[3/4] bigtop git commit: BIGTOP-2279. Add provisioner into
release-assembly.xml
Posted by ev...@apache.org.
BIGTOP-2279. Add provisioner into release-assembly.xml
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/9d9f086b
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/9d9f086b
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/9d9f086b
Branch: refs/heads/BIGTOP-2253
Commit: 9d9f086b5bba80fd04ce6bdc201595a07ac601a2
Parents: c384439
Author: Evans Ye <ev...@apache.org>
Authored: Wed Apr 13 02:56:54 2016 +0800
Committer: Evans Ye <ev...@apache.org>
Committed: Sat Apr 23 08:23:49 2016 +0000
----------------------------------------------------------------------
src/assembly/release-assembly.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/9d9f086b/src/assembly/release-assembly.xml
----------------------------------------------------------------------
diff --git a/src/assembly/release-assembly.xml b/src/assembly/release-assembly.xml
index 71c1121..0d2a323 100644
--- a/src/assembly/release-assembly.xml
+++ b/src/assembly/release-assembly.xml
@@ -37,6 +37,7 @@
<include>gradle*</include>
<include>docs/**</include>
<include>docker/**</include>
+ <include>provisioner/**</include>
<include>src/**</include>
<include>test/**</include>
<include>*.bom</include>