You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ev...@apache.org on 2015/04/01 18:00:17 UTC
[1/2] bigtop git commit: BIGTOP-1770. Let the user specify which JDK
package to be installed in vagrantconfig.yaml
Repository: bigtop
Updated Branches:
refs/heads/master c4ebc72b7 -> a5b1ad7d1
BIGTOP-1770. Let the user specify which JDK package to be installed in vagrantconfig.yaml
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/ad784a9a
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/ad784a9a
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/ad784a9a
Branch: refs/heads/master
Commit: ad784a9a0b8e192a1209259319676454e59091d8
Parents: c4ebc72
Author: Evans Ye <ev...@apache.org>
Authored: Mon Mar 23 07:36:05 2015 +0000
Committer: Evans Ye <ev...@apache.org>
Committed: Wed Apr 1 15:59:20 2015 +0000
----------------------------------------------------------------------
bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh | 3 ++-
bigtop-deploy/vm/vagrant-puppet-docker/provision.sh | 4 +---
bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml | 1 +
.../vm/vagrant-puppet-docker/vagrantconfig_debian.yaml | 1 +
bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 6 +++++-
bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml | 1 +
6 files changed, 11 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad784a9a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
index 2661861..d6e95bf 100755
--- a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
@@ -33,6 +33,7 @@ create() {
hadoop_head_node=(`echo "hostname -f" |vagrant ssh ${nodes[0]} |tail -n 1`)
repo=$(get-yaml-config repo)
components="[`echo $(get-yaml-config components) | sed 's/ /, /g'`]"
+ jdk=$(get-yaml-config jdk)
distro=$(get-yaml-config distro)
enable_local_repo=$(get-yaml-config enable_local_repo)
@@ -40,7 +41,7 @@ create() {
for node in ${nodes[*]}; do
(
echo "/bigtop-home/bigtop-deploy/vm/utils/setup-env-$distro.sh $enable_local_repo" |vagrant ssh $node
- echo "/vagrant/provision.sh $hadoop_head_node $repo \"$components\"" |vagrant ssh $node
+ echo "/vagrant/provision.sh $hadoop_head_node $repo \"$components\" $jdk" |vagrant ssh $node
) &
done
wait
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad784a9a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh b/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
index 9dbaa56..48cb968 100755
--- a/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/provision.sh
@@ -26,10 +26,8 @@ ln -s /vagrant/hosts /etc/hosts
# Prepare puppet configuration file
if [ -f /etc/debian_version ] ; then
apt-get -y install puppet-module-puppetlabs-stdlib
- jdk="openjdk-7-jdk"
else
cd /etc/puppet/modules && puppet module install puppetlabs/stdlib
- jdk="java-1.7.0-openjdk-devel.x86_64"
fi
mkdir -p /etc/puppet/hieradata
@@ -39,6 +37,6 @@ cat > /etc/puppet/hieradata/site.yaml << EOF
bigtop::hadoop_head_node: $1
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
bigtop::bigtop_repo_uri: $2
-bigtop::jdk_package_name: $jdk
hadoop_cluster_node::cluster_components: $3
+bigtop::jdk_package_name: $4
EOF
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad784a9a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
index 312bb96..fa55671 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
@@ -13,3 +13,4 @@ namenode_ui_port: "50070"
yarn_ui_port: "8088"
hbase_ui_port: "60010"
enable_local_repo: false
+jdk: "java-1.7.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad784a9a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
index 10a1740..008c15c 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
@@ -13,3 +13,4 @@ namenode_ui_port: "50070"
yarn_ui_port: "8088"
hbase_ui_port: "60010"
enable_local_repo: false
+jdk: "openjdk-7-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad784a9a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
index 2f5413c..9cf0476 100755
--- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
@@ -42,6 +42,10 @@ run_smoke_tests = CONF['run_smoke_tests']
# Force success - not worried if this step fails, since we generally only use it for development.
enable_local_repo = CONF['enable_local_repo']
puts "vagrant conf local repo enabled: #{enable_local_repo}"
+
+# JDK package name
+jdk = CONF['jdk']
+
# master node hostname
bigtop_master = "bigtop1.vagrant"
@@ -58,8 +62,8 @@ cat > /etc/puppet/hieradata/site.yaml << EOF
bigtop::hadoop_head_node: #{bigtop_master}
hadoop::hadoop_storage_dirs: [/data/1, /data/2]
bigtop::bigtop_repo_uri: #{repo}
-bigtop::jdk_package_name: java-1.7.0-openjdk-devel.x86_64
hadoop_cluster_node::cluster_components: #{components}
+bigtop::jdk_package_name: #{jdk}
EOF
SCRIPT
http://git-wip-us.apache.org/repos/asf/bigtop/blob/ad784a9a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
index 7554d12..33ef53b 100644
--- a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
@@ -7,3 +7,4 @@ components: [hadoop, yarn]
enable_local_repo: false
run_smoke_tests: false
distro: centos
+jdk: "java-1.7.0-openjdk-devel.x86_64"
[2/2] bigtop git commit: BIGTOP-1771. Let the user specify what
components to be tested when running smoke-tests by Docker/VM provisioner
Posted by ev...@apache.org.
BIGTOP-1771. Let the user specify what components to be tested when running smoke-tests by Docker/VM provisioner
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a5b1ad7d
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a5b1ad7d
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a5b1ad7d
Branch: refs/heads/master
Commit: a5b1ad7d1a692a25c5960d6ac2971dce5feb73e4
Parents: ad784a9
Author: Evans Ye <ev...@apache.org>
Authored: Thu Mar 26 03:07:51 2015 +0000
Committer: Evans Ye <ev...@apache.org>
Committed: Wed Apr 1 15:59:23 2015 +0000
----------------------------------------------------------------------
bigtop-deploy/vm/utils/smoke-tests.sh | 8 +++++---
bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh | 3 ++-
bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml | 1 +
.../vm/vagrant-puppet-docker/vagrantconfig_debian.yaml | 1 +
bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile | 8 +++++++-
bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml | 3 ++-
6 files changed, 18 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/utils/smoke-tests.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/utils/smoke-tests.sh b/bigtop-deploy/vm/utils/smoke-tests.sh
index 581e5fb..dff51a5 100755
--- a/bigtop-deploy/vm/utils/smoke-tests.sh
+++ b/bigtop-deploy/vm/utils/smoke-tests.sh
@@ -1,5 +1,6 @@
#!/bin/bash
HCFS_USER="hdfs"
+SMOKE_TESTS=${1:-mapreduce,pig}
export HADOOP_CONF_DIR=/etc/hadoop/conf/
export BIGTOP_HOME=/bigtop-home/
@@ -7,6 +8,7 @@ export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
export HIVE_HOME=/usr/lib/hive/
export PIG_HOME=/usr/lib/pig/
export FLUME_HOME=/usr/lib/flume/
+export SQOOP_HOME=/usr/lib/sqoop/
export HIVE_CONF_DIR=/etc/hive/conf/
export JAVA_HOME="/usr/lib/jvm/java-openjdk/"
export MAHOUT_HOME="/usr/lib/mahout"
@@ -17,8 +19,8 @@ su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
if [ -f /etc/debian_version ] ; then
- apt-get -y install pig hive flume mahout
+ apt-get -y install pig hive flume mahout sqoop
else
- yum install -y pig hive flume mahout
+ yum install -y pig hive flume mahout sqoop
fi
-cd /bigtop-home/bigtop-tests/smoke-tests && ./gradlew clean compileGroovy test -Dsmoke.tests=mapreduce,pig --info
+cd /bigtop-home/bigtop-tests/smoke-tests && ./gradlew clean compileGroovy test -Dsmoke.tests=$SMOKE_TESTS --info
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
index d6e95bf..63c18fd 100755
--- a/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/docker-hadoop.sh
@@ -64,7 +64,8 @@ provision() {
smoke-tests() {
nodes=(`vagrant status |grep running |grep -v image |awk '{print $1}'`)
- echo "/bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh" |vagrant ssh ${nodes[0]}
+ smoke_test_components="`echo $(get-yaml-config smoke_test_components) | sed 's/ /,/g'`"
+ echo "/bigtop-home/bigtop-deploy/vm/utils/smoke-tests.sh \"$smoke_test_components\"" |vagrant ssh ${nodes[0]}
}
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
index fa55671..5eeeb45 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig.yaml
@@ -13,4 +13,5 @@ namenode_ui_port: "50070"
yarn_ui_port: "8088"
hbase_ui_port: "60010"
enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
jdk: "java-1.7.0-openjdk-devel.x86_64"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
index 008c15c..012fd36 100644
--- a/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-docker/vagrantconfig_debian.yaml
@@ -13,4 +13,5 @@ namenode_ui_port: "50070"
yarn_ui_port: "8088"
hbase_ui_port: "60010"
enable_local_repo: false
+smoke_test_components: [mapreduce, pig]
jdk: "openjdk-7-jdk"
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
index 9cf0476..5384b8c 100755
--- a/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/Vagrantfile
@@ -38,6 +38,9 @@ components = CONF['components']
# Whether to run smoke tests
run_smoke_tests = CONF['run_smoke_tests']
+# Smoke test Components to run
+smoke_test_components = CONF['smoke_test_components'].join(',')
+
# This is a update to allow dev packages
# Force success - not worried if this step fails, since we generally only use it for development.
enable_local_repo = CONF['enable_local_repo']
@@ -120,7 +123,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
if run_smoke_tests then
if i==num_instances then
puts "creating provisioner directive for running tests"
- bigtop.vm.provision "shell", path: "../utils/smoke-tests.sh"
+ bigtop.vm.provision :shell do |shell|
+ shell.path = "../utils/smoke-tests.sh"
+ shell.args = ["#{smoke_test_components}"]
+ end
else
puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}"
end
http://git-wip-us.apache.org/repos/asf/bigtop/blob/a5b1ad7d/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
index 33ef53b..0b3b84d 100644
--- a/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
+++ b/bigtop-deploy/vm/vagrant-puppet-vm/vagrantconfig.yaml
@@ -3,8 +3,9 @@ number_cpus: 1
box: "puppetlabs/centos-6.6-64-nocm"
repo: "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"
num_instances: 1
+distro: centos
components: [hadoop, yarn]
enable_local_repo: false
run_smoke_tests: false
-distro: centos
+smoke_test_components: [mapredcue, pig]
jdk: "java-1.7.0-openjdk-devel.x86_64"