You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by ja...@apache.org on 2014/10/08 03:54:17 UTC
git commit: BIGTOP-1449. Add RC Testing to vagrant recipe.
Repository: bigtop
Updated Branches:
refs/heads/master cff10fbe2 -> 04802cc59
BIGTOP-1449. Add RC Testing to vagrant recipe.
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/04802cc5
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/04802cc5
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/04802cc5
Branch: refs/heads/master
Commit: 04802cc5960c06108219e1ec886c895367798f7b
Parents: cff10fb
Author: jay@apache.org <ja...@apache.org>
Authored: Tue Sep 30 21:46:40 2014 -0400
Committer: jay@apache.org <jayunit100>
Committed: Tue Oct 7 21:53:38 2014 -0400
----------------------------------------------------------------------
bigtop-deploy/vm/vagrant-puppet/Vagrantfile | 32 ++++++++++++++-------
bigtop-deploy/vm/vagrant-puppet/provision.sh | 9 ++++--
bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh | 21 ++++++++++++++
bigtop-tests/smoke-tests/README | 18 ++++++++++--
bigtop-tests/smoke-tests/build.gradle | 14 +++++----
5 files changed, 74 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/04802cc5/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
index c6825ad..ffe23a9 100755
--- a/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
+++ b/bigtop-deploy/vm/vagrant-puppet/Vagrantfile
@@ -20,14 +20,16 @@ VAGRANTFILE_API_VERSION = "2"
Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
vmbox = "puppetlab-centos-64-nocm"
+ # Repository
+ # Example for testing a Release candidate.
+ repo = "http://bigtop01.cloudera.org:8080/view/Releases/job/Bigtop-0.8.0/label=centos6/6/artifact/output/"
+ # repo = "http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64"
+
# Virtual box implementation
vmboxurl = "http://puppet-vagrant-boxes.puppetlabs.com/centos-64-x64-vbox4210-nocm.box"
# number of instances
- num_instances = 3
-
- # the bigtop puppet recipes
- bigtop_puppet = "../../puppet"
+ num_instances = 1
# manage /etc/hosts by hostmanager plugin(https://github.com/smdahlen/vagrant-hostmanager)
# use vagrant-cachier to cache packages at local(https://github.com/fgrehm/vagrant-cachier)
@@ -56,29 +58,39 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
end
bigtop.vm.provider :virtualbox do |vb|
- vb.customize ["modifyvm", :id, "--memory", "1024"]
+ vb.customize ["modifyvm", :id, "--memory", "10024"]
end
bigtop.vm.network :private_network, ip: bigtop_ip
bigtop.vm.hostname = bigtop_hostname
- bigtop.vm.synced_folder bigtop_puppet, "/bigtop-puppet"
+ # three levels up is the bigtop "home" directory.
+ # the current directory has puppet recipes which we need for provisioning.
+ bigtop.vm.synced_folder "../../../", "/bigtop-home"
# shell provisioner to setup basic vm environment
bigtop.vm.provision :shell do |shell|
shell.path = "provision.sh"
- shell.args = bigtop_master
+ shell.args = [ bigtop_master , repo ]
end
bigtop.vm.provision :hostmanager
# deploy Apache Hadoop and it's ecosystem
bigtop.vm.provision :puppet do |puppet|
- puppet.module_path = bigtop_puppet + "/modules"
- puppet.manifests_path = bigtop_puppet + "/manifests"
+ puppet.module_path = "../../puppet/modules/"
+ puppet.manifests_path = "../../puppet/manifests/"
puppet.manifest_file = "site.pp"
- puppet.options = "--debug --confdir=/bigtop-puppet"
+ puppet.options = "--debug --confdir=/bigtop-home/bigtop-deploy/puppet/"
end
+
+ if i==num_instances then
+ puts "creating provisioner directive for running tests"
+ bigtop.vm.provision "shell", path: "smoke-tests.sh"
+ else
+ puts "Not creating provisioner directive for tests yet... only on vm #{i} of #{num_instances}"
+ end
+
end
end
http://git-wip-us.apache.org/repos/asf/bigtop/blob/04802cc5/bigtop-deploy/vm/vagrant-puppet/provision.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/provision.sh b/bigtop-deploy/vm/vagrant-puppet/provision.sh
index 1cf3724..fa0571b 100755
--- a/bigtop-deploy/vm/vagrant-puppet/provision.sh
+++ b/bigtop-deploy/vm/vagrant-puppet/provision.sh
@@ -23,11 +23,13 @@ service iptables stop
chkconfig iptables off
cat /dev/null > /etc/hosts
+echo "Bigtop yum repo = $2"
+
# Prepare puppet configuration file
-cat > /bigtop-puppet/config/site.csv << EOF
+cat > /bigtop-home/bigtop-deploy/puppet/config/site.csv << EOF
hadoop_head_node,$1
hadoop_storage_dirs,/data/1,/data/2
-bigtop_yumrepo_uri,http://bigtop.s3.amazonaws.com/releases/0.7.0/redhat/6/x86_64
+bigtop_yumrepo_uri,$2
jdk_package_name,java-1.7.0-openjdk-devel.x86_64
components,hadoop,hbase,yarn,mapred-app
EOF
@@ -39,3 +41,6 @@ mkdir -p /data/{1,2}
yum -y install rng-tools
sed -i.bak 's/EXTRAOPTIONS=\"\"/EXTRAOPTIONS=\"-r \/dev\/urandom\"/' /etc/sysconfig/rngd
service rngd start
+
+echo "Now installing gradle"
+cd /bigtop-home && puppet apply --modulepath=./ -e "include bigtop_toolchain::gradle" # alias gradle=/usr/local/gradle/bin/gradle
http://git-wip-us.apache.org/repos/asf/bigtop/blob/04802cc5/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh
----------------------------------------------------------------------
diff --git a/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh b/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh
new file mode 100755
index 0000000..d02c128
--- /dev/null
+++ b/bigtop-deploy/vm/vagrant-puppet/smoke-tests.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+HCFS_USER="hdfs"
+gradle="/usr/local/gradle/bin/gradle"
+
+export HADOOP_CONF_DIR=/etc/hadoop/conf/
+export BIGTOP_HOME=/bigtop-home/
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapreduce/
+export HIVE_HOME=/usr/lib/hive/
+export PIG_HOME=/usr/lib/pig/
+export FLUME_HOME=/usr/lib/flume/
+export HIVE_CONF_DIR=/etc/hive/conf/
+export JAVA_HOME="/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.65.x86_64/"
+export MAHOUT_HOME="/usr/lib/mahout"
+export ITEST="0.7.0"
+
+su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant'
+su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root'
+
+yum install -y pig hive flume mahout
+cd /bigtop-home/bigtop-tests/smoke-tests && $gradle compileGroovy test -Dsmoke.tests=pig --info
http://git-wip-us.apache.org/repos/asf/bigtop/blob/04802cc5/bigtop-tests/smoke-tests/README
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/README b/bigtop-tests/smoke-tests/README
index c8ac345..f682b61 100644
--- a/bigtop-tests/smoke-tests/README
+++ b/bigtop-tests/smoke-tests/README
@@ -16,14 +16,28 @@ So overall, this framework makes testing with apache bigtop easy as well as fun!
## Running
+You need to export several environment variables. For example,
+```
+export HADOOP_CONF_DIR=/etc/hadoop/conf/
+export BIGTOP_HOME=/bigtop-home/
+export HADOOP_MAPRED_HOME=/usr/lib/hadoop-mapred/
+export HIVE_HOME=/usr/lib/hive/
+export HIVE_CONF_DIR=/etc/hive/conf/
+export JAVA_HOME="/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.65.x86_64/"
+export MAHOUT_HOME="/usr/lib/mahout"
+export ITEST="0.7.0"
+```
+
+Then, simply invoke the tests including the smoke.tests you want to run, as described below.
+
Each directory is a gradle "subproject" mapping to an ecosystem component.
If you specify, for example, "flume", then the flume test runs. To test hive as well,
you would specify "flume,hive", and so on. An example is below:
```
- gradle compileGroovy test -Dsmoke-tests=flume,hive --info
+ gradle compileGroovy test -Dsmoke.tests=flume,hive --info
```
-The --info option is a critical feature here: You can easily debug your tests this way,
+The --info option is a nice feature to have here: You can easily debug your tests this way,
just make sure you have log4j.properties in your conf/ directory locally.
## Adding more tests
http://git-wip-us.apache.org/repos/asf/bigtop/blob/04802cc5/bigtop-tests/smoke-tests/build.gradle
----------------------------------------------------------------------
diff --git a/bigtop-tests/smoke-tests/build.gradle b/bigtop-tests/smoke-tests/build.gradle
index 7907600..44a52e8 100644
--- a/bigtop-tests/smoke-tests/build.gradle
+++ b/bigtop-tests/smoke-tests/build.gradle
@@ -27,8 +27,6 @@ subprojects {
ext.hadoopVersion = '2.3.0'
//Note you need a snapshot itest to run this.
- //if you dont have it, change to 0.7.0.
- ext.itestVersion = '0.8.0-SNAPSHOT'
ext.doExclude = { String filename ->
//print("Exclude? ${filename} ... ")
def keep_this_test = true;
@@ -67,9 +65,13 @@ subprojects {
}
/**
- * In general, java_home should be declared in all cases.
+ * Two important environment variables.
+ * java_home should be declared in all cases.
+ * itest can equal = 0.7.0, 0.8.0-SNAPSHOT,
*/
- checkEnv(["JAVA_HOME"])
-test.dependsOn compileGroovy
-compileGroovy.dependsOn clean
+ checkEnv(["JAVA_HOME","ITEST"])
+ ext.itestVersion = System.getenv("ITEST")
+
+ test.dependsOn compileGroovy
+ compileGroovy.dependsOn clean
}