You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2015/02/10 22:56:23 UTC
[1/3] bigtop git commit: BIGTOP-1502. Improve puppet deployment for
new gridgain component
Repository: bigtop
Updated Branches:
refs/heads/master f506701c8 -> 459e663bc
BIGTOP-1502. Improve puppet deployment for new gridgain component
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/459e663b
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/459e663b
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/459e663b
Branch: refs/heads/master
Commit: 459e663bcd403ff5f3b185e5abbc3d67dcc4df85
Parents: 098f67f
Author: Konstantin Boudnik <co...@apache.org>
Authored: Mon Feb 9 21:33:47 2015 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Tue Feb 10 13:56:05 2015 -0800
----------------------------------------------------------------------
bigtop-deploy/puppet/README.md | 4 ++
bigtop-deploy/puppet/manifests/cluster.pp | 4 ++
.../modules/gridgain-hadoop/manifests/init.pp | 28 +++++++++
.../gridgain-hadoop/templates/core-site.xml | 65 ++++++++++++++++++++
.../gridgain-hadoop/templates/mapred-site.xml | 62 +++++++++++++++++++
5 files changed, 163 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/459e663b/bigtop-deploy/puppet/README.md
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/README.md b/bigtop-deploy/puppet/README.md
index edab477..d29cefb 100644
--- a/bigtop-deploy/puppet/README.md
+++ b/bigtop-deploy/puppet/README.md
@@ -124,3 +124,7 @@ And run the following on those nodes:
# EOF
# puppet apply -d --modulepath="bigtop-deploy/puppet/modules:/etc/puppet/modules" bigtop-deploy/puppet/manifests/site.pp
</pre>
+
+When gridgain-hadoop accelerator is deployed the client configs are placed under
+/etc/hadoop/gridgain.client.conf. All one needs to do to run Mapreduce jobs on gridgain-hadoop grid
+is to set HADOOP_CONF_DIR=/etc/hadoop/gridgain.client.conf in the client session.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/459e663b/bigtop-deploy/puppet/manifests/cluster.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index d4bae8a..c862948 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -64,6 +64,10 @@ class hadoop_worker_node (
include hadoop-hbase::server
}
+ if ($all or "gridgain-hadoop" in $components) {
+ gridgain-hadoop::server { "gridgain-hadoop-node": }
+ }
+
### If mapred is not installed, yarn can fail.
### So, when we install yarn, we also need mapred for now.
### This dependency should be cleaned up eventually.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/459e663b/bigtop-deploy/puppet/modules/gridgain-hadoop/manifests/init.pp
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/gridgain-hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/gridgain-hadoop/manifests/init.pp
index 7088eca..c130832 100644
--- a/bigtop-deploy/puppet/modules/gridgain-hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/gridgain-hadoop/manifests/init.pp
@@ -28,6 +28,34 @@ class gridgain-hadoop {
require => Package["gridgain-hadoop"],
}
+ file { "/etc/hadoop/gridgain.client.conf":
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0755',
+ require => Package["gridgain-hadoop-service"],
+ }
+ file { "/etc/hadoop/gridgain.client.conf/core-site.xml":
+ content => template('gridgain-hadoop/core-site.xml'),
+ require => [File["/etc/hadoop/gridgain.client.conf"]],
+ }
+ file {
+ "/etc/hadoop/gridgain.client.conf/mapred-site.xml":
+ content => template('gridgain-hadoop/mapred-site.xml'),
+ require => [File["/etc/hadoop/gridgain.client.conf"]],
+ }
+## let's make sure that gridgain-hadoop libs are linked properly
+ file {'/usr/lib/hadoop/lib/gridgain-core.jar':
+ ensure => link,
+ target => '/usr/lib/gridgain-hadoop/libs/gridgain-core.jar',
+ require => [Package["gridgain-hadoop-service"]],
+ }
+ file {'/usr/lib/hadoop/lib/gridgain-hadoop.jar':
+ ensure => link,
+ target => '/usr/lib/gridgain-hadoop/libs/gridgain-hadoop/gridgain-hadoop.jar',
+ require => [Package["gridgain-hadoop-service"]],
+ }
+
service { "gridgain-hadoop":
ensure => running,
require => [ Package["gridgain-hadoop", "gridgain-hadoop-service"], File["/etc/default/gridgain-hadoop"] ],
http://git-wip-us.apache.org/repos/asf/bigtop/blob/459e663b/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/core-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/core-site.xml b/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/core-site.xml
new file mode 100644
index 0000000..7e39015
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/core-site.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!--
+ This template file contains settings needed to run Apache Hadoop jobs
+ with GridGain's distributed in-memory file system GGFS.
+
+ You can replace '$HADOOP_HOME/etc/hadoop/core-site.xml' file with this one
+ to work with GridGain GGFS nodes running on localhost (these local nodes can be
+ a part of distributed cluster though). To work with file system on remote
+ hosts you need to change the host of file system URI to any host running
+ GridGain's GGFS node.
+
+ Note that GridGain jars must be in Apache Hadoop client classpath to work
+ with this configuration.
+
+ Run script '$GRIDGAIN_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup.
+-->
+
+<configuration>
+ <!--
+ Set default file system to GGFS instance named "ggfs" configured in GridGain.
+ -->
+ <property>
+ <name>fs.defaultFS</name>
+ <value>ggfs://ggfs@localhost</value>
+ </property>
+
+ <!--
+ Set Hadoop 1.* file system implementation class for GGFS.
+ -->
+ <property>
+ <name>fs.ggfs.impl</name>
+ <value>org.gridgain.grid.ggfs.hadoop.v1.GridGgfsHadoopFileSystem</value>
+ </property>
+
+ <!--
+ Set Hadoop 2.* file system implementation class for GGFS.
+ -->
+ <property>
+ <name>fs.AbstractFileSystem.ggfs.impl</name>
+ <value>org.gridgain.grid.ggfs.hadoop.v2.GridGgfsHadoopFileSystem</value>
+ </property>
+
+ <!--
+ Disallow data node replacement since it does not make sense for GridGain's GGFS nodes.
+ -->
+ <property>
+ <name>dfs.client.block.write.replace-datanode-on-failure.policy</name>
+ <value>NEVER</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/bigtop/blob/459e663b/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/mapred-site.xml
----------------------------------------------------------------------
diff --git a/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/mapred-site.xml
new file mode 100644
index 0000000..d2be8eb
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/gridgain-hadoop/templates/mapred-site.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!--
+ This template file contains settings needed to run Apache Hadoop jobs
+ with GridGain In-Memory Accelerator.
+
+ You can replace '$HADOOP_HOME/etc/hadoop/mapred-site.xml' file with this one
+ to run jobs on localhost (local node can be a part of distributed cluster though).
+ To run jobs on remote host you have to change jobtracker address to the REST address
+ of any running GridGain node.
+
+ Note that GridGain jars must be in Apache Hadoop client classpath to work
+ with this configuration.
+
+ Run script '$GRIDGAIN_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup.
+-->
+
+<configuration>
+ <!--
+ Framework name must be set to 'gridgain'.
+ -->
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>gridgain</value>
+ </property>
+
+ <!--
+ Job tracker address must be set to the REST address of any running GridGain node.
+ -->
+ <property>
+ <name>mapreduce.jobtracker.address</name>
+ <value>localhost:11211</value>
+ </property>
+
+ <!-- Parameters for job tuning. -->
+ <!--
+ <property>
+ <name>mapreduce.job.reduces</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <name>mapreduce.job.maps</name>
+ <value>4</value>
+ </property>
+ -->
+
+</configuration>
[2/3] bigtop git commit: BIGTOP-1659. gridgain-hadoop packages need
to provide unversioned softlinks for accelerator jar files
Posted by co...@apache.org.
BIGTOP-1659. gridgain-hadoop packages need to provide unversioned softlinks for accelerator jar files
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/098f67f5
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/098f67f5
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/098f67f5
Branch: refs/heads/master
Commit: 098f67f5ad07377ac44b92ac3fed270b6239cb5a
Parents: 258c97c
Author: Konstantin Boudnik <co...@apache.org>
Authored: Mon Feb 9 21:32:53 2015 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Tue Feb 10 13:56:05 2015 -0800
----------------------------------------------------------------------
.../src/common/gridgain-hadoop/install_gridgain.sh | 10 ++++++++++
bigtop-packages/src/deb/gridgain-hadoop/rules | 1 +
.../src/rpm/gridgain-hadoop/SPECS/gridgain-hadoop.spec | 4 ++--
3 files changed, 13 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/098f67f5/bigtop-packages/src/common/gridgain-hadoop/install_gridgain.sh
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/common/gridgain-hadoop/install_gridgain.sh b/bigtop-packages/src/common/gridgain-hadoop/install_gridgain.sh
index 213f227..18b7fc1 100644
--- a/bigtop-packages/src/common/gridgain-hadoop/install_gridgain.sh
+++ b/bigtop-packages/src/common/gridgain-hadoop/install_gridgain.sh
@@ -122,6 +122,16 @@ cp -a $UNZIP_DIR/config/* $PREFIX/$CONF_DIR
cp -ra $UNZIP_DIR/bin/* $PREFIX/$BIN_DIR
ln -s $ETC_DIR/conf $PREFIX/$LIB_DIR/config
+# Create unversion softlinks to the main libraries
+for dir in $PREFIX/$LIB_DIR/libs $PREFIX/$LIB_DIR/libs/gridgain-hadoop ; do
+ (cd $dir &&
+ for j in gridgain-*.jar; do
+ if [[ $j =~ gridgain-(.*)-${GRIDGAIN_HADOOP_VERSION}.jar ]]; then
+ name=${BASH_REMATCH[1]}
+ ln -s $j gridgain-$name.jar
+ fi
+ done)
+done
wrapper=$PREFIX/usr/bin/gridgain-hadoop
mkdir -p `dirname $wrapper`
http://git-wip-us.apache.org/repos/asf/bigtop/blob/098f67f5/bigtop-packages/src/deb/gridgain-hadoop/rules
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/deb/gridgain-hadoop/rules b/bigtop-packages/src/deb/gridgain-hadoop/rules
index 979c5d7..0d1f52e 100644
--- a/bigtop-packages/src/deb/gridgain-hadoop/rules
+++ b/bigtop-packages/src/deb/gridgain-hadoop/rules
@@ -35,6 +35,7 @@ override_dh_auto_build:
override_dh_auto_install:
cp debian/gridgain-hadoop.default debian/${gridgain-hadoop_pkg_name}/etc/default/${gridgain-hadoop_pkg_name}
+ env HADOOP_VERSION=${GRIDGAIN_HADOOP_BASE_VERSION} \
bash -x debian/install_gridgain.sh \
--build-dir=target \
--conf-dir=/etc/${gridgain-hadoop_pkg_name}/conf.dist \
http://git-wip-us.apache.org/repos/asf/bigtop/blob/098f67f5/bigtop-packages/src/rpm/gridgain-hadoop/SPECS/gridgain-hadoop.spec
----------------------------------------------------------------------
diff --git a/bigtop-packages/src/rpm/gridgain-hadoop/SPECS/gridgain-hadoop.spec b/bigtop-packages/src/rpm/gridgain-hadoop/SPECS/gridgain-hadoop.spec
index a70457e..86c1200 100644
--- a/bigtop-packages/src/rpm/gridgain-hadoop/SPECS/gridgain-hadoop.spec
+++ b/bigtop-packages/src/rpm/gridgain-hadoop/SPECS/gridgain-hadoop.spec
@@ -139,11 +139,11 @@ Documentation for GridGain platform
%setup -n gridgain-%{vcs_tag}
%build
-env gridgain_hadoop_VERSION=%{version} bash %{SOURCE1}
+bash %{SOURCE1}
%install
%__rm -rf $RPM_BUILD_ROOT
-bash %{SOURCE2} \
+env GRIDGAIN_HADOOP_VERSION=%{version} bash %{SOURCE2} \
--build-dir=target \
--doc-dir=%{doc_gridgain} \
--conf-dir=%{etc_gridgain_conf_dist} \
[3/3] bigtop git commit: BIGTOP-1501. come up with a way to use
Jenkins plugins/Groovy console to create jobs
Posted by co...@apache.org.
BIGTOP-1501. come up with a way to use Jenkins plugins/Groovy console to create jobs
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/258c97c6
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/258c97c6
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/258c97c6
Branch: refs/heads/master
Commit: 258c97c69e9a326da9e40ebb398d675f0919bc3c
Parents: f506701
Author: Roman Shaposhnik <rv...@apache.org>
Authored: Sun Dec 14 22:02:33 2014 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Tue Feb 10 13:56:05 2015 -0800
----------------------------------------------------------------------
bigtop-ci/jenkins/README | 31 ++++
bigtop-ci/jenkins/jobsCreator.groovy | 233 ++++++++++++++++++++++++++++++
2 files changed, 264 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/258c97c6/bigtop-ci/jenkins/README
----------------------------------------------------------------------
diff --git a/bigtop-ci/jenkins/README b/bigtop-ci/jenkins/README
new file mode 100644
index 0000000..963f814
--- /dev/null
+++ b/bigtop-ci/jenkins/README
@@ -0,0 +1,31 @@
+Bigtop's approach to running CI off of Jenkins is simple: we use
+Groovy DSL Job plugin extensively so that Jenkins configuration
+can be as stateless as possible. All we require from a Jenkins
+admin is that the following plugins are installed:
+ * Amazon EC2 Plugin
+ * Credentials Plugin
+ * Disk Usage Plugin
+ * DynamicAxis Plugin
+ * Git Plugin
+ * Gradle plugin
+ * Hudson Extended Read Permission Plugin
+ * Job DSL Plugin
+ * Mailer
+ * Matrix Authorization Strategy Plugin
+ * Matrix Project Plugin
+ * Matrix Reloaded Plugin
+ * Parameterized Trigger Plugin
+ * SSH Credentials Plugin
+ * SSH Slaves plugin
+ * Timestamper
+
+Once you have all the above plugins installed, all you have to do in
+order to start Bigtop CI is to create one Bootstrap job. Pick a type
+of 'Build a free-style software project' when asked by Jenkins and
+hook it up to Bigtop's Git repo. Then select 'Process Job DSLs' as
+one and only build step, pointing it at
+ bigtop-ci/jenkins/jobsCreator.groovy
+as a DSL script that needs to be looked up in the filesystem.
+
+That is it! You only need to run it now in order to have all
+the required Jenkins jobs.
http://git-wip-us.apache.org/repos/asf/bigtop/blob/258c97c6/bigtop-ci/jenkins/jobsCreator.groovy
----------------------------------------------------------------------
diff --git a/bigtop-ci/jenkins/jobsCreator.groovy b/bigtop-ci/jenkins/jobsCreator.groovy
new file mode 100644
index 0000000..36c0f3e
--- /dev/null
+++ b/bigtop-ci/jenkins/jobsCreator.groovy
@@ -0,0 +1,233 @@
+// FIXME: it would be nice to extract the following from bigtop.mk on the fly
+def bigtopComponents = ["bigtop-groovy", "bigtop-jsvc", "bigtop-tomcat", "bigtop-utils",
+ "zookeeper", "hadoop", "hbase", "hive", "pig", "crunch", "datafu",
+ "flume", "giraph", "gridgain-hadoop", "mahout", "oozie", "phoenix",
+ "solr", "spark", "sqoop", "tachyon", "whirr"]
+// FIXME: it would be nice to extract the following from some static configuration file
+def targetOS = ["fedora-20", "opensuse-12.3", "ubuntu-14.04"]
+def gitUrl = "https://git-wip-us.apache.org/repos/asf/bigtop.git"
+def gitBranch = "master"
+def dockerLabel = "docker"
+def jobPrefix="Bigtop"
+def pkgTestsuites =["TestPackagesBasicsWithRM", "TestPackagesPseudoDistributedServices",
+ "TestPackagesPseudoDistributedDependency", "TestPackagesPseudoDistributedFileContents",
+ "TestPackagesPseudoDistributedWithRM", "TestPackagesBasics"]
+
+job {
+ name "${jobPrefix}-${gitBranch}-All"
+ description "Top level job that kicks off everything for a complete Bigtop CI run on branch ${gitBranch}"
+ logRotator(7 /*days to keep */, 10 /* # of builds */, 7 /*days to keep */, 10 /* # of builds */)
+ label('master')
+ triggers {
+ cron("0 3 * * *")
+ }
+
+ steps {
+ downstreamParameterized {
+ trigger("${jobPrefix}-${gitBranch}-" + bigtopComponents.join("-pkg,${jobPrefix}-${gitBranch}-") + "-pkg",
+ 'UNSTABLE_OR_BETTER', true, ["buildStepFailure": "FAILURE", "failure": "FAILURE", "unstable": "UNSTABLE"]) {
+ currentBuild()
+ }
+
+ trigger("${jobPrefix}-${gitBranch}-Repository",
+ 'UNSTABLE_OR_BETTER', true, ["buildStepFailure": "FAILURE", "failure": "FAILURE", "unstable": "UNSTABLE"]) {
+ currentBuild()
+ }
+
+ trigger("${jobPrefix}-${gitBranch}-Packagetest",
+ 'UNSTABLE_OR_BETTER', true, ["buildStepFailure": "FAILURE", "failure": "FAILURE", "unstable": "UNSTABLE"]) {
+ currentBuild()
+ }
+ }
+ }
+}
+
+job(type: Matrix) {
+ name "${jobPrefix}-${gitBranch}-Repository"
+ description "Top level job that creates final repository from packages built off of ${gitBranch} branch for all the matrix"
+ logRotator(2 /*days to keep */, 2 /* # of builds */, 2 /*days to keep */, 2 /* # of builds */)
+ label('master')
+
+ steps {
+ shell('''
+#!/bin/bash -ex
+env
+rm -rf * /var/tmp/* || :
+
+# By default Jenkins uses an internal (unresolvable) EC2 DNS name
+JOB_URL="${JOB_URL/#*:/http://bigtop01.cloudera.org:}"
+
+export PROJECTS="''' + bigtopComponents.join(' ') + '''"
+
+mkdir -p packages
+pushd packages
+ for project in $PROJECTS; do
+ mkdir -p ${project}
+ pushd ${project}
+ wget "http://bigtop01.cloudera.org:8080/job/${jobPrefix}-${gitBranch}-${project}-pkg/TARGET_OS=${TARGET_OS},slaves=docker/lastSuccessfulBuild/artifact/*zip*/archive.zip"
+ unzip archive.zip
+ rm archive.zip
+ popd
+ done
+popd
+
+if [ -n "find packages -iname '*.src.rpm'`" ] ; then
+ mkdir -p repo/{RPMS,SRPMS}
+ mv `find packages -iname "*.src.rpm"` repo/SRPMS/
+ mv `find packages -iname "*.rpm"` repo/RPMS/
+
+ pushd repo
+ createrepo .
+ popd
+ cat > repo/bigtop.repo << __EOT__
+[bigtop]
+name=Bigtop
+enabled=1
+gpgcheck=0
+type=NONE
+baseurl=${JOB_URL}/lastSuccessfulBuild/artifact/repo/
+__EOT__
+
+
+else
+ mkdir -p repo/conf
+ cat > repo/conf/distributions <<__EOT__
+Origin: Bigtop
+Label: Bigtop
+Suite: stable
+Codename: bigtop
+Version: 0.3
+Architectures: i386 amd64 source
+Components: contrib
+Description: Bigtop
+__EOT__
+ for i in `find packages -name \\*.changes` ; do
+ reprepro -Vb repo include bigtop $i
+ done
+ echo "deb ${JOB_URL}/lastSuccessfulBuild/artifact/repo/ bigtop contrib" > repo/bigtop.list
+fi
+ ''')
+ }
+
+ publishers {
+ archiveArtifacts('repo/**/*')
+ }
+
+ axes {
+ text('TARGET_OS', targetOS)
+ label('slaves', dockerLabel)
+ }
+}
+
+job(type: Matrix) {
+ name "${jobPrefix}-${gitBranch}-Packagetest"
+ description "Runs smoke tests on all packages built off of ${gitBranch} branch for all the matrix"
+ logRotator(4 /*days to keep */, 4 /* # of builds */, 4 /*days to keep */, 4 /* # of builds */)
+ label('master')
+
+ parameters {
+ choiceParam("PKG_SUITE", pkgTestsuites)
+ }
+
+ scm {
+ git { node -> // is hudson.plugins.git.GitSCM
+ // node / gitConfigName('Bigtop')
+ // node / gitConfigEmail('dev@bigtop.apache.org')
+
+ remote {
+ name(gitBranch)
+ url(gitUrl)
+ }
+ branch(gitBranch)
+ }
+ }
+
+ steps {
+ shell('''
+#!/bin/bash
+
+# Working around SuSE madness
+rm -f /etc/zypp/repos.d/*Cloud* || :
+
+export HADOOP_HOME=/usr/lib/hadoop
+export HADOOP_CONF_DIR=/etc/hadoop/conf
+export REPO_FILE_URL="http://bigtop01.cloudera.org:8080/view/Bigtop-trunk/job/Bigtop-trunk-Repository/label=${label/-slave/}/lastSuccessfulBuild/artifact/repo/bigtop"
+export REPO_KEY_URL="http://archive.apache.org/dist/incubator/bigtop/bigtop-0.3.0-incubating/repos/GPG-KEY-bigtop"
+
+if [ "$label" = "precise-slave" -o "$label" = "quetzal-slave" -o "$label" = "trusty-slave" ]; then
+ REPO_FILE_URL="${REPO_FILE_URL}.list"
+else
+ REPO_FILE_URL="${REPO_FILE_URL}.repo"
+fi
+
+docker run -u `id -u` -e HOME=/var/lib/jenkins -e BIGTOP_BUILD_STAMP=.${BUILD_NUMBER} \\
+ -v `pwd`/build/home:/var/lib/jenkins \\
+ -v `pwd`:/ws bigtop/slaves:$TARGET_OS \\
+ bash -c '. /etc/profile.d/bigtop.sh; cd /ws ; mvn \\
+ -f bigtop-tests/test-execution/package/pom.xml \\
+ clean verify \\
+ -Dbigtop.repo.file.url="'${REPO_FILE_URL}'" \\
+ -Dorg.apache.bigtop.itest.log4j.level=TRACE \\
+ -Dlog4j.debug=true \\
+ -Dorg.apache.maven-failsafe-plugin.testInclude="**/'${PKG_SUITE}'.*"
+ ''')
+ }
+
+ publishers {
+ archiveJunit('**/bigtop-tests/test-execution/package/target/failsafe-reports/*.xml') {
+ retainLongStdout()
+ }
+ }
+
+ axes {
+ text('TARGET_OS', targetOS)
+ label('slaves', dockerLabel)
+ }
+}
+
+bigtopComponents.each { comp->
+ job(type: Matrix) {
+ println comp
+ name "${jobPrefix}-${gitBranch}-${comp}-pkg"
+ description "Builds packages on every platform of the matrix according to the ${gitBranch} branch"
+ logRotator(7 /*days to keep */, 10 /* # of builds */, 7 /*days to keep */, 10 /* # of builds */)
+ label('master')
+
+ scm {
+ git { node -> // is hudson.plugins.git.GitSCM
+ // node / gitConfigName('Bigtop')
+ // node / gitConfigEmail('dev@bigtop.apache.org')
+
+ remote {
+ name(gitBranch)
+ url(gitUrl)
+ }
+ branch(gitBranch)
+ }
+ }
+
+// export JAVA_OPTS="-Xmx1536m -Xms256m -XX:MaxPermSize=256m"
+// export MAVEN_OPTS="-Xmx1536m -Xms256m -XX:MaxPermSize=256m"
+//
+// if [ -e /etc/SuSE-release ] ; then
+// export LDFLAGS="-lrt"
+// fi
+ steps {
+ shell('''
+ mkdir -p build/home || :
+ docker run -u `id -u` -e HOME=/var/lib/jenkins -e BIGTOP_BUILD_STAMP=.${BUILD_NUMBER} \\
+ -v `pwd`/build/home:/var/lib/jenkins \\
+ -v `pwd`:/ws bigtop/slaves:$TARGET_OS \\
+ bash -c '. /etc/profile.d/bigtop.sh; cd /ws ; gradle ''' + "'\"${comp}-pkg\"")
+ }
+
+ publishers {
+ archiveArtifacts('output/**/*/*')
+ }
+
+ axes {
+ text('TARGET_OS', targetOS)
+ label('slaves', dockerLabel)
+ }
+ }
+}