You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by co...@apache.org on 2014/12/04 21:17:56 UTC
[1/2] bigtop git commit: BIGTOP-1558. Make gradle easier to navigate
Repository: bigtop
Updated Branches:
refs/heads/master bee6ae3e1 -> eff797efd
BIGTOP-1558. Make gradle easier to navigate
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/eff797ef
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/eff797ef
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/eff797ef
Branch: refs/heads/master
Commit: eff797efd6c9b3260f74e8257e96d4a29d5f3558
Parents: 9559313
Author: Konstantin Boudnik <co...@apache.org>
Authored: Wed Dec 3 17:12:17 2014 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Dec 4 12:16:59 2014 -0800
----------------------------------------------------------------------
build.gradle | 15 +++----
packages.gradle | 118 +++++++++++++++++++++++++++++++--------------------
2 files changed, 80 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/eff797ef/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 33ec6bd..ff776bf 100644
--- a/build.gradle
+++ b/build.gradle
@@ -25,6 +25,8 @@ version = '0.9.0-SNAPSHOT'
description = """Bigtop"""
+def TESTARTIFACTS_GROUP = 'test artifacts'
+
sourceCompatibility = 1.6
targetCompatibility = 1.6
@@ -49,11 +51,9 @@ task installCommon(type:Exec) {
workingDir "."
commandLine 'mvn clean install -f bigtop-tests/test-execution/common/pom.xml'.split(" ")
}
-task installAllLocalArtifacts() {
-
-}
-task installArtifact() {
-
+task installAllLocalArtifacts (
+ description: "Prepare and locally install all test artifacts",
+ group: TESTARTIFACTS_GROUP) {
}
/**
@@ -72,8 +72,7 @@ def artifactToInstall = {
artifactCollection.add(file.name)
}
artifactCollection.each { artifact ->
- task "install-${artifact}" << {
- description 'Installs ${artifact} artifact with Maven'
+ task "install-${artifact}" (description: "Installs ${artifact} artifact with Maven", group: TESTARTIFACTS_GROUP) << {
def final PATH = "${ARTIFACT_DIR}/$artifact/pom.xml"
def final WRAPPER = "mvn clean install -f " + PATH
exec {
@@ -103,4 +102,4 @@ buildscript {
dependencies {
classpath 'de.undercouch:gradle-download-task:1.0'
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/bigtop/blob/eff797ef/packages.gradle
----------------------------------------------------------------------
diff --git a/packages.gradle b/packages.gradle
index e80cd0a..95dc375 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -30,6 +30,8 @@ apply plugin: 'java'
apply plugin: 'groovy'
apply plugin: 'download-task'
+def PACKAGES_GROUP = 'package'
+
final String VERBOSE = "verbose"
final String BOM = "bigtop.mk"
@@ -74,15 +76,16 @@ def getDate() {
new Date().format('E, dd MMM yyyy HH:mm:ss Z')
}
-task showHelp << {
+task "packages-help" (description: "All package build related tasks information", group: PACKAGES_GROUP) << {
targets.each { target ->
println (target + "\n\t[" + tasks.findAll { alltask -> alltask.name.startsWith(target)}*.name.join(", ") + "]")
}
}
def genTasks = { target, variable ->
- task "${target}-download" (dependsOn: "${target}_vardefines") << {
- description "Download $target artifacts"
+ Task t = task "${target}-download" (dependsOn: "${target}_vardefines",
+ description: "Download $target artifacts",
+ group: PACKAGES_GROUP) << {
def final TARBALL_SRC = BOM_map[variable + '_TARBALL_SRC']
def final DOWNLOAD_DST = BOM_map[variable + '_DOWNLOAD_DST']
@@ -126,12 +129,13 @@ def genTasks = { target, variable ->
}
touchTargetFile(BOM_map[variable + '_TARGET_DL'])
}
- task "${target}-tar" (dependsOn: ["${target}_vardefines", "${target}-download"]) << {
+ task "${target}-tar" (dependsOn: ["${target}_vardefines", "${target}-download"],
+ description: "Preparing a tarball for $target artifacts",
+ group: PACKAGES_GROUP) << {
if (new File(BOM_map[variable + '_TARGET_TAR'])?.exists()) {
println "\tNothing to do. Exiting..."
return
}
- description "Preparing a tarball for $target artifacts"
def final TAR_DIR = BOM_map[variable + '_TAR_DIR']
def final TARBALL_SRC = BOM_map[variable + '_TARBALL_SRC'] ?: ""
def final DOWNLOAD_DST = BOM_map[variable + '_DOWNLOAD_DST'] ?: ""
@@ -195,12 +199,13 @@ def genTasks = { target, variable ->
}
// Keeping the reference to deb task to be used later for correct sequencing
- Task tdeb = task "$target-deb" (dependsOn: "${target}-sdeb") << {
+ Task tdeb = task "$target-deb"(dependsOn: "${target}-sdeb",
+ description: "Buildind DEB for $target artifacts",
+ group: PACKAGES_GROUP) << {
if (new File(BOM_map[variable + '_TARGET_DEB'])?.exists()) {
println "\tNothing to do. Exiting..."
return
}
- description "Buildind DEB for $target artifacts"
def final PKG_NAME = BOM_map[variable + '_PKG_NAME']
def final PKG_RELEASE = BOM_map[variable + '_PKG_RELEASE']
def final PKG_VERSION = BOM_map[variable + '_PKG_VERSION']
@@ -242,12 +247,14 @@ def genTasks = { target, variable ->
// Guarantee that tasks are ran in the order set by BOM file
if (targets.size() > 0)
tdeb.mustRunAfter "${targets.get(targets.size() - 1)}-deb"
- task "$target-sdeb" (dependsOn: ["${target}_vardefines", "${target}-tar"]) << {
+ task "$target-sdeb" (dependsOn: ["${target}_vardefines", "${target}-tar"],
+ description: "Buildind SDEB for $target artifacts",
+ group: PACKAGES_GROUP
+ ) << {
if (new File(BOM_map[variable + '_TARGET_SDEB'])?.exists()) {
println "\tNothing to do. Exiting..."
return
}
- description "Buildind SDEB for $target artifacts"
def final PKG_BUILD_DIR = BOM_map[variable + '_BUILD_DIR']
def final NAME = BOM_map[variable + '_NAME']
def final PKG_NAME = BOM_map[variable + '_PKG_NAME']
@@ -301,12 +308,13 @@ def genTasks = { target, variable ->
}
touchTargetFile(BOM_map[variable + '_TARGET_SDEB'])
}
- task "$target-apt" (dependsOn: "$target-deb") << {
+ task "$target-apt" (dependsOn: "$target-deb",
+ description: "Creating APT repository for $target packages",
+ group: PACKAGES_GROUP) << {
if (new File(BOM_map[variable + '_TARGET_APT'])?.exists()) {
println "\tNothing to do. Exiting..."
return
}
- description "Creating APT repository for $target packages"
def final PKG_NAME = BOM_map[variable + '_NAME']
def final PKG_RELEASE = BOM_map[variable + '_PKG_RELEASE']
def final PKG_VERSION = BOM_map[variable + '_PKG_VERSION']
@@ -328,12 +336,13 @@ def genTasks = { target, variable ->
}
// Keeping the reference to task to be used later for correct sequencing
- Task trpm = task "$target-rpm" (dependsOn: ["${target}-srpm"]) << {
+ Task trpm = task "$target-rpm" (dependsOn: ["${target}-srpm"],
+ description: "Buildind RPM for $target artifacts",
+ group: PACKAGES_GROUP) << {
if (new File(BOM_map[variable + '_TARGET_RPM'])?.exists()) {
println "\tNothing to do. Exiting..."
return
}
- description "Buildind RPM for $target artifacts"
def final PKG_BUILD_DIR = BOM_map[variable + '_BUILD_DIR']
def final NAME = BOM_map[variable + '_NAME']
def final PKG_NAME = BOM_map[variable + '_PKG_NAME']
@@ -364,12 +373,13 @@ def genTasks = { target, variable ->
// Guarantee that tasks are ran in the order set by BOM file
if (targets.size() > 0)
trpm.mustRunAfter "${targets.get(targets.size() - 1)}-rpm"
- task "$target-srpm" (dependsOn: ["${target}_vardefines" , "${target}-tar"]) << {
+ task "$target-srpm" (dependsOn: ["${target}_vardefines" , "${target}-tar"],
+ description: "Buildind SRPM for $target artifacts",
+ group: PACKAGES_GROUP) << {
if (new File(BOM_map[variable + '_TARGET_SRPM'])?.exists()) {
println "\tNothing to do. Exiting..."
return
}
- description "Buildind SRPM for $target artifacts"
def final NAME = BOM_map[variable + '_NAME']
def final PKG_NAME = BOM_map[variable + '_PKG_NAME']
def final PKG_NAME_FOR_PKG = BOM_map[variable + '_NAME'].replaceAll("-", "_")
@@ -419,19 +429,20 @@ def genTasks = { target, variable ->
}
touchTargetFile(BOM_map[variable + '_TARGET_SRPM'])
}
- task "$target-yum" (dependsOn: "$target-rpm") << {
+ task "$target-yum" (dependsOn: "$target-rpm",
+ description: "Creating YUM repository for $target packages",
+ group: PACKAGES_GROUP) << {
if (new File(BOM_map[variable + '_TARGET_YUM'])?.exists()) {
println "\tNothing to do. Exiting..."
return
}
- description "Creating YUM repository for $target packages"
exec {
workingDir BUILD_DIR
commandLine "createrepo -o $OUTPUT_DIR $OUTPUT_DIR".split(' ')
}
touchTargetFile(BOM_map["${variable}_TARGET_YUM"])
}
- task "$target-version" << {
+ task "$target-version" (description: "Show version of $target component", group: PACKAGES_GROUP) << {
println "Base: ${BOM_map[variable + '_BASE_VERSION']}"
}
task "${target}_vardefines" << {
@@ -475,7 +486,9 @@ def genTasks = { target, variable ->
}
}
- task "$target-info" (dependsOn: "${target}_vardefines") << {
+ task "$target-info" (dependsOn: "${target}_vardefines",
+ description: "Info about $target component build",
+ group: PACKAGES_GROUP) << {
println "Info for package $target"
println " Will download from URL: ${BOM_map[variable + '_DOWNLOAD_URL']}"
println " To destination file: ${BOM_map[variable + '_DOWNLOAD_DST']}"
@@ -486,17 +499,15 @@ def genTasks = { target, variable ->
println "Version: " + BOM_map[variable + '_BASE_VERSION']
//TODO more about stamping
}
- task "$target-relnotes" << {
- description "Preparing release notes for $target. No yet implemented!!!"
+ task "$target-relnotes" (description: "Preparing release notes for $target. No yet implemented!!!", group: PACKAGES_GROUP)<< {
}
- task "$target-clean" (dependsOn: "${target}_vardefines") << {
- def blDir = BOM_map[variable + '_BUILD_DIR']
- def outDir = BOM_map[variable + '_OUTPUT_DIR']
- description "Removing $target component $blDir and $outDir"
- delete(blDir)
+ task "$target-clean" (dependsOn: "${target}_vardefines",
+ description: "Removing $target component ${BOM_map[variable + '_BUILD_DIR']} and ${BOM_map[variable + '_OUTPUT_DIR']}",
+ group: PACKAGES_GROUP) << {
+ delete(BOM_map[variable + '_BUILD_DIR'])
delete(BOM_map[variable + '_OUTPUT_DIR'])
}
- task "$target-help" << {
+ task "$target-help" (description: "List of available tasks for $target", group: PACKAGES_GROUP) << {
println (target + "\n\t[" + tasks.findAll { alltask -> alltask.name.startsWith(target)}*.name.join(", ") + "]")
}
}
@@ -543,28 +554,45 @@ project.afterEvaluate {
BOM_map[BIGTOP_BOM] = bomVersions
if (System.getProperty(VERBOSE))println "BIGTOP_BOM:\n${BOM_map[BIGTOP_BOM]}"
// Putting all targets of different types into one common target
- task "srpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-srpm")}*.name) << { }
- task "rpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-rpm")}*.name) << { }
- task "yum" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-yum")}*.name) << { }
- task "sdeb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-sdeb")}*.name) << { }
- task "deb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-deb")}*.name) << { }
- task "apt" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-apt")}*.name) << { }
-
- task clean (overwrite: true, dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-clean")}*.name) << {
- description "Cleaning all components' build and output directories. Overwrites standard clean task"
+ task "srpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-srpm")}*.name,
+ description: "Build all SRPM packages for the stack components",
+ group: PACKAGES_GROUP
+ ) << { }
+ task "rpm" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-rpm")}*.name,
+ description: "Build all RPM packages for the stack",
+ group: PACKAGES_GROUP
+ ) << { }
+ task "yum" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-yum")}*.name,
+ description: "Create YUM repository for all pre-built RPM packages",
+ group: PACKAGES_GROUP
+ ) << { }
+ task "sdeb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-sdeb")}*.name,
+ description: "Build all SDEB packages for the stack components",
+ group: PACKAGES_GROUP
+ ) << { }
+ task "deb" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-deb")}*.name,
+ description: "Build all DEB packages for the stack components",
+ group: PACKAGES_GROUP
+ ) << { }
+ task "apt" (dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-apt")}*.name,
+ description: "Create APT repository for all pre-built DEB packages",
+ group: PACKAGES_GROUP
+ ) << { }
+
+ task clean (overwrite: true, dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-clean")}*.name,
+ description: "Cleaning all components' build and output directories. Overwrites standard clean task",
+ group: PACKAGES_GROUP) << {
}
- task allclean (dependsOn: clean ) << {
- description "Removing $BUILD_DIR, $OUTPUT_DIR, and $DIST_DIR"
+ task allclean (dependsOn: clean,
+ description: "Removing $BUILD_DIR, $OUTPUT_DIR, and $DIST_DIR",
+ group: PACKAGES_GROUP) << {
delete (BUILD_DIR)
delete (OUTPUT_DIR)
delete (DIST_DIR)
}
- task realclean (dependsOn: allclean) << {
- description "Removing $BUILD_DIR, $OUTPUT_DIR, $DIST_DIR, and $DL_DIR"
+ task realclean (dependsOn: allclean,
+ description: "Removing $BUILD_DIR, $OUTPUT_DIR, $DIST_DIR, and $DL_DIR",
+ group: PACKAGES_GROUP) << {
delete (DL_DIR)
}
-
- task "all" (dependsOn: [srpm, sdeb])
-}
-
-defaultTasks 'showHelp'
+}
\ No newline at end of file
[2/2] bigtop git commit: BIGTOP-1534. Update README.md to reflect new
build system and other changes
Posted by co...@apache.org.
BIGTOP-1534. Update README.md to reflect new build system and other changes
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo
Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/95593137
Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/95593137
Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/95593137
Branch: refs/heads/master
Commit: 955931375786f58549c1cfea7f7cb89b1151d3dc
Parents: bee6ae3
Author: Konstantin Boudnik <co...@apache.org>
Authored: Wed Dec 3 15:21:18 2014 -0800
Committer: Konstantin Boudnik <co...@apache.org>
Committed: Thu Dec 4 12:16:59 2014 -0800
----------------------------------------------------------------------
README.md | 32 +++++++++++++++++++++++++-------
1 file changed, 25 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bigtop/blob/95593137/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 17b7a4a..de51707 100644
--- a/README.md
+++ b/README.md
@@ -88,18 +88,32 @@ WARNING: since testing packages requires installing them on a live system it is
*smokes/hadoop/pom.xml*
*smokes/hive/pom.xml*
*... and so on.*
-
-* Step 1: Build the smokes with snapshots. This ensures that all transitive dependencies etc.. are in your repo
+
+* New way (with Gradle build in place)
+ * Step 1: install smoke tests for one or more components
+ * Example 1:
+
+ gradle installTestArtifacts
+
+ * Example 2: Installing just Hadoop-specific smoke tests
+
+ gradle install-hadoop
+
+ * Step 2: Run the the smoke tests on your cluster (see Step 3 and/or Step 4 below)
+
+
+* Old Way
+ * Step 1: Build the smokes with snapshots. This ensures that all transitive dependencies etc.. are in your repo
mvn clean install -DskipTests -DskipITs -DperformRelease -f ./bigtop-test-framework/pom.xml
mvn clean install -DskipTests -DskipITs -DperformRelease -f ./test-artifacts/pom.xml
-* Step 2: Now, rebuild in "offline" mode. This will make sure that your local changes to bigtop are embeded in the changes.
+ * Step 2: Now, rebuild in "offline" mode. This will make sure that your local changes to bigtop are embeded in the changes.
mvn clean install -DskipTests -DskipITs -DperformRelease -o -nsu -f ./bigtop-test-framework/pom.xml
mvn clean install -DskipTests -DskipITs -DperformRelease -o -nsu -f ./bigtop-tests/test-artifacts/pom.xml
-* Step 3: Now, you can run the smoke tests on your cluster.
+ * Step 3: Now, you can run the smoke tests on your cluster.
* Example 1: Running all the smoke tests with TRACE level logging (shows std out from each mr job).
mvn clean verify -Dorg.apache.bigtop.itest.log4j.level=TRACE -f ./bigtop/bigtop-tests/test-execution/smokes/pom.xml
@@ -130,14 +144,18 @@ __On all systems, Building Apache Bigtop requires the following tools__
* All systems need these tools installed to build bigtop:
- Java JDK 1.6, Apache Forrest 0.8, Apache Ant, Apache Maven, git, subversion, autoconf, automake, liblzo2-dev, libz-dev, sharutils, libfuse-dev, libssl-dev
+ Java JDK/OpenJDK7, Apache Forrest 0.8, Apache Ant, Apache Maven, git, subversion, autoconf, automake, liblzo2-dev, libz-dev, sharutils, libfuse-dev, libssl-dev
* Additionally, some details for specific linux versions :
* __Debian__ based distros need these packages : build-essential dh-make debhelper devscripts, reprepro
* __openSUSE 11.4__ needs these packages : relaxngDatatypedocbook-utils docbook-simple, fuse-devel, docbook5, docbook5-xsl-stylesheets, libxml2-devel, xmlformat, xmlto, libxslt, libopenssl-devel
-* __Building packages__ : `make [component-name]-[rpm|deb]`
-* __Building local YUM/APT repositories__ : `make [component-name]-[yum|apt]`
+ Preferred way of setting up needed dependencies: the list in this README file might be outdated.
+ The best source of data are bigtop_toolchain recipes that would configure all your development
+ environment from a single Puppet run.
+
+* __Building packages__ : `gradle [component-name]-[rpm|deb]`
+* __Building local YUM/APT repositories__ : `gradle [component-name]-[yum|apt]`
For Developers: Building and modifying the web site