You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by da...@apache.org on 2017/02/03 00:39:18 UTC
[1/2] beam git commit: Move website jobs to main Beam repo to reduce
duplication.
Repository: beam
Updated Branches:
refs/heads/master c0910fdf4 -> 1533d7013
Move website jobs to main Beam repo to reduce duplication.
Signed-off-by: Jason Kuster <ja...@google.com>
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/5dc83f57
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/5dc83f57
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/5dc83f57
Branch: refs/heads/master
Commit: 5dc83f577e7b4275475556bf3bd0212bf69e3d35
Parents: c0910fd
Author: Jason Kuster <ja...@google.com>
Authored: Wed Feb 1 16:39:40 2017 -0800
Committer: Davor Bonaci <da...@google.com>
Committed: Thu Feb 2 16:38:47 2017 -0800
----------------------------------------------------------------------
.jenkins/common_job_properties.groovy | 40 +++++++++----
...job_beam_PostCommit_Java_MavenInstall.groovy | 2 +-
...ostCommit_Java_RunnableOnService_Apex.groovy | 2 +-
...ommit_Java_RunnableOnService_Dataflow.groovy | 2 +-
...stCommit_Java_RunnableOnService_Flink.groovy | 2 +-
...ommit_Java_RunnableOnService_Gearpump.groovy | 4 +-
...stCommit_Java_RunnableOnService_Spark.groovy | 2 +-
.../job_beam_PostCommit_Python_Verify.groovy | 2 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 2 +-
.../job_beam_PreCommit_Website_Stage.groovy | 62 ++++++++++++++++++++
.jenkins/job_beam_PreCommit_Website_Test.groovy | 47 +++++++++++++++
.../job_beam_Release_NightlySnapshot.groovy | 2 +-
.jenkins/job_seed.groovy | 11 ++--
13 files changed, 154 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/common_job_properties.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/common_job_properties.groovy b/.jenkins/common_job_properties.groovy
index f1f2b7f..eee1cda 100644
--- a/.jenkins/common_job_properties.groovy
+++ b/.jenkins/common_job_properties.groovy
@@ -20,14 +20,27 @@
// common properties that are shared among all Jenkins projects.
class common_job_properties {
- // Sets common top-level job properties.
- static def setTopLevelJobProperties(def context,
- def default_branch = 'master',
- def default_timeout = 100) {
+ // Sets common top-level job properties for website repository jobs.
+ static def setTopLevelWebsiteJobProperties(def context) {
+ setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30)
+ }
+
+ // Sets common top-level job properties for main repository jobs.
+ static def setTopLevelMainJobProperties(def context,
+ def default_branch = 'master') {
+ setTopLevelJobProperties(context, 'beam', default_branch, 100)
+ }
+
+ // Sets common top-level job properties. Should be accessed through one of the
+ // above methods to protect jobs from internal details of param defaults.
+ private static def setTopLevelJobProperties(def context,
+ def repository_name,
+ def default_branch,
+ def default_timeout) {
// GitHub project.
context.properties {
- githubProjectUrl('https://github.com/apache/beam/')
+ githubProjectUrl('https://github.com/apache/' + repository_name + '/')
}
// Set JDK version.
@@ -46,7 +59,7 @@ class common_job_properties {
context.scm {
git {
remote {
- url('https://github.com/apache/beam.git')
+ url('https://github.com/apache/' + repository_name + '.git')
refspec('+refs/heads/*:refs/remotes/origin/* ' +
'+refs/pull/*:refs/remotes/origin/pr/*')
}
@@ -83,10 +96,11 @@ class common_job_properties {
}
}
- // Sets the pull request build trigger.
- static def setPullRequestBuildTrigger(def context,
- def commitStatusContext,
- def successComment = '--none--') {
+ // Sets the pull request build trigger. Accessed through precommit methods
+ // below to insulate callers from internal parameter defaults.
+ private static def setPullRequestBuildTrigger(def context,
+ def commitStatusContext,
+ def successComment = '--none--') {
context.triggers {
githubPullRequest {
admins(['asfbot'])
@@ -150,9 +164,11 @@ class common_job_properties {
}
// Sets common config for PreCommit jobs.
- static def setPreCommit(def context, comment) {
+ static def setPreCommit(def context,
+ def commitStatusName,
+ def successComment = '--none--') {
// Set pull request build trigger.
- setPullRequestBuildTrigger(context, comment)
+ setPullRequestBuildTrigger(context, commitStatusName, successComment)
}
// Sets common config for PostCommit jobs.
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
index 31257e2..a288a84 100644
--- a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
@@ -29,7 +29,7 @@ mavenJob('beam_PostCommit_Java_MavenInstall') {
concurrentBuild()
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
index 232c94e..f657596 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Apex.groovy
@@ -23,7 +23,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Apex') {
description('Runs the RunnableOnService suite on the Apex runner.')
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
index 44136ad..f849c83 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Dataflow.groovy
@@ -26,7 +26,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Dataflow') {
previousNames('beam_PostCommit_RunnableOnService_GoogleCloudDataflow')
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
index 5a5863f..c0caa51 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Flink.groovy
@@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Flink') {
previousNames('beam_PostCommit_RunnableOnService_FlinkLocal')
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
index f07bbc6..1e35017 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Gearpump.groovy
@@ -26,7 +26,9 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Gearpump') {
previousNames('beam_PostCommit_RunnableOnService_GearpumpLocal')
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate, 'gearpump-runner')
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate,
+ 'gearpump-runner')
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
index 875c6e6..97f6a70 100644
--- a/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
+++ b/.jenkins/job_beam_PostCommit_Java_RunnableOnService_Spark.groovy
@@ -25,7 +25,7 @@ mavenJob('beam_PostCommit_Java_RunnableOnService_Spark') {
previousNames('beam_PostCommit_RunnableOnService_SparkLocal')
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Set maven parameters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PostCommit_Python_Verify.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Python_Verify.groovy b/.jenkins/job_beam_PostCommit_Python_Verify.groovy
index 7f40a15..d92958c 100644
--- a/.jenkins/job_beam_PostCommit_Python_Verify.groovy
+++ b/.jenkins/job_beam_PostCommit_Python_Verify.groovy
@@ -25,7 +25,7 @@ job('beam_PostCommit_Python_Verify') {
previousNames('beam_PostCommit_PythonVerify')
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Sets that this is a PostCommit job.
common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
index 153df14..ff2e476 100644
--- a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
+++ b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
@@ -29,7 +29,7 @@ mavenJob('beam_PreCommit_Java_MavenInstall') {
concurrentBuild()
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Set Maven parameters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PreCommit_Website_Stage.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Website_Stage.groovy b/.jenkins/job_beam_PreCommit_Website_Stage.groovy
new file mode 100644
index 0000000..b63d40a
--- /dev/null
+++ b/.jenkins/job_beam_PreCommit_Website_Stage.groovy
@@ -0,0 +1,62 @@
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Stage') {
+ description('Stages the pull requests proposed for the Apache Beam ' +
+ 'website to a temporary location to ease reviews.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+ // Set pull request build trigger.
+ common_job_properties.setPreCommit(
+ delegate,
+ 'Jenkins: automatic staging of pull requests',
+ '\nJenkins built the site at commit id ${ghprbActualCommit} with ' +
+ 'Jekyll and staged it [here](http://apache-beam-website-pull-' +
+ 'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' +
+ 'Happy reviewing.\n\nNote that any previous site has been deleted. ' +
+ 'This staged site will be automatically deleted after its TTL ' +
+ 'expires. Push any commit to the pull request branch or re-trigger ' +
+ 'the build to get it staged again.')
+
+ steps {
+ // Run the following shell script as a build step.
+ shell '''
+ # Install RVM.
+ gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+ 409B6B1796C275462A1703113804BB82D39DC0E3
+ \\curl -sSL https://get.rvm.io | bash
+ source /home/jenkins/.rvm/scripts/rvm
+
+ # Install Ruby.
+ RUBY_VERSION_NUM=2.3.0
+ rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+ # Install Bundler gem
+ PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+ GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+ gem install bundler --user-install
+
+ # Install all needed gems.
+ bundle install --path ~/.gem/
+
+ # Remove current site if it exists.
+ GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/"
+ gsutil -m rm -r -f ${GCS_PATH} || true
+
+ # Build the new site with the baseurl specified.
+ rm -fr ./content/
+ bundle exec jekyll build --baseurl=/${ghprbPullId}
+
+ # Install BeautifulSoup HTML Parser for python.
+ pip install --user beautifulsoup4
+
+ # Fix links on staged website.
+ python .jenkins/append_index_html_to_internal_links.py
+
+ # Upload the new site.
+ gsutil -m cp -R ./content/* ${GCS_PATH}
+ '''.stripIndent().trim()
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_PreCommit_Website_Test.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Website_Test.groovy b/.jenkins/job_beam_PreCommit_Website_Test.groovy
new file mode 100644
index 0000000..ada089f
--- /dev/null
+++ b/.jenkins/job_beam_PreCommit_Website_Test.groovy
@@ -0,0 +1,47 @@
+import common_job_properties
+
+// Defines a job.
+job('beam_PreCommit_Website_Test') {
+ description('Runs tests on the pull requests proposed for the Apache Beam ' +
+ 'website.')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelWebsiteJobProperties(delegate)
+
+ // Execute concurrent builds. Multiple builds of this project may be executed
+ // in parallel. This is safe because this build does not require exclusive
+ // access to any shared resources.
+ concurrentBuild()
+
+ // Set pull request build trigger.
+ common_job_properties.setPreCommit(
+ delegate,
+ 'Jenkins: test website (dead links, etc.)')
+
+ steps {
+ // Run the following shell script as a build step.
+ shell '''
+ # Install RVM.
+ gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
+ 409B6B1796C275462A1703113804BB82D39DC0E3
+ \\curl -sSL https://get.rvm.io | bash
+ source /home/jenkins/.rvm/scripts/rvm
+
+ # Install Ruby.
+ RUBY_VERSION_NUM=2.3.0
+ rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
+
+ # Install Bundler gem
+ PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
+ GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
+ gem install bundler --user-install
+
+ # Install all needed gems.
+ bundle install --path ~/.gem/
+
+ # Build the new site and test it.
+ rm -fr ./content/
+ bundle exec rake test
+ '''.stripIndent().trim()
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_beam_Release_NightlySnapshot.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_Release_NightlySnapshot.groovy b/.jenkins/job_beam_Release_NightlySnapshot.groovy
index a19f8ff..f2c3ff0 100644
--- a/.jenkins/job_beam_Release_NightlySnapshot.groovy
+++ b/.jenkins/job_beam_Release_NightlySnapshot.groovy
@@ -28,7 +28,7 @@ mavenJob('beam_Release_NightlySnapshot') {
concurrentBuild()
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// Set maven paramaters.
common_job_properties.setMavenConfig(delegate)
http://git-wip-us.apache.org/repos/asf/beam/blob/5dc83f57/.jenkins/job_seed.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_seed.groovy b/.jenkins/job_seed.groovy
index fcdd564..b830f88 100644
--- a/.jenkins/job_seed.groovy
+++ b/.jenkins/job_seed.groovy
@@ -19,13 +19,14 @@
import common_job_properties
// Defines the seed job, which creates or updates all other Jenkins projects.
-job('beam_SeedJob_Main') {
- description('Automatically configures all Apache Beam main repo Jenkins ' +
- 'projects based on Jenkins DSL groovy files checked into the ' +
- 'code repository.')
+job('beam_SeedJob') {
+ description('Automatically configures all Apache Beam Jenkins projects based' +
+ ' on Jenkins DSL groovy files checked into the code repository.')
+
+ previousNames('beam_SeedJob_Main')
// Set common parameters.
- common_job_properties.setTopLevelJobProperties(delegate)
+ common_job_properties.setTopLevelMainJobProperties(delegate)
// This is a post-commit job that runs once per day, not for every push.
common_job_properties.setPostCommit(
[2/2] beam git commit: This closes #1897
Posted by da...@apache.org.
This closes #1897
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/1533d701
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/1533d701
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/1533d701
Branch: refs/heads/master
Commit: 1533d70137a77db497ba6dcebcb421c1988ec1df
Parents: c0910fd 5dc83f5
Author: Davor Bonaci <da...@google.com>
Authored: Thu Feb 2 16:39:00 2017 -0800
Committer: Davor Bonaci <da...@google.com>
Committed: Thu Feb 2 16:39:00 2017 -0800
----------------------------------------------------------------------
.jenkins/common_job_properties.groovy | 40 +++++++++----
...job_beam_PostCommit_Java_MavenInstall.groovy | 2 +-
...ostCommit_Java_RunnableOnService_Apex.groovy | 2 +-
...ommit_Java_RunnableOnService_Dataflow.groovy | 2 +-
...stCommit_Java_RunnableOnService_Flink.groovy | 2 +-
...ommit_Java_RunnableOnService_Gearpump.groovy | 4 +-
...stCommit_Java_RunnableOnService_Spark.groovy | 2 +-
.../job_beam_PostCommit_Python_Verify.groovy | 2 +-
.../job_beam_PreCommit_Java_MavenInstall.groovy | 2 +-
.../job_beam_PreCommit_Website_Stage.groovy | 62 ++++++++++++++++++++
.jenkins/job_beam_PreCommit_Website_Test.groovy | 47 +++++++++++++++
.../job_beam_Release_NightlySnapshot.groovy | 2 +-
.jenkins/job_seed.groovy | 11 ++--
13 files changed, 154 insertions(+), 26 deletions(-)
----------------------------------------------------------------------