You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2017/09/28 21:53:58 UTC
[2/2] beam git commit: Updated Pipeline job for Jenkins PreCommit
Updated Pipeline job for Jenkins PreCommit
Signed-off-by: Jason Kuster <ja...@google.com>
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d4323e98
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d4323e98
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d4323e98
Branch: refs/heads/master
Commit: d4323e98c0e287236169627e972c88e5bb1f1d53
Parents: 0724f1c
Author: Jason Kuster <ja...@google.com>
Authored: Wed Jun 28 16:22:52 2017 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Sep 28 14:53:29 2017 -0700
----------------------------------------------------------------------
.test-infra/jenkins/PreCommit_Pipeline.groovy | 89 +++++++++
.../jenkins/common_job_properties.groovy | 185 ++++++++++++++-----
.test-infra/jenkins/job_beam_Java_Build.groovy | 82 ++++++++
.../jenkins/job_beam_Java_CodeHealth.groovy | 39 ++++
.../job_beam_Java_IntegrationTest.groovy | 63 +++++++
.../jenkins/job_beam_Java_UnitTest.groovy | 49 +++++
.../jenkins/job_beam_PreCommit_Pipeline.groovy | 84 +++++++++
.../jenkins/job_beam_Python_UnitTest.groovy | 40 ++++
8 files changed, 584 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/PreCommit_Pipeline.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/PreCommit_Pipeline.groovy b/.test-infra/jenkins/PreCommit_Pipeline.groovy
new file mode 100644
index 0000000..9abf39d
--- /dev/null
+++ b/.test-infra/jenkins/PreCommit_Pipeline.groovy
@@ -0,0 +1,89 @@
+#!groovy
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import hudson.model.Result
+
+int NO_BUILD = -1
+
+// These are args for the GitHub Pull Request Builder (ghprb) Plugin. Providing these arguments is
+// necessary due to a bug in the ghprb plugin where environment variables are not correctly passed
+// to jobs downstream of a Pipeline job.
+// Tracked by https://github.com/jenkinsci/ghprb-plugin/issues/572.
+List<Object> ghprbArgs = [
+ string(name: 'ghprbGhRepository', value: "${ghprbGhRepository}"),
+ string(name: 'ghprbActualCommit', value: "${ghprbActualCommit}"),
+ string(name: 'ghprbPullId', value: "${ghprbPullId}")
+]
+
+// This argument is the commit at which to build.
+List<Object> commitArg = [string(name: 'sha1', value: "origin/pr/${ghprbPullId}/head")]
+
+int javaBuildNum = NO_BUILD
+
+// This (and the below) define "Stages" of a pipeline. These stages run serially, and inside can
+// have "parallel" blocks which execute several work steps concurrently. This work is limited to
+// simple operations -- more complicated operations need to be performed on an actual node. In this
+// case we are using the pipeline to trigger downstream builds.
+stage('Build') {
+ parallel (
+ java: {
+ def javaBuild = build job: 'beam_Java_Build', parameters: commitArg + ghprbArgs
+ if(javaBuild.getResult() == Result.SUCCESS.toString()) {
+ javaBuildNum = javaBuild.getNumber()
+ }
+ },
+ python_unit: { // Python doesn't have a build phase, so we include this here.
+ build job: 'beam_Python_UnitTest', parameters: commitArg + ghprbArgs
+ }
+ )
+}
+
+// This argument is provided to downstream jobs so they know from which build to pull artifacts.
+javaBuildArg = [string(name: 'buildNum', value: "${javaBuildNum}")]
+javaUnitPassed = false
+
+stage('Unit Test / Code Health') {
+ parallel (
+ java_unit: {
+ if(javaBuildNum != NO_BUILD) {
+ def javaTest = build job: 'beam_Java_UnitTest', parameters: javaBuildArg + ghprbArgs
+ if(javaTest.getResult() == Result.SUCCESS.toString()) {
+ javaUnitPassed = true
+ }
+ }
+ },
+ java_codehealth: {
+ if(javaBuildNum != NO_BUILD) {
+ build job: 'beam_Java_CodeHealth', parameters: javaBuildArg + ghprbArgs
+ }
+ }
+ )
+}
+
+stage('Integration Test') {
+ parallel (
+ // Not gated on codehealth because codehealth shouldn't affect whether tests provide useful
+ // signal.
+ java_integration: {
+ if(javaUnitPassed) {
+ build job: 'beam_Java_IntegrationTest', parameters: javaBuildArg + ghprbArgs
+ }
+ }
+ )
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/common_job_properties.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
index 70534c6..87763a2 100644
--- a/.test-infra/jenkins/common_job_properties.groovy
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -22,8 +22,27 @@
// http://groovy-lang.org/style-guide.html
class common_job_properties {
+ static void setSCM(def context, String repositoryName) {
+ context.scm {
+ git {
+ remote {
+ // Double quotes here mean ${repositoryName} is interpolated.
+ github("apache/${repositoryName}")
+ // Single quotes here mean that ${ghprbPullId} is not interpolated and instead passed
+ // through to Jenkins where it refers to the environment variable.
+ refspec('+refs/heads/*:refs/remotes/origin/* ' +
+ '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*')
+ }
+ branch('${sha1}')
+ extensions {
+ cleanAfterCheckout()
+ }
+ }
+ }
+ }
+
// Sets common top-level job properties for website repository jobs.
- static void setTopLevelWebsiteJobProperties(context,
+ static void setTopLevelWebsiteJobProperties(def context,
String branch = 'asf-site') {
setTopLevelJobProperties(
context,
@@ -34,7 +53,7 @@ class common_job_properties {
}
// Sets common top-level job properties for main repository jobs.
- static void setTopLevelMainJobProperties(context,
+ static void setTopLevelMainJobProperties(def context,
String branch = 'master',
int timeout = 100,
String jenkinsExecutorLabel = 'beam') {
@@ -48,7 +67,7 @@ class common_job_properties {
// Sets common top-level job properties. Accessed through one of the above
// methods to protect jobs from internal details of param defaults.
- private static void setTopLevelJobProperties(context,
+ private static void setTopLevelJobProperties(def context,
String repositoryName,
String defaultBranch,
String jenkinsExecutorLabel,
@@ -71,19 +90,7 @@ class common_job_properties {
}
// Source code management.
- context.scm {
- git {
- remote {
- url('https://github.com/apache/' + repositoryName + '.git')
- refspec('+refs/heads/*:refs/remotes/origin/* ' +
- '+refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*')
- }
- branch('${sha1}')
- extensions {
- cleanAfterCheckout()
- }
- }
- }
+ setSCM(context, repositoryName)
context.parameters {
// This is a recommended setup if you want to run the job manually. The
@@ -141,41 +148,19 @@ class common_job_properties {
delegate.context("Jenkins: " + commitStatusContext)
}
- /*
- This section is disabled, because of jenkinsci/ghprb-plugin#417 issue.
- For the time being, an equivalent configure section below is added.
-
// Comment messages after build completes.
buildStatus {
completedStatus('SUCCESS', successComment)
completedStatus('FAILURE', '--none--')
completedStatus('ERROR', '--none--')
}
- */
}
}
}
-
- // Comment messages after build completes.
- context.configure {
- def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message(successComment)
- result('SUCCESS')
- }
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message('--none--')
- result('ERROR')
- }
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message('--none--')
- result('FAILURE')
- }
- }
}
// Sets common config for Maven jobs.
- static void setMavenConfig(context, mavenInstallation='Maven 3.3.3') {
+ static void setMavenConfig(context, String mavenInstallation='Maven 3.3.3') {
context.mavenInstallation(mavenInstallation)
context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true')
context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS')
@@ -236,10 +221,19 @@ class common_job_properties {
}
}
+ static def mapToArgString(LinkedHashMap<String, String> inputArgs) {
+ List argList = []
+ inputArgs.each({
+ // FYI: Replacement only works with double quotes.
+ key, value -> argList.add("--$key=$value")
+ })
+ return argList.join(' ')
+ }
+
// Configures the argument list for performance tests, adding the standard
// performance test job arguments.
private static def genPerformanceArgs(def argMap) {
- def standard_args = [
+ LinkedHashMap<String, String> standardArgs = [
project: 'apache-beam-testing',
dpb_log_level: 'INFO',
maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn',
@@ -248,13 +242,8 @@ class common_job_properties {
official: 'true'
]
// Note: in case of key collision, keys present in ArgMap win.
- def joined_args = standard_args.plus(argMap)
- def argList = []
- joined_args.each({
- // FYI: Replacement only works with double quotes.
- key, value -> argList.add("--$key=$value")
- })
- return argList.join(' ')
+ LinkedHashMap<String, String> joinedArgs = standardArgs.plus(argMap)
+ return mapToArgString(joinedArgs)
}
// Adds the standard performance test job steps.
@@ -273,4 +262,106 @@ class common_job_properties {
shell("python PerfKitBenchmarker/pkb.py $pkbArgs")
}
}
+
+ /**
+ * Sets properties for all jobs which are run by a pipeline top-level (maven) job.
+ * @param context The delegate from the top level of a MavenJob.
+ * @param jobTimeout How long (in minutes) to wait for the job to finish.
+ * @param descriptor A short string identifying the job, e.g. "Java Unit Test".
+ */
+ static def setPipelineJobProperties(def context, int jobTimeout, String descriptor) {
+ context.parameters {
+ stringParam(
+ 'ghprbGhRepository',
+ 'N/A',
+ 'Repository name for use by ghprb plugin.')
+ stringParam(
+ 'ghprbActualCommit',
+ 'N/A',
+ 'Commit ID for use by ghprb plugin.')
+ stringParam(
+ 'ghprbPullId',
+ 'N/A',
+ 'PR # for use by ghprb plugin.')
+
+ }
+
+ // Set JDK version.
+ context.jdk('JDK 1.8 (latest)')
+
+ // Restrict this project to run only on Jenkins executors as specified
+ context.label('beam')
+
+ // Execute concurrent builds if necessary.
+ context.concurrentBuild()
+
+ context.wrappers {
+ timeout {
+ absolute(jobTimeout)
+ abortBuild()
+ }
+ credentialsBinding {
+ string("COVERALLS_REPO_TOKEN", "beam-coveralls-token")
+ }
+ downstreamCommitStatus {
+ delegate.context("Jenkins: ${descriptor}")
+ triggeredStatus("${descriptor} Pending")
+ startedStatus("Running ${descriptor}")
+ statusUrl()
+ completedStatus('SUCCESS', "${descriptor} Passed")
+ completedStatus('FAILURE', "${descriptor} Failed")
+ completedStatus('ERROR', "Error Executing ${descriptor}")
+ }
+ // Set SPARK_LOCAL_IP for spark tests.
+ environmentVariables {
+ env('SPARK_LOCAL_IP', '127.0.0.1')
+ }
+ }
+
+ // Set Maven parameters.
+ setMavenConfig(context)
+ }
+
+ /**
+ * Sets job properties common to pipeline jobs which are responsible for being the root of a
+ * build tree. Downstream jobs should pull artifacts from these jobs.
+ * @param context The delegate from the top level of a MavenJob.
+ */
+ static def setPipelineBuildJobProperties(def context) {
+ context.properties {
+ githubProjectUrl('https://github.com/apache/beam/')
+ }
+
+ context.parameters {
+ stringParam(
+ 'sha1',
+ 'master',
+ 'Commit id or refname (e.g. origin/pr/9/head) you want to build.')
+ }
+
+ // Source code management.
+ setSCM(context, 'beam')
+ }
+
+ /**
+ * Sets common job parameters for jobs which consume artifacts built for them by an upstream job.
+ * @param context The delegate from the top level of a MavenJob.
+ * @param jobName The job from which to copy artifacts.
+ */
+ static def setPipelineDownstreamJobProperties(def context, String jobName) {
+ context.parameters {
+ stringParam(
+ 'buildNum',
+ 'N/A',
+ "Build number of ${jobName} to copy from.")
+ }
+
+ context.preBuildSteps {
+ copyArtifacts(jobName) {
+ buildSelector {
+ buildNumber('${buildNum}')
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/job_beam_Java_Build.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_Java_Build.groovy b/.test-infra/jenkins/job_beam_Java_Build.groovy
new file mode 100644
index 0000000..704d7eb
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_Build.groovy
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which builds artifacts for downstream jobs to consume.
+mavenJob('beam_Java_Build') {
+ description('Builds Beam Java SDK and archives artifacts. Meant to be run as part of a pipeline.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Build")
+ // Set standard properties for a pipeline job which needs to pull from GitHub instead of an
+ // upstream job.
+ common_job_properties.setPipelineBuildJobProperties(delegate)
+
+ configure { project ->
+ // The CopyArtifact plugin doesn't support the job DSL so we have to configure it manually.
+ project / 'properties' / 'hudson.plugins.copyartifact.CopyArtifactPermissionProperty' / 'projectNameList' {
+ 'string' "beam_*"
+ }
+ // The Build Discarder also doesn't support the job DSL in the right way so we have to configure it manually.
+ // -1 indicates that a property is "infinite".
+ project / 'properties' / 'jenkins.model.BuildDiscarderProperty' / 'strategy'(class:'hudson.tasks.LogRotator') {
+ 'daysToKeep'(-1)
+ 'numToKeep'(-1)
+ 'artifactDaysToKeep'(1)
+ 'artifactNumToKeep'(-1)
+ }
+ }
+
+ // Construct Maven goals for this job.
+ profiles = [
+ 'direct-runner',
+ 'dataflow-runner',
+ 'spark-runner',
+ 'flink-runner',
+ 'apex-runner'
+ ]
+ args = [
+ '-B',
+ '-e',
+ "-P${profiles.join(',')}",
+ 'clean',
+ 'install',
+ "-pl '!sdks/python,!sdks/java/javadoc'",
+ '-DskipTests',
+ '-Dcheckstyle.skip',
+ ]
+ goals(args.join(' '))
+
+ // This job publishes artifacts so that downstream jobs can use them.
+ publishers {
+ archiveArtifacts {
+ pattern('.repository/org/apache/beam/**/*')
+ pattern('.test-infra/**/*')
+ pattern('.github/**/*')
+ pattern('examples/**/*')
+ pattern('runners/**/*')
+ pattern('sdks/**/*')
+ pattern('target/**/*')
+ pattern('pom.xml')
+ exclude('examples/**/*.jar,runners/**/*.jar,sdks/**/*.jar,target/**/*.jar')
+ onlyIfSuccessful()
+ defaultExcludes()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
new file mode 100644
index 0000000..41a4536
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_CodeHealth.groovy
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which runs the Beam code health checks.
+mavenJob('beam_Java_CodeHealth') {
+ description('Runs Java code health checks. Meant to be run as part of a pipeline.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Code Health")
+ // This job runs downstream of the beam_Java_Build job and gets artifacts from that job.
+ common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build')
+
+ args = [
+ '-B',
+ '-e',
+ "-pl '!sdks/python'",
+ 'checkstyle:check',
+ 'findbugs:check',
+ 'org.apache.rat:apache-rat-plugin:check',
+ ]
+ goals(args.join(' '))
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
new file mode 100644
index 0000000..56daf73
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_IntegrationTest.groovy
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which runs the set of precommit integration tests.
+mavenJob('beam_Java_IntegrationTest') {
+ description('Runs Java Failsafe integration tests. Designed to be run as part of a pipeline.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Integration Tests")
+ // Set standard properties for a job which pulls artifacts from an upstream job.
+ common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build')
+
+ // Profiles to activate in order to ensure runners are available at test time.
+ profiles = [
+ 'jenkins-precommit',
+ 'direct-runner',
+ 'dataflow-runner',
+ 'spark-runner',
+ 'flink-runner',
+ 'apex-runner'
+ ]
+ // In the case of the precommit integration tests, we are currently only running the integration
+ // tests in the examples directory. By directly invoking failsafe with an execution name (which we
+ // do in order to avoid building artifacts again) we are required to enumerate each execution we
+ // want to run, something which is feasible in this case.
+ examples_integration_executions = [
+ 'apex-runner-integration-tests',
+ 'dataflow-runner-integration-tests',
+ 'dataflow-runner-integration-tests-streaming',
+ 'direct-runner-integration-tests',
+ 'flink-runner-integration-tests',
+ 'spark-runner-integration-tests',
+ ]
+ // Arguments to provide Maven.
+ args = [
+ '-B',
+ '-e',
+ "-P${profiles.join(',')}",
+ "-pl examples/java",
+ ]
+ // This adds executions for each of the failsafe invocations listed above to the list of goals.
+ examples_integration_executions.each({
+ value -> args.add("failsafe:integration-test@${value}")
+ })
+ goals(args.join(' '))
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_Java_UnitTest.groovy b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
new file mode 100644
index 0000000..8f03900
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Java_UnitTest.groovy
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java Jenkins job which runs the current set of standard unit tests.
+mavenJob('beam_Java_UnitTest') {
+ description('Runs Java Surefire unit tests. Designed to be run by a pipeline job.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 30, "Java Unit Tests")
+ // Set standard properties for a job which pulls artifacts from an upstream job.
+ common_job_properties.setPipelineDownstreamJobProperties(delegate, 'beam_Java_Build')
+
+ // Construct Maven goals for this job.
+ profiles = [
+ 'direct-runner',
+ 'dataflow-runner',
+ 'spark-runner',
+ 'flink-runner',
+ 'apex-runner'
+ ]
+ args = [
+ '-B',
+ '-e',
+ "-P${profiles.join(',')}",
+ 'surefire:test@default-test',
+ 'coveralls:report', // TODO: Will this work? Can't verify on my own Jenkins due to no coveralls.
+ "-pl '!sdks/python'",
+ '-DrepoToken=$COVERALLS_REPO_TOKEN',
+ '-DpullRequest=$ghprbPullId',
+ ]
+ goals(args.join(' '))
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy
new file mode 100644
index 0000000..2e1ea55
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Pipeline.groovy
@@ -0,0 +1,84 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job owns the overall execution of the precommit pipeline. The actual pipeline code is in
+// Precommit_Pipeline.groovy.
+pipelineJob('beam_PreCommit_Pipeline') {
+ description('PreCommit Pipeline Job. Owns overall lifecycle of PreCommit tests.')
+
+ properties {
+ githubProjectUrl('https://github.com/apache/beam/')
+ }
+
+ parameters {
+ // Allow building at a specific commit.
+ stringParam(
+ 'commit',
+ 'master',
+ 'Commit id or refname (e.g. origin/pr/9/head) you want to build.')
+ }
+
+ wrappers {
+ // Set a timeout appropriate for the precommit tests.
+ timeout {
+ absolute(120)
+ abortBuild()
+ }
+ }
+
+ // Restrict this project to run only on Jenkins executors as specified
+ label('beam')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ triggers {
+ githubPullRequest {
+ admins(['asfbot'])
+ useGitHubHooks()
+ orgWhitelist(['apache'])
+ allowMembersOfWhitelistedOrgsAsAdmin()
+ permitAll()
+ // Remove once Pipeline Build is default.
+ triggerPhrase('Run PreCommit Pipeline')
+ onlyTriggerPhrase()
+ displayBuildErrorsOnDownstreamBuilds()
+ extensions {
+ commitStatus {
+ context("Jenkins: PreCommit Pipeline")
+ }
+ buildStatus {
+ completedStatus('SUCCESS', '--none--')
+ completedStatus('FAILURE', '--none--')
+ completedStatus('ERROR', '--none--')
+ }
+ }
+ }
+ }
+
+ definition {
+ cpsScm {
+ // Source code management.
+ common_job_properties.setSCM(delegate, 'beam')
+ scriptPath('.test-infra/jenkins/PreCommit_Pipeline.groovy')
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/d4323e98/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_Python_UnitTest.groovy b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
new file mode 100644
index 0000000..89701d4
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_Python_UnitTest.groovy
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Python Jenkins job which runs a maven install, and the current set of precommit
+// tests.
+mavenJob('beam_Python_UnitTest') {
+ description('Runs Python unit tests on a specific commit. Designed to be run by a pipeline job.')
+
+ // Set standard properties for a job which is part of a pipeline.
+ common_job_properties.setPipelineJobProperties(delegate, 35, "Python Unit Tests")
+ // Set standard properties for a pipeline job which needs to pull from GitHub instead of an
+ // upstream job.
+ common_job_properties.setPipelineBuildJobProperties(delegate)
+
+ // Construct Maven goals for this job.
+ args = [
+ '-B',
+ '-e',
+ 'clean install',
+ '-pl sdks/python',
+ ]
+ goals(args.join(' '))
+}