You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by tg...@apache.org on 2017/04/12 20:52:46 UTC
[6/6] beam git commit: Move travis/jenkins folders in a test-infra
folder
Move travis/jenkins folders in a test-infra folder
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/66b20af9
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/66b20af9
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/66b20af9
Branch: refs/heads/master
Commit: 66b20af9fed36ff419ca6da3de4b1b1a02e66464
Parents: 1788cef
Author: Stephen Sisk <si...@google.com>
Authored: Fri Apr 7 15:57:33 2017 -0700
Committer: Thomas Groh <tg...@google.com>
Committed: Wed Apr 12 13:52:28 2017 -0700
----------------------------------------------------------------------
.jenkins/common_job_properties.groovy | 261 -------------------
.../job_beam_PerformanceTests_Dataflow.groovy | 43 ---
.jenkins/job_beam_PerformanceTests_JDBC.groovy | 60 -----
.jenkins/job_beam_PerformanceTests_Spark.groovy | 44 ----
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 ---
..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 ----
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 ----
...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 ---
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 ----
...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 ----
.../job_beam_PostCommit_Python_Verify.groovy | 55 ----
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 ---
.../job_beam_PreCommit_Website_Stage.groovy | 80 ------
.jenkins/job_beam_PreCommit_Website_Test.groovy | 65 -----
.../job_beam_Release_NightlySnapshot.groovy | 45 ----
.jenkins/job_seed.groovy | 53 ----
.../jenkins/common_job_properties.groovy | 261 +++++++++++++++++++
.../job_beam_PerformanceTests_Dataflow.groovy | 43 +++
.../job_beam_PerformanceTests_JDBC.groovy | 60 +++++
.../job_beam_PerformanceTests_Spark.groovy | 44 ++++
...job_beam_PostCommit_Java_MavenInstall.groovy | 42 +++
..._PostCommit_Java_ValidatesRunner_Apex.groovy | 48 ++++
...tCommit_Java_ValidatesRunner_Dataflow.groovy | 45 ++++
...PostCommit_Java_ValidatesRunner_Flink.groovy | 43 +++
...tCommit_Java_ValidatesRunner_Gearpump.groovy | 49 ++++
...PostCommit_Java_ValidatesRunner_Spark.groovy | 44 ++++
.../job_beam_PostCommit_Python_Verify.groovy | 55 ++++
.../job_beam_PreCommit_Java_MavenInstall.groovy | 42 +++
.../job_beam_PreCommit_Website_Stage.groovy | 80 ++++++
.../job_beam_PreCommit_Website_Test.groovy | 65 +++++
.../job_beam_Release_NightlySnapshot.groovy | 45 ++++
.test-infra/jenkins/job_seed.groovy | 53 ++++
.test-infra/travis/README.md | 23 ++
.test-infra/travis/settings.xml | 33 +++
.test-infra/travis/test_wordcount.sh | 125 +++++++++
.travis.yml | 4 +-
.travis/README.md | 23 --
.travis/settings.xml | 33 ---
.travis/test_wordcount.sh | 125 ---------
39 files changed, 1202 insertions(+), 1202 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/common_job_properties.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/common_job_properties.groovy b/.jenkins/common_job_properties.groovy
deleted file mode 100644
index ee10281..0000000
--- a/.jenkins/common_job_properties.groovy
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Contains functions that help build Jenkins projects. Functions typically set
-// common properties that are shared among all Jenkins projects.
-// Code in this directory should conform to the Groovy style guide.
-// http://groovy-lang.org/style-guide.html
-class common_job_properties {
-
- // Sets common top-level job properties for website repository jobs.
- static void setTopLevelWebsiteJobProperties(context) {
- setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30)
- }
-
- // Sets common top-level job properties for main repository jobs.
- static void setTopLevelMainJobProperties(context,
- String defaultBranch = 'master',
- int defaultTimeout = 100) {
- setTopLevelJobProperties(context, 'beam', defaultBranch, defaultTimeout)
- }
-
- // Sets common top-level job properties. Accessed through one of the above
- // methods to protect jobs from internal details of param defaults.
- private static void setTopLevelJobProperties(context,
- String repositoryName,
- String defaultBranch,
- int defaultTimeout) {
-
- // GitHub project.
- context.properties {
- githubProjectUrl('https://github.com/apache/' + repositoryName + '/')
- }
-
- // Set JDK version.
- context.jdk('JDK 1.8 (latest)')
-
- // Restrict this project to run only on Jenkins executors dedicated to the
- // Apache Beam project.
- context.label('beam')
-
- // Discard old builds. Build records are only kept up to this number of days.
- context.logRotator {
- daysToKeep(14)
- }
-
- // Source code management.
- context.scm {
- git {
- remote {
- url('https://github.com/apache/' + repositoryName + '.git')
- refspec('+refs/heads/*:refs/remotes/origin/* ' +
- '+refs/pull/*:refs/remotes/origin/pr/*')
- }
- branch('${sha1}')
- extensions {
- cleanAfterCheckout()
- pruneBranches()
- }
- }
- }
-
- context.parameters {
- // This is a recommended setup if you want to run the job manually. The
- // ${sha1} parameter needs to be provided, and defaults to the main branch.
- stringParam(
- 'sha1',
- defaultBranch,
- 'Commit id or refname (eg: origin/pr/9/head) you want to build.')
- }
-
- context.wrappers {
- // Abort the build if it's stuck for more minutes than specified.
- timeout {
- absolute(defaultTimeout)
- abortBuild()
- }
-
- // Set SPARK_LOCAL_IP for spark tests.
- environmentVariables {
- env('SPARK_LOCAL_IP', '127.0.0.1')
- }
- credentialsBinding {
- string("COVERALLS_REPO_TOKEN", "beam-coveralls-token")
- }
- }
- }
-
- // Sets the pull request build trigger. Accessed through precommit methods
- // below to insulate callers from internal parameter defaults.
- private static void setPullRequestBuildTrigger(context,
- String commitStatusContext,
- String successComment = '--none--',
- String prTriggerPhrase = '') {
- context.triggers {
- githubPullRequest {
- admins(['asfbot'])
- useGitHubHooks()
- orgWhitelist(['apache'])
- allowMembersOfWhitelistedOrgsAsAdmin()
- permitAll()
- // prTriggerPhrase is the argument which gets set when we want to allow
- // post-commit builds to run against pending pull requests. This block
- // overrides the default trigger phrase with the new one. Setting this
- // will disable automatic invocation of this build; the phrase will be
- // required to start it.
- if (prTriggerPhrase) {
- triggerPhrase(prTriggerPhrase)
- onlyTriggerPhrase()
- }
-
- extensions {
- commitStatus {
- // This is the name that will show up in the GitHub pull request UI
- // for this Jenkins project.
- delegate.context("Jenkins: " + commitStatusContext)
- }
-
- /*
- This section is disabled, because of jenkinsci/ghprb-plugin#417 issue.
- For the time being, an equivalent configure section below is added.
-
- // Comment messages after build completes.
- buildStatus {
- completedStatus('SUCCESS', successComment)
- completedStatus('FAILURE', '--none--')
- completedStatus('ERROR', '--none--')
- }
- */
- }
- }
- }
-
- // Comment messages after build completes.
- context.configure {
- def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message(successComment)
- result('SUCCESS')
- }
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message('--none--')
- result('ERROR')
- }
- messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
- message('--none--')
- result('FAILURE')
- }
- }
- }
-
- // Sets common config for Maven jobs.
- static void setMavenConfig(context) {
- context.mavenInstallation('Maven 3.3.3')
- context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true')
- context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS')
- // The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable
- // tiered compilation to make the JVM startup times faster during the tests.
- context.mavenOpts('-XX:+TieredCompilation')
- context.mavenOpts('-XX:TieredStopAtLevel=1')
- context.rootPOM('pom.xml')
- // Use a repository local to the workspace for better isolation of jobs.
- context.localRepository(LocalRepositoryLocation.LOCAL_TO_WORKSPACE)
- // Disable archiving the built artifacts by default, as this is slow and flaky.
- // We can usually recreate them easily, and we can also opt-in individual jobs
- // to artifact archiving.
- context.archivingDisabled(true)
- }
-
- // Sets common config for PreCommit jobs.
- static void setPreCommit(context,
- String commitStatusName,
- String successComment = '--none--') {
- // Set pull request build trigger.
- setPullRequestBuildTrigger(context, commitStatusName, successComment)
- }
-
- // Enable triggering postcommit runs against pull requests. Users can comment the trigger phrase
- // specified in the postcommit job and have the job run against their PR to run
- // tests not in the presubmit suite for additional confidence.
- static void enablePhraseTriggeringFromPullRequest(context,
- String commitStatusName,
- String prTriggerPhrase) {
- setPullRequestBuildTrigger(
- context,
- commitStatusName,
- '--none--',
- prTriggerPhrase)
- }
-
- // Sets common config for PostCommit jobs.
- static void setPostCommit(context,
- String buildSchedule = '0 */6 * * *',
- boolean triggerEveryPush = true,
- String notifyAddress = 'commits@beam.apache.org',
- boolean emailIndividuals = true) {
- // Set build triggers
- context.triggers {
- // By default runs every 6 hours.
- cron(buildSchedule)
- if (triggerEveryPush) {
- githubPush()
- }
- }
-
- context.publishers {
- // Notify an email address for each failed build (defaults to commits@).
- mailer(notifyAddress, false, emailIndividuals)
- }
- }
-
- // Configures the argument list for performance tests, adding the standard
- // performance test job arguments.
- private static def genPerformanceArgs(def argMap) {
- def standard_args = [
- project: 'apache-beam-testing',
- dpb_log_level: 'INFO',
- maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn',
- bigquery_table: 'beam_performance.pkb_results',
- // Publishes results with official tag, for use in dashboards.
- official: 'true'
- ]
- // Note: in case of key collision, keys present in ArgMap win.
- def joined_args = standard_args.plus(argMap)
- def argList = []
- joined_args.each({
- // FYI: Replacement only works with double quotes.
- key, value -> argList.add("--$key=$value")
- })
- return argList.join(' ')
- }
-
- // Adds the standard performance test job steps.
- static def buildPerformanceTest(def context, def argMap) {
- def pkbArgs = genPerformanceArgs(argMap)
- context.steps {
- // Clean up environment.
- shell('rm -rf PerfKitBenchmarker')
- // Clone appropriate perfkit branch
- shell('git clone https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git')
- // Install job requirements.
- shell('pip install --user -r PerfKitBenchmarker/requirements.txt')
- // Launch performance test.
- shell("python PerfKitBenchmarker/pkb.py $pkbArgs")
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PerformanceTests_Dataflow.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PerformanceTests_Dataflow.groovy b/.jenkins/job_beam_PerformanceTests_Dataflow.groovy
deleted file mode 100644
index 51c73f3..0000000
--- a/.jenkins/job_beam_PerformanceTests_Dataflow.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Beam performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_Dataflow'){
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- 'commits@beam.apache.org',
- false)
-
- def argMap = [
- benchmarks: 'dpb_wordcount_benchmark',
- dpb_dataflow_staging_location: 'gs://temp-storage-for-perf-tests/staging',
- dpb_wordcount_input: 'dataflow-samples/shakespeare/kinglear.txt',
- config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataflow'
- ]
-
- common_job_properties.buildPerformanceTest(delegate, argMap)
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PerformanceTests_JDBC.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PerformanceTests_JDBC.groovy b/.jenkins/job_beam_PerformanceTests_JDBC.groovy
deleted file mode 100644
index 8e581c2..0000000
--- a/.jenkins/job_beam_PerformanceTests_JDBC.groovy
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Beam performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_JDBC'){
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- 'commits@beam.apache.org',
- false)
-
- def pipelineArgs = [
- tempRoot: 'gs://temp-storage-for-end-to-end-tests',
- project: 'apache-beam-testing',
- postgresServerName: '10.36.0.11',
- postgresUsername: 'postgres',
- postgresDatabaseName: 'postgres',
- postgresPassword: 'uuinkks',
- postgresSsl: 'false'
- ]
- def pipelineArgList = []
- pipelineArgs.each({
- key, value -> pipelineArgList.add("--$key=$value")
- })
- def pipelineArgsJoined = pipelineArgList.join(',')
-
- def argMap = [
- benchmarks: 'beam_integration_benchmark',
- beam_it_module: 'sdks/java/io/jdbc',
- beam_it_args: pipelineArgsJoined,
- beam_it_class: 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
- // Profile is located in $BEAM_ROOT/sdks/java/io/pom.xml.
- beam_it_profile: 'io-it'
- ]
-
- common_job_properties.buildPerformanceTest(delegate, argMap)
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PerformanceTests_Spark.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PerformanceTests_Spark.groovy b/.jenkins/job_beam_PerformanceTests_Spark.groovy
deleted file mode 100644
index ba719bf..0000000
--- a/.jenkins/job_beam_PerformanceTests_Spark.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Beam performance tests on PerfKit Benchmarker.
-job('beam_PerformanceTests_Spark'){
- // Set default Beam job properties.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Run job in postcommit every 6 hours, don't trigger every push, and
- // don't email individual committers.
- common_job_properties.setPostCommit(
- delegate,
- '0 */6 * * *',
- false,
- 'commits@beam.apache.org',
- false)
-
- def argMap = [
- benchmarks: 'dpb_wordcount_benchmark',
- // There are currently problems uploading to Dataproc, so we use a file
- // already present on the machines as input.
- dpb_wordcount_input: '/etc/hosts',
- config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataproc'
- ]
-
- common_job_properties.buildPerformanceTest(delegate, argMap)
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
deleted file mode 100644
index a288a84..0000000
--- a/.jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the Java postcommit tests, including the suite of integration
-// tests.
-mavenJob('beam_PostCommit_Java_MavenInstall') {
- description('Runs postcommit tests on the Java SDK.')
-
- previousNames('beam_PostCommit_MavenVerify')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Maven goals for this job.
- goals('-B -e -P release,dataflow-runner clean install coveralls:report -DrepoToken=$COVERALLS_REPO_TOKEN -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
deleted file mode 100644
index c16a1e2..0000000
--- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Apex runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') {
- description('Runs the ValidatesRunner suite on the Apex runner.')
- previousNames('beam_PostCommit_Java_RunnableOnService_Apex')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Apex Runner ValidatesRunner Tests',
- 'Run Apex ValidatesRunner')
-
- // Maven goals for this job.
- goals('''clean verify --projects runners/apex \
- --also-make \
- --batch-mode \
- --errors \
- --activate-profiles validates-runner-tests \
- --activate-profiles local-validates-runner-tests''')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
deleted file mode 100644
index 33235ff..0000000
--- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Dataflow
-// runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') {
- description('Runs the ValidatesRunner suite on the Dataflow runner.')
- previousNames('beam_PostCommit_Java_RunnableOnService_Dataflow')
-
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Google Cloud Dataflow Runner ValidatesRunner Tests',
- 'Run Dataflow ValidatesRunner')
-
- // Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner", "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
deleted file mode 100644
index 411106d..0000000
--- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Flink runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Flink') {
- description('Runs the ValidatesRunner suite on the Flink runner.')
- previousNames('beam_PostCommit_Java_RunnableOnService_Flink')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Flink Runner ValidatesRunner Tests',
- 'Run Flink ValidatesRunner')
-
- // Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/flink/runner -Plocal-validates-runner-tests -Pvalidates-runner-tests')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
deleted file mode 100644
index 1348a19..0000000
--- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Gearpump
-// runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
- description('Runs the ValidatesRunner suite on the Gearpump runner.')
-
- previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(
- delegate,
- 'gearpump-runner')
-
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
-
- // Sets that this is a PostCommit job.
- // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
- // In post-commit this job triggers only on SCM changes.
- common_job_properties.setPostCommit(delegate, '0 5 31 2 *')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Gearpump Runner ValidatesRunner Tests',
- 'Run Gearpump ValidatesRunner')
-
- // Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy b/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
deleted file mode 100644
index 9fbc219..0000000
--- a/.jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job runs the suite of ValidatesRunner tests against the Spark runner.
-mavenJob('beam_PostCommit_Java_ValidatesRunner_Spark') {
- description('Runs the ValidatesRunner suite on the Spark runner.')
-
- previousNames('beam_PostCommit_Java_RunnableOnService_Spark')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Set maven parameters.
- common_job_properties.setMavenConfig(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate)
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Apache Spark Runner ValidatesRunner Tests',
- 'Run Spark ValidatesRunner')
-
- // Maven goals for this job.
- goals('-B -e clean verify -am -pl runners/spark -Pvalidates-runner-tests -Plocal-validates-runner-tests -Dspark.ui.enabled=false')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PostCommit_Python_Verify.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PostCommit_Python_Verify.groovy b/.jenkins/job_beam_PostCommit_Python_Verify.groovy
deleted file mode 100644
index 28cf77e..0000000
--- a/.jenkins/job_beam_PostCommit_Python_Verify.groovy
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This job defines the Python postcommit tests.
-job('beam_PostCommit_Python_Verify') {
- description('Runs postcommit tests on the Python SDK.')
-
- previousNames('beam_PostCommit_PythonVerify')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Sets that this is a PostCommit job.
- common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Python SDK PostCommit Tests',
- 'Run Python PostCommit')
-
- // Allow the test to only run on particular nodes
- // TODO(BEAM-1817): Remove once the tests can run on all nodes
- parameters {
- nodeParam('TEST_HOST') {
- description('select test host as either beam1, 2 or 3')
- defaultNodes(['beam3'])
- allowedNodes(['beam1', 'beam2', 'beam3'])
- trigger('multiSelectionDisallowed')
- eligibility('IgnoreOfflineNodeEligibility')
- }
- }
-
- // Execute shell command to test Python SDK.
- steps {
- shell('bash sdks/python/run_postcommit.sh')
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
deleted file mode 100644
index 3718551..0000000
--- a/.jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the Java precommit which runs a maven install, and the current set
-// of precommit tests.
-mavenJob('beam_PreCommit_Java_MavenInstall') {
- description('Runs an install of the current GitHub Pull Request.')
-
- previousNames('beam_PreCommit_MavenVerify')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Set Maven parameters.
- common_job_properties.setMavenConfig(delegate)
-
- // Sets that this is a PreCommit job.
- common_job_properties.setPreCommit(delegate, 'Maven clean install')
-
- // Maven goals for this job.
- goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PreCommit_Website_Stage.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Website_Stage.groovy b/.jenkins/job_beam_PreCommit_Website_Stage.groovy
deleted file mode 100644
index 7c64f11..0000000
--- a/.jenkins/job_beam_PreCommit_Website_Stage.groovy
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// Defines a job.
-job('beam_PreCommit_Website_Stage') {
- description('Stages the pull requests proposed for the Apache Beam ' +
- 'website to a temporary location to ease reviews.')
-
- // Set common parameters.
- common_job_properties.setTopLevelWebsiteJobProperties(delegate)
-
- // Set pull request build trigger.
- common_job_properties.setPreCommit(
- delegate,
- 'Automatic staging of pull requests',
- '\nJenkins built the site at commit id ${ghprbActualCommit} with ' +
- 'Jekyll and staged it [here](http://apache-beam-website-pull-' +
- 'requests.storage.googleapis.com/${ghprbPullId}/index.html). ' +
- 'Happy reviewing.\n\nNote that any previous site has been deleted. ' +
- 'This staged site will be automatically deleted after its TTL ' +
- 'expires. Push any commit to the pull request branch or re-trigger ' +
- 'the build to get it staged again.')
-
- steps {
- // Run the following shell script as a build step.
- shell '''
- # Install RVM.
- gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
- 409B6B1796C275462A1703113804BB82D39DC0E3
- \\curl -sSL https://get.rvm.io | bash
- source /home/jenkins/.rvm/scripts/rvm
-
- # Install Ruby.
- RUBY_VERSION_NUM=2.3.0
- rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
-
- # Install Bundler gem
- PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
- GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
- gem install bundler --user-install
-
- # Install all needed gems.
- bundle install --path ~/.gem/
-
- # Remove current site if it exists.
- GCS_PATH="gs://apache-beam-website-pull-requests/${ghprbPullId}/"
- gsutil -m rm -r -f ${GCS_PATH} || true
-
- # Build the new site with the baseurl specified.
- rm -fr ./content/
- bundle exec jekyll build --baseurl=/${ghprbPullId}
-
- # Install BeautifulSoup HTML Parser for python.
- pip install --user beautifulsoup4
-
- # Fix links on staged website.
- python .jenkins/append_index_html_to_internal_links.py
-
- # Upload the new site.
- gsutil -m cp -R ./content/* ${GCS_PATH}
- '''.stripIndent().trim()
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_PreCommit_Website_Test.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_PreCommit_Website_Test.groovy b/.jenkins/job_beam_PreCommit_Website_Test.groovy
deleted file mode 100644
index 421b58a..0000000
--- a/.jenkins/job_beam_PreCommit_Website_Test.groovy
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// Defines a job.
-job('beam_PreCommit_Website_Test') {
- description('Runs tests on the pull requests proposed for the Apache Beam ' +
- 'website.')
-
- // Set common parameters.
- common_job_properties.setTopLevelWebsiteJobProperties(delegate)
-
- // Execute concurrent builds. Multiple builds of this project may be executed
- // in parallel. This is safe because this build does not require exclusive
- // access to any shared resources.
- concurrentBuild()
-
- // Set pull request build trigger.
- common_job_properties.setPreCommit(
- delegate,
- 'Test website (dead links, etc.)')
-
- steps {
- // Run the following shell script as a build step.
- shell '''
- # Install RVM.
- gpg --keyserver hkp://keys.gnupg.net --recv-keys \\
- 409B6B1796C275462A1703113804BB82D39DC0E3
- \\curl -sSL https://get.rvm.io | bash
- source /home/jenkins/.rvm/scripts/rvm
-
- # Install Ruby.
- RUBY_VERSION_NUM=2.3.0
- rvm install ruby $RUBY_VERSION_NUM --autolibs=read-only
-
- # Install Bundler gem
- PATH=~/.gem/ruby/$RUBY_VERSION_NUM/bin:$PATH
- GEM_PATH=~/.gem/ruby/$RUBY_VERSION_NUM/:$GEM_PATH
- gem install bundler --user-install
-
- # Install all needed gems.
- bundle install --path ~/.gem/
-
- # Build the new site and test it.
- rm -fr ./content/
- bundle exec rake test
- '''.stripIndent().trim()
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_beam_Release_NightlySnapshot.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_beam_Release_NightlySnapshot.groovy b/.jenkins/job_beam_Release_NightlySnapshot.groovy
deleted file mode 100644
index f2c3ff0..0000000
--- a/.jenkins/job_beam_Release_NightlySnapshot.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// This is the nightly snapshot build -- we use this to deploy a daily snapshot
-// to https://repository.apache.org/content/groups/snapshots/org/apache/beam.
-// Runs the postsubmit suite before deploying.
-mavenJob('beam_Release_NightlySnapshot') {
- description('Runs a mvn clean deploy of the nightly snapshot.')
-
- // Execute concurrent builds if necessary.
- concurrentBuild()
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // Set maven paramaters.
- common_job_properties.setMavenConfig(delegate)
-
- // This is a post-commit job that runs once per day, not for every push.
- common_job_properties.setPostCommit(
- delegate,
- '0 7 * * *',
- false,
- 'dev@beam.apache.org')
-
- // Maven goals for this job.
- goals('-B -e clean deploy -P release,dataflow-runner -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'')
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.jenkins/job_seed.groovy
----------------------------------------------------------------------
diff --git a/.jenkins/job_seed.groovy b/.jenkins/job_seed.groovy
deleted file mode 100644
index 7e5272b..0000000
--- a/.jenkins/job_seed.groovy
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import common_job_properties
-
-// Defines the seed job, which creates or updates all other Jenkins projects.
-job('beam_SeedJob') {
- description('Automatically configures all Apache Beam Jenkins projects based' +
- ' on Jenkins DSL groovy files checked into the code repository.')
-
- previousNames('beam_SeedJob_Main')
-
- // Set common parameters.
- common_job_properties.setTopLevelMainJobProperties(delegate)
-
- // This is a post-commit job that runs once per day, not for every push.
- common_job_properties.setPostCommit(
- delegate,
- '0 6 * * *',
- false,
- 'dev@beam.apache.org')
-
- // Allows triggering this build against pull requests.
- common_job_properties.enablePhraseTriggeringFromPullRequest(
- delegate,
- 'Seed Job',
- 'Run Seed Job')
-
- steps {
- dsl {
- // A list or a glob of other groovy files to process.
- external('.jenkins/job_*.groovy')
-
- // If a job is removed from the script, disable it (rather than deleting).
- removeAction('DISABLE')
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/common_job_properties.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/common_job_properties.groovy b/.test-infra/jenkins/common_job_properties.groovy
new file mode 100644
index 0000000..ee10281
--- /dev/null
+++ b/.test-infra/jenkins/common_job_properties.groovy
@@ -0,0 +1,261 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Contains functions that help build Jenkins projects. Functions typically set
+// common properties that are shared among all Jenkins projects.
+// Code in this directory should conform to the Groovy style guide.
+// http://groovy-lang.org/style-guide.html
+class common_job_properties {
+
+ // Sets common top-level job properties for website repository jobs.
+ static void setTopLevelWebsiteJobProperties(context) {
+ setTopLevelJobProperties(context, 'beam-site', 'asf-site', 30)
+ }
+
+ // Sets common top-level job properties for main repository jobs.
+ static void setTopLevelMainJobProperties(context,
+ String defaultBranch = 'master',
+ int defaultTimeout = 100) {
+ setTopLevelJobProperties(context, 'beam', defaultBranch, defaultTimeout)
+ }
+
+ // Sets common top-level job properties. Accessed through one of the above
+ // methods to protect jobs from internal details of param defaults.
+ private static void setTopLevelJobProperties(context,
+ String repositoryName,
+ String defaultBranch,
+ int defaultTimeout) {
+
+ // GitHub project.
+ context.properties {
+ githubProjectUrl('https://github.com/apache/' + repositoryName + '/')
+ }
+
+ // Set JDK version.
+ context.jdk('JDK 1.8 (latest)')
+
+ // Restrict this project to run only on Jenkins executors dedicated to the
+ // Apache Beam project.
+ context.label('beam')
+
+ // Discard old builds. Build records are only kept up to this number of days.
+ context.logRotator {
+ daysToKeep(14)
+ }
+
+ // Source code management.
+ context.scm {
+ git {
+ remote {
+ url('https://github.com/apache/' + repositoryName + '.git')
+ refspec('+refs/heads/*:refs/remotes/origin/* ' +
+ '+refs/pull/*:refs/remotes/origin/pr/*')
+ }
+ branch('${sha1}')
+ extensions {
+ cleanAfterCheckout()
+ pruneBranches()
+ }
+ }
+ }
+
+ context.parameters {
+ // This is a recommended setup if you want to run the job manually. The
+ // ${sha1} parameter needs to be provided, and defaults to the main branch.
+ stringParam(
+ 'sha1',
+ defaultBranch,
+ 'Commit id or refname (eg: origin/pr/9/head) you want to build.')
+ }
+
+ context.wrappers {
+ // Abort the build if it's stuck for more minutes than specified.
+ timeout {
+ absolute(defaultTimeout)
+ abortBuild()
+ }
+
+ // Set SPARK_LOCAL_IP for spark tests.
+ environmentVariables {
+ env('SPARK_LOCAL_IP', '127.0.0.1')
+ }
+ credentialsBinding {
+ string("COVERALLS_REPO_TOKEN", "beam-coveralls-token")
+ }
+ }
+ }
+
+ // Sets the pull request build trigger. Accessed through precommit methods
+ // below to insulate callers from internal parameter defaults.
+ private static void setPullRequestBuildTrigger(context,
+ String commitStatusContext,
+ String successComment = '--none--',
+ String prTriggerPhrase = '') {
+ context.triggers {
+ githubPullRequest {
+ admins(['asfbot'])
+ useGitHubHooks()
+ orgWhitelist(['apache'])
+ allowMembersOfWhitelistedOrgsAsAdmin()
+ permitAll()
+ // prTriggerPhrase is the argument which gets set when we want to allow
+ // post-commit builds to run against pending pull requests. This block
+ // overrides the default trigger phrase with the new one. Setting this
+ // will disable automatic invocation of this build; the phrase will be
+ // required to start it.
+ if (prTriggerPhrase) {
+ triggerPhrase(prTriggerPhrase)
+ onlyTriggerPhrase()
+ }
+
+ extensions {
+ commitStatus {
+ // This is the name that will show up in the GitHub pull request UI
+ // for this Jenkins project.
+ delegate.context("Jenkins: " + commitStatusContext)
+ }
+
+ /*
+ This section is disabled, because of jenkinsci/ghprb-plugin#417 issue.
+ For the time being, an equivalent configure section below is added.
+
+ // Comment messages after build completes.
+ buildStatus {
+ completedStatus('SUCCESS', successComment)
+ completedStatus('FAILURE', '--none--')
+ completedStatus('ERROR', '--none--')
+ }
+ */
+ }
+ }
+ }
+
+ // Comment messages after build completes.
+ context.configure {
+ def messages = it / triggers / 'org.jenkinsci.plugins.ghprb.GhprbTrigger' / extensions / 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildStatus' / messages
+ messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
+ message(successComment)
+ result('SUCCESS')
+ }
+ messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
+ message('--none--')
+ result('ERROR')
+ }
+ messages << 'org.jenkinsci.plugins.ghprb.extensions.comments.GhprbBuildResultMessage' {
+ message('--none--')
+ result('FAILURE')
+ }
+ }
+ }
+
+ // Sets common config for Maven jobs.
+ static void setMavenConfig(context) {
+ context.mavenInstallation('Maven 3.3.3')
+ context.mavenOpts('-Dorg.slf4j.simpleLogger.showDateTime=true')
+ context.mavenOpts('-Dorg.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd\\\'T\\\'HH:mm:ss.SSS')
+ // The -XX:+TieredCompilation -XX:TieredStopAtLevel=1 JVM options enable
+ // tiered compilation to make the JVM startup times faster during the tests.
+ context.mavenOpts('-XX:+TieredCompilation')
+ context.mavenOpts('-XX:TieredStopAtLevel=1')
+ context.rootPOM('pom.xml')
+ // Use a repository local to the workspace for better isolation of jobs.
+ context.localRepository(LocalRepositoryLocation.LOCAL_TO_WORKSPACE)
+ // Disable archiving the built artifacts by default, as this is slow and flaky.
+ // We can usually recreate them easily, and we can also opt-in individual jobs
+ // to artifact archiving.
+ context.archivingDisabled(true)
+ }
+
+ // Sets common config for PreCommit jobs.
+ static void setPreCommit(context,
+ String commitStatusName,
+ String successComment = '--none--') {
+ // Set pull request build trigger.
+ setPullRequestBuildTrigger(context, commitStatusName, successComment)
+ }
+
+ // Enable triggering postcommit runs against pull requests. Users can comment the trigger phrase
+ // specified in the postcommit job and have the job run against their PR to run
+ // tests not in the presubmit suite for additional confidence.
+ static void enablePhraseTriggeringFromPullRequest(context,
+ String commitStatusName,
+ String prTriggerPhrase) {
+ setPullRequestBuildTrigger(
+ context,
+ commitStatusName,
+ '--none--',
+ prTriggerPhrase)
+ }
+
+ // Sets common config for PostCommit jobs.
+ static void setPostCommit(context,
+ String buildSchedule = '0 */6 * * *',
+ boolean triggerEveryPush = true,
+ String notifyAddress = 'commits@beam.apache.org',
+ boolean emailIndividuals = true) {
+ // Set build triggers
+ context.triggers {
+ // By default runs every 6 hours.
+ cron(buildSchedule)
+ if (triggerEveryPush) {
+ githubPush()
+ }
+ }
+
+ context.publishers {
+ // Notify an email address for each failed build (defaults to commits@).
+ mailer(notifyAddress, false, emailIndividuals)
+ }
+ }
+
+ // Configures the argument list for performance tests, adding the standard
+ // performance test job arguments.
+ private static def genPerformanceArgs(def argMap) {
+ def standard_args = [
+ project: 'apache-beam-testing',
+ dpb_log_level: 'INFO',
+ maven_binary: '/home/jenkins/tools/maven/latest/bin/mvn',
+ bigquery_table: 'beam_performance.pkb_results',
+ // Publishes results with official tag, for use in dashboards.
+ official: 'true'
+ ]
+ // Note: in case of key collision, keys present in ArgMap win.
+ def joined_args = standard_args.plus(argMap)
+ def argList = []
+ joined_args.each({
+ // FYI: Replacement only works with double quotes.
+ key, value -> argList.add("--$key=$value")
+ })
+ return argList.join(' ')
+ }
+
+ // Adds the standard performance test job steps.
+ static def buildPerformanceTest(def context, def argMap) {
+ def pkbArgs = genPerformanceArgs(argMap)
+ context.steps {
+ // Clean up environment.
+ shell('rm -rf PerfKitBenchmarker')
+ // Clone appropriate perfkit branch
+ shell('git clone https://github.com/GoogleCloudPlatform/PerfKitBenchmarker.git')
+ // Install job requirements.
+ shell('pip install --user -r PerfKitBenchmarker/requirements.txt')
+ // Launch performance test.
+ shell("python PerfKitBenchmarker/pkb.py $pkbArgs")
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
new file mode 100644
index 0000000..51c73f3
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_Dataflow.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the Beam performance tests on PerfKit Benchmarker.
+job('beam_PerformanceTests_Dataflow'){
+ // Set default Beam job properties.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Run job in postcommit every 6 hours, don't trigger every push, and
+ // don't email individual committers.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 */6 * * *',
+ false,
+ 'commits@beam.apache.org',
+ false)
+
+ def argMap = [
+ benchmarks: 'dpb_wordcount_benchmark',
+ dpb_dataflow_staging_location: 'gs://temp-storage-for-perf-tests/staging',
+ dpb_wordcount_input: 'dataflow-samples/shakespeare/kinglear.txt',
+ config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataflow'
+ ]
+
+ common_job_properties.buildPerformanceTest(delegate, argMap)
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
new file mode 100644
index 0000000..8e581c2
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_JDBC.groovy
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the Beam performance tests on PerfKit Benchmarker.
+job('beam_PerformanceTests_JDBC'){
+ // Set default Beam job properties.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Run job in postcommit every 6 hours, don't trigger every push, and
+ // don't email individual committers.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 */6 * * *',
+ false,
+ 'commits@beam.apache.org',
+ false)
+
+ def pipelineArgs = [
+ tempRoot: 'gs://temp-storage-for-end-to-end-tests',
+ project: 'apache-beam-testing',
+ postgresServerName: '10.36.0.11',
+ postgresUsername: 'postgres',
+ postgresDatabaseName: 'postgres',
+ postgresPassword: 'uuinkks',
+ postgresSsl: 'false'
+ ]
+ def pipelineArgList = []
+ pipelineArgs.each({
+ key, value -> pipelineArgList.add("--$key=$value")
+ })
+ def pipelineArgsJoined = pipelineArgList.join(',')
+
+ def argMap = [
+ benchmarks: 'beam_integration_benchmark',
+ beam_it_module: 'sdks/java/io/jdbc',
+ beam_it_args: pipelineArgsJoined,
+ beam_it_class: 'org.apache.beam.sdk.io.jdbc.JdbcIOIT',
+ // Profile is located in $BEAM_ROOT/sdks/java/io/pom.xml.
+ beam_it_profile: 'io-it'
+ ]
+
+ common_job_properties.buildPerformanceTest(delegate, argMap)
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy b/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy
new file mode 100644
index 0000000..ba719bf
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PerformanceTests_Spark.groovy
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the Beam performance tests on PerfKit Benchmarker.
+job('beam_PerformanceTests_Spark'){
+ // Set default Beam job properties.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Run job in postcommit every 6 hours, don't trigger every push, and
+ // don't email individual committers.
+ common_job_properties.setPostCommit(
+ delegate,
+ '0 */6 * * *',
+ false,
+ 'commits@beam.apache.org',
+ false)
+
+ def argMap = [
+ benchmarks: 'dpb_wordcount_benchmark',
+ // There are currently problems uploading to Dataproc, so we use a file
+ // already present on the machines as input.
+ dpb_wordcount_input: '/etc/hosts',
+ config_override: 'dpb_wordcount_benchmark.dpb_service.service_type=dataproc'
+ ]
+
+ common_job_properties.buildPerformanceTest(delegate, argMap)
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
new file mode 100644
index 0000000..a288a84
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_MavenInstall.groovy
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the Java postcommit tests, including the suite of integration
+// tests.
+mavenJob('beam_PostCommit_Java_MavenInstall') {
+ description('Runs postcommit tests on the Java SDK.')
+
+ previousNames('beam_PostCommit_MavenVerify')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(delegate)
+
+ // Maven goals for this job.
+ goals('-B -e -P release,dataflow-runner clean install coveralls:report -DrepoToken=$COVERALLS_REPO_TOKEN -DskipITs=false -DintegrationTestPipelineOptions=\'[ "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-end-to-end-tests", "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner" ]\'')
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
new file mode 100644
index 0000000..c16a1e2
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Apex.groovy
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the suite of ValidatesRunner tests against the Apex runner.
+mavenJob('beam_PostCommit_Java_ValidatesRunner_Apex') {
+ description('Runs the ValidatesRunner suite on the Apex runner.')
+ previousNames('beam_PostCommit_Java_RunnableOnService_Apex')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(delegate)
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Apache Apex Runner ValidatesRunner Tests',
+ 'Run Apex ValidatesRunner')
+
+ // Maven goals for this job.
+ goals('''clean verify --projects runners/apex \
+ --also-make \
+ --batch-mode \
+ --errors \
+ --activate-profiles validates-runner-tests \
+ --activate-profiles local-validates-runner-tests''')
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
new file mode 100644
index 0000000..33235ff
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Dataflow.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the suite of ValidatesRunner tests against the Dataflow
+// runner.
+mavenJob('beam_PostCommit_Java_ValidatesRunner_Dataflow') {
+ description('Runs the ValidatesRunner suite on the Dataflow runner.')
+ previousNames('beam_PostCommit_Java_RunnableOnService_Dataflow')
+
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate, 'master', 120)
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(delegate)
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Google Cloud Dataflow Runner ValidatesRunner Tests',
+ 'Run Dataflow ValidatesRunner')
+
+ // Maven goals for this job.
+ goals('-B -e clean verify -am -pl runners/google-cloud-dataflow-java -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=org.apache.beam.runners.dataflow.testing.TestDataflowRunner", "--project=apache-beam-testing", "--tempRoot=gs://temp-storage-for-validates-runner-tests/" ]\'')
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
new file mode 100644
index 0000000..411106d
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Flink.groovy
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the suite of ValidatesRunner tests against the Flink runner.
+mavenJob('beam_PostCommit_Java_ValidatesRunner_Flink') {
+ description('Runs the ValidatesRunner suite on the Flink runner.')
+ previousNames('beam_PostCommit_Java_RunnableOnService_Flink')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(delegate)
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Apache Flink Runner ValidatesRunner Tests',
+ 'Run Flink ValidatesRunner')
+
+ // Maven goals for this job.
+ goals('-B -e clean verify -am -pl runners/flink/runner -Plocal-validates-runner-tests -Pvalidates-runner-tests')
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
new file mode 100644
index 0000000..1348a19
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Gearpump.groovy
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the suite of ValidatesRunner tests against the Gearpump
+// runner.
+mavenJob('beam_PostCommit_Java_ValidatesRunner_Gearpump') {
+ description('Runs the ValidatesRunner suite on the Gearpump runner.')
+
+ previousNames('beam_PostCommit_Java_RunnableOnService_Gearpump')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(
+ delegate,
+ 'gearpump-runner')
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ // 0 5 31 2 * will run on Feb 31 (i.e. never) according to job properties.
+ // In post-commit this job triggers only on SCM changes.
+ common_job_properties.setPostCommit(delegate, '0 5 31 2 *')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Apache Gearpump Runner ValidatesRunner Tests',
+ 'Run Gearpump ValidatesRunner')
+
+ // Maven goals for this job.
+ goals('-B -e clean verify -am -pl runners/gearpump -DforkCount=0 -DvalidatesRunnerPipelineOptions=\'[ "--runner=TestGearpumpRunner", "--streaming=false" ]\'')
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
new file mode 100644
index 0000000..9fbc219
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Java_ValidatesRunner_Spark.groovy
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job runs the suite of ValidatesRunner tests against the Spark runner.
+mavenJob('beam_PostCommit_Java_ValidatesRunner_Spark') {
+ description('Runs the ValidatesRunner suite on the Spark runner.')
+
+ previousNames('beam_PostCommit_Java_RunnableOnService_Spark')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Set maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(delegate)
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Apache Spark Runner ValidatesRunner Tests',
+ 'Run Spark ValidatesRunner')
+
+ // Maven goals for this job.
+ goals('-B -e clean verify -am -pl runners/spark -Pvalidates-runner-tests -Plocal-validates-runner-tests -Dspark.ui.enabled=false')
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy b/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy
new file mode 100644
index 0000000..28cf77e
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PostCommit_Python_Verify.groovy
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This job defines the Python postcommit tests.
+job('beam_PostCommit_Python_Verify') {
+ description('Runs postcommit tests on the Python SDK.')
+
+ previousNames('beam_PostCommit_PythonVerify')
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Sets that this is a PostCommit job.
+ common_job_properties.setPostCommit(delegate, '0 3-22/6 * * *')
+
+ // Allows triggering this build against pull requests.
+ common_job_properties.enablePhraseTriggeringFromPullRequest(
+ delegate,
+ 'Python SDK PostCommit Tests',
+ 'Run Python PostCommit')
+
+ // Allow the test to only run on particular nodes
+ // TODO(BEAM-1817): Remove once the tests can run on all nodes
+ parameters {
+ nodeParam('TEST_HOST') {
+ description('select test host as either beam1, 2 or 3')
+ defaultNodes(['beam3'])
+ allowedNodes(['beam1', 'beam2', 'beam3'])
+ trigger('multiSelectionDisallowed')
+ eligibility('IgnoreOfflineNodeEligibility')
+ }
+ }
+
+ // Execute shell command to test Python SDK.
+ steps {
+ shell('bash sdks/python/run_postcommit.sh')
+ }
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/66b20af9/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
new file mode 100644
index 0000000..3718551
--- /dev/null
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_MavenInstall.groovy
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import common_job_properties
+
+// This is the Java precommit which runs a maven install, and the current set
+// of precommit tests.
+mavenJob('beam_PreCommit_Java_MavenInstall') {
+ description('Runs an install of the current GitHub Pull Request.')
+
+ previousNames('beam_PreCommit_MavenVerify')
+
+ // Execute concurrent builds if necessary.
+ concurrentBuild()
+
+ // Set common parameters.
+ common_job_properties.setTopLevelMainJobProperties(delegate)
+
+ // Set Maven parameters.
+ common_job_properties.setMavenConfig(delegate)
+
+ // Sets that this is a PreCommit job.
+ common_job_properties.setPreCommit(delegate, 'Maven clean install')
+
+ // Maven goals for this job.
+ goals('-B -e -Prelease,include-runners,jenkins-precommit,direct-runner,dataflow-runner,spark-runner,flink-runner,apex-runner -DrepoToken=$COVERALLS_REPO_TOKEN -DpullRequest=$ghprbPullId help:effective-settings clean install coveralls:report')
+}