You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lg...@apache.org on 2019/03/19 13:11:47 UTC
[beam] branch master updated: [BEAM-6703] Added a Jenkins job
definition for Java 11 enabled Dataflow harness (#7999)
This is an automated email from the ASF dual-hosted git repository.
lgajowy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c1ff555 [BEAM-6703] Added a Jenkins job definition for Java 11 enabled Dataflow harness (#7999)
c1ff555 is described below
commit c1ff555e92e2a502e10a2b851a4d00650bda9645
Author: Michal Walenia <32...@users.noreply.github.com>
AuthorDate: Tue Mar 19 14:11:37 2019 +0100
[BEAM-6703] Added a Jenkins job definition for Java 11 enabled Dataflow harness (#7999)
---
.test-infra/jenkins/README.md | 1 +
...mit_Java_ValidatesRunner_Dataflow_Java11.groovy | 46 ++++++++++++++++++++++
runners/google-cloud-dataflow-java/build.gradle | 31 +++++++++++++++
3 files changed, 78 insertions(+)
diff --git a/.test-infra/jenkins/README.md b/.test-infra/jenkins/README.md
index 278b9a0..f478d31 100644
--- a/.test-infra/jenkins/README.md
+++ b/.test-infra/jenkins/README.md
@@ -57,6 +57,7 @@ Beam Jenkins overview page: [link](https://builds.apache.org/view/A-D/view/Beam/
| beam_PostCommit_Java_PortabilityApi | [cron](https://builds.apache.org/job/beam_PostCommit_Java_PortabilityApi/), [phrase](https://builds.apache.org/job/beam_PostCommit_Java_PortabilityApi_PR/) | `Run Java PortabilityApi PostCommit` | [](https://builds.apache.org/job/beam_PostCommit_Java_PortabilityApi) |
| beam_PostCommit_Java_ValidatesRunner_Apex | [cron](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Apex/), [phrase](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Apex_PR/) | `Run Apex ValidatesRunner` | [](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Apex) |
| beam_PostCommit_Java_ValidatesRunner_Dataflow | [cron](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow/), [phrase](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow_PR/) | `Run Dataflow ValidatesRunner` | [](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Dataflow) |
+| beam_PostCommit_Java11_ValidatesRunner_Dataflow | [cron](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_Dataflow/), [phrase](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_Dataflow_PR/) | `Run Dataflow ValidatesRunner Java 11` | [](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_Dataflow) |
| beam_PostCommit_Java11_ValidatesRunner_PortabilityApi_Dataflow | [cron](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_PortabilityApi_Dataflow/), [phrase](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_PortabilityApi_Dataflow_PR/) | `Run Dataflow PortabilityApi ValidatesRunner with Java 11` | [](https://builds.apache.org/job [...]
| beam_PostCommit_Java11_ValidatesRunner_Direct | [cron](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_Direct), [phrase](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_Direct_PR) | `Run Direct ValidatesRunner in Java 11` | [](https://builds.apache.org/job/beam_PostCommit_Java11_ValidatesRunner_Direct) |
| beam_PostCommit_Java_ValidatesRunner_Flink | [cron](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink/), [phrase](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink_PR/) | `Run Flink ValidatesRunner` | [](https://builds.apache.org/job/beam_PostCommit_Java_ValidatesRunner_Flink) |
diff --git a/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy
new file mode 100644
index 0000000..4e6da6a
--- /dev/null
+++ b/.test-infra/jenkins/job_PostCommit_Java_ValidatesRunner_Dataflow_Java11.groovy
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
+
+
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Java11_ValidatesRunner_Dataflow',
+ 'Run Dataflow ValidatesRunner Java 11', 'Google Cloud Dataflow Runner ValidatesRunner Tests On Java 11', this) {
+
+ description('Runs the ValidatesRunner suite on the Dataflow runner with Java 11 worker harness.')
+
+ commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 180)
+
+ publishers {
+ archiveJunit('**/build/test-results/**/*.xml')
+ }
+
+ steps {
+ gradle {
+ rootBuildScriptDir(commonJobProperties.checkoutDir)
+ tasks(':beam-runners-google-cloud-dataflow-java:validatesJava11Runner')
+ // Increase parallel worker threads above processor limit since most time is
+ // spent waiting on Dataflow jobs. ValidatesRunner tests on Dataflow are slow
+ // because each one launches a Dataflow job with about 3 mins of overhead.
+ // 3 x num_cores strikes a good balance between maxing out parallelism without
+ // overloading the machines.
+ commonJobProperties.setGradleSwitches(delegate, 3 * Runtime.runtime.availableProcessors())
+ }
+ }
+}
diff --git a/runners/google-cloud-dataflow-java/build.gradle b/runners/google-cloud-dataflow-java/build.gradle
index 0aaa7e4..596ad7b 100644
--- a/runners/google-cloud-dataflow-java/build.gradle
+++ b/runners/google-cloud-dataflow-java/build.gradle
@@ -177,6 +177,31 @@ task validatesRunnerLegacyWorkerTest(type: Test) {
}
}
+task validatesRunnerLegacyWorkerJava11Test(type: Test) {
+ group = "Verification"
+ dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
+ "--runner=TestDataflowRunner",
+ "--project=${dataflowProject}",
+ "--tempRoot=${dataflowValidatesTempRoot}",
+ "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
+ ])
+
+ systemProperty "java.specification.version", "11"
+
+ // Increase test parallelism up to the number of Gradle workers. By default this is equal
+ // to the number of CPU cores, but can be increased by setting --max-workers=N.
+ maxParallelForks Integer.MAX_VALUE
+ classpath = configurations.validatesRunner
+ testClassesDirs = files(project(":beam-sdks-java-core").sourceSets.test.output.classesDirs)
+ useJUnit {
+ includeCategories 'org.apache.beam.sdk.testing.ValidatesRunner'
+ commonExcludeCategories.each {
+ excludeCategories it
+ }
+ }
+}
+
// Push docker images to a container registry for use within tests.
// NB: Tasks which consume docker images from the registry should depend on this
// task directly ('dependsOn buildAndPushDockerContainer'). This ensures the correct
@@ -282,6 +307,12 @@ task validatesRunner {
dependsOn validatesRunnerLegacyWorkerTest
}
+task validatesJava11Runner {
+ group = 'Verification'
+ description 'Verifies Dataflow runner with Java 11 container harness'
+ dependsOn validatesRunnerLegacyWorkerJava11Test
+}
+
task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test) {
group = "Verification"
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"