You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2017/11/21 23:45:32 UTC
[2/3] beam git commit: Add precommit integration tests to
examples/java
Add precommit integration tests to examples/java
Also make Jenkins run a common root task which enumerates all the tasks that it should run.
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/81e22da1
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/81e22da1
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/81e22da1
Branch: refs/heads/master
Commit: 81e22da1f1805f6a6614a23e558f62b9aebfc0a5
Parents: e9d746a
Author: Lukasz Cwik <lc...@google.com>
Authored: Mon Nov 20 22:35:57 2017 -0800
Committer: Luke Cwik <lc...@google.com>
Committed: Tue Nov 21 15:44:39 2017 -0800
----------------------------------------------------------------------
.../job_beam_PreCommit_Java_GradleBuild.groovy | 13 +--
build.gradle | 15 +++
build_rules.gradle | 6 +-
examples/java/build.gradle | 100 ++++++++++++++++++-
4 files changed, 118 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/81e22da1/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
----------------------------------------------------------------------
diff --git a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
index 6610a62..15921f3 100644
--- a/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
+++ b/.test-infra/jenkins/job_beam_PreCommit_Java_GradleBuild.groovy
@@ -32,11 +32,6 @@ job('beam_PreCommit_Java_GradleBuild') {
'master',
240)
- def root_projects = [
- ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core',
- ':beam-runners-parent:beam-runners-direct-java',
- ':beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution',
- ]
def gradle_switches = [
// Continue the build even if there is a failure to show as many potential failures as possible.
'--continue',
@@ -44,17 +39,13 @@ job('beam_PreCommit_Java_GradleBuild') {
'--rerun-tasks',
]
- def gradle_command_line = './gradlew ' + gradle_switches.join(' ') + ' ' + root_projects.join(':buildNeeded ') + ' ' + root_projects.join(':buildDependents ')
+ def gradle_command_line = './gradlew ' + gradle_switches.join(' ') + ' :javaPreCommit'
// Sets that this is a PreCommit job.
common_job_properties.setPreCommit(delegate, gradle_command_line, 'Run Java Gradle PreCommit')
-
steps {
gradle {
rootBuildScriptDir(common_job_properties.checkoutDir)
- for (String root_project : root_projects) {
- tasks(root_project + ':buildNeeded')
- tasks(root_project + ':buildDependents')
- }
+ tasks(':javaPreCommit')
for (String gradle_switch : gradle_switches) {
switches(gradle_switch)
}
http://git-wip-us.apache.org/repos/asf/beam/blob/81e22da1/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 2d55701..d4181f4 100644
--- a/build.gradle
+++ b/build.gradle
@@ -262,3 +262,18 @@ rat {
]
}
check.dependsOn rat
+
+// Define a root Java PreCommit task simplifying what is needed
+// to be specified on the commandline when executing locally.
+def javaPreCommitRoots = [
+ ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core",
+ ":beam-runners-parent:beam-runners-direct-java",
+ ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-fn-execution",
+]
+task javaPreCommit() {
+ for (String javaPreCommitRoot : javaPreCommitRoots) {
+ dependsOn javaPreCommitRoot + ":buildNeeded"
+ dependsOn javaPreCommitRoot + ":buildDependents"
+ }
+ dependsOn ":beam-examples-parent:beam-examples-java:preCommit"
+}
http://git-wip-us.apache.org/repos/asf/beam/blob/81e22da1/build_rules.gradle
----------------------------------------------------------------------
diff --git a/build_rules.gradle b/build_rules.gradle
index abb1c26..453beb2 100644
--- a/build_rules.gradle
+++ b/build_rules.gradle
@@ -140,7 +140,7 @@ ext.applyJavaNature = {
options.encoding = "UTF-8"
}
- // Configure the test tasks to always use JUnit and also configure the set of tests executed
+ // Configure the default test tasks set of tests executed
// to match the equivalent set that is executed by the maven-surefire-plugin.
// See http://maven.apache.org/components/surefire/maven-surefire-plugin/test-mojo.html
test {
@@ -148,6 +148,10 @@ ext.applyJavaNature = {
include "**/*Test.class"
include "**/*Tests.class"
include "**/*TestCase.class"
+ }
+
+ // Configure all test tasks to use JUnit
+ tasks.withType(Test) {
useJUnit { }
}
http://git-wip-us.apache.org/repos/asf/beam/blob/81e22da1/examples/java/build.gradle
----------------------------------------------------------------------
diff --git a/examples/java/build.gradle b/examples/java/build.gradle
index 6386001..501c743 100644
--- a/examples/java/build.gradle
+++ b/examples/java/build.gradle
@@ -16,6 +16,8 @@
* limitations under the License.
*/
+import groovy.json.JsonOutput
+
apply from: project(":").file("build_rules.gradle")
applyJavaNature()
@@ -30,6 +32,18 @@ description = "Apache Beam :: Examples :: Java"
*/
evaluationDependsOn(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-google-cloud-platform")
+/** Define the list of runners which execute a precommit test. */
+def preCommitRunners = ["apexRunner", "dataflowRunner", "dataflowStreamingRunner", "directRunner", "flinkRunner", "sparkRunner"]
+for (String runner : preCommitRunners) {
+ configurations.create(runner + "PreCommit")
+}
+configurations.sparkRunnerPreCommit {
+ // Ban certain dependencies to prevent a StackOverflow within Spark
+ // because JUL -> SLF4J -> JUL, and similarly JDK14 -> SLF4J -> JDK14
+ exclude group: "org.slf4j", module: "jul-to-slf4j"
+ exclude group: "org.slf4j", module: "slf4j-jdk14"
+}
+
dependencies {
compile library.java.guava
shadow project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-core", configuration: "shadow")
@@ -50,10 +64,44 @@ dependencies {
shadow library.java.slf4j_api
shadow project(path: ":beam-runners-parent:beam-runners-direct-java", configuration: "shadow")
shadow library.java.slf4j_jdk14
- testCompile project(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-google-cloud-platform").sourceSets.test.output
- testCompile library.java.hamcrest_core
- testCompile library.java.junit
- testCompile library.java.mockito_core
+ shadowTest project(":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-google-cloud-platform").sourceSets.test.output
+ shadowTest library.java.hamcrest_core
+ shadowTest library.java.junit
+ shadowTest library.java.mockito_core
+
+ // Add dependencies for the PreCommit configurations
+ // For each runner a project level dependency on the examples project.
+ for (String runner : preCommitRunners) {
+ delegate.add(runner + "PreCommit", project(path: ":beam-examples-parent:beam-examples-java", configuration: "shadow"))
+ delegate.add(runner + "PreCommit", project(path: ":beam-examples-parent:beam-examples-java", configuration: "shadowTest"))
+ }
+ apexRunnerPreCommit project(path: ":beam-runners-parent:beam-runners-apex", configuration: "shadow")
+ dataflowRunnerPreCommit project(path: ":beam-runners-parent:beam-runners-google-cloud-dataflow-java", configuration: "shadow")
+ dataflowStreamingRunnerPreCommit project(path: ":beam-runners-parent:beam-runners-google-cloud-dataflow-java", configuration: "shadow")
+ directRunnerPreCommit project(path: ":beam-runners-parent:beam-runners-direct-java", configuration: "shadow")
+ flinkRunnerPreCommit project(path: ":beam-runners-parent:beam-runners-flink_2.10", configuration: "shadow")
+ sparkRunnerPreCommit project(path: ":beam-runners-parent:beam-runners-spark", configuration: "shadow")
+ sparkRunnerPreCommit project(path: ":beam-sdks-parent:beam-sdks-java-parent:beam-sdks-java-io-parent:beam-sdks-java-io-hadoop-file-system", configuration: "shadow")
+ sparkRunnerPreCommit library.java.spark_streaming
+ sparkRunnerPreCommit library.java.spark_core
+}
+
+// Create a shaded test jar.
+task shadowTestJar(type: com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar) {
+ classifier = "shaded-tests"
+ from sourceSets.test.output
+ configurations = [project.configurations.testRuntime]
+ dependencies {
+ exclude(".*")
+ include(dependency(library.java.guava))
+ }
+ relocate("com.google.common", "org.apache.beam." + project.name.replace("-", ".") + ".repackaged.com.google.common") {
+ // com.google.common is too generic, need to exclude guava-testlib
+ exclude "com.google.common.collect.testing.**"
+ exclude "com.google.common.escape.testing.**"
+ exclude "com.google.common.testing.**"
+ exclude "com.google.common.util.concurrent.testing.**"
+ }
}
task packageTests(type: Jar) {
@@ -62,3 +110,47 @@ task packageTests(type: Jar) {
}
artifacts.archives packageTests
+artifacts {
+ shadowTest shadowTestJar
+}
+
+/*
+ * Create a ${runner}PreCommit task for each runner which runs a set
+ * of integration tests for WordCount and WindowedWordCount.
+ */
+def preCommitRunnerClass = [
+ apexRunner: "org.apache.beam.runners.apex.TestApexRunner",
+ dataflowRunner: "TestDataflowRunner",
+ dataflowStreamingRunner: "TestDataflowRunner",
+ directRunner: "org.apache.beam.runners.direct.DirectRunner",
+ flinkRunner: "org.apache.beam.runners.flink.TestFlinkRunner",
+ sparkRunner: "org.apache.beam.runners.spark.TestSparkRunner",
+]
+def preCommitAdditionalFlags = [
+ dataflowStreamingRunner: [ "--streaming=true" ],
+]
+for (String runner : preCommitRunners) {
+ tasks.create(name: runner + "PreCommit", type: Test) {
+ def preCommitBeamTestPipelineOptions = [
+ "--project=apache-beam-testing",
+ "--tempRoot=gs://temp-storage-for-end-to-end-tests",
+ "--runner=" + preCommitRunnerClass[runner],
+ ] + preCommitAdditionalFlags[runner]
+ classpath = configurations."${runner}PreCommit"
+ include "**/WordCountIT.class"
+ if (!"sparkRunner".equals(runner)) {
+ include "**/WindowedWordCountIT.class"
+ }
+ forkEvery 1
+ maxParallelForks 4
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+ }
+}
+
+/* Define a common precommit task which depends on all the individual precommits. */
+task preCommit() {
+ for (String runner : preCommitRunners) {
+ dependsOn runner + "PreCommit"
+ }
+}
+