You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by lc...@apache.org on 2018/10/29 21:10:52 UTC
[beam] branch master updated: [BEAM-5800] Support WordCountIT using
fn-api worker (#6760)
This is an automated email from the ASF dual-hosted git repository.
lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 1570194 [BEAM-5800] Support WordCountIT using fn-api worker (#6760)
1570194 is described below
commit 1570194229a99d839a085e0d7e7186bc2eaf54dc
Author: Boyuan Zhang <36...@users.noreply.github.com>
AuthorDate: Mon Oct 29 14:10:44 2018 -0700
[BEAM-5800] Support WordCountIT using fn-api worker (#6760)
* Support WordCountIT using fn-api worker
* Clean up docker image after test finished
---
.../job_PreCommit_Java_PortabilityApi.groovy | 37 +++++++++++++
build.gradle | 6 ++-
.../examples/build.gradle | 62 +++++++++++++++++++++-
3 files changed, 103 insertions(+), 2 deletions(-)
diff --git a/.test-infra/jenkins/job_PreCommit_Java_PortabilityApi.groovy b/.test-infra/jenkins/job_PreCommit_Java_PortabilityApi.groovy
new file mode 100644
index 0000000..5dda694
--- /dev/null
+++ b/.test-infra/jenkins/job_PreCommit_Java_PortabilityApi.groovy
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import PrecommitJobBuilder
+
+PrecommitJobBuilder builder = new PrecommitJobBuilder(
+ scope: this,
+ nameBase: 'Java PortabilityApi',
+ gradleTask: ':javaPreCommitPortabilityApi',
+ triggerPathPatterns: [
+ '^model/.*$',
+ '^sdks/java/.*$',
+ '^runners/google-cloud-dataflow-java/worker.*$',
+ '^examples/java/.*$',
+ '^release/.*$',
+ ]
+)
+builder.build {
+ publishers {
+ archiveJunit('**/build/test-results/**/*.xml')
+ }
+}
diff --git a/build.gradle b/build.gradle
index f36328a..357865a 100644
--- a/build.gradle
+++ b/build.gradle
@@ -188,7 +188,6 @@ task javaPreCommit() {
dependsOn ":beam-model-job-management:build"
dependsOn ":beam-model-fn-execution:build"
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:build"
- dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:build"
dependsOn ":beam-sdks-java-core:buildNeeded"
dependsOn ":beam-sdks-java-core:buildDependents"
dependsOn ":beam-examples-java:preCommit"
@@ -196,6 +195,11 @@ task javaPreCommit() {
dependsOn ":beam-sdks-java-javadoc:allJavadoc"
}
+task javaPreCommitPortabilityApi() {
+ dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:build"
+ dependsOn ":beam-runners-google-cloud-dataflow-java-examples:preCommitPortabilityApi"
+}
+
task javaPostCommit() {
dependsOn ":javaPreCommit"
dependsOn ":beam-runners-google-cloud-dataflow-java:postCommit"
diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle
index 6009ff9..bee66be 100644
--- a/runners/google-cloud-dataflow-java/examples/build.gradle
+++ b/runners/google-cloud-dataflow-java/examples/build.gradle
@@ -24,6 +24,8 @@ applyJavaNature(publish: false)
// its sourceSets.test.output directly.
evaluationDependsOn(":beam-examples-java")
evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-legacy-worker")
+evaluationDependsOn(":beam-runners-google-cloud-dataflow-java-fn-api-worker")
+evaluationDependsOn(":beam-sdks-java-container")
configurations { dataflowRunnerPreCommit }
@@ -35,8 +37,13 @@ dependencies {
def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
def gcsTempRoot = project.findProperty('gcsTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests/'
+def dockerImageRoot = project.findProperty('dockerImageRoot') ?: "us.gcr.io/${gcpProject}/java-examples-it"
+def dockerImageContainer = "${dockerImageRoot}/java"
+def dockerTag = new Date().format('yyyyMMddHHmmss')
+def dockerImageName = "${dockerImageContainer}:${dockerTag}"
+def defaultDockerImageName = containerImageName(name: "java")
-task preCommit(type: Test) {
+task preCommitLegacyWorker(type: Test) {
dependsOn ":beam-runners-google-cloud-dataflow-java-legacy-worker:shadowJar"
def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-legacy-worker").shadowJar.archivePath
@@ -56,3 +63,56 @@ task preCommit(type: Test) {
maxParallelForks 4
systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
}
+
+task buildAndPushDockerContainer() {
+ dependsOn ":beam-sdks-java-container:docker"
+ doLast {
+ exec {
+ commandLine "docker", "tag", "${defaultDockerImageName}", "${dockerImageName}"
+ }
+ exec {
+ commandLine "gcloud", "docker", "--", "push", "${dockerImageContainer}"
+ }
+ }
+}
+
+task preCommitFnApiWorker(type: Test) {
+ dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
+ dependsOn buildAndPushDockerContainer
+
+ def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":beam-runners-google-cloud-dataflow-java-fn-api-worker").shadowJar.archivePath
+ def preCommitBeamTestPipelineOptions = [
+ "--project=${gcpProject}",
+ "--tempRoot=${gcsTempRoot}",
+ "--runner=TestDataflowRunner",
+ "--dataflowWorkerJar=${dataflowWorkerJar}",
+ "--workerHarnessContainerImage=${dockerImageName}",
+ "--experiments=beam_fn_api",
+ ]
+ testClassesDirs = files(project(":beam-examples-java").sourceSets.test.output.classesDirs)
+ include "**/WordCountIT.class"
+ include "**/WindowedWordCountIT.class"
+ forkEvery 1
+ maxParallelForks 4
+ systemProperty "beamTestPipelineOptions", JsonOutput.toJson(preCommitBeamTestPipelineOptions)
+ useJUnit {
+ excludeCategories 'org.apache.beam.sdk.testing.StreamingIT'
+ }
+}
+
+task preCommit() {
+ dependsOn preCommitLegacyWorker
+}
+
+task preCommitPortabilityApi() {
+ dependsOn preCommitFnApiWorker
+ // Clean up built images
+ doLast {
+ exec {
+ commandLine "docker", "rmi", "${dockerImageName}"
+ }
+ exec {
+ commandLine "gcloud", "--quiet", "container", "images", "delete", "${dockerImageName}"
+ }
+ }
+}