You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ib...@apache.org on 2019/11/08 22:16:16 UTC
[beam] branch master updated: [BEAM-8512] Add integration tests for
flink_runner.py
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new d692d2f [BEAM-8512] Add integration tests for flink_runner.py
new 40383d8 Merge pull request #9998 from ibzib/flink-runner-test
d692d2f is described below
commit d692d2fbe1a040fb392b0fb8a2d484b843aa6e79
Author: Kyle Weaver <kc...@google.com>
AuthorDate: Tue Nov 5 11:59:29 2019 -0800
[BEAM-8512] Add integration tests for flink_runner.py
---
build.gradle | 4 ++++
.../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy | 13 ++++++++-----
sdks/python/test-suites/portable/py2/build.gradle | 7 +++++++
sdks/python/test-suites/portable/py35/build.gradle | 7 +++++++
sdks/python/test-suites/portable/py36/build.gradle | 7 +++++++
sdks/python/test-suites/portable/py37/build.gradle | 7 +++++++
6 files changed, 40 insertions(+), 5 deletions(-)
diff --git a/build.gradle b/build.gradle
index 133543f..abc0cb5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -227,22 +227,26 @@ task python2PostCommit() {
dependsOn ":sdks:python:test-suites:direct:py2:directRunnerIT"
dependsOn ":sdks:python:test-suites:direct:py2:hdfsIntegrationTest"
dependsOn ":sdks:python:test-suites:direct:py2:mongodbioIT"
+ dependsOn ":sdks:python:test-suites:portable:py2:postCommitPy2"
}
task python35PostCommit() {
dependsOn ":sdks:python:test-suites:dataflow:py35:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py35:postCommitIT"
+ dependsOn ":sdks:python:test-suites:portable:py35:postCommitPy35"
}
task python36PostCommit() {
dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT"
+ dependsOn ":sdks:python:test-suites:portable:py36:postCommitPy36"
}
task python37PostCommit() {
dependsOn ":sdks:python:test-suites:dataflow:py37:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py37:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py37:hdfsIntegrationTest"
+ dependsOn ":sdks:python:test-suites:portable:py37:postCommitPy37"
}
task portablePythonPreCommit() {
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 04f96eb..3ea3643 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1899,11 +1899,12 @@ class BeamModulePlugin implements Plugin<Project> {
}
}
- def addPortableWordCountTask = { boolean isStreaming ->
- project.task('portableWordCount' + (isStreaming ? 'Streaming' : 'Batch')) {
+ def addPortableWordCountTask = { boolean isStreaming, String runner ->
+ project.task('portableWordCount' + (runner.equals("PortableRunner") ? "" : runner) + (isStreaming ? 'Streaming' : 'Batch')) {
dependsOn = ['installGcpTest']
mustRunAfter = [
':runners:flink:1.9:job-server-container:docker',
+ ':runners:flink:1.9:job-server:shadowJar',
':sdks:python:container:py2:docker',
':sdks:python:container:py35:docker',
':sdks:python:container:py36:docker',
@@ -1914,7 +1915,7 @@ class BeamModulePlugin implements Plugin<Project> {
def options = [
"--input=/etc/profile",
"--output=/tmp/py-wordcount-direct",
- "--runner=PortableRunner",
+ "--runner=${runner}",
"--experiments=worker_threads=100",
"--parallelism=2",
"--shutdown_sources_on_final_watermark",
@@ -1953,8 +1954,10 @@ class BeamModulePlugin implements Plugin<Project> {
}
project.ext.addPortableWordCountTasks = {
->
- addPortableWordCountTask(false)
- addPortableWordCountTask(true)
+ addPortableWordCountTask(false, "PortableRunner")
+ addPortableWordCountTask(true, "PortableRunner")
+ addPortableWordCountTask(false, "FlinkRunner")
+ addPortableWordCountTask(true, "FlinkRunner")
}
}
}
diff --git a/sdks/python/test-suites/portable/py2/build.gradle b/sdks/python/test-suites/portable/py2/build.gradle
index 2b95296..3c1548d 100644
--- a/sdks/python/test-suites/portable/py2/build.gradle
+++ b/sdks/python/test-suites/portable/py2/build.gradle
@@ -34,6 +34,13 @@ task preCommitPy2() {
dependsOn portableWordCountStreaming
}
+task postCommitPy2() {
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:flink:1.9:job-server:shadowJar'
+ dependsOn portableWordCountFlinkRunnerBatch
+ dependsOn portableWordCountFlinkRunnerStreaming
+}
+
// TODO: Move the rest of this file into ../common.gradle.
// Before running this, you need to:
diff --git a/sdks/python/test-suites/portable/py35/build.gradle b/sdks/python/test-suites/portable/py35/build.gradle
index 42667c7..1b2cb4f 100644
--- a/sdks/python/test-suites/portable/py35/build.gradle
+++ b/sdks/python/test-suites/portable/py35/build.gradle
@@ -30,3 +30,10 @@ task preCommitPy35() {
dependsOn portableWordCountBatch
dependsOn portableWordCountStreaming
}
+
+task postCommitPy35() {
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:flink:1.9:job-server:shadowJar'
+ dependsOn portableWordCountFlinkRunnerBatch
+ dependsOn portableWordCountFlinkRunnerStreaming
+}
diff --git a/sdks/python/test-suites/portable/py36/build.gradle b/sdks/python/test-suites/portable/py36/build.gradle
index d536d14..475e110 100644
--- a/sdks/python/test-suites/portable/py36/build.gradle
+++ b/sdks/python/test-suites/portable/py36/build.gradle
@@ -30,3 +30,10 @@ task preCommitPy36() {
dependsOn portableWordCountBatch
dependsOn portableWordCountStreaming
}
+
+task postCommitPy36() {
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:flink:1.9:job-server:shadowJar'
+ dependsOn portableWordCountFlinkRunnerBatch
+ dependsOn portableWordCountFlinkRunnerStreaming
+}
diff --git a/sdks/python/test-suites/portable/py37/build.gradle b/sdks/python/test-suites/portable/py37/build.gradle
index da57c93..912b316 100644
--- a/sdks/python/test-suites/portable/py37/build.gradle
+++ b/sdks/python/test-suites/portable/py37/build.gradle
@@ -30,3 +30,10 @@ task preCommitPy37() {
dependsOn portableWordCountBatch
dependsOn portableWordCountStreaming
}
+
+task postCommitPy37() {
+ dependsOn 'setupVirtualenv'
+ dependsOn ':runners:flink:1.9:job-server:shadowJar'
+ dependsOn portableWordCountFlinkRunnerBatch
+ dependsOn portableWordCountFlinkRunnerStreaming
+}