You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ma...@apache.org on 2019/05/30 22:53:42 UTC
[beam] branch master updated: [BEAM-6623] Complete
DataflowValidatesRunner test coverage for Python 3. (#8721)
This is an automated email from the ASF dual-hosted git repository.
markliu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3f8ee2b [BEAM-6623] Complete DataflowValidatesRunner test coverage for Python 3. (#8721)
3f8ee2b is described below
commit 3f8ee2b2eddeda94c0e363788fc790b507c891d0
Author: tvalentyn <tv...@users.noreply.github.com>
AuthorDate: Thu May 30 15:53:25 2019 -0700
[BEAM-6623] Complete DataflowValidatesRunner test coverage for Python 3. (#8721)
* [BEAM-6623] add DataflowValidatesRunner tests for python 3.6 and 3.7
---
...PostCommit_Python_ValidatesRunner_Dataflow.groovy | 10 ++++++++--
build.gradle | 1 -
project-mappings | 10 ++++++----
sdks/python/test-suites/dataflow/py35/build.gradle | 20 ++++++++++++++++++++
sdks/python/test-suites/dataflow/py36/build.gradle | 20 ++++++++++++++++++++
sdks/python/test-suites/dataflow/py37/build.gradle | 20 ++++++++++++++++++++
6 files changed, 74 insertions(+), 7 deletions(-)
diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
index cfdd01b..28bda61 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesRunner_Dataflow.groovy
@@ -32,8 +32,14 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_VR_Dataflow', 'Run Python
steps {
gradle {
rootBuildScriptDir(commonJobProperties.checkoutDir)
- tasks(':beam-sdks-python:validatesRunnerBatchTests')
- tasks(':beam-sdks-python:validatesRunnerStreamingTests')
+ tasks(':sdks:python:validatesRunnerBatchTests')
+ tasks(':sdks:python:validatesRunnerStreamingTests')
+ tasks(':sdks:python:test-suites:dataflow:py35:validatesRunnerBatchTests')
+ tasks(':sdks:python:test-suites:dataflow:py36:validatesRunnerBatchTests')
+ tasks(':sdks:python:test-suites:dataflow:py37:validatesRunnerBatchTests')
+ tasks(':sdks:python:test-suites:dataflow:py35:validatesRunnerStreamingTests')
+ tasks(':sdks:python:test-suites:dataflow:py36:validatesRunnerStreamingTests')
+ tasks(':sdks:python:test-suites:dataflow:py37:validatesRunnerStreamingTests')
commonJobProperties.setGradleSwitches(delegate)
}
}
diff --git a/build.gradle b/build.gradle
index 9e9fba1..6741833 100644
--- a/build.gradle
+++ b/build.gradle
@@ -204,7 +204,6 @@ task python3PostCommit() {
dependsOn ":sdks:python:test-suites:dataflow:py35:postCommitIT"
dependsOn ":sdks:python:test-suites:dataflow:py36:postCommitIT"
dependsOn ":sdks:python:test-suites:dataflow:py37:postCommitIT"
- dependsOn ":sdks:python:test-suites:dataflow:py35:validatesRunnerBatchTests"
dependsOn ":sdks:python:test-suites:direct:py35:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py36:postCommitIT"
dependsOn ":sdks:python:test-suites:direct:py37:postCommitIT"
diff --git a/project-mappings b/project-mappings
index a211dcf..c786301 100644
--- a/project-mappings
+++ b/project-mappings
@@ -2,14 +2,16 @@
:beam-vendor-sdks-java-extensions-protobuf :vendor:sdks-java-extensions-protobuf
:beam-vendor-guava-20_0 :vendor:guava-20_0
:beam-vendor-grpc-1_13_1 :vendor:grpc-1_13_1
-:beam-sdks-python-test-suites-tox-py36 :sdks:python:test-suites:tox:py37
-:beam-sdks-python-test-suites-tox-py36 :sdks:python:test-suites:tox:py36
-:beam-sdks-python-test-suites-tox-py35 :sdks:python:test-suites:tox:py35
:beam-sdks-python-test-suites-direct-py35 :sdks:python:test-suites:direct:py35
:beam-sdks-python-test-suites-direct-py36 :sdks:python:test-suites:direct:py36
+:beam-sdks-python-test-suites-direct-py37 :sdks:python:test-suites:direct:py37
+:beam-sdks-python-test-suites-dataflow :sdks:python:test-suites:dataflow
:beam-sdks-python-test-suites-dataflow-py35 :sdks:python:test-suites:dataflow:py35
:beam-sdks-python-test-suites-dataflow-py36 :sdks:python:test-suites:dataflow:py36
-:beam-sdks-python-test-suites-dataflow :sdks:python:test-suites:dataflow
+:beam-sdks-python-test-suites-dataflow-py37 :sdks:python:test-suites:dataflow:py37
+:beam-sdks-python-test-suites-tox-py35 :sdks:python:test-suites:tox:py35
+:beam-sdks-python-test-suites-tox-py36 :sdks:python:test-suites:tox:py36
+:beam-sdks-python-test-suites-tox-py37 :sdks:python:test-suites:tox:py37
:beam-sdks-python-load-tests :sdks:python:apache_beam:testing:load_tests
:beam-sdks-python-container-py3 :sdks:python:container:py3
:beam-sdks-python-container :sdks:python:container
diff --git a/sdks/python/test-suites/dataflow/py35/build.gradle b/sdks/python/test-suites/dataflow/py35/build.gradle
index c5b21d2..4e6360a 100644
--- a/sdks/python/test-suites/dataflow/py35/build.gradle
+++ b/sdks/python/test-suites/dataflow/py35/build.gradle
@@ -70,3 +70,23 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
}
}
}
+
+task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) {
+ dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+ def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+ doLast {
+ // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"]
+ def argMap = ["test_opts": testOpts,
+ "streaming": "true",
+ "sdk_location": "${project.buildDir}/apache-beam.tar.gz",
+ "worker_jar": dataflowWorkerJar]
+ def cmdArgs = project.mapToArgString(argMap)
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}
diff --git a/sdks/python/test-suites/dataflow/py36/build.gradle b/sdks/python/test-suites/dataflow/py36/build.gradle
index 9440d6c..e60f548 100644
--- a/sdks/python/test-suites/dataflow/py36/build.gradle
+++ b/sdks/python/test-suites/dataflow/py36/build.gradle
@@ -70,3 +70,23 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
}
}
}
+
+task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) {
+ dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+ def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+ doLast {
+ // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"]
+ def argMap = ["test_opts": testOpts,
+ "streaming": "true",
+ "sdk_location": "${project.buildDir}/apache-beam.tar.gz",
+ "worker_jar": dataflowWorkerJar]
+ def cmdArgs = project.mapToArgString(argMap)
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}
diff --git a/sdks/python/test-suites/dataflow/py37/build.gradle b/sdks/python/test-suites/dataflow/py37/build.gradle
index cc73204..d80deeb 100644
--- a/sdks/python/test-suites/dataflow/py37/build.gradle
+++ b/sdks/python/test-suites/dataflow/py37/build.gradle
@@ -70,3 +70,23 @@ task validatesRunnerBatchTests(dependsOn: ['installGcpTest', 'sdist']) {
}
}
}
+
+task validatesRunnerStreamingTests(dependsOn: ['installGcpTest', 'sdist']) {
+ dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
+
+ def dataflowWorkerJar = project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
+
+ doLast {
+ // TODO(BEAM-3544,BEAM-5025): Disable tests with 'sickbay-streaming' tag.
+ def testOpts = basicTestOpts + ["--attr=ValidatesRunner,!sickbay-streaming"]
+ def argMap = ["test_opts": testOpts,
+ "streaming": "true",
+ "sdk_location": "${project.buildDir}/apache-beam.tar.gz",
+ "worker_jar": dataflowWorkerJar]
+ def cmdArgs = project.mapToArgString(argMap)
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh $cmdArgs"
+ }
+ }
+}