You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ma...@apache.org on 2019/04/23 21:14:51 UTC
[beam] branch master updated: [BEAM-6908] Support Python3
performance benchmarks - part 2
This is an automated email from the ASF dual-hosted git repository.
markliu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 43be51f [BEAM-6908] Support Python3 performance benchmarks - part 2
new c97e391 Merge pull request #8324 from markflyhigh/py3-perf
43be51f is described below
commit 43be51f1d64ffe3ae856570ac68348ab99ea5946
Author: Mark Liu <ma...@google.com>
AuthorDate: Tue Apr 16 14:28:38 2019 -0700
[BEAM-6908] Support Python3 performance benchmarks - part 2
---
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 57 ++++++++++++++++++++++
.../apache_beam/io/gcp/gcsio_integration_test.py | 4 +-
sdks/python/build.gradle | 31 +-----------
sdks/python/scripts/run_integration_test.sh | 25 +++++-----
sdks/python/test-suites/dataflow/py3/build.gradle | 1 +
5 files changed, 75 insertions(+), 43 deletions(-)
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 6614954..4b6c75d 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -235,6 +235,24 @@ class BeamModulePlugin implements Plugin<Project> {
String itModule = System.getProperty('itModule')
}
+ // Reads and contains all necessary performance test parameters
+ class PythonPerformanceTestConfiguration {
+ // Fully qualified name of the test to run.
+ String tests = System.getProperty('tests')
+
+ // Attribute tag that can filter the test set.
+ String attribute = System.getProperty('attr')
+
+ // Extra test options pass to nose.
+ String[] extraTestOptions = ["--nocapture"]
+
+ // Name of Cloud KMS encryption key to use in some tests.
+ String kmsKeyName = System.getProperty('kmsKeyName')
+
+ // Pipeline options to be used for pipeline invocation.
+ String pipelineOptions = System.getProperty('pipelineOptions', '')
+ }
+
// A class defining the set of configurable properties accepted by containerImageName.
class ContainerImageNameConfiguration {
String root = null // Sets the docker repository root (optional).
@@ -1733,6 +1751,45 @@ class BeamModulePlugin implements Plugin<Project> {
outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/")
}
}
+
+ // Run single or a set of integration tests with provided test options and pipeline options.
+ project.ext.enablePythonPerformanceTest = {
+
+ // Use the implicit it parameter of the closure to handle zero argument or one argument map calls.
+ // See: http://groovy-lang.org/closures.html#implicit-it
+ def config = it ? it as PythonPerformanceTestConfiguration : new PythonPerformanceTestConfiguration()
+
+ project.task('integrationTest') {
+ dependsOn 'installGcpTest'
+ dependsOn 'sdist'
+
+ doLast {
+ def argMap = [:]
+
+ // Build test options that configures test environment and framework
+ def testOptions = []
+ if (config.tests)
+ testOptions += "--tests=$config.tests"
+ if (config.attribute)
+ testOptions += "--attr=$config.attribute"
+ testOptions.addAll(config.extraTestOptions)
+ argMap["test_opts"] = testOptions
+
+ // Build pipeline options that configures pipeline job
+ if (config.pipelineOptions)
+ argMap["pipeline_opts"] = config.pipelineOptions
+ if (config.kmsKeyName)
+ argMap["kms_key_name"] = config.kmsKeyName
+
+ def cmdArgs = project.mapToArgString(argMap)
+ def runScriptsDir = "${pythonRootDir}/scripts"
+ project.exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && ${runScriptsDir}/run_integration_test.sh ${cmdArgs}"
+ }
+ }
+ }
+ }
}
}
}
diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
index 18a367d..8312f07 100644
--- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
+++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py
@@ -32,8 +32,8 @@ permissions for the key specified in --kms_key_name.
To run these tests manually:
./gradlew beam-sdks-python:integrationTest \
- -Ptests=apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest \
- -PkmsKeyName=KMS_KEY_NAME
+ -Dtests=apache_beam.io.gcp.gcsio_integration_test:GcsIOIntegrationTest \
+ -DkmsKeyName=KMS_KEY_NAME
"""
from __future__ import absolute_import
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index 3c9cd93..495a724 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -20,6 +20,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
plugins { id 'org.apache.beam.module' }
applyPythonNature()
+enablePythonPerformanceTest()
/*************************************************************************************************/
@@ -197,36 +198,6 @@ def portableWordCountTask(name, streaming) {
}
}
-// Run single or a set of integration tests with provided test options and pipeline options.
-task integrationTest(dependsOn: ['installGcpTest', 'sdist']) {
- doLast {
- def argMap = [:]
-
- // Build test options that configures test environment and framework
- def testOptions = basicTestOpts
- if (project.hasProperty('testOptions'))
- // Customize test options by adding more nose flags to -Ptest_opt in commandline
- testOptions = project.property('testOptions')
- if (project.hasProperty('attr'))
- testOptions += ["--attr=${project.property('attr')}"]
- if (project.hasProperty('tests'))
- testOptions += ["--tests=${project.property('tests')}"]
- argMap["test_opts"] = testOptions
-
- // Build pipeline options that configures pipeline job
- if (project.hasProperty('pipelineOptions'))
- argMap["pipeline_opts"] = project.property('pipelineOptions')
- if (project.hasProperty('kmsKeyName'))
- argMap["kms_key_name"] = project.property('kmsKeyName')
-
- def cmdArgs = project.mapToArgString(argMap)
- exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_integration_test.sh $cmdArgs"
- }
- }
-}
-
// Run PostCommit integration tests on default runner (TestDataflowRunner)
task postCommitIT(dependsOn: ['installGcpTest', 'sdist']) {
dependsOn ":beam-runners-google-cloud-dataflow-java-fn-api-worker:shadowJar"
diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh
index af9b965..ba72a14 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -153,20 +153,23 @@ set -o errexit
###########################################################################
-# Build pipeline options if not provided in --pipeline_opts from commandline
-if [[ -z $PIPELINE_OPTS ]]; then
+# Check that the script is running in a known directory.
+if [[ $PWD != *sdks/python* ]]; then
+ echo 'Unable to locate Apache Beam Python SDK root directory'
+ exit 1
+fi
- # Check that the script is running in a known directory.
- if [[ $PWD != *sdks/python* ]]; then
- echo 'Unable to locate Apache Beam Python SDK root directory'
- exit 1
- fi
+# Go to the Apache Beam Python SDK root
+if [[ $PWD != *sdks/python ]]; then
+ cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
+fi
- # Go to the Apache Beam Python SDK root
- if [[ $PWD != *sdks/python ]]; then
- cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
- fi
+
+###########################################################################
+# Build pipeline options if not provided in --pipeline_opts from commandline
+
+if [[ -z $PIPELINE_OPTS ]]; then
# Create a tarball if not exists
if [[ $(find ${SDK_LOCATION} 2> /dev/null) ]]; then
diff --git a/sdks/python/test-suites/dataflow/py3/build.gradle b/sdks/python/test-suites/dataflow/py3/build.gradle
index 66cc985..7d31cb3 100644
--- a/sdks/python/test-suites/dataflow/py3/build.gradle
+++ b/sdks/python/test-suites/dataflow/py3/build.gradle
@@ -18,6 +18,7 @@
apply plugin: org.apache.beam.gradle.BeamModulePlugin
applyPythonNature()
+enablePythonPerformanceTest()
// Required to setup a Python 3 virtualenv.
pythonVersion = '3.5'