You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2019/03/14 16:20:18 UTC
[beam] 01/01: Revert "[BEAM-6527] Use Gradle to parallel Python tox
tests"
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch revert-7675-py3-tox-2
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 8ae6ebb25753277930632b1605542fc2d91e2aa1
Author: Ahmet Altay <aa...@gmail.com>
AuthorDate: Thu Mar 14 09:20:02 2019 -0700
Revert "[BEAM-6527] Use Gradle to parallel Python tox tests"
---
build.gradle | 5 +-
.../org/apache/beam/gradle/BeamModulePlugin.groovy | 41 +---------------
sdks/python/build.gradle | 56 ++++++++++++++++++++--
sdks/python/scripts/generate_pydoc.sh | 2 +-
sdks/python/scripts/run_integration_test.sh | 2 +-
sdks/python/scripts/run_mini_py3lint.sh | 2 +-
sdks/python/scripts/run_pylint.sh | 2 +-
sdks/python/scripts/run_pylint_2to3.sh | 2 +-
sdks/python/scripts/run_tox.sh | 17 ++-----
sdks/python/scripts/run_tox_cleanup.sh | 2 +-
sdks/python/test-suites/tox/py35/build.gradle | 53 --------------------
sdks/python/test-suites/tox/py36/build.gradle | 34 -------------
sdks/python/tox.ini | 2 +-
settings.gradle | 4 --
14 files changed, 67 insertions(+), 157 deletions(-)
diff --git a/build.gradle b/build.gradle
index e34eaa2..4c47a26 100644
--- a/build.gradle
+++ b/build.gradle
@@ -191,10 +191,7 @@ task goIntegrationTests() {
}
task pythonPreCommit() {
- dependsOn ":beam-sdks-python:preCommitPy2"
- dependsOn ":beam-sdks-python-test-suites-tox-py35:preCommitPy35"
- dependsOn ":beam-sdks-python-test-suites-tox-py36:preCommitPy36"
- dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT"
+ dependsOn ":beam-sdks-python:preCommit"
}
task pythonPostCommit() {
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index beb27e6..7c50a52 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -1611,38 +1611,16 @@ class BeamModulePlugin implements Plugin<Project> {
outputs.dirs(project.ext.envdir)
}
- def pythonSdkDeps = project.files(
- project.fileTree(
- dir: "${project.rootDir}",
- include: ['model/**', 'sdks/python/**'],
- // Exclude temporary directories used in build and test.
- exclude: [
- 'sdks/python/build/**',
- 'sdks/python/dist/**',
- 'sdks/python/target/**',
- 'sdks/python/test-suites/**',
- ])
- )
- def copiedSrcRoot = "${project.buildDir}/srcs"
-
project.configurations { distConfig }
project.task('sdist', dependsOn: 'setupVirtualenv') {
doLast {
- // Copy sdk sources to an isolated directory
- project.copy {
- from pythonSdkDeps
- into copiedSrcRoot
- }
-
- // Build artifact
project.exec {
executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedSrcRoot}/sdks/python && python setup.py sdist --formats zip,gztar --dist-dir ${project.buildDir}"
+ args '-c', ". ${project.ext.envdir}/bin/activate && cd ${pythonRootDir} && python setup.py sdist --keep-temp --formats zip,gztar --dist-dir ${project.buildDir}"
}
def collection = project.fileTree("${project.buildDir}"){ include '**/*.tar.gz' exclude '**/apache-beam.tar.gz'}
println "sdist archive name: ${collection.singleFile}"
-
// we need a fixed name for the artifact
project.copy { from collection.singleFile; into "${project.buildDir}"; rename { 'apache-beam.tar.gz' } }
}
@@ -1668,7 +1646,7 @@ class BeamModulePlugin implements Plugin<Project> {
project.exec {
executable 'sh'
args '-c', "if [ -e ${activate} ]; then " +
- ". ${activate} && cd ${pythonRootDir} && python setup.py clean; " +
+ ". ${activate} && python ${pythonRootDir}/setup.py clean; " +
"fi"
}
project.delete project.buildDir // Gradle build directory
@@ -1695,21 +1673,6 @@ class BeamModulePlugin implements Plugin<Project> {
}
return argList.join(' ')
}
-
- project.ext.toxTask = { name, tox_env ->
- project.tasks.create(name) {
- dependsOn = ['sdist']
- doLast {
- def copiedPyRoot = "${copiedSrcRoot}/sdks/python"
- project.exec {
- executable 'sh'
- args '-c', ". ${project.ext.envdir}/bin/activate && cd ${copiedPyRoot} && scripts/run_tox.sh $tox_env ${project.buildDir}/apache-beam.tar.gz"
- }
- }
- inputs.files pythonSdkDeps
- outputs.files project.fileTree(dir: "${pythonRootDir}/target/.tox/${tox_env}/log/")
- }
- }
}
}
}
diff --git a/sdks/python/build.gradle b/sdks/python/build.gradle
index f13eb63..b83e614 100644
--- a/sdks/python/build.gradle
+++ b/sdks/python/build.gradle
@@ -38,8 +38,34 @@ build.dependsOn buildPython
/*************************************************************************************************/
-// Unit tests for Python 2
-// See Python 3 tests in test-suites/tox
+// Unit testing
+
+def pythonSdkDeps = files(
+ fileTree(dir: 'apache_beam', includes: ['**/*.py', '**/*.pyx', '**/*.pxd']),
+ fileTree(dir: 'apache_beam/testing/data'),
+ fileTree(dir: "${project.rootDir}/model"),
+ fileTree(dir: 'scripts'),
+ ".pylintrc",
+ "MANIFEST.in",
+ "gen_protos.py",
+ "setup.cfg",
+ "setup.py",
+ "test_config.py",
+ "tox.ini")
+
+def toxTask = {
+ name, tox_env -> tasks.create(name) {
+ dependsOn = ['setupVirtualenv']
+ doLast {
+ exec {
+ executable 'sh'
+ args '-c', ". ${project.ext.envdir}/bin/activate && ./scripts/run_tox.sh $tox_env"
+ }
+ }
+ inputs.files pythonSdkDeps
+ outputs.files fileTree(dir: "${project.rootDir}/sdks/python/target/.tox/${tox_env}/log/")
+ }
+}
task lint {}
check.dependsOn lint
@@ -50,12 +76,24 @@ lint.dependsOn lintPy27
toxTask "lintPy27_3", "py27-lint3"
lint.dependsOn lintPy27_3
+toxTask "lintPy35", "py35-lint"
+lint.dependsOn lintPy35
+
toxTask "testPy2Gcp", "py27-gcp"
test.dependsOn testPy2Gcp
+toxTask "testPy35Gcp", "py35-gcp"
+test.dependsOn testPy35Gcp
+
toxTask "testPython2", "py27"
test.dependsOn testPython2
+toxTask "testPython35", "py35"
+test.dependsOn testPython35
+
+toxTask "testPython36", "py36"
+test.dependsOn testPython36
+
toxTask "testPy2Cython", "py27-cython"
test.dependsOn testPy2Cython
// Ensure that testPy2Cython runs exclusively to other tests. This line is not
@@ -63,17 +101,29 @@ test.dependsOn testPy2Cython
// project.
testPy2Cython.mustRunAfter testPython2, testPy2Gcp
+toxTask "testPy35Cython", "py35-cython"
+test.dependsOn testPy35Cython
+// Ensure that testPy35Cython runs exclusively to other tests. This line is not
+// actually required, since gradle doesn't do parallel execution within a
+// project.
+testPy35Cython.mustRunAfter testPython35, testPy35Gcp
+
toxTask "docs", "docs"
assemble.dependsOn docs
toxTask "cover", "cover"
-task preCommitPy2() {
+task preCommit() {
dependsOn "docs"
dependsOn "testPy2Cython"
+ dependsOn "testPy35Cython"
dependsOn "testPython2"
+ dependsOn "testPython35"
+ dependsOn "testPython36"
dependsOn "testPy2Gcp"
+ dependsOn "testPy35Gcp"
dependsOn "lint"
+ dependsOn ":beam-sdks-python-test-suites-dataflow:preCommitIT"
}
task portablePreCommit() {
diff --git a/sdks/python/scripts/generate_pydoc.sh b/sdks/python/scripts/generate_pydoc.sh
index a9bffe3..4929c5e 100755
--- a/sdks/python/scripts/generate_pydoc.sh
+++ b/sdks/python/scripts/generate_pydoc.sh
@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
fi
# Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
diff --git a/sdks/python/scripts/run_integration_test.sh b/sdks/python/scripts/run_integration_test.sh
index f0c01ed..273f963 100755
--- a/sdks/python/scripts/run_integration_test.sh
+++ b/sdks/python/scripts/run_integration_test.sh
@@ -163,7 +163,7 @@ if [[ -z $PIPELINE_OPTS ]]; then
fi
# Go to the Apache Beam Python SDK root
- if [[ $PWD != *sdks/python ]]; then
+ if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
diff --git a/sdks/python/scripts/run_mini_py3lint.sh b/sdks/python/scripts/run_mini_py3lint.sh
index 27ca3ce..0729c7b 100755
--- a/sdks/python/scripts/run_mini_py3lint.sh
+++ b/sdks/python/scripts/run_mini_py3lint.sh
@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
fi
# Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
diff --git a/sdks/python/scripts/run_pylint.sh b/sdks/python/scripts/run_pylint.sh
index 2814b4f..17f6aa7 100755
--- a/sdks/python/scripts/run_pylint.sh
+++ b/sdks/python/scripts/run_pylint.sh
@@ -30,7 +30,7 @@ if [[ $PWD != *sdks/python* ]]; then
fi
# Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
diff --git a/sdks/python/scripts/run_pylint_2to3.sh b/sdks/python/scripts/run_pylint_2to3.sh
index 0853b09..2957dd9 100755
--- a/sdks/python/scripts/run_pylint_2to3.sh
+++ b/sdks/python/scripts/run_pylint_2to3.sh
@@ -29,7 +29,7 @@ if [[ $PWD != *sdks/python* ]]; then
fi
# Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
diff --git a/sdks/python/scripts/run_tox.sh b/sdks/python/scripts/run_tox.sh
index e799042..24ccb20 100755
--- a/sdks/python/scripts/run_tox.sh
+++ b/sdks/python/scripts/run_tox.sh
@@ -24,10 +24,9 @@
###########################################################################
# Usage check.
-if [[ $# < 1 || $# > 2 ]]; then
- printf "Usage: \n$> ./scripts/run_tox.sh <tox_environment> [<sdk_location>]"
+if [[ $# != 1 ]]; then
+ printf "Usage: \n$> ./scripts/run_tox.sh <tox_environment>"
printf "\n\ttox_environment: [required] Tox environment to run the test in.\n"
- printf "\n\tsdk_location: [optional] SDK tarball artifact location.\n"
exit 1
fi
@@ -38,19 +37,11 @@ if [[ $PWD != *sdks/python* ]]; then
fi
# Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
-# Used in tox.ini to isolate toxworkdir of each environment.
-export ENV_NAME=.tox-$1
-
-if [[ ! -z $2 ]]; then
- tox -c tox.ini --recreate -e $1 --installpkg $2
-else
- tox -c tox.ini --recreate -e $1
-fi
-
+tox -c tox.ini --recreate -e $1
exit_code=$?
# Retry once for the specific exit code 245.
if [[ $exit_code == 245 ]]; then
diff --git a/sdks/python/scripts/run_tox_cleanup.sh b/sdks/python/scripts/run_tox_cleanup.sh
index be44095..87081da 100755
--- a/sdks/python/scripts/run_tox_cleanup.sh
+++ b/sdks/python/scripts/run_tox_cleanup.sh
@@ -26,7 +26,7 @@ if [[ $PWD != *sdks/python* ]]; then
fi
# Go to the Apache Beam Python SDK root
-if [[ $PWD != *sdks/python ]]; then
+if [[ "*sdks/python" != $PWD ]]; then
cd $(pwd | sed 's/sdks\/python.*/sdks\/python/')
fi
diff --git a/sdks/python/test-suites/tox/py35/build.gradle b/sdks/python/test-suites/tox/py35/build.gradle
deleted file mode 100644
index 5f86fe9..0000000
--- a/sdks/python/test-suites/tox/py35/build.gradle
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Unit tests for Python 3.5
- */
-
-plugins { id 'org.apache.beam.module' }
-applyPythonNature()
-
-// Required to setup a Python 3 virtualenv.
-project.ext.python3 = true
-
-task lint {}
-check.dependsOn lint
-
-toxTask "lintPy35", "py35-lint"
-lint.dependsOn lintPy35
-
-toxTask "testPython35", "py35"
-test.dependsOn testPython35
-
-toxTask "testPy35Gcp", "py35-gcp"
-test.dependsOn testPy35Gcp
-
-toxTask "testPy35Cython", "py35-cython"
-test.dependsOn testPy35Cython
-// Ensure that testPy35Cython runs exclusively to other tests. This line is not
-// actually required, since gradle doesn't do parallel execution within a
-// project.
-testPy35Cython.mustRunAfter testPython35, testPy35Gcp
-
-task preCommitPy35() {
- dependsOn "testPython35"
- dependsOn "testPy35Gcp"
- dependsOn "testPy35Cython"
- dependsOn "lint"
-}
diff --git a/sdks/python/test-suites/tox/py36/build.gradle b/sdks/python/test-suites/tox/py36/build.gradle
deleted file mode 100644
index 1bdfb78..0000000
--- a/sdks/python/test-suites/tox/py36/build.gradle
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Unit tests for Python 3.6
- */
-
-plugins { id 'org.apache.beam.module' }
-applyPythonNature()
-
-// Required to setup a Python 3 virtualenv.
-project.ext.python3 = true
-
-toxTask "testPython36", "py36"
-test.dependsOn testPython36
-
-task preCommitPy36() {
- dependsOn "testPython36"
-}
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 2b765f8..cd19a6b 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -18,7 +18,7 @@
[tox]
# new environments will be excluded by default unless explicitly added to envlist.
envlist = py27,py35,py36,py27-{gcp,cython,lint,lint3},py35-{gcp,cython,lint},docs
-toxworkdir = {toxinidir}/target/{env:ENV_NAME:.tox}
+toxworkdir = {toxinidir}/target/.tox
[pycodestyle]
# Disable all errors and warnings except for the ones related to blank lines.
diff --git a/settings.gradle b/settings.gradle
index c453272..b8fc633 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -207,10 +207,6 @@ include "beam-sdks-python-test-suites-dataflow-py3"
project(":beam-sdks-python-test-suites-dataflow-py3").dir = file("sdks/python/test-suites/dataflow/py3")
include "beam-sdks-python-test-suites-direct-py3"
project(":beam-sdks-python-test-suites-direct-py3").dir = file("sdks/python/test-suites/direct/py3")
-include "beam-sdks-python-test-suites-tox-py35"
-project(":beam-sdks-python-test-suites-tox-py35").dir = file("sdks/python/test-suites/tox/py35")
-include "beam-sdks-python-test-suites-tox-py36"
-project(":beam-sdks-python-test-suites-tox-py36").dir = file("sdks/python/test-suites/tox/py36")
include "beam-sdks-python-load-tests"
project(":beam-sdks-python-load-tests").dir = file("sdks/python/apache_beam/testing/load_tests")
include "beam-vendor-grpc-1_13_1"