You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bh...@apache.org on 2021/10/01 00:15:58 UTC
[beam] branch master updated: [BEAM-12957] Add support for pyarrow
5.x (#15588)
This is an automated email from the ASF dual-hosted git repository.
bhulette pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new e79750d [BEAM-12957] Add support for pyarrow 5.x (#15588)
e79750d is described below
commit e79750d2b5464cbcbe3b8fc49ff9d0ddd754d02e
Author: Brian Hulette <bh...@google.com>
AuthorDate: Thu Sep 30 17:15:09 2021 -0700
[BEAM-12957] Add support for pyarrow 5.x (#15588)
* Allow pyarrow 5
* Create pyarrow test tasks programmatically
* precommit depends on pyarrow-x tasks
* Add posargs, TODO
---
sdks/python/setup.py | 2 +-
sdks/python/test-suites/tox/common.gradle | 4 +---
sdks/python/test-suites/tox/py38/build.gradle | 16 ++++++----------
sdks/python/tox.ini | 6 ++++--
4 files changed, 12 insertions(+), 16 deletions(-)
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index d9d7591..34f96c1 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -148,7 +148,7 @@ REQUIRED_PACKAGES = [
'pymongo>=3.8.0,<4.0.0',
'oauth2client>=2.0.1,<5',
'protobuf>=3.12.2,<4',
- 'pyarrow>=0.15.1,<5.0.0',
+ 'pyarrow>=0.15.1,<6.0.0',
'pydot>=1.2.0,<2',
'python-dateutil>=2.8.0,<3',
'pytz>=2018.3',
diff --git a/sdks/python/test-suites/tox/common.gradle b/sdks/python/test-suites/tox/common.gradle
index af74725..285a8a7 100644
--- a/sdks/python/test-suites/tox/common.gradle
+++ b/sdks/python/test-suites/tox/common.gradle
@@ -33,9 +33,7 @@ test.dependsOn "testPy38CloudCoverage"
project.task("preCommitPy${pythonVersionSuffix}") {
// Generates coverage reports only once, in Py38, to remove duplicated work
if (pythonVersionSuffix.equals('38')) {
- dependsOn = ["testPy38CloudCoverage", "testPy38Cython",
- "testPy38pyarrow-0", "testPy38pyarrow-1", "testPy38pyarrow-2",
- "testPy38pyarrow-3", "testPy38pyarrow-4"]
+ dependsOn = ["testPy38CloudCoverage", "testPy38Cython"]
} else {
dependsOn = ["testPy${pythonVersionSuffix}Cloud", "testPy${pythonVersionSuffix}Cython"]
}
diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle
index 8497d57..6e54a86 100644
--- a/sdks/python/test-suites/tox/py38/build.gradle
+++ b/sdks/python/test-suites/tox/py38/build.gradle
@@ -34,16 +34,12 @@ apply from: "../common.gradle"
// TODO(BEAM-8954): Remove this once tox uses isolated builds.
testPy38Cython.mustRunAfter testPython38, testPy38CloudCoverage
-toxTask "testPy38pyarrow-0", "py38-pyarrow-0"
-toxTask "testPy38pyarrow-1", "py38-pyarrow-1"
-toxTask "testPy38pyarrow-2", "py38-pyarrow-2"
-toxTask "testPy38pyarrow-3", "py38-pyarrow-3"
-toxTask "testPy38pyarrow-4", "py38-pyarrow-4"
-test.dependsOn "testPy38pyarrow-0"
-test.dependsOn "testPy38pyarrow-1"
-test.dependsOn "testPy38pyarrow-2"
-test.dependsOn "testPy38pyarrow-3"
-test.dependsOn "testPy38pyarrow-4"
+(0..5).each {version ->
+ // Create a test task for each major version of pyarrow
+ toxTask "testPy38pyarrow-$version", "py38-pyarrow-$version"
+ test.dependsOn "testPy38pyarrow-$version"
+ preCommitPy38.dependsOn "testPy38pyarrow-$version"
+}
toxTask "whitespacelint", "whitespacelint"
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index 1c28708..f3946cb 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -233,7 +233,7 @@ extras = test
commands =
{toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}
-[testenv:py{36,37,38}-pyarrow-{0,1,2,3,4}]
+[testenv:py{36,37,38}-pyarrow-{0,1,2,3,4,5}]
deps =
0: pyarrow>=0.15.1,<0.18.0
1: pyarrow>=1,<2
@@ -243,7 +243,9 @@ deps =
{0,1,2}: numpy<1.20.0
3: pyarrow>=3,<4
4: pyarrow>=4,<5
+ 5: pyarrow>=5,<6
commands =
# Log pyarrow and numpy version for debugging
/bin/sh -c "pip freeze | grep -E '(pyarrow|numpy)'"
- {toxinidir}/scripts/run_pytest.sh {envname} '-m uses_pyarrow'
+ # TODO(BEAM-12985): Running run_pytest.sh with -m causes us to run all the tests twice
+ {toxinidir}/scripts/run_pytest.sh {envname} "{posargs}" '-m uses_pyarrow'