You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by an...@apache.org on 2024/01/10 16:51:26 UTC
(beam) branch master updated: Add support for pandas 2.0 (#28636)
This is an automated email from the ASF dual-hosted git repository.
anandinguva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0a368056904 Add support for pandas 2.0 (#28636)
0a368056904 is described below
commit 0a368056904ef62e947b354f1f44bd53506fb4e5
Author: caneff <ca...@google.com>
AuthorDate: Wed Jan 10 11:51:17 2024 -0500
Add support for pandas 2.0 (#28636)
* Add support for pandas 2.0
* Fix pyarrow tests for pandas 2 compat
* Fix tox.ini maybe
---
sdks/python/apache_beam/examples/dataframe/flight_delays.py | 2 +-
sdks/python/setup.py | 6 +++---
sdks/python/test-suites/tox/py38/build.gradle | 4 ++++
sdks/python/tox.ini | 10 +++++++++-
4 files changed, 17 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/examples/dataframe/flight_delays.py b/sdks/python/apache_beam/examples/dataframe/flight_delays.py
index c2c467b39ea..4ba4b77fa9d 100644
--- a/sdks/python/apache_beam/examples/dataframe/flight_delays.py
+++ b/sdks/python/apache_beam/examples/dataframe/flight_delays.py
@@ -44,7 +44,7 @@ def get_mean_delay_at_top_airports(airline_df):
top_airports = total.nlargest(10, keep='all').dropna()
at_top_airports = airline_df['arrival_airport'].isin(
top_airports.index.values)
- return airline_df[at_top_airports].mean()
+ return airline_df[at_top_airports].mean(numeric_only=True)
def input_date(date):
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index e8c93ce8548..db783fea234 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -166,7 +166,7 @@ else:
# Exclude 1.5.0 and 1.5.1 because of
# https://github.com/pandas-dev/pandas/issues/45725
dataframe_dependency = [
- 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<1.6;python_version>="3.8"',
+ 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.1;python_version>="3.8"',
]
def find_by_ext(root_dir, ext):
@@ -337,14 +337,14 @@ if __name__ == '__main__':
# Pinning docutils as a workaround for Sphinx issue:
# https://github.com/sphinx-doc/sphinx/issues/9727
'docutils==0.17.1',
- 'pandas<2.0.0',
+ 'pandas<2.1.0',
],
'test': [
'docstring-parser>=0.15,<1.0',
'freezegun>=0.3.12',
'joblib>=1.0.1',
'mock>=1.0.1,<6.0.0',
- 'pandas<2.0.0',
+ 'pandas<2.1.0',
'parameterized>=0.7.1,<0.10.0',
'pyhamcrest>=1.9,!=1.10.0,<3.0.0',
'pyyaml>=3.12,<7.0.0',
diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle
index 1e03b505808..ea34eec1fae 100644
--- a/sdks/python/test-suites/tox/py38/build.gradle
+++ b/sdks/python/test-suites/tox/py38/build.gradle
@@ -89,6 +89,10 @@ toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}"
test.dependsOn "testPy38pandas-15"
preCommitPyCoverage.dependsOn "testPy38pandas-15"
+toxTask "testPy38pandas-20", "py38-pandas-20", "${posargs}"
+test.dependsOn "testPy38pandas-20"
+preCommitPyCoverage.dependsOn "testPy38pandas-20"
+
// Create a test task for each minor version of pytorch
toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}"
test.dependsOn "testPy38pytorch-19"
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index dbe90c084af..7306a6774bb 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -266,12 +266,18 @@ extras = test
commands =
bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}
+
[testenv:py{38,39,310}-pyarrow-{3,4,5,6,7,8,9}]
deps =
+ # Pandas 2 minimum for pyarrow is 7
3: pyarrow>=3,<4
+ 3: pandas<2
4: pyarrow>=4,<5
+ 4: pandas<2
5: pyarrow>=5,<6
+ 5: pandas<2
6: pyarrow>=6,<7
+ 6: pandas<2
7: pyarrow>=7,<8
8: pyarrow>=8,<9
9: pyarrow>=9,<10
@@ -296,11 +302,13 @@ commands =
/bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
-[testenv:py{38,39,310,311}-pandas-{14,15}]
+[testenv:py{38,39,310,311}-pandas-{14,15,20}]
deps =
14: pandas>=1.4.3,<1.5.0
# Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
15: pandas>=1.5.2,<1.6.0
+ 20: pandas>=2.0.0,<2.1.0
+ 20: pyarrow>=7
commands =
# Log pandas and numpy version for debugging
/bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"