You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by an...@apache.org on 2024/01/10 16:51:26 UTC

(beam) branch master updated: Add support for pandas 2.0 (#28636)

This is an automated email from the ASF dual-hosted git repository.

anandinguva pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 0a368056904 Add support for pandas 2.0 (#28636)
0a368056904 is described below

commit 0a368056904ef62e947b354f1f44bd53506fb4e5
Author: caneff <ca...@google.com>
AuthorDate: Wed Jan 10 11:51:17 2024 -0500

    Add support for pandas 2.0 (#28636)
    
    * Add support for pandas 2.0
    
    * Fix pyarrow tests for pandas 2 compat
    
    * Fix tox.ini maybe
---
 sdks/python/apache_beam/examples/dataframe/flight_delays.py |  2 +-
 sdks/python/setup.py                                        |  6 +++---
 sdks/python/test-suites/tox/py38/build.gradle               |  4 ++++
 sdks/python/tox.ini                                         | 10 +++++++++-
 4 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/sdks/python/apache_beam/examples/dataframe/flight_delays.py b/sdks/python/apache_beam/examples/dataframe/flight_delays.py
index c2c467b39ea..4ba4b77fa9d 100644
--- a/sdks/python/apache_beam/examples/dataframe/flight_delays.py
+++ b/sdks/python/apache_beam/examples/dataframe/flight_delays.py
@@ -44,7 +44,7 @@ def get_mean_delay_at_top_airports(airline_df):
   top_airports = total.nlargest(10, keep='all').dropna()
   at_top_airports = airline_df['arrival_airport'].isin(
       top_airports.index.values)
-  return airline_df[at_top_airports].mean()
+  return airline_df[at_top_airports].mean(numeric_only=True)
 
 
 def input_date(date):
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index e8c93ce8548..db783fea234 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -166,7 +166,7 @@ else:
 # Exclude 1.5.0 and 1.5.1 because of
 # https://github.com/pandas-dev/pandas/issues/45725
 dataframe_dependency = [
-    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<1.6;python_version>="3.8"',
+    'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.1;python_version>="3.8"',
 ]
 
 def find_by_ext(root_dir, ext):
@@ -337,14 +337,14 @@ if __name__ == '__main__':
               # Pinning docutils as a workaround for Sphinx issue:
               # https://github.com/sphinx-doc/sphinx/issues/9727
               'docutils==0.17.1',
-              'pandas<2.0.0',
+              'pandas<2.1.0',
           ],
           'test': [
               'docstring-parser>=0.15,<1.0',
               'freezegun>=0.3.12',
               'joblib>=1.0.1',
               'mock>=1.0.1,<6.0.0',
-              'pandas<2.0.0',
+              'pandas<2.1.0',
               'parameterized>=0.7.1,<0.10.0',
               'pyhamcrest>=1.9,!=1.10.0,<3.0.0',
               'pyyaml>=3.12,<7.0.0',
diff --git a/sdks/python/test-suites/tox/py38/build.gradle b/sdks/python/test-suites/tox/py38/build.gradle
index 1e03b505808..ea34eec1fae 100644
--- a/sdks/python/test-suites/tox/py38/build.gradle
+++ b/sdks/python/test-suites/tox/py38/build.gradle
@@ -89,6 +89,10 @@ toxTask "testPy38pandas-15", "py38-pandas-15", "${posargs}"
 test.dependsOn "testPy38pandas-15"
 preCommitPyCoverage.dependsOn "testPy38pandas-15"
 
+toxTask "testPy38pandas-20", "py38-pandas-20", "${posargs}"
+test.dependsOn "testPy38pandas-20"
+preCommitPyCoverage.dependsOn "testPy38pandas-20"
+
 // Create a test task for each minor version of pytorch
 toxTask "testPy38pytorch-19", "py38-pytorch-19", "${posargs}"
 test.dependsOn "testPy38pytorch-19"
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index dbe90c084af..7306a6774bb 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -266,12 +266,18 @@ extras = test
 commands =
   bash {toxinidir}/scripts/pytest_validates_runner.sh {envname} {toxinidir}/apache_beam/runners/portability/spark_runner_test.py {posargs}
 
+
 [testenv:py{38,39,310}-pyarrow-{3,4,5,6,7,8,9}]
 deps =
+  # Pandas 2 minimum for pyarrow is 7
   3: pyarrow>=3,<4
+  3: pandas<2
   4: pyarrow>=4,<5
+  4: pandas<2
   5: pyarrow>=5,<6
+  5: pandas<2
   6: pyarrow>=6,<7
+  6: pandas<2
   7: pyarrow>=7,<8
   8: pyarrow>=8,<9
   9: pyarrow>=9,<10
@@ -296,11 +302,13 @@ commands =
   /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret'
 
 
-[testenv:py{38,39,310,311}-pandas-{14,15}]
+[testenv:py{38,39,310,311}-pandas-{14,15,20}]
 deps =
   14: pandas>=1.4.3,<1.5.0
   # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725
   15: pandas>=1.5.2,<1.6.0
+  20: pandas>=2.0.0,<2.1.0
+  20: pyarrow>=7
 commands =
   # Log pandas and numpy version for debugging
   /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"