diff --git a/.github/trigger_files/beam_PostCommit_Python_Dependency.json b/.github/trigger_files/beam_PostCommit_Python_Dependency.json index 0c0e8ca6a12b..1aa32de42b22 100644 --- a/.github/trigger_files/beam_PostCommit_Python_Dependency.json +++ b/.github/trigger_files/beam_PostCommit_Python_Dependency.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 8 + "modification": 6 } diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 703047cb6ef1..9a00fe03ea07 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -31,7 +31,7 @@ requires = [ # Avoid https://github.com/pypa/virtualenv/issues/2006 "distlib==0.4.4", # Numpy headers - "numpy>=1.14.3,<2.5.0", # Update setup.py as well. + "numpy>=1.23.2,<2.6.0", # Update setup.py as well. # having cython here will create wheels that are platform dependent. "cython>=3.2.6,<4", ## deps for generating external transform wrappers: diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 3279282fc6b9..621df4cf0cde 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -156,12 +156,11 @@ def cythonize(*args, **kwargs): 'pyarrow>=14.0.1,<26.0.0', ] -# Exclude pandas<=1.4.2 since it doesn't work with numpy 1.24.x. -# Exclude 1.5.0 and 1.5.1 because of -# https://github.com/pandas-dev/pandas/issues/45725 # must update the below "docs" and "test" for extras_require dataframe_dependency = [ - 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.4', + # pandas 2.2.2 is the first version that works with numpy 2.0.0+ + # TODO(https://github.com/apache/beam/issues/40709): support pandas 3+ + 'pandas>=2.2.2,<2.4', ] milvus_dependency = ['pymilvus>=2.5.10,<3.0.0'] @@ -435,7 +434,9 @@ def get_portability_package_data(): 'jsonpickle>=3.0.4,<5.0.0', # numpy can have breaking changes in minor versions. # Use a strict upper bound. - 'numpy>=1.14.3,<2.5.0', # Update pyproject.toml as well. + # Increment the lower bound when the minimum Python version is + # updated. + 'numpy>=1.23.2,<2.6.0', # Update pyproject.toml as well. 'objsize>=0.6.1,<0.8.0', 'packaging>=22.0', 'pillow>=12.1.1,<13', @@ -490,7 +491,7 @@ def get_portability_package_data(): 'docstring-parser>=0.15,<1.0', 'docutils>=0.18.1', 'markdown', - 'pandas<2.4.0', + 'pandas<2.6.0', 'openai', 'virtualenv-clone>=0.5,<1.0', ], @@ -501,7 +502,7 @@ def get_portability_package_data(): 'jinja2>=3.0,<3.2', 'joblib>=1.0.1', 'mock>=1.0.1,<6.0.0', - 'pandas<2.4.0', + 'pandas<2.6.0', 'parameterized>=0.7.1,<0.10.0', 'pydot>=1.2.0,<2', 'pyhamcrest>=1.9,!=1.10.0,<3.0.0', diff --git a/sdks/python/test-suites/tox/py311/build.gradle b/sdks/python/test-suites/tox/py311/build.gradle index 3228469cd6bd..fe1202cc0d32 100644 --- a/sdks/python/test-suites/tox/py311/build.gradle +++ b/sdks/python/test-suites/tox/py311/build.gradle @@ -84,30 +84,15 @@ toxTask "testPy311pyarrow-25", "py311-pyarrow-25", "${posargs}" test.dependsOn "testPy311pyarrow-25" postCommitPyDep.dependsOn "testPy311pyarrow-25" -// Create a test task for each supported minor version of pandas -toxTask "testPy311pandas-14", "py311-pandas-14", "${posargs}" -test.dependsOn "testPy311pandas-14" -postCommitPyDep.dependsOn "testPy311pandas-14" - -toxTask "testPy311pandas-15", "py311-pandas-15", "${posargs}" -test.dependsOn "testPy311pandas-15" -postCommitPyDep.dependsOn "testPy311pandas-15" - -toxTask "testPy311pandas-20", "py311-pandas-20", "${posargs}" -test.dependsOn "testPy311pandas-20" -postCommitPyDep.dependsOn "testPy311pandas-20" - -// TODO(https://github.com/apache/beam/issues/31192): Add below suites -// after dependency compat tests suite switches to Python 3.9 or we add -// Python 2.2 support. - -// toxTask "testPy311pandas-21", "py311-pandas-21", "${posargs}" -// test.dependsOn "testPy311pandas-21" -// postCommitPyDep.dependsOn "testPy311pandas-21" - -// toxTask "testPy311pandas-22", "py311-pandas-22", "${posargs}" -// test.dependsOn "testPy311pandas-22" -// postCommitPyDep.dependsOn "testPy311pandas-22" +// Create a test task for each supported minor version of pandas. +// Keep in sync with [testenv:py311-pandas-...] in tox.ini. +toxTask "testPy311pandas-22", "py311-pandas-22", "${posargs}" +test.dependsOn "testPy311pandas-22" +postCommitPyDep.dependsOn "testPy311pandas-22" + +toxTask "testPy311pandas-23", "py311-pandas-23", "${posargs}" +test.dependsOn "testPy311pandas-23" +postCommitPyDep.dependsOn "testPy311pandas-23" // TODO(https://github.com/apache/beam/issues/30908): Revise what are we testing diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index b579f91fd2eb..4d0b1903b130 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -365,16 +365,14 @@ commands = /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py311-pandas-{14,15,20}] +[testenv:py311-pandas-{22,23}] +# Keep in line with dataframe_dependency in setup.py. pandas 2.2.2 is the first +# release compatible with numpy 2. deps = - 14: pandas>=1.4.3,<1.5.0 - 14: numpy>=1.14.3,<1.27.0 - # Exclude 1.5.0 and 1.5.1 because of https://github.com/pandas-dev/pandas/issues/45725 - 15: pandas>=1.5.2,<1.6.0 - 15: numpy>=1.14.3,<1.27.0 - 20: pandas>=2.0.0,<2.1.0 - 20: pyarrow>=7 - 20: numpy>=1.14.3,<1.27.0 + # Lower bound of the supported range; also keeps numpy 1.x covered. + 22: pandas>=2.2.2,<2.3.0 + 22: numpy>=1.23.2,<2.0.0 + 23: pandas>=2.3.0,<2.4.0 commands = # Log pandas and numpy version for debugging /bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"