From b6260927807c5d7c17940582f816ea1cfe298f9c Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 21 Oct 2026 18:25:00 +0000 Subject: [PATCH 1/4] Update pytorch dependencies --- .../beam_PostCommit_Python_Dependency.json | 2 +- sdks/python/setup.py | 9 ++-- .../python/test-suites/tox/py311/build.gradle | 27 ++++------- .../python/test-suites/tox/py312/build.gradle | 4 -- .../python/test-suites/tox/py314/build.gradle | 8 ++++ sdks/python/tox.ini | 47 +++---------------- 6 files changed, 29 insertions(+), 68 deletions(-) diff --git a/.github/trigger_files/beam_PostCommit_Python_Dependency.json b/.github/trigger_files/beam_PostCommit_Python_Dependency.json index 16209484727a..9d28ab31a100 100644 --- a/.github/trigger_files/beam_PostCommit_Python_Dependency.json +++ b/.github/trigger_files/beam_PostCommit_Python_Dependency.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "modification": 4 + "modification": 7 } diff --git a/sdks/python/setup.py b/sdks/python/setup.py index a983db5395ed..ecd168a668a9 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -198,8 +198,8 @@ def cythonize(*args, **kwargs): # Drop this cap once transformers updates the CLIP call site to # `cls_token=` or tokenizers reinstates `cls=` as a deprecation alias. 'tokenizers<0.23', - # Avoid torch 2.12.0+ which fails to run unit tests with segfault. - 'torch<2.12.0', + # Keep in line with the py*-pytorch-* environments in tox.ini. + 'torch>=2.3.0,<2.15.0', # Match tested transformers range. 'transformers>=4.28.0,<4.56.0', # Keep tokenizers compatible with this transformers range. @@ -648,7 +648,7 @@ def get_portability_package_data(): # in https://github.com/apache/beam/blob/master/sdks/python/tox.ini # For more info, see # https://docs.google.com/document/d/1c84Gc-cZRCfrU8f7kWGsNR2o8oSRjCM-dGHO9KvPWPw/edit?usp=sharing - 'torch': ['torch>=1.9.0,<2.8.0'], + 'torch': ['torch>=2.3.0,<2.15.0'], 'tensorflow': [ 'tensorflow>=2.12rc1,<2.22', # tensorflow transitive dep 'absl-py>=0.12.0' @@ -656,8 +656,7 @@ def get_portability_package_data(): 'transformers': [ 'transformers>=4.28.0,<4.56.0', 'tensorflow>=2.12.0', - # Avoid torch 2.12.0+ which fails to run unit tests with segfault - 'torch>=1.9.0,<2.12.0' + 'torch>=2.3.0,<2.15.0' ], 'ml_cpu': [ 'tensorflow>=2.12.0', diff --git a/sdks/python/test-suites/tox/py311/build.gradle b/sdks/python/test-suites/tox/py311/build.gradle index a2b0f0d5e235..7c12d4cf8444 100644 --- a/sdks/python/test-suites/tox/py311/build.gradle +++ b/sdks/python/test-suites/tox/py311/build.gradle @@ -111,24 +111,15 @@ postCommitPyDep.dependsOn "testPy311pandas-20" // TODO(https://github.com/apache/beam/issues/30908): Revise what are we testing -// Create a test task for each minor version of pytorch - -toxTask "testPy311pytorch-111", "py311-pytorch-111", "${posargs}" -test.dependsOn "testPy311pytorch-111" -postCommitPyDep.dependsOn "testPy311pytorch-111" - -toxTask "testPy311pytorch-112", "py311-pytorch-112", "${posargs}" -test.dependsOn "testPy311pytorch-112" -postCommitPyDep.dependsOn "testPy311pytorch-112" - -toxTask "testPy311pytorch-113", "py311-pytorch-113", "${posargs}" -test.dependsOn "testPy311pytorch-113" -postCommitPyDep.dependsOn "testPy311pytorch-113" - -// run on precommit -toxTask "testPy311pytorch-200", "py311-pytorch-200", "${posargs}" -test.dependsOn "testPy311pytorch-200" -postCommitPyDep.dependsOn "testPy311pytorch-200" +// Create a test task for the lower and upper bounds of supported pytorch. +// Keep in sync with [testenv:py{311,314}-pytorch-...] in tox.ini. +toxTask "testPy311pytorch-23", "py311-pytorch-23", "${posargs}" +test.dependsOn "testPy311pytorch-23" +postCommitPyDep.dependsOn "testPy311pytorch-23" + +toxTask "testPy311pytorch-latest", "py311-pytorch-latest", "${posargs}" +test.dependsOn "testPy311pytorch-latest" +postCommitPyDep.dependsOn "testPy311pytorch-latest" toxTask "testPy311tft-113", "py311-tft-113", "${posargs}" test.dependsOn "testPy311tft-113" diff --git a/sdks/python/test-suites/tox/py312/build.gradle b/sdks/python/test-suites/tox/py312/build.gradle index a8f2ac7fa5cb..c5f1774ca46b 100644 --- a/sdks/python/test-suites/tox/py312/build.gradle +++ b/sdks/python/test-suites/tox/py312/build.gradle @@ -30,10 +30,6 @@ pythonVersion = '3.12' project.tasks.register("postCommitPyDep") {} -toxTask "testPy312pytorch-271", "py312-pytorch-271", "${posargs}" -test.dependsOn "testPy312pytorch-271" -postCommitPyDep.dependsOn "testPy312pytorch-271" - toxTask "testPy312tensorflow-216", "py312-tensorflow-216", "${posargs}" test.dependsOn "testPy312tensorflow-216" postCommitPyDep.dependsOn "testPy312tensorflow-216" diff --git a/sdks/python/test-suites/tox/py314/build.gradle b/sdks/python/test-suites/tox/py314/build.gradle index cf05b96223d5..999c0198481e 100644 --- a/sdks/python/test-suites/tox/py314/build.gradle +++ b/sdks/python/test-suites/tox/py314/build.gradle @@ -30,3 +30,11 @@ project.tasks.register("postCommitPyDep") {} apply from: "../common.gradle" +def posargs = project.findProperty("posargs") ?: "" + +// Latest supported pytorch. torch>=2.10 is required for Python 3.14. +// Keep in sync with [testenv:py{311,314}-pytorch-...] in tox.ini. +toxTask "testPy314pytorch-latest", "py314-pytorch-latest", "${posargs}" +test.dependsOn "testPy314pytorch-latest" +postCommitPyDep.dependsOn "testPy314pytorch-latest" + diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index c88e5e2fea6a..d0a8db40de05 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -386,53 +386,20 @@ deps = commands = bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py' -[testenv:py311-pytorch-{19,110,111,112,113}] +[testenv:py{311,314}-pytorch-{23,latest}] +# Keep in line with the torch bounds in setup.py. torch<2.3 is not compatible +# with numpy 2, so numpy is intentionally left unpinned. torch 2.3 has no +# Python 3.14 wheels, so only py314-pytorch-latest is used for 3.14. deps = - 19: torch>=1.9.0,<1.10.0 - 110: torch>=1.10.0,<1.11.0 - 111: torch>=1.11.0,<1.12.0 - 112: torch>=1.12.0,<1.13.0 - 113: torch>=1.13.0,<1.14.0 - numpy==1.26.4 -extras = test,gcp -# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. -setenv = -commands = - # Log torch version for debugging - /bin/sh -c "pip freeze | grep -E torch" - # Run all PyTorch<2 unit tests - # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. - /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' - -[testenv:py311-pytorch-200] -deps = - 200: - torch>=2.0.0,<2.1.0 - mpmath==1.3.0 - numpy==1.26.4 -extras = test,gcp -# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. -setenv = -commands = - # Log torch version for debugging - /bin/sh -c "pip freeze | grep -E torch" - # Run all PyTorch>=2 unit tests - # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. - /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' - -[testenv:py{311,312}-pytorch-271] -deps = - 200: - torch>=2.7.1,<2.8.0 - mpmath==1.3.0 - numpy==1.26.4 + 23: torch>=2.3.0,<2.4.0 + latest: torch>=2.14.0,<2.15.0 extras = test,gcp # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. setenv = commands = # Log torch version for debugging /bin/sh -c "pip freeze | grep -E torch" - # Run all PyTorch>=2 unit tests + # Run all PyTorch unit tests # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' From 8b87eef695cc4f0d1408c19afd00d249851806e8 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 21 Oct 2026 18:38:51 +0000 Subject: [PATCH 2/4] split out 3.11 and 3.14 test env definitions --- .../python/test-suites/tox/py311/build.gradle | 2 +- .../python/test-suites/tox/py314/build.gradle | 2 +- sdks/python/tox.ini | 20 ++++++++++++++++--- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/sdks/python/test-suites/tox/py311/build.gradle b/sdks/python/test-suites/tox/py311/build.gradle index d360f88a85a3..1d39e0ef18f6 100644 --- a/sdks/python/test-suites/tox/py311/build.gradle +++ b/sdks/python/test-suites/tox/py311/build.gradle @@ -112,7 +112,7 @@ postCommitPyDep.dependsOn "testPy311pandas-20" // TODO(https://github.com/apache/beam/issues/30908): Revise what are we testing // Create a test task for the lower and upper bounds of supported pytorch. -// Keep in sync with [testenv:py{311,314}-pytorch-...] in tox.ini. +// Keep in sync with [testenv:py311-pytorch-...] in tox.ini. toxTask "testPy311pytorch-23", "py311-pytorch-23", "${posargs}" test.dependsOn "testPy311pytorch-23" postCommitPyDep.dependsOn "testPy311pytorch-23" diff --git a/sdks/python/test-suites/tox/py314/build.gradle b/sdks/python/test-suites/tox/py314/build.gradle index 999c0198481e..448907a3f837 100644 --- a/sdks/python/test-suites/tox/py314/build.gradle +++ b/sdks/python/test-suites/tox/py314/build.gradle @@ -33,7 +33,7 @@ apply from: "../common.gradle" def posargs = project.findProperty("posargs") ?: "" // Latest supported pytorch. torch>=2.10 is required for Python 3.14. -// Keep in sync with [testenv:py{311,314}-pytorch-...] in tox.ini. +// Keep in sync with [testenv:py314-pytorch-latest] in tox.ini. toxTask "testPy314pytorch-latest", "py314-pytorch-latest", "${posargs}" test.dependsOn "testPy314pytorch-latest" postCommitPyDep.dependsOn "testPy314pytorch-latest" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index cae064f9d046..9d1c452154aa 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -386,10 +386,9 @@ deps = commands = bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py' -[testenv:py{311,314}-pytorch-{23,latest}] +[testenv:py311-pytorch-{23,latest}] # Keep in line with the torch bounds in setup.py. torch<2.3 is not compatible -# with numpy 2, so numpy is intentionally left unpinned. torch 2.3 has no -# Python 3.14 wheels, so only py314-pytorch-latest is used for 3.14. +# with numpy 2, so numpy is intentionally left unpinned. deps = 23: torch>=2.3.0,<2.4.0 latest: torch>=2.14.0,<2.15.0 @@ -403,6 +402,21 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' +[testenv:py314-pytorch-latest] +# Keep in line with py311-pytorch-latest above. torch>=2.10 is required for +# Python 3.14, so the torch 2.3 lower-bound env only runs on Python 3.11. +deps = + torch>=2.14.0,<2.15.0 +extras = test,gcp +# Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. +setenv = +commands = + # Log torch version for debugging + /bin/sh -c "pip freeze | grep -E torch" + # Run all PyTorch unit tests + # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. + /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' + # TODO(https://github.com/apache/beam/issues/25796) - uncomment onnx tox task in tox/py311/build.gradle once onnx supports protobuf 4.x.x [testenv:py311-onnx-113] # TODO(https://github.com/apache/beam/issues/25443) From 991d911ef7d8ef3d03253b797ee5b5bce34bde67 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Thu, 22 Oct 2026 18:27:47 +0000 Subject: [PATCH 3/4] use 214 instead of latest --- sdks/python/test-suites/tox/py311/build.gradle | 6 +++--- sdks/python/test-suites/tox/py314/build.gradle | 8 ++++---- sdks/python/tox.ini | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/sdks/python/test-suites/tox/py311/build.gradle b/sdks/python/test-suites/tox/py311/build.gradle index f61553699d04..6fad4aeadee3 100644 --- a/sdks/python/test-suites/tox/py311/build.gradle +++ b/sdks/python/test-suites/tox/py311/build.gradle @@ -117,9 +117,9 @@ toxTask "testPy311pytorch-23", "py311-pytorch-23", "${posargs}" test.dependsOn "testPy311pytorch-23" postCommitPyDep.dependsOn "testPy311pytorch-23" -toxTask "testPy311pytorch-latest", "py311-pytorch-latest", "${posargs}" -test.dependsOn "testPy311pytorch-latest" -postCommitPyDep.dependsOn "testPy311pytorch-latest" +toxTask "testPy311pytorch-214", "py311-pytorch-214", "${posargs}" +test.dependsOn "testPy311pytorch-214" +postCommitPyDep.dependsOn "testPy311pytorch-214" toxTask "testPy311tft-113", "py311-tft-113", "${posargs}" test.dependsOn "testPy311tft-113" diff --git a/sdks/python/test-suites/tox/py314/build.gradle b/sdks/python/test-suites/tox/py314/build.gradle index 448907a3f837..d2a8f85aa6d7 100644 --- a/sdks/python/test-suites/tox/py314/build.gradle +++ b/sdks/python/test-suites/tox/py314/build.gradle @@ -33,8 +33,8 @@ apply from: "../common.gradle" def posargs = project.findProperty("posargs") ?: "" // Latest supported pytorch. torch>=2.10 is required for Python 3.14. -// Keep in sync with [testenv:py314-pytorch-latest] in tox.ini. -toxTask "testPy314pytorch-latest", "py314-pytorch-latest", "${posargs}" -test.dependsOn "testPy314pytorch-latest" -postCommitPyDep.dependsOn "testPy314pytorch-latest" +// Keep in sync with [testenv:py314-pytorch-214] in tox.ini. +toxTask "testPy314pytorch-214", "py314-pytorch-214", "${posargs}" +test.dependsOn "testPy314pytorch-214" +postCommitPyDep.dependsOn "testPy314pytorch-214" diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini index 33c930c6d344..5450a2a391bb 100644 --- a/sdks/python/tox.ini +++ b/sdks/python/tox.ini @@ -389,12 +389,12 @@ deps = commands = bash {toxinidir}/scripts/run_pytest.sh {envname} 'apache_beam/ml/transforms apache_beam/examples/snippets/transforms/elementwise/mltransform_test.py' -[testenv:py311-pytorch-{23,latest}] +[testenv:py311-pytorch-{23,214}] # Keep in line with the torch bounds in setup.py. torch<2.3 is not compatible # with numpy 2, so numpy is intentionally left unpinned. deps = 23: torch>=2.3.0,<2.4.0 - latest: torch>=2.14.0,<2.15.0 + 214: torch>=2.14.0,<2.15.0 extras = test,gcp # Don't set TMPDIR to avoid "AF_UNIX path too long" errors in certain tests. setenv = @@ -405,7 +405,7 @@ commands = # Allow exit code 5 (no tests run) so that we can run this command safely on arbitrary subdirectories. /bin/sh -c 'pytest -o junit_suite_name={envname} --junitxml=pytest_{envname}.xml -n 6 -m uses_pytorch {posargs}; ret=$?; [ $ret = 5 ] && exit 0 || exit $ret' -[testenv:py314-pytorch-latest] +[testenv:py314-pytorch-214] # Keep in line with py311-pytorch-latest above. torch>=2.10 is required for # Python 3.14, so the torch 2.3 lower-bound env only runs on Python 3.11. deps = From 8387134de97a9150f8d1f62dcbd6efd41acbf2c7 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Fri, 23 Oct 2026 13:37:17 +0000 Subject: [PATCH 4/4] Pull down torch upper bound in transformers to avoid crash issue --- sdks/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdks/python/setup.py b/sdks/python/setup.py index 1a4ef206acd1..513ff7e47a64 100644 --- a/sdks/python/setup.py +++ b/sdks/python/setup.py @@ -666,7 +666,7 @@ def get_portability_package_data(): 'transformers': [ 'transformers>=4.28.0,<4.56.0', 'tensorflow>=2.12.0', - 'torch>=2.3.0,<2.15.0' + 'torch>=2.3.0,<2.12.0' ], 'ml_cpu': [ 'tensorflow>=2.12.0',