From 1e151f0c91fb4e481f3f377eea4daa51b69089e1 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 15 Oct 2026 21:20:34 -0400 Subject: [PATCH 1/7] Skip read_gbq tests when apitools is unavailable --- sdks/python/apache_beam/dataframe/io_it_test.py | 7 +++---- sdks/python/apache_beam/dataframe/io_test.py | 11 +++-------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/sdks/python/apache_beam/dataframe/io_it_test.py b/sdks/python/apache_beam/dataframe/io_it_test.py index da88ffb54760..9f750e2ff58c 100644 --- a/sdks/python/apache_beam/dataframe/io_it_test.py +++ b/sdks/python/apache_beam/dataframe/io_it_test.py @@ -33,13 +33,12 @@ _LOGGER = logging.getLogger(__name__) try: - from google.api_core.exceptions import GoogleAPICallError + from apitools.base.py.exceptions import HttpError except ImportError: - GoogleAPICallError = None + HttpError = None -@unittest.skipIf( - GoogleAPICallError is None, 'GCP dependencies are not installed') +@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed') class ReadUsingReadGbqTests(unittest.TestCase): @pytest.mark.it_postcommit def test_ReadGbq(self): diff --git a/sdks/python/apache_beam/dataframe/io_test.py b/sdks/python/apache_beam/dataframe/io_test.py index 051a85b379f7..d9723e1e5bb5 100644 --- a/sdks/python/apache_beam/dataframe/io_test.py +++ b/sdks/python/apache_beam/dataframe/io_test.py @@ -46,9 +46,9 @@ from apache_beam.testing.util import equal_to try: - from google.api_core.exceptions import GoogleAPICallError + from apitools.base.py.exceptions import HttpError except ImportError: - GoogleAPICallError = None + HttpError = None # Get major, minor version PD_VERSION = tuple(map(int, pd.__version__.split('.')[0:2])) @@ -492,15 +492,10 @@ def test_double_write(self): set(self.read_all_lines(output + 'out2.csv*'))) -@unittest.skipIf( - GoogleAPICallError is None, 'GCP dependencies are not installed') +@unittest.skipIf(HttpError is None, 'GCP dependencies are not installed') class ReadGbqTransformTests(unittest.TestCase): @mock.patch.object(BigQueryWrapper, 'get_table') def test_bad_schema_public_api_direct_read(self, get_table): - try: - bigquery.TableFieldSchema - except AttributeError: - raise ValueError('Please install GCP Dependencies.') fields = [ bigquery.TableFieldSchema(name='stn', type='DOUBLE', mode="NULLABLE"), bigquery.TableFieldSchema(name='temp', type='FLOAT64', mode="REPEATED"), From 96c2ddf088f5468cec8dbc397133fa3ecb84b4c6 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 15 Oct 2026 21:31:32 -0400 Subject: [PATCH 2/7] Trigger failed postcommit workflow. --- .github/trigger_files/beam_PostCommit_Python_Versions.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/trigger_files/beam_PostCommit_Python_Versions.json b/.github/trigger_files/beam_PostCommit_Python_Versions.json index 8b2c8c445c1f..b5704c67ef1c 100644 --- a/.github/trigger_files/beam_PostCommit_Python_Versions.json +++ b/.github/trigger_files/beam_PostCommit_Python_Versions.json @@ -1,4 +1,4 @@ { "comment": "Modify this file in a trivial way to cause this test suite to run", - "revision": 5 + "revision": 6 } From 2c6cc81d6a4d4a78fe6bdc7374197ce510df2658 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 15 Oct 2026 22:36:57 -0400 Subject: [PATCH 3/7] Skip TestRedisCache when Docker is unavailable requestresponse_it_test.py used to be skipped in non-GCP envs only because google-api-core was missing. Now that it comes in through cloud-sql-python-connector 1.23.0, the tests run on macOS and Windows runners, which have no Docker daemon. --- sdks/python/apache_beam/io/requestresponse_it_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sdks/python/apache_beam/io/requestresponse_it_test.py b/sdks/python/apache_beam/io/requestresponse_it_test.py index 072a1dc27b9f..e45e4f3cf1b2 100644 --- a/sdks/python/apache_beam/io/requestresponse_it_test.py +++ b/sdks/python/apache_beam/io/requestresponse_it_test.py @@ -33,6 +33,7 @@ # pylint: disable=ungrouped-imports try: + import docker from testcontainers.redis import RedisContainer from apache_beam.io.requestresponse import Caller @@ -206,6 +207,15 @@ def __exit__(self, exc_type, exc_val, exc_tb): @pytest.mark.uses_testcontainer class TestRedisCache(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + try: + client = docker.from_env() + client.ping() + client.close() + except Exception as e: # pylint: disable=broad-except + raise unittest.SkipTest(f'Docker is not available: {e}') + def setUp(self) -> None: self.retries = 5 self._start_container() From de72dd69fddb69846aa93c6f2b61814f6010a64c Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Thu, 15 Oct 2026 23:09:59 -0400 Subject: [PATCH 4/7] Fix cloudsql_it_test.py similar to requestresponse_it_test.py --- .../transforms/enrichment_handlers/cloudsql_it_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py index 04db85a75c29..52b951f5f087 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py @@ -33,6 +33,7 @@ # pylint: disable=ungrouped-imports try: + import docker from sqlalchemy import VARCHAR from sqlalchemy import Column from sqlalchemy import Engine @@ -589,6 +590,13 @@ def setUpClass(cls): # Skip setup for the base class. raise unittest.SkipTest("Base class - no db_adapter defined") + try: + client = docker.from_env() + client.ping() + client.close() + except Exception as e: # pylint: disable=broad-except + raise unittest.SkipTest(f'Docker is not available: {e}') + # Type hint data from subclasses. cls._db_adapter: DatabaseTypeAdapter From 991b8044342f203be44de983f55b60407a33ae35 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 16 Oct 2026 00:20:16 -0400 Subject: [PATCH 5/7] Add require_linux_docker() and use it in testcontainer tests --- .../apache_beam/io/requestresponse_it_test.py | 9 +---- sdks/python/apache_beam/testing/test_utils.py | 25 +++++++++++++ .../apache_beam/testing/test_utils_test.py | 37 +++++++++++++++++++ .../enrichment_handlers/cloudsql_it_test.py | 9 +---- 4 files changed, 66 insertions(+), 14 deletions(-) diff --git a/sdks/python/apache_beam/io/requestresponse_it_test.py b/sdks/python/apache_beam/io/requestresponse_it_test.py index e45e4f3cf1b2..4e4c2b638441 100644 --- a/sdks/python/apache_beam/io/requestresponse_it_test.py +++ b/sdks/python/apache_beam/io/requestresponse_it_test.py @@ -30,10 +30,10 @@ from apache_beam.coders import coders from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.test_utils import require_linux_docker # pylint: disable=ungrouped-imports try: - import docker from testcontainers.redis import RedisContainer from apache_beam.io.requestresponse import Caller @@ -209,12 +209,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): class TestRedisCache(unittest.TestCase): @classmethod def setUpClass(cls) -> None: - try: - client = docker.from_env() - client.ping() - client.close() - except Exception as e: # pylint: disable=broad-except - raise unittest.SkipTest(f'Docker is not available: {e}') + require_linux_docker() def setUp(self) -> None: self.retries = 5 diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py index 425b4c6dfac4..ffe6331b789e 100644 --- a/sdks/python/apache_beam/testing/test_utils.py +++ b/sdks/python/apache_beam/testing/test_utils.py @@ -31,6 +31,7 @@ import struct import sys import tempfile +import unittest import numpy as np @@ -186,6 +187,30 @@ def block_imports(*packages): sys.modules.update(hidden_modules) +def require_linux_docker(): + """Raises ``unittest.SkipTest`` unless Docker can run Linux containers. + + Intended for tests that start containers via testcontainers, e.g. from + ``setUpClass``. Such tests need a reachable Docker daemon running Linux + containers. Some environments have no daemon at all (e.g. GitHub macOS + runners), while others only run Windows containers (e.g. GitHub Windows + runners). + """ + try: + import docker + client = docker.from_env() + try: + os_type = client.info().get('OSType') + finally: + client.close() + except Exception as e: # pylint: disable=broad-except + raise unittest.SkipTest(f'Docker is not available: {e}') + if os_type != 'linux': + raise unittest.SkipTest( + f'Docker daemon runs {os_type} containers; ' + 'Linux containers are required.') + + @retry.with_exponential_backoff( num_retries=3, retry_filter=retry.retry_on_beam_io_error_filter) def delete_files(file_paths): diff --git a/sdks/python/apache_beam/testing/test_utils_test.py b/sdks/python/apache_beam/testing/test_utils_test.py index 9c379cf35aab..3b99687c1158 100644 --- a/sdks/python/apache_beam/testing/test_utils_test.py +++ b/sdks/python/apache_beam/testing/test_utils_test.py @@ -126,6 +126,43 @@ def test_block_imports_only_blocks_given_packages(self): self.assertEqual(attempted_imports, []) +class RequireLinuxDockerTest(unittest.TestCase): + def _fake_docker(self, os_type=None, error=None): + fake_docker = mock.MagicMock() + client = fake_docker.from_env.return_value + if error: + client.info.side_effect = error + else: + client.info.return_value = {'OSType': os_type} + return fake_docker, client + + def test_linux_docker_does_not_skip(self): + fake_docker, client = self._fake_docker(os_type='linux') + with mock.patch.dict(sys.modules, {'docker': fake_docker}): + utils.require_linux_docker() + client.close.assert_called_once() + + def test_windows_docker_skips(self): + fake_docker, client = self._fake_docker(os_type='windows') + with mock.patch.dict(sys.modules, {'docker': fake_docker}): + with self.assertRaisesRegex(unittest.SkipTest, 'windows containers'): + utils.require_linux_docker() + client.close.assert_called_once() + + def test_unreachable_docker_skips(self): + fake_docker, client = self._fake_docker( + error=ConnectionError('no such file')) + with mock.patch.dict(sys.modules, {'docker': fake_docker}): + with self.assertRaisesRegex(unittest.SkipTest, 'Docker is not available'): + utils.require_linux_docker() + client.close.assert_called_once() + + def test_missing_docker_sdk_skips(self): + with utils.block_imports('docker'): + with self.assertRaisesRegex(unittest.SkipTest, 'Docker is not available'): + utils.require_linux_docker() + + class LCGeneratorTest(unittest.TestCase): Generator = utils.LCGenerator diff --git a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py index 52b951f5f087..644dd56978bc 100644 --- a/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py +++ b/sdks/python/apache_beam/transforms/enrichment_handlers/cloudsql_it_test.py @@ -28,12 +28,12 @@ import apache_beam as beam from apache_beam.coders import coders from apache_beam.testing.test_pipeline import TestPipeline +from apache_beam.testing.test_utils import require_linux_docker from apache_beam.testing.util import assert_that from apache_beam.testing.util import equal_to # pylint: disable=ungrouped-imports try: - import docker from sqlalchemy import VARCHAR from sqlalchemy import Column from sqlalchemy import Engine @@ -590,12 +590,7 @@ def setUpClass(cls): # Skip setup for the base class. raise unittest.SkipTest("Base class - no db_adapter defined") - try: - client = docker.from_env() - client.ping() - client.close() - except Exception as e: # pylint: disable=broad-except - raise unittest.SkipTest(f'Docker is not available: {e}') + require_linux_docker() # Type hint data from subclasses. cls._db_adapter: DatabaseTypeAdapter From 0fa1d39530f9d1869f6fe5075213a52052f676a2 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 16 Oct 2026 09:00:23 -0400 Subject: [PATCH 6/7] Fix testcontainers timeout config in conftest.py testcontainers 4.x reads testcontainers_config (max_tries * sleep_time), not waiting_utils.config, and ignores TC_TIMEOUT/TC_SLEEP_TIME. CI ended up with a 15s timeout instead of 300s, so MySQL containers timed out. --- sdks/python/conftest.py | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/sdks/python/conftest.py b/sdks/python/conftest.py index 92d90769fb15..0b2db2649a99 100644 --- a/sdks/python/conftest.py +++ b/sdks/python/conftest.py @@ -18,11 +18,11 @@ """Pytest configuration and custom hooks.""" import gc +import math import os import sys import threading import time -from types import SimpleNamespace import pytest @@ -150,6 +150,32 @@ def enhance_mock_stability(): time.sleep(0.05) +def _configure_testcontainers_timeouts(): + """Applies TC_* environment settings to testcontainers' global config. + + testcontainers 4.x computes its wait timeout as ``max_tries * sleep_time`` + from ``testcontainers_config``. It reads ``TC_MAX_TRIES`` and + ``TC_POOLING_INTERVAL`` itself, but not ``TC_TIMEOUT`` or ``TC_SLEEP_TIME``, + which Beam's CI workflows set. Translate them here so that the effective + timeout is at least ``TC_TIMEOUT`` seconds. + """ + from testcontainers.core.config import testcontainers_config + + sleep_time = float( + os.getenv("TC_SLEEP_TIME") or os.getenv("TC_POOLING_INTERVAL") or "1") + timeout = float(os.getenv("TC_TIMEOUT", "120")) + max_tries = int(os.getenv("TC_MAX_TRIES", "0")) + + testcontainers_config.sleep_time = sleep_time + testcontainers_config.max_tries = max( + max_tries, math.ceil(timeout / sleep_time)) + print( + "Set testcontainers max_tries=%d, sleep_time=%s (timeout=%ss)" % ( + testcontainers_config.max_tries, + testcontainers_config.sleep_time, + testcontainers_config.timeout)) + + def pytest_configure(config): """Saves options added in pytest_addoption for later use. This is necessary since pytest-xdist workers do not have the @@ -159,14 +185,8 @@ def pytest_configure(config): # for the entire test session. print("\n--- Applying global testcontainers timeout configuration ---") try: - from testcontainers.core import waiting_utils - waiting_utils.config = SimpleNamespace( - timeout=int(os.getenv("TC_TIMEOUT", "120")), - max_tries=int(os.getenv("TC_MAX_TRIES", "120")), - sleep_time=float(os.getenv("TC_SLEEP_TIME", "1")), - ) - print("Successfully set waiting utils config") - except ModuleNotFoundError: + _configure_testcontainers_timeouts() + except ImportError: print("The testcontainers library is not installed.") TestPipeline.pytest_test_pipeline_options = config.getoption( From f19f3b4ad4c918c97b32e4c5980593e4085911e2 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 16 Oct 2026 09:19:31 -0400 Subject: [PATCH 7/7] Fix lint: avoid bare unittest import in test_utils.py run_lint.sh requires files with `import unittest` to call unittest.main(); import SkipTest directly instead. --- sdks/python/apache_beam/testing/test_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/testing/test_utils.py b/sdks/python/apache_beam/testing/test_utils.py index ffe6331b789e..ee953a902a0d 100644 --- a/sdks/python/apache_beam/testing/test_utils.py +++ b/sdks/python/apache_beam/testing/test_utils.py @@ -31,7 +31,7 @@ import struct import sys import tempfile -import unittest +from unittest import SkipTest import numpy as np @@ -204,9 +204,9 @@ def require_linux_docker(): finally: client.close() except Exception as e: # pylint: disable=broad-except - raise unittest.SkipTest(f'Docker is not available: {e}') + raise SkipTest(f'Docker is not available: {e}') if os_type != 'linux': - raise unittest.SkipTest( + raise SkipTest( f'Docker daemon runs {os_type} containers; ' 'Linux containers are required.')