From 20ac7b0fd5ce1a11f3a7aec1406710f45edd3862 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 18 Sep 2026 10:21:40 -0400 Subject: [PATCH 1/5] Add tests to reproduce first --- .../python/apache_beam/io/filesystems_test.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/sdks/python/apache_beam/io/filesystems_test.py b/sdks/python/apache_beam/io/filesystems_test.py index 1ea7c34d9f4f..e4818d9a935a 100644 --- a/sdks/python/apache_beam/io/filesystems_test.py +++ b/sdks/python/apache_beam/io/filesystems_test.py @@ -30,8 +30,12 @@ import mock from apache_beam.io import localfilesystem +from apache_beam.io.aws import s3filesystem +from apache_beam.io.aws import s3io from apache_beam.io.filesystem import BeamIOError from apache_beam.io.filesystems import FileSystems +from apache_beam.io.gcp import gcsfilesystem +from apache_beam.io.gcp import gcsio def _gen_fake_join(separator): @@ -63,9 +67,27 @@ def test_get_filesystem(self): isinstance( FileSystems.get_filesystem('c:\\abc\\def'), localfilesystem.LocalFileSystem)) + self.assertTrue( + isinstance( + FileSystems.get_filesystem('gs://bucket/file'), + gcsfilesystem.GCSFileSystem)) + self.assertTrue( + isinstance( + FileSystems.get_filesystem('s3://bucket/file'), + s3filesystem.S3FileSystem)) with self.assertRaises(ValueError): FileSystems.get_filesystem('error://abc/def') + def test_missing_cloud_dependencies_raise_runtime_error(self): + with mock.patch.object(gcsio, 'GCS_INSTALLED', False): + with self.assertRaisesRegex(RuntimeError, + 'GCP dependencies are not installed'): + gcsio.GcsIO() + with mock.patch.object(s3io, 'BOTO3_INSTALLED', False): + with self.assertRaisesRegex(RuntimeError, + 'AWS dependencies are not installed'): + s3io.S3IO(options={}) + @mock.patch('apache_beam.io.localfilesystem.os') def test_unix_path_join(self, *unused_mocks): # Test joining of Unix paths. From 024c524424af62cc0b84a39cadf587c02955109a Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 18 Sep 2026 10:22:37 -0400 Subject: [PATCH 2/5] Make GCP storage dependencies optional on import in gcsio --- sdks/python/apache_beam/io/gcp/gcsio.py | 42 ++++++++++++++++++------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index 179525de2cb0..c74467854eed 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -35,22 +35,33 @@ from typing import Optional from typing import Union -from google.api_core.exceptions import Conflict -from google.api_core.exceptions import RetryError -from google.cloud import storage -from google.cloud.exceptions import NotFound -from google.cloud.exceptions import from_http_response -from google.cloud.storage.fileio import BlobReader -from google.cloud.storage.fileio import BlobWriter -from google.cloud.storage.retry import DEFAULT_RETRY - from apache_beam import version as beam_version from apache_beam.internal.gcp import auth -from apache_beam.io.gcp import gcsio_retry from apache_beam.metrics.metric import Metrics from apache_beam.options.pipeline_options import GoogleCloudOptions from apache_beam.options.pipeline_options import PipelineOptions +try: + # pylint: disable=wrong-import-order, wrong-import-position + # pylint: disable=ungrouped-imports + from google.api_core.exceptions import Conflict + from google.api_core.exceptions import RetryError + from google.cloud import storage + from google.cloud.exceptions import NotFound + from google.cloud.exceptions import from_http_response + from google.cloud.storage.fileio import BlobReader + from google.cloud.storage.fileio import BlobWriter + from google.cloud.storage.retry import DEFAULT_RETRY + + from apache_beam.io.gcp import gcsio_retry + GCS_INSTALLED = True +except ImportError: + GCS_INSTALLED = False + storage = None # type: ignore + BlobReader = object # type: ignore + BlobWriter = object # type: ignore + DEFAULT_RETRY = None # type: ignore + __all__ = ['GcsIO', 'create_storage_client'] _LOGGER = logging.getLogger(__name__) @@ -202,13 +213,18 @@ class GcsIO(object): """Google Cloud Storage I/O client.""" def __init__( self, - storage_client: Optional[storage.Client] = None, + storage_client: Optional['storage.Client'] = None, pipeline_options: Optional[Union[dict, PipelineOptions]] = None) -> None: if pipeline_options is None: pipeline_options = PipelineOptions() elif isinstance(pipeline_options, dict): pipeline_options = PipelineOptions.from_dictionary(pipeline_options) if storage_client is None: + if not GCS_INSTALLED: + message = ( + 'GCP dependencies are not installed, and no alternative ' + 'client was provided to GcsIO.') + raise RuntimeError(message) storage_client = create_storage_client(pipeline_options) google_cloud_options = pipeline_options.view_as(GoogleCloudOptions) @@ -220,7 +236,9 @@ def __init__( self.client = storage_client self._rewrite_cb = None self.bucket_to_project_number = {} - self._storage_client_retry = gcsio_retry.get_retry(pipeline_options) + self._storage_client_retry = ( + gcsio_retry.get_retry(pipeline_options) + if GCS_INSTALLED else None) self._use_blob_generation = getattr( google_cloud_options, 'enable_gcsio_blob_generation', False) From 3b9b124bd0d5cc8f2d7d4fb48a3553790cc24c35 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 18 Sep 2026 10:31:21 -0400 Subject: [PATCH 3/5] Fix BOTO3_INSTALLED detection in s3io when boto3 is missing --- sdks/python/apache_beam/io/aws/s3io.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/aws/s3io.py b/sdks/python/apache_beam/io/aws/s3io.py index 534e63eab514..69cc9f9989e7 100644 --- a/sdks/python/apache_beam/io/aws/s3io.py +++ b/sdks/python/apache_beam/io/aws/s3io.py @@ -36,7 +36,8 @@ try: # pylint: disable=wrong-import-order, wrong-import-position - # pylint: disable=ungrouped-imports + # pylint: disable=ungrouped-imports, unused-import + import boto3 from apache_beam.io.aws.clients.s3 import boto3_client BOTO3_INSTALLED = True except ImportError: From 387357cce89b2c100747e994a6eb71db49a3172c Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 18 Sep 2026 10:56:17 -0400 Subject: [PATCH 4/5] Reformat --- sdks/python/apache_beam/io/gcp/gcsio.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sdks/python/apache_beam/io/gcp/gcsio.py b/sdks/python/apache_beam/io/gcp/gcsio.py index c74467854eed..fc9a0cc1fd42 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio.py +++ b/sdks/python/apache_beam/io/gcp/gcsio.py @@ -237,8 +237,7 @@ def __init__( self._rewrite_cb = None self.bucket_to_project_number = {} self._storage_client_retry = ( - gcsio_retry.get_retry(pipeline_options) - if GCS_INSTALLED else None) + gcsio_retry.get_retry(pipeline_options) if GCS_INSTALLED else None) self._use_blob_generation = getattr( google_cloud_options, 'enable_gcsio_blob_generation', False) From 035c636a7f500b8137b219091f9d39653365c148 Mon Sep 17 00:00:00 2001 From: Shunping Huang Date: Fri, 18 Sep 2026 11:50:20 -0400 Subject: [PATCH 5/5] Fix broken tests. --- .../examples/cookbook/group_with_coder_test.py | 2 ++ .../transforms/elementwise/runinference_test.py | 3 +++ .../io/external/xlang_bigqueryio_it_test.py | 3 +++ .../io/external/xlang_snowflakeio_it_test.py | 3 +++ .../io/gcp/gcsfilesystem_integration_test.py | 3 ++- .../apache_beam/io/gcp/gcsio_integration_test.py | 2 ++ .../apache_beam/ml/inference/onnx_inference_test.py | 3 +++ .../apache_beam/ml/inference/pytorch_inference_test.py | 3 +++ .../ml/inference/sklearn_inference_it_test.py | 3 +++ .../ml/inference/tensorrt_inference_test.py | 3 +++ sdks/python/apache_beam/options/pipeline_options.py | 10 ++++++++++ .../apache_beam/options/pipeline_options_test.py | 4 ++-- sdks/python/apache_beam/runners/render.py | 2 ++ 13 files changed, 41 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py index 9cf36e70e45a..90b4829906ad 100644 --- a/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py +++ b/sdks/python/apache_beam/examples/cookbook/group_with_coder_test.py @@ -32,6 +32,8 @@ # Protect against environments where gcsio library is not available. try: from apache_beam.io.gcp import gcsio + if not gcsio.GCS_INSTALLED: + gcsio = None except ImportError: gcsio = None diff --git a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/runinference_test.py b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/runinference_test.py index 8dd46c659f39..c65e54ca54e4 100644 --- a/sdks/python/apache_beam/examples/snippets/transforms/elementwise/runinference_test.py +++ b/sdks/python/apache_beam/examples/snippets/transforms/elementwise/runinference_test.py @@ -39,7 +39,10 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports, unused-import try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + if not GCS_INSTALLED: + raise unittest.SkipTest('GCP dependencies are not installed') except ImportError: raise unittest.SkipTest('GCP dependencies are not installed') diff --git a/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py b/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py index 49725d54e990..c7b8568595f4 100644 --- a/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py +++ b/sdks/python/apache_beam/io/external/xlang_bigqueryio_it_test.py @@ -44,7 +44,10 @@ # pylint: disable=wrong-import-order, wrong-import-position try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsio import GcsIO + if not GCS_INSTALLED: + GcsIO = None except ImportError: GcsIO = None diff --git a/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py b/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py index f78175a8696d..110d385c265d 100644 --- a/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py +++ b/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py @@ -61,7 +61,10 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + if not GCS_INSTALLED: + GCSFileSystem = None except ImportError: GCSFileSystem = None # pylint: enable=wrong-import-order, wrong-import-position, ungrouped-imports diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py index 3cefb6ca30d8..824c11169dba 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py @@ -39,8 +39,9 @@ from apache_beam.testing.test_pipeline import TestPipeline try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem - fs_not_available = False + fs_not_available = not GCS_INSTALLED except ImportError: fs_not_available = True # type: ignore diff --git a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py index a68b1179faf0..b86fb4e85455 100644 --- a/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsio_integration_test.py @@ -52,6 +52,8 @@ try: from apache_beam.io.gcp import gcsio + if not gcsio.GCS_INSTALLED: + gcsio = None # type: ignore except ImportError: gcsio = None # type: ignore diff --git a/sdks/python/apache_beam/ml/inference/onnx_inference_test.py b/sdks/python/apache_beam/ml/inference/onnx_inference_test.py index 61e8c983c9d3..37d667c3a9e3 100644 --- a/sdks/python/apache_beam/ml/inference/onnx_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/onnx_inference_test.py @@ -55,7 +55,10 @@ raise unittest.SkipTest('Onnx dependencies are not installed') try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + if not GCS_INSTALLED: + GCSFileSystem = None # type: ignore except ImportError: GCSFileSystem = None # type: ignore diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py b/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py index 8efec14c865f..2a3e23f68075 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py @@ -49,7 +49,10 @@ raise unittest.SkipTest('PyTorch dependencies are not installed') try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + if not GCS_INSTALLED: + GCSFileSystem = None # type: ignore except ImportError: GCSFileSystem = None # type: ignore diff --git a/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py b/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py index c82b1698981b..7ad94884b1b0 100644 --- a/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py @@ -32,7 +32,10 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports, unused-import try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + if not GCS_INSTALLED: + raise unittest.SkipTest('GCP dependencies are not installed') except ImportError: raise unittest.SkipTest('GCP dependencies are not installed') diff --git a/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py b/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py index 48931eee399a..e7cf6b229ee8 100644 --- a/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py @@ -45,7 +45,10 @@ raise unittest.SkipTest('TensorRT dependencies are not installed') try: + from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + if not GCS_INSTALLED: + GCSFileSystem = None # type: ignore except ImportError: GCSFileSystem = None # type: ignore diff --git a/sdks/python/apache_beam/options/pipeline_options.py b/sdks/python/apache_beam/options/pipeline_options.py index 9c460f7678e4..98e3dea38d2c 100644 --- a/sdks/python/apache_beam/options/pipeline_options.py +++ b/sdks/python/apache_beam/options/pipeline_options.py @@ -1212,6 +1212,11 @@ def _create_default_gcs_bucket(self): except ImportError: _LOGGER.warning('Unable to create default GCS bucket.') return None + if not gcsio.GCS_INSTALLED: + _LOGGER.warning( + 'Unable to create default GCS bucket because GCP dependencies are ' + 'not installed.') + return None bucket = gcsio.get_or_create_default_gcs_bucket(self) if bucket: return 'gs://%s/' % bucket.id @@ -1229,6 +1234,11 @@ def _warn_if_soft_delete_policy_enabled(self, arg_name): gcs_path = getattr(self, arg_name, None) try: from apache_beam.io.gcp import gcsio + if not gcsio.GCS_INSTALLED: + _LOGGER.warning( + 'Unable to check soft delete policy because GCP dependencies are ' + 'not installed.') + return if gcsio.GcsIO().is_soft_delete_enabled(gcs_path): logger.log_first_n( logging.WARN, diff --git a/sdks/python/apache_beam/options/pipeline_options_test.py b/sdks/python/apache_beam/options/pipeline_options_test.py index fbdaf25f0e8b..d9e1d850ec7f 100644 --- a/sdks/python/apache_beam/options/pipeline_options_test.py +++ b/sdks/python/apache_beam/options/pipeline_options_test.py @@ -47,8 +47,8 @@ _LOGGER = logging.getLogger(__name__) try: - import apache_beam.io.gcp.gcsio # pylint: disable=unused-import - has_gcsio = True + from apache_beam.io.gcp.gcsio import GCS_INSTALLED + has_gcsio = GCS_INSTALLED except ImportError: has_gcsio = False diff --git a/sdks/python/apache_beam/runners/render.py b/sdks/python/apache_beam/runners/render.py index 9f37e0201d94..17fd922782eb 100644 --- a/sdks/python/apache_beam/runners/render.py +++ b/sdks/python/apache_beam/runners/render.py @@ -75,6 +75,8 @@ try: from apache_beam.io.gcp import gcsio + if not gcsio.GCS_INSTALLED: + gcsio = None # type: ignore except ImportError: gcsio = None # type: ignore