From 36ec9abff9dddc0108689970806675b37c29e701 Mon Sep 17 00:00:00 2001 From: jrmccluskey Date: Wed, 23 Sep 2026 15:03:21 +0000 Subject: [PATCH] Clean up ruff breakage findings --- sdks/python/apache_beam/coders/row_coder_test.py | 6 +++--- sdks/python/apache_beam/io/aws/s3io.py | 1 + .../io/external/xlang_snowflakeio_it_test.py | 2 +- .../io/gcp/gcsfilesystem_integration_test.py | 2 +- .../apache_beam/io/gcp/pubsub_io_perf_test.py | 2 +- .../ml/inference/onnx_inference_test.py | 2 +- .../ml/inference/pytorch_inference_test.py | 2 +- .../ml/inference/sklearn_inference_it_test.py | 2 +- .../ml/inference/tensorrt_inference_test.py | 2 +- .../kafka_streams_java_job_server_test.py | 2 +- .../apache_beam/testing/pubsub_test_context.py | 2 +- .../testing/pubsub_test_context_test.py | 3 ++- sdks/python/apache_beam/transforms/sql_test.py | 2 +- sdks/python/apache_beam/transforms/util.py | 4 ++-- sdks/python/apache_beam/transforms/util_test.py | 6 +++--- .../typehints/native_type_compatibility_test.py | 2 +- sdks/python/apache_beam/utils/secret.py | 14 ++++++++------ .../tensorrt_runinference/build_test_engines.py | 6 +++--- 18 files changed, 33 insertions(+), 29 deletions(-) diff --git a/sdks/python/apache_beam/coders/row_coder_test.py b/sdks/python/apache_beam/coders/row_coder_test.py index 92b01241c14a..bd1307ff992e 100644 --- a/sdks/python/apache_beam/coders/row_coder_test.py +++ b/sdks/python/apache_beam/coders/row_coder_test.py @@ -515,9 +515,9 @@ def test_batch_encode_decode(self): def test_row_coder_with_tuples(self): class TupleRecord(typing.NamedTuple): key: str - fixed_tuple: typing.Tuple[str, int] - var_tuple: typing.Tuple[int, ...] - homo_tuple: typing.Tuple[str, str] + fixed_tuple: tuple[str, int] + var_tuple: tuple[int, ...] + homo_tuple: tuple[str, str] coder = RowCoder(typing_to_runner_api(TupleRecord).row_type.schema) record = TupleRecord("k1", ("hello", 42), (1, 2, 3), ("a", "b")) diff --git a/sdks/python/apache_beam/io/aws/s3io.py b/sdks/python/apache_beam/io/aws/s3io.py index 69cc9f9989e7..c42b022f2c7d 100644 --- a/sdks/python/apache_beam/io/aws/s3io.py +++ b/sdks/python/apache_beam/io/aws/s3io.py @@ -38,6 +38,7 @@ # pylint: disable=wrong-import-order, wrong-import-position # pylint: disable=ungrouped-imports, unused-import import boto3 + from apache_beam.io.aws.clients.s3 import boto3_client BOTO3_INSTALLED = True except ImportError: diff --git a/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py b/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py index 110d385c265d..70293f863f47 100644 --- a/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py +++ b/sdks/python/apache_beam/io/external/xlang_snowflakeio_it_test.py @@ -61,8 +61,8 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports try: - from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + from apache_beam.io.gcp.gcsio import GCS_INSTALLED if not GCS_INSTALLED: GCSFileSystem = None except ImportError: diff --git a/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py b/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py index 824c11169dba..ab6b65e93ec1 100644 --- a/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py +++ b/sdks/python/apache_beam/io/gcp/gcsfilesystem_integration_test.py @@ -39,8 +39,8 @@ from apache_beam.testing.test_pipeline import TestPipeline try: - from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + from apache_beam.io.gcp.gcsio import GCS_INSTALLED fs_not_available = not GCS_INSTALLED except ImportError: fs_not_available = True # type: ignore diff --git a/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py b/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py index e22ab61e8529..64ce82597200 100644 --- a/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py +++ b/sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py @@ -56,11 +56,11 @@ from apache_beam.options.pipeline_options import StandardOptions from apache_beam.testing.load_tests.load_test import LoadTest from apache_beam.testing.load_tests.load_test_metrics_utils import MeasureTime +from apache_beam.testing.pubsub_test_context import TestPubsubContext from apache_beam.testing.synthetic_pipeline import SyntheticSource from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.transforms import trigger from apache_beam.transforms import window -from apache_beam.testing.pubsub_test_context import TestPubsubContext # pylint: disable=wrong-import-order, wrong-import-position try: diff --git a/sdks/python/apache_beam/ml/inference/onnx_inference_test.py b/sdks/python/apache_beam/ml/inference/onnx_inference_test.py index 37d667c3a9e3..c301f98169fd 100644 --- a/sdks/python/apache_beam/ml/inference/onnx_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/onnx_inference_test.py @@ -55,8 +55,8 @@ raise unittest.SkipTest('Onnx dependencies are not installed') try: - from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + from apache_beam.io.gcp.gcsio import GCS_INSTALLED if not GCS_INSTALLED: GCSFileSystem = None # type: ignore except ImportError: diff --git a/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py b/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py index 2a3e23f68075..f0870b2f9b24 100644 --- a/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/pytorch_inference_test.py @@ -49,8 +49,8 @@ raise unittest.SkipTest('PyTorch dependencies are not installed') try: - from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + from apache_beam.io.gcp.gcsio import GCS_INSTALLED if not GCS_INSTALLED: GCSFileSystem = None # type: ignore except ImportError: diff --git a/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py b/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py index 7ad94884b1b0..c472fe39f19e 100644 --- a/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py +++ b/sdks/python/apache_beam/ml/inference/sklearn_inference_it_test.py @@ -32,8 +32,8 @@ # pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports, unused-import try: - from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + from apache_beam.io.gcp.gcsio import GCS_INSTALLED if not GCS_INSTALLED: raise unittest.SkipTest('GCP dependencies are not installed') except ImportError: diff --git a/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py b/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py index ea2b9e4f8d4d..461014a023f0 100644 --- a/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py +++ b/sdks/python/apache_beam/ml/inference/tensorrt_inference_test.py @@ -45,8 +45,8 @@ raise unittest.SkipTest('TensorRT dependencies are not installed') try: - from apache_beam.io.gcp.gcsio import GCS_INSTALLED from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem + from apache_beam.io.gcp.gcsio import GCS_INSTALLED if not GCS_INSTALLED: GCSFileSystem = None # type: ignore except ImportError: diff --git a/sdks/python/apache_beam/runners/portability/kafka_streams_java_job_server_test.py b/sdks/python/apache_beam/runners/portability/kafka_streams_java_job_server_test.py index 61f47155dd1a..d5acdd052111 100644 --- a/sdks/python/apache_beam/runners/portability/kafka_streams_java_job_server_test.py +++ b/sdks/python/apache_beam/runners/portability/kafka_streams_java_job_server_test.py @@ -25,8 +25,8 @@ from apache_beam.options import pipeline_options from apache_beam.runners.portability.kafka_streams_runner import KafkaStreamsJarJobServer -from apache_beam.utils import subprocess_server from apache_beam.runners.portability.kafka_streams_runner import KafkaStreamsRunner +from apache_beam.utils import subprocess_server class KafkaStreamsTestPipelineOptions(pipeline_options.PipelineOptions): diff --git a/sdks/python/apache_beam/testing/pubsub_test_context.py b/sdks/python/apache_beam/testing/pubsub_test_context.py index 508d8d9b2fb2..4fd7f2c28621 100644 --- a/sdks/python/apache_beam/testing/pubsub_test_context.py +++ b/sdks/python/apache_beam/testing/pubsub_test_context.py @@ -16,8 +16,8 @@ # import inspect -import time import logging +import time logger = logging.getLogger(__name__) diff --git a/sdks/python/apache_beam/testing/pubsub_test_context_test.py b/sdks/python/apache_beam/testing/pubsub_test_context_test.py index 51fea039f33a..c99eac1fc56d 100644 --- a/sdks/python/apache_beam/testing/pubsub_test_context_test.py +++ b/sdks/python/apache_beam/testing/pubsub_test_context_test.py @@ -17,7 +17,8 @@ import logging import unittest -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock +from unittest.mock import patch # Import the renamed class from apache_beam.testing.pubsub_test_context import TestPubsubContext diff --git a/sdks/python/apache_beam/transforms/sql_test.py b/sdks/python/apache_beam/transforms/sql_test.py index 0d3ac699a053..420a4121e35b 100644 --- a/sdks/python/apache_beam/transforms/sql_test.py +++ b/sdks/python/apache_beam/transforms/sql_test.py @@ -58,7 +58,7 @@ def __eq__(self, other): coders.registry.register_coder(UserTypeRow, coders.RowCoder) TupleRow = typing.NamedTuple( - "TupleRow", [("id", int), ("coords", typing.Tuple[str, int])]) + "TupleRow", [("id", int), ("coords", tuple[str, int])]) coders.registry.register_coder(TupleRow, coders.RowCoder) diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py index 60295f68a92b..30f8dc01c458 100644 --- a/sdks/python/apache_beam/transforms/util.py +++ b/sdks/python/apache_beam/transforms/util.py @@ -82,9 +82,9 @@ from apache_beam.utils import shared from apache_beam.utils import windowed_value from apache_beam.utils.annotations import deprecated -from apache_beam.utils.secret import Secret -from apache_beam.utils.secret import GcpSecret from apache_beam.utils.secret import GcpHsmGeneratedSecret +from apache_beam.utils.secret import GcpSecret +from apache_beam.utils.secret import Secret from apache_beam.utils.sharded_key import ShardedKey from apache_beam.utils.timestamp import Timestamp diff --git a/sdks/python/apache_beam/transforms/util_test.py b/sdks/python/apache_beam/transforms/util_test.py index 446fe68e594c..509f2f2c8975 100644 --- a/sdks/python/apache_beam/transforms/util_test.py +++ b/sdks/python/apache_beam/transforms/util_test.py @@ -72,9 +72,6 @@ from apache_beam.transforms.core import FlatMapTuple from apache_beam.transforms.trigger import AfterCount from apache_beam.transforms.trigger import Repeatedly -from apache_beam.utils.secret import GcpHsmGeneratedSecret -from apache_beam.utils.secret import GcpSecret -from apache_beam.utils.secret import Secret from apache_beam.transforms.util import _BatchSizeEstimator from apache_beam.transforms.util import _GlobalWindowsBatchingDoFn from apache_beam.transforms.window import FixedWindows @@ -88,6 +85,9 @@ from apache_beam.typehints.sharded_key_type import ShardedKeyType from apache_beam.utils import proto_utils from apache_beam.utils import timestamp +from apache_beam.utils.secret import GcpHsmGeneratedSecret +from apache_beam.utils.secret import GcpSecret +from apache_beam.utils.secret import Secret from apache_beam.utils.timestamp import MAX_TIMESTAMP from apache_beam.utils.timestamp import MIN_TIMESTAMP from apache_beam.utils.windowed_value import PANE_INFO_UNKNOWN diff --git a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py index 8694cd08430b..01ad3652250a 100644 --- a/sdks/python/apache_beam/typehints/native_type_compatibility_test.py +++ b/sdks/python/apache_beam/typehints/native_type_compatibility_test.py @@ -33,6 +33,7 @@ from apache_beam.options.pipeline_options import PipelineOptions from apache_beam.options.pipeline_options_context import scoped_pipeline_options from apache_beam.typehints import typehints +from apache_beam.typehints.native_type_compatibility import _safe_issubclass from apache_beam.typehints.native_type_compatibility import convert_builtin_to_typing from apache_beam.typehints.native_type_compatibility import convert_to_beam_type from apache_beam.typehints.native_type_compatibility import convert_to_beam_types @@ -41,7 +42,6 @@ from apache_beam.typehints.native_type_compatibility import convert_typing_to_builtin from apache_beam.typehints.native_type_compatibility import is_any from apache_beam.typehints.native_type_compatibility import match_dataclass_for_row -from apache_beam.typehints.native_type_compatibility import _safe_issubclass _TestNamedTuple = typing.NamedTuple( '_TestNamedTuple', [('age', int), ('name', bytes)]) diff --git a/sdks/python/apache_beam/utils/secret.py b/sdks/python/apache_beam/utils/secret.py index c9c13f1d60f3..9e2772f70e96 100644 --- a/sdks/python/apache_beam/utils/secret.py +++ b/sdks/python/apache_beam/utils/secret.py @@ -22,7 +22,9 @@ import logging import os import warnings -from typing import Any, Dict, Optional, Union +from typing import Any +from typing import Optional +from typing import Union _LOGGER = logging.getLogger(__name__) @@ -211,7 +213,7 @@ def __init__(self, version_name: str): self._version_name = version_name @classmethod - def from_dict(cls, spec_dict: Dict[str, str]) -> 'GcpSecret': + def from_dict(cls, spec_dict: dict[str, str]) -> 'GcpSecret': """Initialize GcpSecret from a dictionary specification.""" allowed_keys = {'version_name', 'name', 'project', 'version'} invalid_keys = set(spec_dict.keys()) - allowed_keys @@ -222,7 +224,7 @@ def from_dict(cls, spec_dict: Dict[str, str]) -> 'GcpSecret': return cls(version_name) @classmethod - def _parse_version_name(cls, spec_dict: Dict[str, str]) -> str: + def _parse_version_name(cls, spec_dict: dict[str, str]) -> str: if "version_name" in spec_dict: return spec_dict["version_name"] @@ -309,7 +311,7 @@ def __eq__(self, other: Any) -> bool: getattr(self, '_job_name', None) == getattr(other, '_job_name', None)) @classmethod - def from_dict(cls, spec_dict: Dict[str, str]) -> 'GcpHsmGeneratedSecret': + def from_dict(cls, spec_dict: dict[str, str]) -> 'GcpHsmGeneratedSecret': """Initialize GcpHsmGeneratedSecret from a dictionary specification.""" allowed_keys = { 'project_id', 'location_id', 'key_ring_id', 'key_id', 'job_name' @@ -453,12 +455,12 @@ def generate_dek(self, dek_size: int = 32) -> bytes: raise RuntimeError(f'Failed to generate DEK with exception {e}') -_SECRET_TYPE_TO_SECRET_MANAGER: Dict[str, str] = { +_SECRET_TYPE_TO_SECRET_MANAGER: dict[str, str] = { "gcpsecret": "GoogleCloudSecretManager", "gcphsmgeneratedsecret": "GoogleCloudHsmGeneratedSecretManager", } -_SECRET_CLASSES: Dict[str, Any] = { +_SECRET_CLASSES: dict[str, Any] = { "googlecloudsecretmanager": "GcpSecret", "googlecloudhsmgeneratedsecretmanager": "GcpHsmGeneratedSecret", } \ No newline at end of file diff --git a/sdks/python/test-suites/containers/tensorrt_runinference/build_test_engines.py b/sdks/python/test-suites/containers/tensorrt_runinference/build_test_engines.py index 606a74df3988..977e19bdd7d7 100644 --- a/sdks/python/test-suites/containers/tensorrt_runinference/build_test_engines.py +++ b/sdks/python/test-suites/containers/tensorrt_runinference/build_test_engines.py @@ -97,8 +97,7 @@ def build_engine(onnx_path, engine_path): def _handler(engine_path, batch_size): - from apache_beam.ml.inference.tensorrt_inference import ( - TensorRTEngineHandlerNumPy) + from apache_beam.ml.inference.tensorrt_inference import TensorRTEngineHandlerNumPy return TensorRTEngineHandlerNumPy( min_batch_size=batch_size, max_batch_size=batch_size, @@ -121,9 +120,10 @@ def verify_ssd(engine_path): The outputs are checked for the shape and ordering the example's PostProcessor indexes by, and for at least one confident detection. """ - from apache_beam.io.filesystems import FileSystems from PIL import Image + from apache_beam.io.filesystems import FileSystems + handler = _handler(engine_path, 1) engine = handler.load_model()