Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions sdks/python/apache_beam/coders/row_coder_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,9 +515,9 @@ def test_batch_encode_decode(self):
def test_row_coder_with_tuples(self):
class TupleRecord(typing.NamedTuple):
key: str
fixed_tuple: typing.Tuple[str, int]
var_tuple: typing.Tuple[int, ...]
homo_tuple: typing.Tuple[str, str]
fixed_tuple: tuple[str, int]
var_tuple: tuple[int, ...]
homo_tuple: tuple[str, str]

coder = RowCoder(typing_to_runner_api(TupleRecord).row_type.schema)
record = TupleRecord("k1", ("hello", 42), (1, 2, 3), ("a", "b"))
Expand Down
1 change: 1 addition & 0 deletions sdks/python/apache_beam/io/aws/s3io.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
# pylint: disable=wrong-import-order, wrong-import-position
# pylint: disable=ungrouped-imports, unused-import
import boto3

from apache_beam.io.aws.clients.s3 import boto3_client
BOTO3_INSTALLED = True
except ImportError:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@

# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports
try:
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
if not GCS_INSTALLED:
GCSFileSystem = None
except ImportError:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@
from apache_beam.testing.test_pipeline import TestPipeline

try:
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
fs_not_available = not GCS_INSTALLED
except ImportError:
fs_not_available = True # type: ignore
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/io/gcp/pubsub_io_perf_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,11 @@
from apache_beam.options.pipeline_options import StandardOptions
from apache_beam.testing.load_tests.load_test import LoadTest
from apache_beam.testing.load_tests.load_test_metrics_utils import MeasureTime
from apache_beam.testing.pubsub_test_context import TestPubsubContext
from apache_beam.testing.synthetic_pipeline import SyntheticSource
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.transforms import trigger
from apache_beam.transforms import window
from apache_beam.testing.pubsub_test_context import TestPubsubContext

# pylint: disable=wrong-import-order, wrong-import-position
try:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@
raise unittest.SkipTest('Onnx dependencies are not installed')

try:
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
if not GCS_INSTALLED:
GCSFileSystem = None # type: ignore
except ImportError:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@
raise unittest.SkipTest('PyTorch dependencies are not installed')

try:
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
if not GCS_INSTALLED:
GCSFileSystem = None # type: ignore
except ImportError:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@

# pylint: disable=wrong-import-order, wrong-import-position, ungrouped-imports, unused-import
try:
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
if not GCS_INSTALLED:
raise unittest.SkipTest('GCP dependencies are not installed')
except ImportError:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@
raise unittest.SkipTest('TensorRT dependencies are not installed')

try:
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
from apache_beam.io.gcp.gcsfilesystem import GCSFileSystem
from apache_beam.io.gcp.gcsio import GCS_INSTALLED
if not GCS_INSTALLED:
GCSFileSystem = None # type: ignore
except ImportError:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,8 @@

from apache_beam.options import pipeline_options
from apache_beam.runners.portability.kafka_streams_runner import KafkaStreamsJarJobServer
from apache_beam.utils import subprocess_server
from apache_beam.runners.portability.kafka_streams_runner import KafkaStreamsRunner
from apache_beam.utils import subprocess_server


class KafkaStreamsTestPipelineOptions(pipeline_options.PipelineOptions):
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/testing/pubsub_test_context.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@
#

import inspect
import time
import logging
import time

logger = logging.getLogger(__name__)

Expand Down
3 changes: 2 additions & 1 deletion sdks/python/apache_beam/testing/pubsub_test_context_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@

import logging
import unittest
from unittest.mock import MagicMock, patch
from unittest.mock import MagicMock
from unittest.mock import patch

# Import the renamed class
from apache_beam.testing.pubsub_test_context import TestPubsubContext
Expand Down
2 changes: 1 addition & 1 deletion sdks/python/apache_beam/transforms/sql_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ def __eq__(self, other):
coders.registry.register_coder(UserTypeRow, coders.RowCoder)

TupleRow = typing.NamedTuple(
"TupleRow", [("id", int), ("coords", typing.Tuple[str, int])])
"TupleRow", [("id", int), ("coords", tuple[str, int])])
coders.registry.register_coder(TupleRow, coders.RowCoder)


Expand Down
4 changes: 2 additions & 2 deletions sdks/python/apache_beam/transforms/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,9 +82,9 @@
from apache_beam.utils import shared
from apache_beam.utils import windowed_value
from apache_beam.utils.annotations import deprecated
from apache_beam.utils.secret import Secret
from apache_beam.utils.secret import GcpSecret
from apache_beam.utils.secret import GcpHsmGeneratedSecret
from apache_beam.utils.secret import GcpSecret
from apache_beam.utils.secret import Secret
from apache_beam.utils.sharded_key import ShardedKey
from apache_beam.utils.timestamp import Timestamp

Expand Down
6 changes: 3 additions & 3 deletions sdks/python/apache_beam/transforms/util_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,6 @@
from apache_beam.transforms.core import FlatMapTuple
from apache_beam.transforms.trigger import AfterCount
from apache_beam.transforms.trigger import Repeatedly
from apache_beam.utils.secret import GcpHsmGeneratedSecret
from apache_beam.utils.secret import GcpSecret
from apache_beam.utils.secret import Secret
from apache_beam.transforms.util import _BatchSizeEstimator
from apache_beam.transforms.util import _GlobalWindowsBatchingDoFn
from apache_beam.transforms.window import FixedWindows
Expand All @@ -88,6 +85,9 @@
from apache_beam.typehints.sharded_key_type import ShardedKeyType
from apache_beam.utils import proto_utils
from apache_beam.utils import timestamp
from apache_beam.utils.secret import GcpHsmGeneratedSecret
from apache_beam.utils.secret import GcpSecret
from apache_beam.utils.secret import Secret
from apache_beam.utils.timestamp import MAX_TIMESTAMP
from apache_beam.utils.timestamp import MIN_TIMESTAMP
from apache_beam.utils.windowed_value import PANE_INFO_UNKNOWN
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
from apache_beam.options.pipeline_options import PipelineOptions
from apache_beam.options.pipeline_options_context import scoped_pipeline_options
from apache_beam.typehints import typehints
from apache_beam.typehints.native_type_compatibility import _safe_issubclass
from apache_beam.typehints.native_type_compatibility import convert_builtin_to_typing
from apache_beam.typehints.native_type_compatibility import convert_to_beam_type
from apache_beam.typehints.native_type_compatibility import convert_to_beam_types
Expand All @@ -41,7 +42,6 @@
from apache_beam.typehints.native_type_compatibility import convert_typing_to_builtin
from apache_beam.typehints.native_type_compatibility import is_any
from apache_beam.typehints.native_type_compatibility import match_dataclass_for_row
from apache_beam.typehints.native_type_compatibility import _safe_issubclass

_TestNamedTuple = typing.NamedTuple(
'_TestNamedTuple', [('age', int), ('name', bytes)])
Expand Down
14 changes: 8 additions & 6 deletions sdks/python/apache_beam/utils/secret.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@
import logging
import os
import warnings
from typing import Any, Dict, Optional, Union
from typing import Any
from typing import Optional
from typing import Union

_LOGGER = logging.getLogger(__name__)

Expand Down Expand Up @@ -211,7 +213,7 @@ def __init__(self, version_name: str):
self._version_name = version_name

@classmethod
def from_dict(cls, spec_dict: Dict[str, str]) -> 'GcpSecret':
def from_dict(cls, spec_dict: dict[str, str]) -> 'GcpSecret':
"""Initialize GcpSecret from a dictionary specification."""
allowed_keys = {'version_name', 'name', 'project', 'version'}
invalid_keys = set(spec_dict.keys()) - allowed_keys
Expand All @@ -222,7 +224,7 @@ def from_dict(cls, spec_dict: Dict[str, str]) -> 'GcpSecret':
return cls(version_name)

@classmethod
def _parse_version_name(cls, spec_dict: Dict[str, str]) -> str:
def _parse_version_name(cls, spec_dict: dict[str, str]) -> str:
if "version_name" in spec_dict:
return spec_dict["version_name"]

Expand Down Expand Up @@ -309,7 +311,7 @@ def __eq__(self, other: Any) -> bool:
getattr(self, '_job_name', None) == getattr(other, '_job_name', None))

@classmethod
def from_dict(cls, spec_dict: Dict[str, str]) -> 'GcpHsmGeneratedSecret':
def from_dict(cls, spec_dict: dict[str, str]) -> 'GcpHsmGeneratedSecret':
"""Initialize GcpHsmGeneratedSecret from a dictionary specification."""
allowed_keys = {
'project_id', 'location_id', 'key_ring_id', 'key_id', 'job_name'
Expand Down Expand Up @@ -453,12 +455,12 @@ def generate_dek(self, dek_size: int = 32) -> bytes:
raise RuntimeError(f'Failed to generate DEK with exception {e}')


_SECRET_TYPE_TO_SECRET_MANAGER: Dict[str, str] = {
_SECRET_TYPE_TO_SECRET_MANAGER: dict[str, str] = {
"gcpsecret": "GoogleCloudSecretManager",
"gcphsmgeneratedsecret": "GoogleCloudHsmGeneratedSecretManager",
}

_SECRET_CLASSES: Dict[str, Any] = {
_SECRET_CLASSES: dict[str, Any] = {
"googlecloudsecretmanager": "GcpSecret",
"googlecloudhsmgeneratedsecretmanager": "GcpHsmGeneratedSecret",
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,7 @@ def build_engine(onnx_path, engine_path):


def _handler(engine_path, batch_size):
from apache_beam.ml.inference.tensorrt_inference import (
TensorRTEngineHandlerNumPy)
from apache_beam.ml.inference.tensorrt_inference import TensorRTEngineHandlerNumPy
return TensorRTEngineHandlerNumPy(
min_batch_size=batch_size,
max_batch_size=batch_size,
Expand All @@ -121,9 +120,10 @@ def verify_ssd(engine_path):
The outputs are checked for the shape and ordering the example's
PostProcessor indexes by, and for at least one confident detection.
"""
from apache_beam.io.filesystems import FileSystems
from PIL import Image

from apache_beam.io.filesystems import FileSystems

handler = _handler(engine_path, 1)
engine = handler.load_model()

Expand Down
Loading