From cd1dc826aaf6174432c163973637b99c00912897 Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Wed, 22 Oct 2025 09:23:19 -0400 Subject: [PATCH 01/11] cache properties from endpoint --- .../properties_from_endpoint.py | 24 ++++--- .../test_properties_from_endpoint.py | 62 ++++++++++++++----- 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py index 1e294bc8ee..47808949ea 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py @@ -7,6 +7,7 @@ from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.retrievers import Retriever +from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.types import Config, StreamSlice @@ -22,19 +23,22 @@ class PropertiesFromEndpoint: config: Config parameters: InitVar[Mapping[str, Any]] + _cached_properties: Optional[List[str]] = None + def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._property_field_path = [ InterpolatedString(string=property_field, parameters=parameters) for property_field in self.property_field_path ] - def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> Iterable[str]: - response_properties = self.retriever.read_records( - records_schema={}, stream_slice=stream_slice - ) - for property_obj in response_properties: - path = [ - node.eval(self.config) if not isinstance(node, str) else node - for node in self._property_field_path - ] - yield dpath.get(property_obj, path, default=[]) # type: ignore # extracted will be a MutableMapping, given input data structure + def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> List[str]: + if self._cached_properties is None: + self._cached_properties = list(map(self._get_property, self.retriever.read_records(records_schema={}, stream_slice=stream_slice))) # type: ignore # extracted will be a MutableMapping, given input data structure + return self._cached_properties + + def _get_property(self, property_obj: Mapping[str, Any]) -> str: + path = [ + node.eval(self.config) if not isinstance(node, str) else node + for node in self._property_field_path + ] + return str(dpath.get(property_obj, path, default=[])) # type: ignore # extracted will be a MutableMapping, given input data structure \ No newline at end of file diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py b/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py index f94925d7dc..d2f8768cf9 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py @@ -44,11 +44,7 @@ def test_get_properties_from_endpoint(): parameters={}, ) - properties = list( - properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) - ) + properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) assert len(properties) == 9 assert properties == expected_properties @@ -89,11 +85,7 @@ def test_get_properties_from_endpoint_with_multiple_field_paths(): parameters={}, ) - properties = list( - properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) - ) + properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) assert len(properties) == 9 assert properties == expected_properties @@ -135,11 +127,51 @@ def test_get_properties_from_endpoint_with_interpolation(): parameters={}, ) - properties = list( - properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) - ) + properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) assert len(properties) == 9 assert properties == expected_properties + + +def test_given_multiple_calls_when_get_properties_from_endpoint_then_only_call_retriever_once(): + retriever = Mock(spec=SimpleRetriever) + retriever.read_records.return_value = iter( + [ + Record(stream_name="players", data={"id": "ace", "value": 1}), + ] + ) + + properties_from_endpoint = PropertiesFromEndpoint( + retriever=retriever, + property_field_path=["value"], + config={}, + parameters={}, + ) + + properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + + assert retriever.read_records.call_count == 1 + + +def test_given_value_is_int_when_get_properties_from_endpoint_then_return_str(): + expected_properties = ["1"] + + retriever = Mock(spec=SimpleRetriever) + retriever.read_records.return_value = iter( + [ + Record(stream_name="players", data={"id": "ace", "value": 1}), + ] + ) + + properties_from_endpoint = PropertiesFromEndpoint( + retriever=retriever, + property_field_path=["value"], + config={}, + parameters={}, + ) + + properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + + assert properties == expected_properties From ad730cf35b20e8a7de7297dec77fec8b52c01f8f Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Wed, 22 Oct 2025 09:37:23 -0400 Subject: [PATCH 02/11] fix typing and add tests --- .../query_properties/property_chunking.py | 2 +- .../query_properties/query_properties.py | 2 +- .../test_property_chunking.py | 24 ++++++++++++++++++- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py b/airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py index 06ade8253f..750088ff2e 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/property_chunking.py @@ -40,7 +40,7 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: ) def get_request_property_chunks( - self, property_fields: Iterable[str], always_include_properties: Optional[List[str]] + self, property_fields: List[str], always_include_properties: Optional[List[str]] ) -> Iterable[List[str]]: if not self.property_limit: single_property_chunk = list(property_fields) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py index 37b26d1713..a70481d286 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py @@ -34,7 +34,7 @@ def get_request_property_chunks( :param stream_slice: The StreamSlice of the current partition being processed during the sync. This is included because subcomponents of QueryProperties can make use of interpolation of the top-level StreamSlice object """ - fields: Union[Iterable[str], List[str]] + fields: List[str] if isinstance(self.property_list, PropertiesFromEndpoint): fields = self.property_list.get_properties_from_endpoint(stream_slice=stream_slice) else: diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py b/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py index 671cc26bc6..3638049283 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py @@ -72,7 +72,6 @@ def test_get_request_property_chunks( property_limit, expected_property_chunks, ): - property_fields = iter(property_fields) property_chunking = PropertyChunking( property_limit_type=property_limit_type, property_limit=property_limit, @@ -104,3 +103,26 @@ def test_get_merge_key(): merge_key = property_chunking.get_merge_key(record=record) assert merge_key == "0" + + +def test_given_single_property_chunk_when_get_request_property_chunks_then_always_include_properties_are_not_added_to_input_list(): + """ + This test is used to validate that we don't manipulate the in-memory values from get_request_property_chunks + """ + property_chunking = PropertyChunking( + property_limit_type=PropertyLimitType.property_count, + property_limit=None, + record_merge_strategy=GroupByKey(key="id", config=CONFIG, parameters={}), + config=CONFIG, + parameters={}, + ) + + property_fields = ["rick", "chelsea", "victoria", "tim", "saxon", "lochlan", "piper"] + list( + property_chunking.get_request_property_chunks( + property_fields=property_fields, + always_include_properties=["id"], + ) + ) + + assert property_fields == ["rick", "chelsea", "victoria", "tim", "saxon", "lochlan", "piper"] \ No newline at end of file From ebac3758d09f3aaa4e059c91144fc9d276561d81 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Wed, 22 Oct 2025 13:47:55 +0000 Subject: [PATCH 03/11] Auto-fix lint and format issues --- .../properties_from_endpoint.py | 9 ++++-- .../test_properties_from_endpoint.py | 28 ++++++++++++++----- .../test_property_chunking.py | 2 +- 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py index 47808949ea..8a3e2157f9 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py @@ -33,7 +33,12 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> List[str]: if self._cached_properties is None: - self._cached_properties = list(map(self._get_property, self.retriever.read_records(records_schema={}, stream_slice=stream_slice))) # type: ignore # extracted will be a MutableMapping, given input data structure + self._cached_properties = list( + map( + self._get_property, + self.retriever.read_records(records_schema={}, stream_slice=stream_slice), + ) + ) # type: ignore # extracted will be a MutableMapping, given input data structure return self._cached_properties def _get_property(self, property_obj: Mapping[str, Any]) -> str: @@ -41,4 +46,4 @@ def _get_property(self, property_obj: Mapping[str, Any]) -> str: node.eval(self.config) if not isinstance(node, str) else node for node in self._property_field_path ] - return str(dpath.get(property_obj, path, default=[])) # type: ignore # extracted will be a MutableMapping, given input data structure \ No newline at end of file + return str(dpath.get(property_obj, path, default=[])) # type: ignore # extracted will be a MutableMapping, given input data structure diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py b/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py index d2f8768cf9..bce4b0c81d 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py @@ -44,7 +44,9 @@ def test_get_properties_from_endpoint(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + properties = properties_from_endpoint.get_properties_from_endpoint( + stream_slice=StreamSlice(cursor_slice={}, partition={}) + ) assert len(properties) == 9 assert properties == expected_properties @@ -85,7 +87,9 @@ def test_get_properties_from_endpoint_with_multiple_field_paths(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + properties = properties_from_endpoint.get_properties_from_endpoint( + stream_slice=StreamSlice(cursor_slice={}, partition={}) + ) assert len(properties) == 9 assert properties == expected_properties @@ -127,7 +131,9 @@ def test_get_properties_from_endpoint_with_interpolation(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + properties = properties_from_endpoint.get_properties_from_endpoint( + stream_slice=StreamSlice(cursor_slice={}, partition={}) + ) assert len(properties) == 9 assert properties == expected_properties @@ -148,9 +154,15 @@ def test_given_multiple_calls_when_get_properties_from_endpoint_then_only_call_r parameters={}, ) - properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) - properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) - properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + properties_from_endpoint.get_properties_from_endpoint( + stream_slice=StreamSlice(cursor_slice={}, partition={}) + ) + properties_from_endpoint.get_properties_from_endpoint( + stream_slice=StreamSlice(cursor_slice={}, partition={}) + ) + properties_from_endpoint.get_properties_from_endpoint( + stream_slice=StreamSlice(cursor_slice={}, partition={}) + ) assert retriever.read_records.call_count == 1 @@ -172,6 +184,8 @@ def test_given_value_is_int_when_get_properties_from_endpoint_then_return_str(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint(stream_slice=StreamSlice(cursor_slice={}, partition={})) + properties = properties_from_endpoint.get_properties_from_endpoint( + stream_slice=StreamSlice(cursor_slice={}, partition={}) + ) assert properties == expected_properties diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py b/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py index 3638049283..7139d7b9cb 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py @@ -125,4 +125,4 @@ def test_given_single_property_chunk_when_get_request_property_chunks_then_alway ) ) - assert property_fields == ["rick", "chelsea", "victoria", "tim", "saxon", "lochlan", "piper"] \ No newline at end of file + assert property_fields == ["rick", "chelsea", "victoria", "tim", "saxon", "lochlan", "piper"] From 0d73b0d5a6fe496cceb118565ed7035868e9932b Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Wed, 22 Oct 2025 09:56:11 -0400 Subject: [PATCH 04/11] mypy --- .../requesters/query_properties/properties_from_endpoint.py | 4 ++-- .../sources/declarative/retrievers/simple_retriever.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py index 8a3e2157f9..6c8e31e046 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py @@ -35,10 +35,10 @@ def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> L if self._cached_properties is None: self._cached_properties = list( map( - self._get_property, + self._get_property, # type: ignore # SimpleRetriever and AsyncRetriever only returns Record. Should we change the return type of Retriever.read_records? self.retriever.read_records(records_schema={}, stream_slice=stream_slice), ) - ) # type: ignore # extracted will be a MutableMapping, given input data structure + ) return self._cached_properties def _get_property(self, property_obj: Mapping[str, Any]) -> str: diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 7e40dc9c1e..001b5ca99a 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -523,7 +523,6 @@ def read_records( """ _slice = stream_slice or StreamSlice(partition={}, cursor_slice={}) # None-check - most_recent_record_from_slice = None record_generator = partial( self._parse_records, stream_slice=stream_slice, From f76ee0871128af57e3a9e89f58dbb1e3f682ed5a Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Wed, 22 Oct 2025 09:56:49 -0400 Subject: [PATCH 05/11] clean up --- .../requesters/query_properties/properties_from_endpoint.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py index 6c8e31e046..6d539cfa13 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py @@ -7,7 +7,6 @@ from airbyte_cdk.sources.declarative.interpolation import InterpolatedString from airbyte_cdk.sources.declarative.retrievers import Retriever -from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.types import Config, StreamSlice From 782a7b16ab32f59794cfffed52d3a4cb931a8444 Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Thu, 23 Oct 2025 15:23:26 -0400 Subject: [PATCH 06/11] remove streamslice from properties_from_endpoint --- .../properties_from_endpoint.py | 4 +-- .../query_properties/query_properties.py | 6 ++-- .../retrievers/simple_retriever.py | 4 +-- .../test_properties_from_endpoint.py | 28 +++++-------------- .../query_properties/test_query_properties.py | 11 ++------ 5 files changed, 15 insertions(+), 38 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py index 6d539cfa13..8d1927d061 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py @@ -30,12 +30,12 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: for property_field in self.property_field_path ] - def get_properties_from_endpoint(self, stream_slice: Optional[StreamSlice]) -> List[str]: + def get_properties_from_endpoint(self) -> List[str]: if self._cached_properties is None: self._cached_properties = list( map( self._get_property, # type: ignore # SimpleRetriever and AsyncRetriever only returns Record. Should we change the return type of Retriever.read_records? - self.retriever.read_records(records_schema={}, stream_slice=stream_slice), + self.retriever.read_records(records_schema={}, stream_slice=StreamSlice(partition={}, cursor_slice={})), ) ) return self._cached_properties diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py index a70481d286..7457e28f92 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py @@ -25,9 +25,7 @@ class QueryProperties: config: Config parameters: InitVar[Mapping[str, Any]] - def get_request_property_chunks( - self, stream_slice: Optional[StreamSlice] = None - ) -> Iterable[List[str]]: + def get_request_property_chunks(self) -> Iterable[List[str]]: """ Uses the defined property_list to fetch the total set of properties dynamically or from a static list and based on the resulting properties, performs property chunking if applicable. @@ -36,7 +34,7 @@ def get_request_property_chunks( """ fields: List[str] if isinstance(self.property_list, PropertiesFromEndpoint): - fields = self.property_list.get_properties_from_endpoint(stream_slice=stream_slice) + fields = self.property_list.get_properties_from_endpoint() else: fields = self.property_list if self.property_list else [] diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 001b5ca99a..826b75eeaf 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -385,9 +385,7 @@ def _read_pages( response = None try: if self.additional_query_properties: - for properties in self.additional_query_properties.get_request_property_chunks( - stream_slice=stream_slice - ): + for properties in self.additional_query_properties.get_request_property_chunks(): stream_slice = StreamSlice( partition=stream_slice.partition or {}, cursor_slice=stream_slice.cursor_slice or {}, diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py b/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py index bce4b0c81d..9132885755 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_properties_from_endpoint.py @@ -44,9 +44,7 @@ def test_get_properties_from_endpoint(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) + properties = properties_from_endpoint.get_properties_from_endpoint() assert len(properties) == 9 assert properties == expected_properties @@ -87,9 +85,7 @@ def test_get_properties_from_endpoint_with_multiple_field_paths(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) + properties = properties_from_endpoint.get_properties_from_endpoint() assert len(properties) == 9 assert properties == expected_properties @@ -131,9 +127,7 @@ def test_get_properties_from_endpoint_with_interpolation(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) + properties = properties_from_endpoint.get_properties_from_endpoint() assert len(properties) == 9 assert properties == expected_properties @@ -154,15 +148,9 @@ def test_given_multiple_calls_when_get_properties_from_endpoint_then_only_call_r parameters={}, ) - properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) - properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) - properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) + properties_from_endpoint.get_properties_from_endpoint() + properties_from_endpoint.get_properties_from_endpoint() + properties_from_endpoint.get_properties_from_endpoint() assert retriever.read_records.call_count == 1 @@ -184,8 +172,6 @@ def test_given_value_is_int_when_get_properties_from_endpoint_then_return_str(): parameters={}, ) - properties = properties_from_endpoint.get_properties_from_endpoint( - stream_slice=StreamSlice(cursor_slice={}, partition={}) - ) + properties = properties_from_endpoint.get_properties_from_endpoint() assert properties == expected_properties diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py b/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py index e673839511..6423fc009e 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py @@ -17,8 +17,6 @@ def test_get_request_property_chunks_static_list_with_chunking(): - stream_slice = StreamSlice(cursor_slice={}, partition={}) - query_properties = QueryProperties( property_list=[ "ace", @@ -43,7 +41,7 @@ def test_get_request_property_chunks_static_list_with_chunking(): parameters={}, ) - property_chunks = list(query_properties.get_request_property_chunks(stream_slice=stream_slice)) + property_chunks = list(query_properties.get_request_property_chunks()) assert len(property_chunks) == 3 assert property_chunks[0] == ["ace", "snake", "santa"] @@ -52,8 +50,6 @@ def test_get_request_property_chunks_static_list_with_chunking(): def test_get_request_property_chunks_static_list_with_always_include_properties(): - stream_slice = StreamSlice(cursor_slice={}, partition={}) - query_properties = QueryProperties( property_list=[ "ace", @@ -78,7 +74,7 @@ def test_get_request_property_chunks_static_list_with_always_include_properties( parameters={}, ) - property_chunks = list(query_properties.get_request_property_chunks(stream_slice=stream_slice)) + property_chunks = list(query_properties.get_request_property_chunks()) assert len(property_chunks) == 3 assert property_chunks[0] == ["zero", "ace", "snake", "santa"] @@ -87,7 +83,6 @@ def test_get_request_property_chunks_static_list_with_always_include_properties( def test_get_request_property_chunks_dynamic_endpoint(): - stream_slice = StreamSlice(cursor_slice={}, partition={}) properties_from_endpoint_mock = Mock(spec=PropertiesFromEndpoint) properties_from_endpoint_mock.get_properties_from_endpoint.return_value = iter( @@ -108,7 +103,7 @@ def test_get_request_property_chunks_dynamic_endpoint(): parameters={}, ) - property_chunks = list(query_properties.get_request_property_chunks(stream_slice=stream_slice)) + property_chunks = list(query_properties.get_request_property_chunks()) assert len(property_chunks) == 2 assert property_chunks[0] == ["alice", "clover", "dio", "k", "luna"] From 1222b182fd42a55014972163a2caedcd2c6b1b24 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Thu, 23 Oct 2025 19:28:12 +0000 Subject: [PATCH 07/11] Auto-fix lint and format issues --- .../requesters/query_properties/properties_from_endpoint.py | 4 +++- .../sources/declarative/retrievers/simple_retriever.py | 4 +++- .../requesters/query_properties/test_query_properties.py | 1 - 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py index 8d1927d061..e6157efc47 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py @@ -35,7 +35,9 @@ def get_properties_from_endpoint(self) -> List[str]: self._cached_properties = list( map( self._get_property, # type: ignore # SimpleRetriever and AsyncRetriever only returns Record. Should we change the return type of Retriever.read_records? - self.retriever.read_records(records_schema={}, stream_slice=StreamSlice(partition={}, cursor_slice={})), + self.retriever.read_records( + records_schema={}, stream_slice=StreamSlice(partition={}, cursor_slice={}) + ), ) ) return self._cached_properties diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 826b75eeaf..a305741079 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -385,7 +385,9 @@ def _read_pages( response = None try: if self.additional_query_properties: - for properties in self.additional_query_properties.get_request_property_chunks(): + for ( + properties + ) in self.additional_query_properties.get_request_property_chunks(): stream_slice = StreamSlice( partition=stream_slice.partition or {}, cursor_slice=stream_slice.cursor_slice or {}, diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py b/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py index 6423fc009e..b00999e60d 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_query_properties.py @@ -83,7 +83,6 @@ def test_get_request_property_chunks_static_list_with_always_include_properties( def test_get_request_property_chunks_dynamic_endpoint(): - properties_from_endpoint_mock = Mock(spec=PropertiesFromEndpoint) properties_from_endpoint_mock.get_properties_from_endpoint.return_value = iter( ["alice", "clover", "dio", "k", "luna", "phi", "quark", "sigma", "tenmyouji"] From d22170d61461634994c430eb2553a756c176407a Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Thu, 23 Oct 2025 15:28:48 -0400 Subject: [PATCH 08/11] update doc --- .../declarative/requesters/query_properties/query_properties.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py index 7457e28f92..5a01adfeeb 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py @@ -29,8 +29,6 @@ def get_request_property_chunks(self) -> Iterable[List[str]]: """ Uses the defined property_list to fetch the total set of properties dynamically or from a static list and based on the resulting properties, performs property chunking if applicable. - :param stream_slice: The StreamSlice of the current partition being processed during the sync. This is included - because subcomponents of QueryProperties can make use of interpolation of the top-level StreamSlice object """ fields: List[str] if isinstance(self.property_list, PropertiesFromEndpoint): From 2adf3eabdf18ef7d1517cc97584750a52d11b08b Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Thu, 23 Oct 2025 16:17:58 -0400 Subject: [PATCH 09/11] code review --- .../sources/declarative/declarative_component_schema.yaml | 2 +- .../requesters/query_properties/properties_from_endpoint.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index a77bc34ea4..d3a62ea94d 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -3325,7 +3325,7 @@ definitions: - ["code", "type"] PropertiesFromEndpoint: title: Properties from Endpoint - description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records. + description: Defines the behavior for fetching the list of properties from an API that will be loaded into the requests to extract records. Note that stream_slices can't be interpolated from this retriever. type: object required: - type diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py index e6157efc47..210e0bc4e3 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/properties_from_endpoint.py @@ -35,9 +35,7 @@ def get_properties_from_endpoint(self) -> List[str]: self._cached_properties = list( map( self._get_property, # type: ignore # SimpleRetriever and AsyncRetriever only returns Record. Should we change the return type of Retriever.read_records? - self.retriever.read_records( - records_schema={}, stream_slice=StreamSlice(partition={}, cursor_slice={}) - ), + self.retriever.read_records(records_schema={}, stream_slice=None), ) ) return self._cached_properties From cf06028be0f0e3c17cbf4cf687c5549fabb65635 Mon Sep 17 00:00:00 2001 From: octavia-squidington-iii Date: Fri, 24 Oct 2025 13:50:25 +0000 Subject: [PATCH 10/11] Auto-fix lint and format issues --- .../sources/declarative/retrievers/simple_retriever.py | 4 +++- .../requesters/query_properties/test_property_chunking.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 826b75eeaf..a305741079 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -385,7 +385,9 @@ def _read_pages( response = None try: if self.additional_query_properties: - for properties in self.additional_query_properties.get_request_property_chunks(): + for ( + properties + ) in self.additional_query_properties.get_request_property_chunks(): stream_slice = StreamSlice( partition=stream_slice.partition or {}, cursor_slice=stream_slice.cursor_slice or {}, diff --git a/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py b/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py index 7b45359652..2c0e7192b1 100644 --- a/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py +++ b/unit_tests/sources/declarative/requesters/query_properties/test_property_chunking.py @@ -175,7 +175,7 @@ def test_given_single_property_chunk_when_get_request_property_chunks_then_alway property_chunking.get_request_property_chunks( property_fields=property_fields, always_include_properties=["id"], - configured_properties=None + configured_properties=None, ) ) From d297e4869960031cc49c0526e01eabe34b6b3619 Mon Sep 17 00:00:00 2001 From: "maxime.c" Date: Fri, 24 Oct 2025 09:59:09 -0400 Subject: [PATCH 11/11] fix docstring --- .../requesters/query_properties/query_properties.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py index 9303751ec7..8bc7f9671b 100644 --- a/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py +++ b/airbyte_cdk/sources/declarative/requesters/query_properties/query_properties.py @@ -34,10 +34,6 @@ def get_request_property_chunks(self) -> Iterable[List[str]]: """ Uses the defined property_list to fetch the total set of properties dynamically or from a static list and based on the resulting properties, performs property chunking if applicable. - :param stream_slice: The StreamSlice of the current partition being processed during the sync. This is included - because subcomponents of QueryProperties can make use of interpolation of the top-level StreamSlice object - :param configured_stream: The customer configured stream being synced which is needed to identify which - record fields to query for and emit. """ fields: List[str] configured_properties = self.property_selector.select() if self.property_selector else None