From ba6844071c558c9a0cb37cde60ea4e5ebf952174 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Tue, 22 Sep 2026 11:33:29 -0700 Subject: [PATCH 01/21] feat(cdk): add WindowReducible protocol and request-window-reduction exceptions Phase 1 of declarative adaptive request-window reduction (airbytehq/airbyte-internal-issues#17174): pure additions, not yet wired into any retriever/factory. - RequestWindowReductionRequiredException / RequestWindowReductionNotSupportedException: a public control-flow exception pair mirroring PageSizeReductionRequiredException, but raisable directly by custom code (not just HttpClient), so exception-driven connectors (Google Ads, Iterable) can opt in without CDK-specific coupling. - WindowReducible protocol: the component that owns datetime slicing (the cursor) is responsible for boundary names, parsing, formatting, granularity, and interval semantics. - ConcurrentCursor.reduce_window(): bisects a StreamSlice along its own boundary fields at the cursor_granularity floor, with an explicit no-progress guard independent of that floor, preserving partition/extra_fields untouched. --- .../retrievers/window_reducible.py | 32 +++ .../sources/streams/concurrent/cursor.py | 67 ++++++ .../request_window_reduction_exception.py | 61 ++++++ .../sources/streams/concurrent/test_cursor.py | 203 ++++++++++++++++++ 4 files changed, 363 insertions(+) create mode 100644 airbyte_cdk/sources/declarative/retrievers/window_reducible.py create mode 100644 airbyte_cdk/sources/streams/http/request_window_reduction_exception.py diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py new file mode 100644 index 0000000000..a603918639 --- /dev/null +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +from typing import List, Optional, Protocol, runtime_checkable + +from airbyte_cdk.sources.types import StreamSlice + + +@runtime_checkable +class WindowReducible(Protocol): + """ + Implemented by a stream slicer/cursor that can replace a failing `StreamSlice` with smaller child slices + covering the same range. + + The component responsible for slicing owns the boundary field names, parsing, output formatting, and + interval semantics (granularity, clamping, comparison), so `reduce_window` lives on that component - not on + a standalone retriever-side parser - and is checked with `isinstance(slicer, WindowReducible)` wherever a + retriever needs to know whether the stream it is reading supports request-window reduction at all. + """ + + def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: + """ + Split `stream_slice` into two or more smaller, non-overlapping child slices that together cover exactly + the same range, preserving `partition` and `extra_fields` unchanged. + + Returns `None` when the slice cannot be split any further - either because it is already at (or below) + the minimum granularity the cursor supports, or because splitting it would not produce children strictly + smaller than the parent (a no-progress guard independent of the granularity check). Callers should treat + `None` as a terminal condition, not retry with the same slice. + """ + ... diff --git a/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte_cdk/sources/streams/concurrent/cursor.py index 11eaad235c..77c614fc3c 100644 --- a/airbyte_cdk/sources/streams/concurrent/cursor.py +++ b/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -607,6 +607,73 @@ def reduce_slice_range(self, stream_slice: StreamSlice) -> StreamSlice: else: return stream_slice + def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: + """ + Split `stream_slice` in half along this cursor's own boundary fields, granularity, and output format. + + Returns `None` when the slice cannot be split any further: either it spans at most one + `cursor_granularity` unit already (the floor), the boundaries are missing/malformed/reversed, or no + `cursor_granularity` was configured at all (there is then no way to know the smallest addressable unit, + or how to keep two children from overlapping at their shared edge). This is a generic implementation of + `WindowReducible` (see `airbyte_cdk.sources.declarative.retrievers.window_reducible`) that reuses the + same parsing/formatting/comparison operations `_split_per_slice_range` already relies on, so it works for + any `CursorValueType`/`GapType` pair this cursor was built with (datetime/timedelta, or int/int), + without a second, duplicate implementation of that logic living outside the cursor. + """ + if not self._cursor_granularity: + return None + + start_field, end_field = self._slice_boundary_fields_wrapper + try: + start_value = self._connector_state_converter.parse_value( + stream_slice.cursor_slice[start_field] + ) + end_value = self._connector_state_converter.parse_value( + stream_slice.cursor_slice[end_field] + ) + except (KeyError, ValueError, TypeError): + return None + + if start_value >= end_value: + # Reversed or degenerate boundaries: nothing safe to split. + return None + + span = end_value - start_value # type: ignore[operator] # concretely a GapType (timedelta/int) at runtime + total_units = span // self._cursor_granularity + 1 # type: ignore[operator] # inclusive count of granularity units + if total_units <= 1: + # Already at the minimum granularity and still rejected; nothing left to split. + return None + + half_units = total_units // 2 + first_end = ( + start_value + (half_units * self._cursor_granularity) - self._cursor_granularity # type: ignore[operator] + ) + second_start = first_end + self._cursor_granularity + + if not (start_value <= first_end < second_start <= end_value): + # No-progress guard, independent of the granularity check above: refuse to return children that + # are not strictly ordered and non-degenerate, rather than risk an identical or overlapping split. + return None + + return [ + StreamSlice( + partition=stream_slice.partition, + cursor_slice={ + start_field: self._connector_state_converter.output_format(start_value), + end_field: self._connector_state_converter.output_format(first_end), + }, + extra_fields=stream_slice.extra_fields, + ), + StreamSlice( + partition=stream_slice.partition, + cursor_slice={ + start_field: self._connector_state_converter.output_format(second_start), + end_field: self._connector_state_converter.output_format(end_value), + }, + extra_fields=stream_slice.extra_fields, + ), + ] + def get_cursor_datetime_from_state( self, stream_state: Mapping[str, Any] ) -> datetime.datetime | None: diff --git a/airbyte_cdk/sources/streams/http/request_window_reduction_exception.py b/airbyte_cdk/sources/streams/http/request_window_reduction_exception.py new file mode 100644 index 0000000000..961b3d3986 --- /dev/null +++ b/airbyte_cdk/sources/streams/http/request_window_reduction_exception.py @@ -0,0 +1,61 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +from typing import Optional + +from airbyte_cdk.models import FailureType +from airbyte_cdk.utils.traced_exception import AirbyteTracedException + + +class RequestWindowReductionRequiredException(AirbyteTracedException): + """ + Signals that the request window backing the current read must be reduced and retried as smaller children. + + This is control flow rather than a failure: `SimpleRetriever.read_records` catches it and replaces the + failing window with ordered, non-overlapping child windows produced by the stream's cursor. Unlike + `PageSizeReductionRequiredException`, which is only ever raised by `HttpClient`, this exception is public + and is meant to be raised directly by custom code - a `CustomRetriever`, a `CustomExtractor`, or code + handling a transport or streamed-decoding failure - so a connector can opt into window reduction for a + failure the built-in `HttpResponseFilter`/`ResponseAction.REDUCE_REQUEST_WINDOW` path cannot see, without + the CDK needing to know about that connector's specific exception types. + + The classifying error's `failure_type` and `error_message` are preserved on this exception so the eventual + terminal failure - if the window cannot be reduced any further - reports the original cause rather than a + generic "please reduce" message. Callers with an underlying exception should chain it the normal Python way, + `raise RequestWindowReductionRequiredException(...) from original_exception`, so the original traceback is + not lost. + """ + + def __init__( + self, + stream_name: Optional[str] = None, + error_message: Optional[str] = None, + failure_type: Optional[FailureType] = None, + ) -> None: + stream = f" of stream {stream_name}" if stream_name else "" + detail = f": {error_message}" if error_message else "" + super().__init__( + internal_message=f"A request window reduction was requested{stream}{detail}", + message=f"The API rejected the current request window{stream} and requires a smaller one. If this message ends a sync, the stream is not set up to reduce its window: add `request_window_reduction` to its retriever, or remove the `REDUCE_REQUEST_WINDOW` action from its error handler.", + failure_type=failure_type or FailureType.config_error, + ) + + +class RequestWindowReductionNotSupportedException(AirbyteTracedException): + """ + Raised when a window reduction is requested but the retriever cannot honor it. + + The factory rejects this at config time wherever it can see the error handler, so reaching this means the + signal came from somewhere it cannot inspect - a custom error handler, a custom requester/retriever raising + `RequestWindowReductionRequiredException` directly, or a stream whose cursor does not implement + `WindowReducible` (for example, a non-datetime or non-incremental slicer). + """ + + def __init__(self, stream_name: Optional[str] = None) -> None: + stream = f"Stream {stream_name}" if stream_name else "The stream" + super().__init__( + internal_message=f"A request window reduction was requested for stream {stream_name} but its retriever defines no `request_window_reduction`, or its cursor does not support window reduction. The action is only supported on a SimpleRetriever whose cursor implements WindowReducible and that defines `request_window_reduction`.", + message=f"{stream} resolves to a request window reduction but is not set up to reduce its window. Add `request_window_reduction` to the stream's retriever (this requires a datetime-based incremental cursor with `cursor_granularity` set), or remove the `REDUCE_REQUEST_WINDOW` action from its error handler.", + failure_type=FailureType.config_error, + ) diff --git a/unit_tests/sources/streams/concurrent/test_cursor.py b/unit_tests/sources/streams/concurrent/test_cursor.py index 13fe1df875..64580ebdd7 100644 --- a/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/unit_tests/sources/streams/concurrent/test_cursor.py @@ -1411,3 +1411,206 @@ def test_final_state_cursor_get_cursor_datetime_from_state_returns_now_for_no_cu result_with_empty_state = cursor.get_cursor_datetime_from_state({}) assert result_with_empty_state is None + + +class ConcurrentCursorReduceWindowTest(TestCase): + """ + Covers ConcurrentCursor.reduce_window(), the WindowReducible implementation backing declarative + `request_window_reduction`. See airbyte_cdk.sources.declarative.retrievers.window_reducible. + """ + + def setUp(self) -> None: + self._message_repository = Mock(spec=MessageRepository) + self._state_manager = Mock(spec=ConnectorStateManager) + + def _cursor( + self, + granularity: Optional[timedelta] = timedelta(seconds=1), + datetime_format: str = "%Y-%m-%dT%H:%M:%SZ", + slice_boundary_fields: Optional[tuple] = _SLICE_BOUNDARY_FIELDS, + ) -> ConcurrentCursor: + return ConcurrentCursor( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {}, + self._message_repository, + self._state_manager, + CustomFormatConcurrentStreamStateConverter( + datetime_format, is_sequential_state=_NOT_SEQUENTIAL + ), + CursorField(_A_CURSOR_FIELD_KEY), + slice_boundary_fields, + datetime(2024, 1, 1, tzinfo=timezone.utc), + lambda: datetime(2024, 2, 1, tzinfo=timezone.utc), + cursor_granularity=granularity, + ) + + def _slice(self, start: str, end: str) -> StreamSlice: + return StreamSlice( + partition={}, + cursor_slice={_LOWER_SLICE_BOUNDARY_FIELD: start, _UPPER_SLICE_BOUNDARY_FIELD: end}, + ) + + def test_given_window_when_reduce_then_split_in_half_at_cursor_granularity_boundary( + self, + ) -> None: + cursor = self._cursor() + + children = cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z")) + + assert children == [ + self._slice("2024-01-01T00:00:00Z", "2024-01-01T11:59:59Z"), + self._slice("2024-01-01T12:00:00Z", "2024-01-01T23:59:59Z"), + ] + + def test_given_odd_number_of_granularity_units_when_reduce_then_children_still_cover_parent_exactly( + self, + ) -> None: + cursor = self._cursor() + + # 3 whole seconds: [00:00:00, 00:00:02] + children = cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:02Z")) + + assert children == [ + self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z"), + self._slice("2024-01-01T00:00:01Z", "2024-01-01T00:00:02Z"), + ] + + def test_given_single_granularity_unit_window_when_reduce_then_return_none(self) -> None: + cursor = self._cursor() + + assert ( + cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z")) + is None + ) + + def test_given_repeated_reduction_when_reduce_then_eventually_reaches_granularity_floor( + self, + ) -> None: + cursor = self._cursor() + current = self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z") + + depth = 0 + while True: + children = cursor.reduce_window(current) + if children is None: + break + # exactly at the boundary + assert ( + children[0].cursor_slice[_UPPER_SLICE_BOUNDARY_FIELD] + < current.cursor_slice[_UPPER_SLICE_BOUNDARY_FIELD] + ) + current = children[0] + depth += 1 + assert depth < 30, "reduce_window did not converge to the granularity floor" + + assert ( + current.cursor_slice[_LOWER_SLICE_BOUNDARY_FIELD] + == current.cursor_slice[_UPPER_SLICE_BOUNDARY_FIELD] + == "2024-01-01T00:00:00Z" + ) + + def test_given_reversed_boundaries_when_reduce_then_return_none(self) -> None: + cursor = self._cursor() + + assert ( + cursor.reduce_window(self._slice("2024-01-01T00:00:05Z", "2024-01-01T00:00:00Z")) + is None + ) + + def test_given_no_cursor_granularity_when_reduce_then_return_none(self) -> None: + cursor = self._cursor(granularity=None) + + assert ( + cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z")) + is None + ) + + def test_given_malformed_boundary_when_reduce_then_return_none(self) -> None: + cursor = self._cursor() + + assert ( + cursor.reduce_window( + StreamSlice( + partition={}, + cursor_slice={ + _LOWER_SLICE_BOUNDARY_FIELD: "not-a-date", + _UPPER_SLICE_BOUNDARY_FIELD: "2024-01-01T23:59:59Z", + }, + ) + ) + is None + ) + + def test_given_missing_boundary_field_when_reduce_then_return_none(self) -> None: + cursor = self._cursor() + + assert ( + cursor.reduce_window(StreamSlice(partition={}, cursor_slice={"only_one_field": "x"})) + is None + ) + + def test_reduce_window_preserves_partition_and_extra_fields(self) -> None: + cursor = self._cursor() + original = StreamSlice( + partition={"parent_id": "123"}, + cursor_slice={ + _LOWER_SLICE_BOUNDARY_FIELD: "2024-01-01T00:00:00Z", + _UPPER_SLICE_BOUNDARY_FIELD: "2024-01-01T23:59:59Z", + }, + extra_fields={"some_extra": "value"}, + ) + + children = cursor.reduce_window(original) + + for child in children: + assert child.partition == {"parent_id": "123"} + assert child.extra_fields == {"some_extra": "value"} + + def test_given_epoch_cursor_when_reduce_then_split_using_epoch_output_format(self) -> None: + cursor = ConcurrentCursor( + _A_STREAM_NAME, + _A_STREAM_NAMESPACE, + {}, + self._message_repository, + self._state_manager, + EpochValueConcurrentStreamStateConverter(is_sequential_state=_NOT_SEQUENTIAL), + CursorField(_A_CURSOR_FIELD_KEY), + _SLICE_BOUNDARY_FIELDS, + 0, + lambda: 1000, + cursor_granularity=timedelta(seconds=1), + ) + + children = cursor.reduce_window( + StreamSlice( + partition={}, + cursor_slice={ + _LOWER_SLICE_BOUNDARY_FIELD: 0, + _UPPER_SLICE_BOUNDARY_FIELD: 99, + }, + ) + ) + + assert children == [ + StreamSlice( + partition={}, + cursor_slice={_LOWER_SLICE_BOUNDARY_FIELD: 0, _UPPER_SLICE_BOUNDARY_FIELD: 49}, + ), + StreamSlice( + partition={}, + cursor_slice={_LOWER_SLICE_BOUNDARY_FIELD: 50, _UPPER_SLICE_BOUNDARY_FIELD: 99}, + ), + ] + + def test_given_multiple_datetime_formats_when_reduce_then_round_trip_format_correctly( + self, + ) -> None: + cursor = self._cursor(datetime_format="%Y-%m-%d %H:%M:%S") + + children = cursor.reduce_window(self._slice("2024-01-01 00:00:00", "2024-01-01 23:59:59")) + + assert children == [ + self._slice("2024-01-01 00:00:00", "2024-01-01 11:59:59"), + self._slice("2024-01-01 12:00:00", "2024-01-01 23:59:59"), + ] From 4e2ada4b97f8f8d2d61c4221f184111323a49768 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Tue, 22 Sep 2026 11:54:31 -0700 Subject: [PATCH 02/21] feat(cdk): add REDUCE_REQUEST_WINDOW response action and dispatch Phase 2 of declarative adaptive request-window reduction (airbytehq/airbyte-internal-issues#17174): the action is now schema-legal and dispatches correctly, but no retriever honors it yet - resolving to it without a configured reducer surfaces as RequestWindowReductionNotSupportedException (a clear, safe failure), added in Phase 1. - declarative_component_schema.yaml (+ hand-synced generated models, since regenerating via the docker-based codegen isn't available here): REDUCE_REQUEST_WINDOW action, RequestWindowReduction schema object (incl. on_partial_response: FAIL | ALLOW_REPLAY), request_window_reduction field on SimpleRetriever. - ResponseAction enum (response_models.py): add REDUCE_REQUEST_WINDOW. - HttpClient._handle_error_resolution: early branch raising RequestWindowReductionRequiredException, preserving the classifying error's failure_type/error_message; auxiliary-request-logging extended so a rejected window doesn't count against the Connector Builder's per-slice page limit. - CompositeErrorHandler: added REDUCE_REQUEST_WINDOW to the short-circuit action list (without this, a later handler resolving to FAIL would silently win over the reduction, exactly the existing REDUCE_PAGE_SIZE bug class this list exists to prevent). --- .../declarative_component_schema.yaml | 59 ++++++++++ .../models/declarative_component_schema.py | 29 +++++ .../error_handlers/composite_error_handler.py | 1 + .../http/error_handlers/response_models.py | 1 + .../sources/streams/http/http_client.py | 44 +++++-- .../test_composite_error_handler.py | 48 ++++++++ .../sources/streams/http/test_http_client.py | 107 ++++++++++++++++++ 7 files changed, 280 insertions(+), 9 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index a02be6fa0e..492174f97f 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2686,6 +2686,7 @@ definitions: - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY - REDUCE_PAGE_SIZE + - REDUCE_REQUEST_WINDOW examples: - SUCCESS - FAIL @@ -2695,6 +2696,7 @@ definitions: - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY - REDUCE_PAGE_SIZE + - REDUCE_REQUEST_WINDOW failure_type: title: Failure Type description: Failure type of traced exception if a response matches the filter. @@ -4261,6 +4263,17 @@ definitions: re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size. "$ref": "#/definitions/PageSizeReduction" + request_window_reduction: + description: >- + Describes how the request window is reduced when an error handler resolves to the + REDUCE_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, + since that is both the smallest window the connector will request and what keeps two child windows + from overlapping at their shared edge. Distinct from page_size_reduction: an API can reject an entire + date/time range as too large even though every individual page of it would be within the API's page + size limit, and no page can be returned to page through in the first place. On such a response the + connector replaces the failing window with two smaller, ordered windows covering the same range and + reads each one fully, with its own pagination state, before checkpointing the original partition. + "$ref": "#/definitions/RequestWindowReduction" ignore_stream_slicer_parameters_on_paginated_requests: description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored. type: boolean @@ -4441,6 +4454,52 @@ definitions: - NEVER - AFTER_SUCCESSFUL_PAGE default: NEVER + RequestWindowReduction: + title: Request Window Reduction + description: >- + Describes how the request window is reduced when an error handler resolves to the REDUCE_REQUEST_WINDOW + action. On such a response, the connector replaces the failing window with two smaller, ordered windows + that together cover the same range, and reads each one fully - with its own pagination state - instead of + retrying the identical request. Only supported on a stream whose incremental_sync cursor defines + cursor_granularity, since that is both the smallest window the connector will ever request and what keeps + the two child windows from overlapping at their shared edge. Distinct from page_size_reduction: this + operates on the window a DatetimeBasedCursor slices by, not on a paginator's page size, and applies even + when the API rejects the request before returning a single page. + type: object + required: + - type + properties: + type: + type: string + enum: [RequestWindowReduction] + on_partial_response: + title: On Partial Response + description: >- + What to do when a REDUCE_REQUEST_WINDOW response arrives after the current window has already + emitted a page or a record. FAIL stops the sync with a configuration error, since reducing and + re-reading the window from scratch would emit those records a second time - this is correct for an + API, such as PayPal's, that always rejects a window before returning its first page. ALLOW_REPLAY + reduces and re-reads the window anyway, accepting at-least-once delivery of the records already + emitted; use it only when the destination or a downstream deduplication step can tolerate duplicate + records, for instance when adopting this action for a transport or streamed-decoding failure that can + occur after some records of the window were already read. + type: string + enum: + - FAIL + - ALLOW_REPLAY + default: FAIL + failure_message: + title: Failure Message + description: >- + Sentence appended to the error message shown to the user when the connector runs out of reductions, + either because the window is already at the cursor's minimum granularity or because reducing it would + not make progress. Use it to tell the user what they can do about it in terms of this specific API, + for instance which filter narrows the query down. Without it the message only states that the API + kept rejecting every window size the connector asked for. + type: string + examples: + - Narrow the sync down by selecting fewer fields on this stream. + - Lower time_window so that each request covers less data. GzipDecoder: title: gzip description: Select 'gzip' for response data that is compressed with gzip. Requires specifying an inner data type/decoder to parse the decompressed data. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index a62c3db2fe..c35aeb5b1d 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -716,6 +716,7 @@ class Action(Enum): RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" + REDUCE_REQUEST_WINDOW = "REDUCE_REQUEST_WINDOW" class FailureType(Enum): @@ -738,6 +739,7 @@ class HttpResponseFilter(BaseModel): "RATE_LIMITED", "REFRESH_TOKEN_THEN_RETRY", "REDUCE_PAGE_SIZE", + "REDUCE_REQUEST_WINDOW", ], title="Action", ) @@ -1471,6 +1473,29 @@ class PageSizeReduction(BaseModel): ) +class OnPartialResponse(Enum): + FAIL = "FAIL" + ALLOW_REPLAY = "ALLOW_REPLAY" + + +class RequestWindowReduction(BaseModel): + type: Literal["RequestWindowReduction"] + on_partial_response: Optional[OnPartialResponse] = Field( + OnPartialResponse.FAIL, + description="What to do when a REDUCE_REQUEST_WINDOW response arrives after the current window has already emitted a page or a record. FAIL stops the sync with a configuration error, since reducing and re-reading the window from scratch would emit those records a second time - this is correct for an API, such as PayPal's, that always rejects a window before returning its first page. ALLOW_REPLAY reduces and re-reads the window anyway, accepting at-least-once delivery of the records already emitted; use it only when the destination or a downstream deduplication step can tolerate duplicate records, for instance when adopting this action for a transport or streamed-decoding failure that can occur after some records of the window were already read.", + title="On Partial Response", + ) + failure_message: Optional[str] = Field( + None, + description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because the window is already at the cursor's minimum granularity or because reducing it would not make progress. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every window size the connector asked for.", + examples=[ + "Narrow the sync down by selecting fewer fields on this stream.", + "Lower time_window so that each request covers less data.", + ], + title="Failure Message", + ) + + class CsvDecoder(BaseModel): type: Literal["CsvDecoder"] encoding: Optional[str] = "utf-8" @@ -3298,6 +3323,10 @@ class SimpleRetriever(BaseModel): None, description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", ) + request_window_reduction: Optional[RequestWindowReduction] = Field( + None, + description="Describes how the request window is reduced when an error handler resolves to the REDUCE_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, since that is both the smallest window the connector will request and what keeps two child windows from overlapping at their shared edge. Distinct from page_size_reduction: an API can reject an entire date/time range as too large even though every individual page of it would be within the API's page size limit, and no page can be returned to page through in the first place. On such a response the connector replaces the failing window with two smaller, ordered windows covering the same range and reads each one fully, with its own pagination state, before checkpointing the original partition.", + ) ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( False, description="If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored.", diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index 5cf9532ec8..708189bbbb 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -72,6 +72,7 @@ def interpret_response( ResponseAction.IGNORE, ResponseAction.RESET_PAGINATION, ResponseAction.REDUCE_PAGE_SIZE, + ResponseAction.REDUCE_REQUEST_WINDOW, ]: return matched_error_resolution diff --git a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py index 10416c1a7e..cec05f6ce8 100644 --- a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py +++ b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py @@ -20,6 +20,7 @@ class ResponseAction(Enum): RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" + REDUCE_REQUEST_WINDOW = "REDUCE_REQUEST_WINDOW" @dataclass diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index caacdd503a..58bc3eef20 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -53,6 +53,9 @@ rate_limit_default_backoff_handler, user_defined_backoff_handler, ) +from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( + RequestWindowReductionRequiredException, +) # Imported from the leaf module rather than the package: `protocols` pulls in nothing from the # CDK, so this import cannot cycle no matter what else lands in `requests_native_auth` -- an @@ -474,20 +477,36 @@ def _send( and self._message_repository is not None ): formatter = log_formatter - # A response resolving to REDUCE_PAGE_SIZE is not a page of the stream: the retriever discards it - # and re-issues the same page with a smaller page size. Logging it as an auxiliary request keeps it - # visible in the Connector Builder while keeping it out of the per-slice page count, which would - # otherwise report "limit reached" on a read that only retried. - log_as_auxiliary = error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE + # A response resolving to REDUCE_PAGE_SIZE or REDUCE_REQUEST_WINDOW is not a page of the stream: + # the retriever discards it and re-issues the request with a smaller page size or a narrower + # window. Logging it as an auxiliary request keeps it visible in the Connector Builder while + # keeping it out of the per-slice page count, which would otherwise report "limit reached" on a + # read that only retried. + log_as_auxiliary = error_resolution.response_action in ( + ResponseAction.REDUCE_PAGE_SIZE, + ResponseAction.REDUCE_REQUEST_WINDOW, + ) self._message_repository.log_message( Level.DEBUG, lambda: _as_auxiliary_request_log( formatter(response), - title=f"Stream '{self._name}' page rejected, retrying with a smaller page size", + title=( + f"Stream '{self._name}' page rejected, retrying with a smaller page size" + if error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE + else f"Stream '{self._name}' request window rejected, retrying with a smaller window" + ), description=( - f"Request for stream '{self._name}' whose response asked for a smaller page. The " - f"same page is requested again with a reduced page size, so this request produced " - f"no records." + ( + f"Request for stream '{self._name}' whose response asked for a smaller page. The " + f"same page is requested again with a reduced page size, so this request produced " + f"no records." + ) + if error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE + else ( + f"Request for stream '{self._name}' whose response asked for a smaller request " + f"window. The window is split and re-read as smaller children, so this request " + f"produced no records." + ) ), ) if log_as_auxiliary @@ -557,6 +576,13 @@ def _handle_error_resolution( stream_name=self._name, error_message=error_resolution.error_message ) + if error_resolution.response_action == ResponseAction.REDUCE_REQUEST_WINDOW: + raise RequestWindowReductionRequiredException( + stream_name=self._name, + error_message=error_resolution.error_message, + failure_type=error_resolution.failure_type, + ) + # Emit stream status RUNNING with the reason RATE_LIMITED to log that the rate limit has been reached if error_resolution.response_action == ResponseAction.RATE_LIMITED: # TODO: Update to handle with message repository when concurrent message repository is ready diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index afc767e5ea..d5e901d670 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -372,3 +372,51 @@ def test_given_reduce_page_size_when_interpret_response_then_stop_at_the_matchin assert error_resolution.response_action == ResponseAction.REDUCE_PAGE_SIZE assert failing_handler.interpret_response.call_count == 0 + + +def test_given_reduce_request_window_when_interpret_response_then_stop_at_the_matching_handler(): + """ + Mirrors test_given_reduce_page_size_when_interpret_response_then_stop_at_the_matching_handler: without the + short circuit, a later handler resolving to FAIL would win over the request window reduction. + """ + reducing_handler = MagicMock() + reducing_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.REDUCE_REQUEST_WINDOW, + failure_type=FailureType.config_error, + ) + failing_handler = MagicMock() + failing_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error + ) + error_handler = CompositeErrorHandler( + error_handlers=[reducing_handler, failing_handler], parameters={} + ) + + error_resolution = error_handler.interpret_response(MagicMock()) + + assert error_resolution.response_action == ResponseAction.REDUCE_REQUEST_WINDOW + assert failing_handler.interpret_response.call_count == 0 + + +def test_given_unrelated_response_error_when_interpret_response_then_behavior_is_unaffected(): + """ + Adding REDUCE_REQUEST_WINDOW to the short-circuit list must not change behavior for actions that were + never in it: RATE_LIMITED still does not short-circuit, so a later handler's FAIL still wins over an + earlier handler's RATE_LIMITED, exactly as before this action existed. + """ + rate_limited_handler = MagicMock() + rate_limited_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.RATE_LIMITED, failure_type=FailureType.transient_error + ) + failing_handler = MagicMock() + failing_handler.interpret_response.return_value = ErrorResolution( + response_action=ResponseAction.FAIL, failure_type=FailureType.system_error + ) + error_handler = CompositeErrorHandler( + error_handlers=[rate_limited_handler, failing_handler], parameters={} + ) + + error_resolution = error_handler.interpret_response(MagicMock()) + + assert error_resolution.response_action == ResponseAction.FAIL + assert failing_handler.interpret_response.call_count == 1 diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 14ddcaf7a6..8d50dd79b9 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -33,6 +33,9 @@ from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( PageSizeReductionRequiredException, ) +from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( + RequestWindowReductionRequiredException, +) from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -1565,3 +1568,107 @@ def test_given_no_reduce_page_size_action_then_log_the_response_as_a_page(): logged = [json.loads(message.log.message) for message in message_repository.consume_queue()] assert [entry["http"].get("is_auxiliary") for entry in logged] == [None] + + +def test_send_raises_request_window_reduction_required_exception_with_reduce_request_window_response_action(): + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={ + 400: ErrorResolution( + ResponseAction.REDUCE_REQUEST_WINDOW, + FailureType.config_error, + "test reduce request window message", + ) + }, + ), + session=mocked_session, + ) + mocked_response = requests.Response() + mocked_response.status_code = 400 + mocked_session.send.return_value = mocked_response + + # the retriever is responsible for reducing and re-reading the window, so the backoff handlers must not retry + with pytest.raises(RequestWindowReductionRequiredException) as exception: + http_client.send_request(http_method="get", url="https://airbyte.io", request_kwargs={}) + + assert http_client._session.send.call_count == 1 + assert "test reduce request window message" in exception.value.internal_message + assert exception.value.message == ( + "The API rejected the current request window of stream test and requires a smaller one. If this " + "message ends a sync, the stream is not set up to reduce its window: add `request_window_reduction` " + "to its retriever, or remove the `REDUCE_REQUEST_WINDOW` action from its error handler." + ) + # the classifying error's own failure_type is preserved rather than defaulting to config_error + assert exception.value.failure_type == FailureType.config_error + + +def test_given_no_classified_failure_type_when_reduce_request_window_then_default_to_config_error(): + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={400: ErrorResolution(ResponseAction.REDUCE_REQUEST_WINDOW, None, None)}, + ), + session=mocked_session, + ) + mocked_response = requests.Response() + mocked_response.status_code = 400 + mocked_session.send.return_value = mocked_response + + with pytest.raises(RequestWindowReductionRequiredException) as exception: + http_client.send_request(http_method="get", url="https://airbyte.io", request_kwargs={}) + + assert exception.value.failure_type == FailureType.config_error + + +def test_given_reduce_request_window_action_then_log_the_response_as_an_auxiliary_request(): + """ + Mirrors test_given_reduce_page_size_action_then_log_the_response_as_an_auxiliary_request: a response + resolving to REDUCE_REQUEST_WINDOW never becomes a page either - the retriever splits the window and + re-reads it - so it must not count against the Connector Builder's per-slice page limit. + """ + message_repository = InMemoryMessageRepository(Level.DEBUG) + mocked_session = MagicMock(spec=requests.Session) + http_client = HttpClient( + name="test", + logger=MagicMock(), + error_handler=HttpStatusErrorHandler( + logger=MagicMock(), + error_mapping={ + 400: ErrorResolution( + ResponseAction.REDUCE_REQUEST_WINDOW, + FailureType.config_error, + "test reduce request window message", + ) + }, + ), + session=mocked_session, + message_repository=message_repository, + ) + mocked_response = requests.Response() + mocked_response.status_code = 400 + mocked_response.request = requests.Request(method="GET", url="https://airbyte.io").prepare() + mocked_session.send.return_value = mocked_response + + with pytest.raises(RequestWindowReductionRequiredException): + http_client.send_request( + http_method="get", + url="https://airbyte.io", + request_kwargs={}, + log_formatter=lambda response: format_http_message( + response, "a title", "a description", "test" + ), + ) + + logged = [json.loads(message.log.message) for message in message_repository.consume_queue()] + assert [entry["http"]["is_auxiliary"] for entry in logged] == [True] + assert logged[0]["http"]["title"] == ( + "Stream 'test' request window rejected, retrying with a smaller window" + ) + assert "no records" in logged[0]["http"]["description"] From 54d384d2c5b280bbde4b60acad57c6f6e1cae0d6 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Tue, 22 Sep 2026 12:27:41 -0700 Subject: [PATCH 03/21] feat(cdk): wire REDUCE_REQUEST_WINDOW into SimpleRetriever and the factory Phase 3 of declarative adaptive request-window reduction (airbytehq/airbyte-internal-issues#17174): the feature is now actually usable from a manifest, completing the shared CDK capability. - SimpleRetriever: request_window_reduction/window_reducer fields + a _read_records_or_reduce_window() recursive wrapper around read_records(). Each recursive call re-enters read_records/_read_pages from scratch for its child slice, so every child gets fresh pagination/retry state for free. Because the recursion lives entirely inside the single generator DeclarativePartition.read() consumes, a failure anywhere in it propagates before PartitionReader.process_partition() ever calls cursor.close_partition() - so a parent is never checkpointed if a descendant fails, with no changes needed to the concurrent orchestration layer. The on_partial_response policy (FAIL default / ALLOW_REPLAY opt-in) makes the "records already emitted before the reduction signal" case an explicit, intentional choice rather than a silent gap or an unconditional hard failure. - ModelToComponentFactory: _create_request_window_reduction()/_validate_request_window_reduction_is_supported() wire request_window_reduction: into SimpleRetriever, resolving window_reducer to the stream's own cursor when it implements WindowReducible. Validation requires cursor_granularity on a DatetimeBasedCursor (checked on the manifest model, not by reaching into the cursor's internals), rejects the same structurally-incompatible combinations page_size_reduction does (query properties, file_uploader, lazy parent reads), and rejects the action on sub-requesters that have no window of their own (login/auth requester, async retriever sub-requesters, file-uploader requester). - Tests across all layers: SimpleRetriever (one-level and nested/asymmetric reduction, both on_partial_response policies, empty children, later-child failure, fresh-pagination-per-child, terminal config_error/transient_error), factory validation/rejection cases, and a full manifest-driven ConcurrentDeclarativeSource read against mocked HTTP that reproduces the PayPal RESULTSET_TOO_LARGE shape end to end (one rejected day-window split into two half-day requests, complete non-duplicated record set). Full fast suite: 4703 passed (4679 baseline + 24 new), 11 skipped, no regressions. ruff/mypy clean. --- .../parsers/model_to_component_factory.py | 174 +++++++ .../retrievers/simple_retriever.py | 115 ++++- .../retrievers/window_reducible.py | 22 + .../test_model_to_component_factory.py | 438 ++++++++++++++++++ .../retrievers/test_simple_retriever.py | 374 +++++++++++++++ .../test_concurrent_declarative_source.py | 112 +++++ 6 files changed, 1220 insertions(+), 15 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 8940f33fa3..4f34f4af82 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -385,6 +385,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( OffsetIncrement as OffsetIncrementModel, ) +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( + OnPartialResponse as OnPartialResponseModel, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( PageIncrement as PageIncrementModel, ) @@ -442,6 +445,9 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( RequestPath as RequestPathModel, ) +from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( + RequestWindowReduction as RequestWindowReductionModel, +) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( ResponseToFileExtractor as ResponseToFileExtractorModel, ) @@ -595,6 +601,11 @@ PageSizeResetPolicy, ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker +from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( + OnPartialResponse, + RequestWindowReduction, + WindowReducible, +) from airbyte_cdk.sources.declarative.schema import ( ComplexFieldType, DefaultSchemaLoader, @@ -1215,6 +1226,9 @@ def create_session_token_authenticator( self._reject_reduce_page_size_action( model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" ) + self._reject_reduce_request_window_action( + model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" + ) decoder = ( self._create_component_from_model(model=model.decoder, config=config) if model.decoder @@ -3692,6 +3706,21 @@ def _get_url(req: Requester) -> str: f"stream's `lazy_read_pointer` for stream {name}." ) + if reads_parent_stream_lazily and ( + model.request_window_reduction + or self._uses_reduce_request_window_action( + getattr(model.requester, "error_handler", None) + ) + ): + # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records + # embedded in the parent's own pages rather than requesting a window of its own, so there is + # nothing here for a reduced child window to be read from. + raise ValueError( + f"`request_window_reduction` and the REDUCE_REQUEST_WINDOW response action are not " + f"supported when reading a parent stream lazily. Remove either the request window " + f"reduction or the parent stream's `lazy_read_pointer` for stream {name}." + ) + if reads_parent_stream_lazily and not bool( self._connector_state_manager.get_stream_state(name, None) ): @@ -3732,6 +3761,9 @@ def _get_url(req: Requester) -> str: ): raise ValueError("PaginationResetLimits are not supported while having record filter.") + request_window_reduction = self._create_request_window_reduction( + model, name, cursor, incremental_sync, query_properties, file_uploader + ) return SimpleRetriever( name=name, paginator=paginator, @@ -3750,6 +3782,10 @@ def _get_url(req: Requester) -> str: page_size_reduction=self._create_page_size_reduction( model, name, query_properties, file_uploader ), + request_window_reduction=request_window_reduction, + window_reducer=( + cursor if request_window_reduction and isinstance(cursor, WindowReducible) else None + ), post_pagination_filter=post_pagination_filter, parameters=model.parameters or {}, ) @@ -4055,6 +4091,139 @@ def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. return False + def _create_request_window_reduction( + self, + model: SimpleRetrieverModel, + name: str, + cursor: Optional[Cursor], + incremental_sync: Optional[ + Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] + ] = None, + query_properties: Optional[QueryProperties] = None, + file_uploader: Optional[DefaultFileUploader] = None, + ) -> Optional[RequestWindowReduction]: + # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define + # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. + error_handler = getattr(model.requester, "error_handler", None) + uses_action = self._uses_reduce_request_window_action(error_handler) + if uses_action and not model.request_window_reduction: + raise ValueError( + f"Stream {name} has a response filter with the REDUCE_REQUEST_WINDOW action but the " + f"retriever does not define `request_window_reduction`. Add a `request_window_reduction` " + f"block to the retriever." + ) + + if not model.request_window_reduction: + return None + + if not uses_action: + # Not raised: a CustomErrorHandler can resolve to REDUCE_REQUEST_WINDOW without us being able to + # see it, and custom code can raise RequestWindowReductionRequiredException directly without + # going through any error handler at all - so the only safe reaction to a block we cannot tie to + # a known trigger is a warning. + LOGGER.warning( + f"Stream {name} defines `request_window_reduction` but no response filter with the " + f"REDUCE_REQUEST_WINDOW action was found on its requester. The request window will never " + f"be reduced unless a custom error handler or custom code resolves to that action." + ) + + self._validate_request_window_reduction_is_supported( + model, name, cursor, incremental_sync, query_properties, file_uploader + ) + + on_partial_response = model.request_window_reduction.on_partial_response + return RequestWindowReduction( + on_partial_response=OnPartialResponse(on_partial_response.value) + if on_partial_response is not None + else OnPartialResponse.FAIL, + failure_message=model.request_window_reduction.failure_message, + ) + + def _validate_request_window_reduction_is_supported( + self, + model: SimpleRetrieverModel, + name: str, + cursor: Optional[Cursor], + incremental_sync: Optional[ + Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] + ] = None, + query_properties: Optional[QueryProperties] = None, + file_uploader: Optional[DefaultFileUploader] = None, + ) -> None: + """ + Request window reduction replaces a failing slice with smaller children derived from the stream's own + cursor, so it only makes sense on a stream whose cursor can do that (see `WindowReducible`) and it is + rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and a + `file_uploader` both mean records of the failing window may already have been emitted by the time the + reduction is requested, from inside the record generator rather than from a page boundary this retriever + controls. + """ + if not isinstance(cursor, WindowReducible): + raise ValueError( + f"`request_window_reduction` requires an incremental cursor that supports window reduction on " + f"stream {name}. Found {type(cursor).__name__ if cursor is not None else 'no cursor'}: this is " + f"only supported today for a `DatetimeBasedCursor` with `cursor_granularity` set and no " + f"partition router combining multiple cursors." + ) + + if not ( + isinstance(incremental_sync, DatetimeBasedCursorModel) + and incremental_sync.cursor_granularity + ): + # `cursor_granularity` is both the smallest window the connector will ever request and what keeps + # two child windows from overlapping at their shared edge, so `WindowReducible.reduce_window` + # cannot split at all without it - checked here, on the manifest model, rather than by reaching + # into the cursor's private state from outside the class it belongs to. + raise ValueError( + f"`request_window_reduction` requires `cursor_granularity` on the `DatetimeBasedCursor` of " + f"stream {name}: without it there is no smallest window to stop splitting at, and no way to " + f"keep two child windows from overlapping at their shared edge." + ) + + if query_properties: + raise ValueError( + f"`request_window_reduction` cannot be used together with query properties on stream {name}. " + f"Records from the earlier property chunks may have already been emitted when a chunk asks to " + f"reduce the window, so reducing and re-reading it could duplicate them." + ) + + if file_uploader: + raise ValueError( + f"`request_window_reduction` cannot be used together with a `file_uploader` on stream {name}. " + f"The file uploader sends one request per record from inside the window's record generator, so " + f"a reduction requested partway through could re-emit records already yielded by it." + ) + + def _reject_reduce_request_window_action(self, requester: Any, description: str) -> None: + """ + `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no window of + their own to reduce, so REDUCE_REQUEST_WINDOW is schema-legal in places where nothing can honor it. Only + the main requester of a `SimpleRetriever` reads a cursor-sliced window, so every other requester is + rejected here rather than surfacing the exception mid-sync as a generic failure. + """ + if requester is None: + return + if self._uses_reduce_request_window_action(getattr(requester, "error_handler", None)): + raise ValueError( + f"The REDUCE_REQUEST_WINDOW response action is not supported on the {description}: only the " + f"main requester of a SimpleRetriever reads a window that can be reduced. Use a different " + f"action on that error handler." + ) + + def _uses_reduce_request_window_action(self, error_handler: Any) -> bool: + if isinstance(error_handler, CompositeErrorHandlerModel): + return any( + self._uses_reduce_request_window_action(nested) + for nested in error_handler.error_handlers + ) + if isinstance(error_handler, DefaultErrorHandlerModel): + return any( + response_filter.action == HttpResponseFilterActionModel.REDUCE_REQUEST_WINDOW + for response_filter in error_handler.response_filters or [] + ) + # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. + return False + def _create_pagination_tracker_factory( self, model: Optional[PaginationResetModel], cursor: Cursor ) -> Callable[[], PaginationTracker]: @@ -4334,6 +4503,10 @@ def create_async_retriever( getattr(model, requester_field, None), f"`{requester_field}` of the AsyncRetriever of stream {name}", ) + self._reject_reduce_request_window_action( + getattr(model, requester_field, None), + f"`{requester_field}` of the AsyncRetriever of stream {name}", + ) def _get_download_retriever( requester: Requester, extractor: RecordExtractor, _decoder: Decoder @@ -4949,6 +5122,7 @@ def create_file_uploader( self, model: FileUploaderModel, config: Config, **kwargs: Any ) -> FileUploader: self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") + self._reject_reduce_request_window_action(model.requester, "requester of a `file_uploader`") name = "File Uploader" requester = self._create_component_from_model( model=model.requester, diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 85871d0eb7..74dce0c077 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -49,6 +49,11 @@ ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever +from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( + OnPartialResponse, + RequestWindowReduction, + WindowReducible, +) from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.source import ExperimentalClassWarning from airbyte_cdk.sources.streams.core import StreamData @@ -59,6 +64,10 @@ from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) +from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( + RequestWindowReductionNotSupportedException, + RequestWindowReductionRequiredException, +) from airbyte_cdk.sources.types import Config, Record, StreamSlice from airbyte_cdk.utils.mapping_helpers import combine_mappings from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -96,6 +105,15 @@ class SimpleRetriever(Retriever): `_read_pages` creates per call, so the retriever and its paginator - both shared by every partition of the stream, read concurrently - stay stateless. When `page_size_reduction` is `None` no reducer is created and `_read_pages` keeps its previous behaviour + request_window_reduction (Optional[RequestWindowReduction]): Policy applied when an error handler + resolves to `ResponseAction.REDUCE_REQUEST_WINDOW`, or when `RequestWindowReductionRequiredException` + is raised directly by custom code. `None` disables request-window reduction entirely: `read_records` + converts the exception into `RequestWindowReductionNotSupportedException` instead of attempting a + reduction it cannot perform. + window_reducer (Optional[WindowReducible]): The component - normally the stream's own cursor - asked to + replace a failing `StreamSlice` with smaller children. `None` has the same effect as + `request_window_reduction` being `None`: reduction is not attempted, and a + `RequestWindowReductionRequiredException` is re-raised as `RequestWindowReductionNotSupportedException`. """ requester: Requester @@ -121,6 +139,8 @@ class SimpleRetriever(Retriever): ) post_pagination_filter: Optional[ClientSideIncrementalRecordFilterDecorator] = None page_size_reduction: Optional[PageSizeReduction] = None + request_window_reduction: Optional[RequestWindowReduction] = None + window_reducer: Optional[WindowReducible] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._paginator = self.paginator or NoPagination(parameters=parameters) @@ -543,27 +563,92 @@ def read_records( :return: The records read from the API source """ _slice = stream_slice or StreamSlice(partition={}, cursor_slice={}) # None-check + yield from self._read_records_or_reduce_window(records_schema, _slice) + def _read_records_or_reduce_window( + self, + records_schema: Mapping[str, Any], + stream_slice: StreamSlice, + ) -> Iterable[StreamData]: + """ + Read `stream_slice` to completion, replacing it with smaller children and recursing into each of them in + turn when a `RequestWindowReductionRequiredException` is raised while reading it. + + Each recursive call re-enters this method - and, through it, `_read_pages` - from scratch for the child + slice it is given, so every child gets its own paginator token, `PaginationTracker`, and + `PageSizeReducer` for free: nothing here needs to reset that state explicitly. Because the whole + recursion lives inside the single generator `DeclarativePartition.read()` consumes, + `PartitionReader.process_partition()` only calls `cursor.close_partition()` once, after this generator is + fully exhausted - so a failure anywhere in the recursion (a child that cannot be read, or a window that + cannot be reduced any further) propagates out without ever checkpointing the original partition. + """ record_generator = partial( self._parse_records, stream_slice=stream_slice, records_schema=records_schema, ) - records: Iterable[Mapping[str, Any]] = self._read_pages(record_generator, _slice) - if self.post_pagination_filter: - # A data feed paginates until it reaches a record older than the cursor, so the page that triggers the stop - # condition still holds already-synced records. Those are filtered here rather than in the record selector - # so that the paginator keeps seeing the whole page: the stop condition is evaluated on the last record of - # the page, which is precisely one of the records being dropped. Two consequences of filtering this late: - # the pagination tracker observes the dropped records, and a `file_uploader` on the record selector has - # already uploaded their files by the time they are dropped. - records = self.post_pagination_filter.filter_records( - records, - # the filter is only used for its cursor comparison, which does not read the stream state - stream_state={}, - stream_slice=_slice, - ) - yield from records + + emitted_any = False + try: + records: Iterable[Mapping[str, Any]] = self._read_pages(record_generator, stream_slice) + if self.post_pagination_filter: + # A data feed paginates until it reaches a record older than the cursor, so the page that triggers the stop + # condition still holds already-synced records. Those are filtered here rather than in the record selector + # so that the paginator keeps seeing the whole page: the stop condition is evaluated on the last record of + # the page, which is precisely one of the records being dropped. Two consequences of filtering this late: + # the pagination tracker observes the dropped records, and a `file_uploader` on the record selector has + # already uploaded their files by the time they are dropped. + records = self.post_pagination_filter.filter_records( + records, + # the filter is only used for its cursor comparison, which does not read the stream state + stream_state={}, + stream_slice=stream_slice, + ) + for record in records: + emitted_any = True + yield record + except RequestWindowReductionRequiredException as exception: + if self.request_window_reduction is None or self.window_reducer is None: + raise RequestWindowReductionNotSupportedException( + stream_name=self.name + ) from exception + + if ( + emitted_any + and self.request_window_reduction.on_partial_response == OnPartialResponse.FAIL + ): + # The reduction is safe only because it re-reads a window whose records were not emitted yet. + # Re-reading it as smaller children here would duplicate the records already yielded above. + raise AirbyteTracedException( + internal_message=f"Stream {self.name} requested a request window reduction after records of the current window had already been emitted", + message=( + f"Stream {self.name} asked to reduce its request window after some records of it were " + f"already read. Reducing and re-reading the window would duplicate those records. Set " + f"`on_partial_response: ALLOW_REPLAY` on `request_window_reduction` if duplicate records " + f"are acceptable for this stream, or move the REDUCE_REQUEST_WINDOW action so it only " + f"triggers before any record of the window has been read." + ), + failure_type=FailureType.config_error, + ) from exception + + children = self.window_reducer.reduce_window(stream_slice) + if children is None: + failure_message = ( + f"The API kept rejecting stream {self.name}'s request window even at the smallest window " + f"its cursor allows, or reducing the window further would not make progress." + ) + if self.request_window_reduction.failure_message: + failure_message = ( + f"{failure_message} {self.request_window_reduction.failure_message}" + ) + raise AirbyteTracedException( + internal_message=f"Stream {self.name} could not reduce its request window {stream_slice} any further", + message=failure_message, + failure_type=exception.failure_type or FailureType.config_error, + ) from exception + + for child in children: + yield from self._read_records_or_reduce_window(records_schema, child) def _parse_records( self, diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index a603918639..d33ec65d76 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -2,11 +2,18 @@ # Copyright (c) 2026 Airbyte, Inc., all rights reserved. # +from dataclasses import dataclass +from enum import Enum from typing import List, Optional, Protocol, runtime_checkable from airbyte_cdk.sources.types import StreamSlice +class OnPartialResponse(Enum): + FAIL = "FAIL" + ALLOW_REPLAY = "ALLOW_REPLAY" + + @runtime_checkable class WindowReducible(Protocol): """ @@ -30,3 +37,18 @@ def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice] `None` as a terminal condition, not retry with the same slice. """ ... + + +@dataclass(frozen=True) +class RequestWindowReduction: + """ + Configuration for the `request_window_reduction` retriever field - created once per stream and shared, + mirroring `PageSizeReduction`. There is no per-partition mutable counterpart to instantiate the way + `PageSizeReducer` is: reducing a window does not accumulate attempts across sibling requests the way + reducing a page size does, since a reduced window is never retried at the same size twice - it is either + split again (recursion) or the sync fails. All state needed to decide that lives in the `StreamSlice` being + read and the `WindowReducible.reduce_window` result for it, so a plain config object is enough. + """ + + on_partial_response: OnPartialResponse = OnPartialResponse.FAIL + failure_message: Optional[str] = None diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index ea3b0cf151..42d82f6e0c 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -205,6 +205,11 @@ PageSizeReduction, PageSizeResetPolicy, ) +from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( + OnPartialResponse, + RequestWindowReduction, + WindowReducible, +) from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( CachingSchemaLoaderDecorator, @@ -7549,6 +7554,439 @@ def test_given_file_uploader_and_page_size_reduction_then_raise(): assert "file_uploader" in str(exception.value) +_REQUEST_WINDOW_REDUCTION_STREAM = """ +type: DeclarativeStream +name: Test +primary_key: id +schema_loader: + type: InlineSchemaLoader + schema: + type: object +{incremental_sync} +retriever: + type: SimpleRetriever + {request_window_reduction} + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/items" + http_method: GET + error_handler: + type: DefaultErrorHandler + response_filters: + - type: HttpResponseFilter + http_codes: [400] + action: {action} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["items"] +""" + +_DATETIME_BASED_CURSOR_WITH_GRANULARITY = """incremental_sync: + type: DatetimeBasedCursor + start_datetime: "2024-01-01T00:00:00Z" + end_datetime: "2024-01-31T00:00:00Z" + step: "P7D" + cursor_field: "updated_at" + cursor_granularity: "PT1S" + datetime_format: "%Y-%m-%dT%H:%M:%SZ\"""" + + +def _request_window_reduction_stream( + request_window_reduction="request_window_reduction:\n type: RequestWindowReduction", + action="REDUCE_REQUEST_WINDOW", + incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, +): + content = _REQUEST_WINDOW_REDUCTION_STREAM.format( + incremental_sync=incremental_sync, + request_window_reduction=request_window_reduction, + action=action, + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + return factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + +def test_given_request_window_reduction_then_create_retriever_with_defaults(): + retriever = get_retriever(_request_window_reduction_stream()) + + assert retriever.request_window_reduction == RequestWindowReduction( + on_partial_response=OnPartialResponse.FAIL, + failure_message=None, + ) + assert isinstance(retriever.window_reducer, WindowReducible) + + +def test_given_request_window_reduction_values_then_create_retriever_with_those_values(): + retriever = get_retriever( + _request_window_reduction_stream( + request_window_reduction=( + "request_window_reduction:\n" + " type: RequestWindowReduction\n" + " on_partial_response: ALLOW_REPLAY\n" + " failure_message: Lower time_window so that each request covers less data." + ) + ) + ) + + assert retriever.request_window_reduction == RequestWindowReduction( + on_partial_response=OnPartialResponse.ALLOW_REPLAY, + failure_message="Lower time_window so that each request covers less data.", + ) + + +def test_given_no_request_window_reduction_then_retriever_has_none(): + retriever = get_retriever( + _request_window_reduction_stream(request_window_reduction="", action="RETRY") + ) + + assert retriever.request_window_reduction is None + assert retriever.window_reducer is None + + +def test_given_reduce_request_window_action_without_request_window_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _request_window_reduction_stream(request_window_reduction="") + + assert "REDUCE_REQUEST_WINDOW" in str(exception.value) + + +def test_given_request_window_reduction_without_reduce_request_window_action_then_warn(caplog): + """ + A `CustomErrorHandler` can resolve to the action without being inspectable, and custom code can raise + `RequestWindowReductionRequiredException` directly without going through any error handler at all - so this + cannot raise. It must not stay silent either: the feature would be dead on a stream that only defines the + block because it expects to need it. + """ + with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): + retriever = get_retriever(_request_window_reduction_stream(action="RETRY")) + + assert retriever.request_window_reduction == RequestWindowReduction() + assert "REDUCE_REQUEST_WINDOW" in caplog.text + + +def test_given_composite_error_handler_with_reduce_request_window_action_then_require_request_window_reduction(): + """The action can be nested in a CompositeErrorHandler, which the guard has to recurse into.""" + stream_definition = { + "type": "DeclarativeStream", + "name": "Test", + "primary_key": "id", + "schema_loader": {"type": "InlineSchemaLoader", "schema": {"type": "object"}}, + "incremental_sync": { + "type": "DatetimeBasedCursor", + "start_datetime": "2024-01-01T00:00:00Z", + "end_datetime": "2024-01-31T00:00:00Z", + "step": "P7D", + "cursor_field": "updated_at", + "cursor_granularity": "PT1S", + "datetime_format": "%Y-%m-%dT%H:%M:%SZ", + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/items", + "http_method": "GET", + "error_handler": { + "type": "CompositeErrorHandler", + "error_handlers": [ + { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [429], + "action": "RATE_LIMITED", + } + ], + }, + { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [400], + "action": "REDUCE_REQUEST_WINDOW", + } + ], + }, + ], + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + }, + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + + assert "REDUCE_REQUEST_WINDOW" in str(exception.value) + + # and the same manifest with the block present builds + stream_definition["retriever"]["request_window_reduction"] = {"type": "RequestWindowReduction"} + retriever = get_retriever( + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + ) + assert retriever.request_window_reduction == RequestWindowReduction() + + +def test_given_no_incremental_sync_and_request_window_reduction_then_raise(): + with pytest.raises(ValueError) as exception: + _request_window_reduction_stream(incremental_sync="") + + assert "cursor_granularity" in str(exception.value) or "WindowReducible" in str(exception.value) + + +def test_given_datetime_based_cursor_without_cursor_granularity_then_raise(): + with pytest.raises(ValueError) as exception: + _request_window_reduction_stream( + incremental_sync=( + 'incremental_sync:\n type: DatetimeBasedCursor\n start_datetime: "2024-01-01T00:00:00Z"\n' + ' end_datetime: "2024-01-31T00:00:00Z"\n step: "P7D"\n cursor_field: "updated_at"\n' + ' datetime_format: "%Y-%m-%dT%H:%M:%SZ"' + ) + ) + + assert "cursor_granularity" in str(exception.value) + + +def test_given_incrementing_count_cursor_and_request_window_reduction_then_raise(): + """ + `IncrementingCountCursor` builds a `ConcurrentCursor` too, but not one carrying the datetime boundary + fields, granularity, or formatting `WindowReducible.reduce_window` needs, so it must be rejected the same + way a stream with no incremental_sync at all is. + """ + with pytest.raises(ValueError) as exception: + _request_window_reduction_stream( + incremental_sync=( + "incremental_sync:\n type: IncrementingCountCursor\n cursor_field: id" + ) + ) + + assert "cursor_granularity" in str(exception.value) + + +def test_given_query_properties_and_request_window_reduction_then_raise(): + content = _REQUEST_WINDOW_REDUCTION_STREAM.format( + incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, + request_window_reduction="request_window_reduction:\n type: RequestWindowReduction", + action="REDUCE_REQUEST_WINDOW", + ).replace( + " http_method: GET\n", + " http_method: GET\n" + " query_properties:\n" + " type: QueryProperties\n" + ' property_list: ["a", "b"]\n', + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "query properties" in str(exception.value) + + +def test_given_file_uploader_and_request_window_reduction_then_raise(): + content = ( + _REQUEST_WINDOW_REDUCTION_STREAM.format( + incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, + request_window_reduction="request_window_reduction:\n type: RequestWindowReduction", + action="REDUCE_REQUEST_WINDOW", + ) + + """file_uploader: + type: FileUploader + requester: + type: HttpRequester + url_base: "https://airbyte.io" + path: "/download" + download_target_extractor: + type: DpathExtractor + field_path: ["url"] +""" + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "file_uploader" in str(exception.value) + + +def test_given_lazy_read_pointer_and_request_window_reduction_then_raise(): + """ + A LazySimpleRetriever reads records embedded in the parent's own pages rather than requesting a window of + its own, so there is nothing for a reduced child window to be read from. + """ + stream_definition = { + "type": "DeclarativeStream", + "name": "items", + "primary_key": [], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object", "properties": {}}, + }, + "retriever": { + "type": "SimpleRetriever", + "request_window_reduction": {"type": "RequestWindowReduction"}, + "requester": { + "type": "HttpRequester", + "url_base": "https://api.test.com", + "path": "parent/{{ stream_partition.parent_id }}/items", + "http_method": "GET", + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [400], + "action": "REDUCE_REQUEST_WINDOW", + } + ], + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["data"]}, + }, + "partition_router": { + "type": "SubstreamPartitionRouter", + "parent_stream_configs": [ + { + "type": "ParentStreamConfig", + "parent_key": "id", + "partition_field": "parent_id", + "lazy_read_pointer": ["items"], + "stream": { + "type": "DeclarativeStream", + "name": "parent", + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object", "properties": {}}, + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://api.test.com", + "path": "/parents", + "http_method": "GET", + }, + "record_selector": { + "type": "RecordSelector", + "extractor": { + "type": "DpathExtractor", + "field_path": ["data"], + }, + }, + }, + }, + } + ], + }, + }, + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + + assert "request_window_reduction" in str(exception.value) + assert "lazily" in str(exception.value) + + +def test_given_login_requester_with_reduce_request_window_action_then_raise(): + """ + Only the main requester of a SimpleRetriever reads a cursor-sliced window, so the action is rejected on a + SessionTokenAuthenticator's login_requester even though it is schema-legal there. + """ + stream_definition = { + "type": "DeclarativeStream", + "name": "Test", + "primary_key": "id", + "schema_loader": {"type": "InlineSchemaLoader", "schema": {"type": "object"}}, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/items", + "http_method": "GET", + "authenticator": { + "type": "SessionTokenAuthenticator", + "login_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/session", + "http_method": "POST", + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [400], + "action": "REDUCE_REQUEST_WINDOW", + } + ], + }, + }, + "session_token_path": ["session_token"], + "expiration_duration": "PT1H", + "request_authentication": { + "type": "ApiKey", + "inject_into": { + "type": "RequestOption", + "inject_into": "header", + "field_name": "X-Session-Token", + }, + }, + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + }, + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, + component_definition=stream_definition, + config={}, + ) + + assert "REDUCE_REQUEST_WINDOW" in str(exception.value) + assert "login_requester" in str(exception.value) + + _REDUCE_PAGE_SIZE_ERROR_HANDLER = { "type": "DefaultErrorHandler", "response_filters": [ diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 1afa3b5fc9..9baa176c5c 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -53,12 +53,21 @@ ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever +from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( + OnPartialResponse, + RequestWindowReduction, + WindowReducible, +) from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( PageSizeReductionRequiredException, ) from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) +from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( + RequestWindowReductionNotSupportedException, + RequestWindowReductionRequiredException, +) from airbyte_cdk.sources.types import Record, StreamSlice from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -2002,3 +2011,368 @@ def test_given_no_data_feed_when_read_records_then_emit_every_record(): ) assert actual_records == page + + +# --- request_window_reduction --- + +A_WINDOW_SLICE = StreamSlice( + cursor_slice={"start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T23:59:59Z"}, + partition={}, +) +A_FIRST_HALF_SLICE = StreamSlice( + cursor_slice={"start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T11:59:59Z"}, + partition={}, +) +A_SECOND_HALF_SLICE = StreamSlice( + cursor_slice={"start_time": "2024-01-01T12:00:00Z", "end_time": "2024-01-01T23:59:59Z"}, + partition={}, +) + + +def _request_window_reduction_retriever( + requester: Requester, + paginator: Paginator, + record_selector: HttpSelector, + window_reducer: WindowReducible, + request_window_reduction: Optional[RequestWindowReduction] = None, +) -> SimpleRetriever: + return SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + request_window_reduction=request_window_reduction or RequestWindowReduction(), + window_reducer=window_reducer, + parameters={}, + config={}, + ) + + +def test_given_request_window_reduction_when_read_records_then_split_and_read_both_children(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + RequestWindowReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert records == [{"id": 1}, {"id": 2}] + window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + assert requester.send_request.call_count == 3 + assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ + A_WINDOW_SLICE, + A_FIRST_HALF_SLICE, + A_SECOND_HALF_SLICE, + ] + + +def test_given_only_one_child_needs_further_reduction_when_read_records_then_recurse_asymmetrically(): + """ + Nested/asymmetric reduction: the first child is itself rejected and split again, while the second + child (read after the first child's full recursive read completes) succeeds on the first try. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + RequestWindowReductionRequiredException(), # top-level window + RequestWindowReductionRequiredException(), # first half, rejected again + [{"id": 1}], # first quarter of the first half + [{"id": 2}], # second quarter of the first half + [{"id": 3}], # second half, succeeds directly + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}], [{"id": 3}]] + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + a_quarter_slice = StreamSlice( + cursor_slice={"start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T05:59:59Z"}, + partition={}, + ) + another_quarter_slice = StreamSlice( + cursor_slice={"start_time": "2024-01-01T06:00:00Z", "end_time": "2024-01-01T11:59:59Z"}, + partition={}, + ) + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.side_effect = [ + [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE], + [a_quarter_slice, another_quarter_slice], + ] + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert records == [{"id": 1}, {"id": 2}, {"id": 3}] + assert window_reducer.reduce_window.call_args_list == [ + ((A_WINDOW_SLICE,),), + ((A_FIRST_HALF_SLICE,),), + ] + assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ + A_WINDOW_SLICE, + A_FIRST_HALF_SLICE, + a_quarter_slice, + another_quarter_slice, + A_SECOND_HALF_SLICE, + ] + + +def test_given_no_request_window_reduction_configured_when_reduction_required_then_raise_not_supported(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = RequestWindowReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + retriever = SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + parameters={}, + config={}, + ) + + with pytest.raises(RequestWindowReductionNotSupportedException): + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + +def test_given_no_window_reducer_when_reduction_required_then_raise_not_supported(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = RequestWindowReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + retriever = SimpleRetriever( + name=A_STREAM_NAME, + primary_key=primary_key, + requester=requester, + record_selector=record_selector, + paginator=paginator, + request_window_reduction=RequestWindowReduction(), + window_reducer=None, + parameters={}, + config={}, + ) + + with pytest.raises(RequestWindowReductionNotSupportedException): + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + +def test_given_records_already_emitted_and_fail_policy_when_reduction_requested_then_raise_config_error(): + """ + A later page of the same window fails after an earlier page already emitted records: with the default + `FAIL` policy, reducing and re-reading the window from scratch would duplicate those records, so this + must raise a hard config_error instead of silently reducing. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [Mock(), RequestWindowReductionRequiredException()] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = {"next_page_token": "page2"} + + window_reducer = Mock(spec=WindowReducible) + + retriever = _request_window_reduction_retriever( + requester, + paginator, + record_selector, + window_reducer, + request_window_reduction=RequestWindowReduction(on_partial_response=OnPartialResponse.FAIL), + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert exception.value.failure_type == FailureType.config_error + window_reducer.reduce_window.assert_not_called() + + +def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_requested_then_reduce_anyway(): + """ + The opt-in `ALLOW_REPLAY` policy accepts at-least-once delivery: unlike the FAIL case above, this must + proceed to split and re-read the window even though the first page already emitted a record. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + Mock(), + RequestWindowReductionRequiredException(), + [{"id": 2}], + [{"id": 3}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}], [{"id": 3}]] + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.side_effect = [{"next_page_token": "page2"}, None, None] + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + + retriever = _request_window_reduction_retriever( + requester, + paginator, + record_selector, + window_reducer, + request_window_reduction=RequestWindowReduction( + on_partial_response=OnPartialResponse.ALLOW_REPLAY + ), + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + # the record from the page emitted before the reduction signal is included once more: ALLOW_REPLAY + # accepts this at-least-once duplication rather than silently dropping it or failing the sync. + assert records == [{"id": 1}, {"id": 2}, {"id": 3}] + window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + + +def test_given_window_reducer_returns_none_when_reduction_requested_then_raise_terminal_error(): + """ + `WindowReducible.reduce_window` returning `None` means the window cannot be split any further - either + the granularity floor was reached or reducing would not make progress. Either way this must terminate + the sync deterministically rather than loop or silently drop the window. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = RequestWindowReductionRequiredException( + failure_type=FailureType.transient_error + ) + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.return_value = None + + retriever = _request_window_reduction_retriever( + requester, + paginator, + record_selector, + window_reducer, + request_window_reduction=RequestWindowReduction( + failure_message="Lower time_window so that each request covers less data." + ), + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + # the classifying error's own failure_type is preserved rather than defaulting to config_error + assert exception.value.failure_type == FailureType.transient_error + assert "Lower time_window so that each request covers less data." in exception.value.message + window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + + +def test_given_empty_children_list_when_reduction_requested_then_emit_no_records_and_do_not_loop(): + requester = Mock(spec=Requester) + requester.send_request.side_effect = RequestWindowReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.return_value = [] + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert records == [] + window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + + +def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates(): + """ + The first child reads fully and its records are emitted before the second child fails; the failure of a + later child must still surface as a real error, not be silently swallowed after the first child's success. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + RequestWindowReductionRequiredException(), + [{"id": 1}], + RequestWindowReductionRequiredException(), + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.return_value = [{"id": 1}] + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.side_effect = [ + [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE], + None, + ] + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + emitted = [] + with pytest.raises(AirbyteTracedException): + for record in retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE): + emitted.append(record) + + # the first child's records were already emitted to the caller before the second child failed + assert emitted == [{"id": 1}] + assert window_reducer.reduce_window.call_args_list == [ + ((A_WINDOW_SLICE,),), + ((A_SECOND_HALF_SLICE,),), + ] + + +def test_given_request_window_reduction_when_read_records_then_each_child_gets_fresh_pagination_state(): + """ + Each recursive call re-enters read_records/_read_pages from scratch for its child slice, so the + paginator's initial token is requested again for every child rather than resuming from wherever the + failed parent window left off. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + RequestWindowReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [[{"id": 1}], [{"id": 2}]] + paginator = _mock_paginator() + paginator.get_initial_token.return_value = "initial token" + paginator.next_page_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert [call.kwargs["next_page_token"] for call in requester.send_request.call_args_list] == [ + {"next_page_token": "initial token"}, + {"next_page_token": "initial token"}, + {"next_page_token": "initial token"}, + ] diff --git a/unit_tests/sources/declarative/test_concurrent_declarative_source.py b/unit_tests/sources/declarative/test_concurrent_declarative_source.py index c8aaca355b..4966dfb8f1 100644 --- a/unit_tests/sources/declarative/test_concurrent_declarative_source.py +++ b/unit_tests/sources/declarative/test_concurrent_declarative_source.py @@ -5167,6 +5167,118 @@ def test_given_reductions_exhausted_when_read_then_emit_a_transient_error(): ) +def _request_window_reduction_manifest(): + return { + "version": "0.34.2", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["Test"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "Test", + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": {"type": "object"}, + }, + "incremental_sync": { + "type": "DatetimeBasedCursor", + "start_datetime": "2024-01-01T00:00:00Z", + "end_datetime": "2024-01-01T23:59:59Z", + "step": "P1D", + "cursor_field": "updated_at", + "cursor_granularity": "PT1S", + "datetime_format": "%Y-%m-%dT%H:%M:%SZ", + "start_time_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "start", + }, + "end_time_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "end", + }, + }, + "retriever": { + "type": "SimpleRetriever", + "request_window_reduction": {"type": "RequestWindowReduction"}, + "requester": { + "type": "HttpRequester", + "url_base": "https://example.org", + "path": "/test", + "authenticator": {"type": "NoAuth"}, + "error_handler": { + "type": "DefaultErrorHandler", + "response_filters": [ + { + "type": "HttpResponseFilter", + "http_codes": [400], + "action": "REDUCE_REQUEST_WINDOW", + }, + ], + }, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + }, + } + ], + "spec": { + "type": "Spec", + "documentation_url": "https://example.org", + "connection_specification": {}, + }, + } + + +def _read_request_window_reduction_source(manifest): + catalog = create_catalog("Test") + source = ConcurrentDeclarativeSource( + source_config=manifest, + config={}, + catalog=catalog, + state=None, + ) + yield from source.read(logger=source.logger, config={}, catalog=catalog, state=[]) + + +def test_given_reduce_request_window_action_when_read_then_split_and_read_both_halves(): + """ + Mirrors the real PayPal `RESULTSET_TOO_LARGE` incident this feature exists to generalize: a request for + the full day is rejected outright, and the connector reads it back as two half-day requests instead, with + a complete, gap-free, non-duplicated record set. + """ + full_window_request = HttpRequest( + "https://example.org/test", + query_params={"start": "2024-01-01T00:00:00Z", "end": "2024-01-01T23:59:59Z"}, + ) + first_half_request = HttpRequest( + "https://example.org/test", + query_params={"start": "2024-01-01T00:00:00Z", "end": "2024-01-01T11:59:59Z"}, + ) + second_half_request = HttpRequest( + "https://example.org/test", + query_params={"start": "2024-01-01T12:00:00Z", "end": "2024-01-01T23:59:59Z"}, + ) + + with HttpMocker() as http_mocker: + http_mocker.get(full_window_request, HttpResponse("", 400)) + http_mocker.get(first_half_request, HttpResponse(json.dumps({"items": [{"id": 1}]}), 200)) + http_mocker.get(second_half_request, HttpResponse(json.dumps({"items": [{"id": 2}]}), 200)) + + messages = list(_read_request_window_reduction_source(_request_window_reduction_manifest())) + + http_mocker.assert_number_of_calls(full_window_request, 1) + http_mocker.assert_number_of_calls(first_half_request, 1) + http_mocker.assert_number_of_calls(second_half_request, 1) + + assert sorted( + message.record.data["id"] for message in messages if message.type == Type.RECORD + ) == [1, 2] + + def test_given_pagination_limit_reached_when_read_then_reset_pagination(): input_config = {} manifest = { From 77d3de1dfce6d03fd94fc74ef8ad6716fdd85ae7 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Tue, 22 Sep 2026 13:27:50 -0700 Subject: [PATCH 04/21] fix(cdk): correct associated_slice on split records, bound split recursion depth Phase 3b of declarative adaptive request-window reduction (airbytehq/airbyte-internal-issues#17174): closes correctness/isolation gaps found while cross-checking the shared capability against #17174's acceptance criteria, before starting PayPal adoption (#17175). - Fixes a real bug: records read from a split child window carried the child's own slice as Record.associated_slice (RecordSelector stamps it from whatever stream_slice a given recursive call used), which ConcurrentCursor.observe() keys its per-partition bookkeeping by. close_partition() always looks that bookkeeping up by the *original* partition's slice, so it could never find it - silently degrading the committed state's most-recently-observed cursor value after a window reduction. For the sequential state format declarative streams use by default, this actually fell back to the window's *start* boundary (not a merely imprecise end-boundary fallback), which would have caused the next sync to re-read the entire already-synced window. Fixed by re-stamping every record yielded from the recursion with the original partition's slice (SimpleRetriever._reassociate_with_original_slice), verified end-to-end via a test that fails with the pre-fix behavior and passes with the fix. - Adds RequestWindowReduction.max_split_depth (schema + generated model + factory wiring), a defense-in-depth bound enforced by SimpleRetriever independent of any WindowReducible's own no-progress guard, so a misbehaving custom cursor that never signals "cannot split further" still fails deterministically instead of recursing without bound. Defaults to 10: a one-year window bisected down to a 12-hour floor needs ~10 halvings, comfortably covering the request-window failures this feature targets in practice (days to a few months), while failing fast otherwise. - New tests: concurrency isolation (two partitions reducing independently on a shared retriever/cursor do not interfere), final-state correctness when the only scheduled partition is split, max-depth exceeded, a non-UTC timezone offset round-trip, and an exception raised during record extraction/decoding rather than at request-send time (the Google Ads/Iterable-style path). Full fast suite: 4712 passed (4703 baseline + 9 new), no regressions beyond pre-existing, change-independent SQLite-cache-locking flakiness in test_read_with_concurrent_and_synchronous_streams* (reproduces identically on main, confirmed via git stash). ruff/mypy clean. --- .../declarative_component_schema.yaml | 20 ++ .../models/declarative_component_schema.py | 7 + .../parsers/model_to_component_factory.py | 3 + .../retrievers/simple_retriever.py | 55 +++- .../retrievers/window_reducible.py | 10 + .../test_model_to_component_factory.py | 4 +- .../retrievers/test_simple_retriever.py | 234 ++++++++++++++++++ .../test_concurrent_declarative_source.py | 53 ++++ .../sources/streams/concurrent/test_cursor.py | 14 ++ 9 files changed, 396 insertions(+), 4 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 492174f97f..7a1e0e652e 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4500,6 +4500,26 @@ definitions: examples: - Narrow the sync down by selecting fewer fields on this stream. - Lower time_window so that each request covers less data. + max_split_depth: + title: Maximum Split Depth + description: >- + Maximum number of times in a row a single request window may be split before the sync fails with a + configuration error. This is a safety bound independent of the cursor's own granularity floor: it + protects against a custom cursor whose reduce_window implementation returns children that do not + actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single + already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world + APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a + one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a + wider window than the request-window failures this feature targets in practice tend to involve (days + to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to + approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning + multiple years, should raise this explicitly. + type: integer + default: 10 + minimum: 1 + examples: + - 10 + - 5 GzipDecoder: title: gzip description: Select 'gzip' for response data that is compressed with gzip. Requires specifying an inner data type/decoder to parse the decompressed data. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index c35aeb5b1d..d906630682 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1494,6 +1494,13 @@ class RequestWindowReduction(BaseModel): ], title="Failure Message", ) + max_split_depth: Optional[int] = Field( + 10, + description="Maximum number of times in a row a single request window may be split before the sync fails with a configuration error. This is a safety bound independent of the cursor's own granularity floor: it protects against a custom cursor whose reduce_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a wider window than the request-window failures this feature targets in practice tend to involve (days to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning multiple years, should raise this explicitly.", + examples=[10, 5], + ge=1, + title="Maximum Split Depth", + ) class CsvDecoder(BaseModel): diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 4f34f4af82..5783251af0 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -4137,6 +4137,9 @@ def _create_request_window_reduction( if on_partial_response is not None else OnPartialResponse.FAIL, failure_message=model.request_window_reduction.failure_message, + max_split_depth=model.request_window_reduction.max_split_depth # type: ignore[arg-type] # the schema defines a default + if model.request_window_reduction.max_split_depth is not None + else 10, ) def _validate_request_window_reduction_is_supported( diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 74dce0c077..4391998886 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -563,12 +563,14 @@ def read_records( :return: The records read from the API source """ _slice = stream_slice or StreamSlice(partition={}, cursor_slice={}) # None-check - yield from self._read_records_or_reduce_window(records_schema, _slice) + yield from self._read_records_or_reduce_window(records_schema, _slice, _slice, depth=0) def _read_records_or_reduce_window( self, records_schema: Mapping[str, Any], stream_slice: StreamSlice, + original_slice: StreamSlice, + depth: int, ) -> Iterable[StreamData]: """ Read `stream_slice` to completion, replacing it with smaller children and recursing into each of them in @@ -581,6 +583,13 @@ def _read_records_or_reduce_window( `PartitionReader.process_partition()` only calls `cursor.close_partition()` once, after this generator is fully exhausted - so a failure anywhere in the recursion (a child that cannot be read, or a window that cannot be reduced any further) propagates out without ever checkpointing the original partition. + + `original_slice` is threaded through the recursion unchanged so every record yielded, however deep the + recursion went to produce it, is re-stamped with the partition's own slice rather than the child slice it + was actually read against - see `_reassociate_with_original_slice`. `depth` bounds the recursion + independently of `WindowReducible.reduce_window`'s own no-progress guard: it is enforced here so a + misbehaving custom cursor cannot recurse indefinitely regardless of what that guard does or does not + catch. """ record_generator = partial( self._parse_records, @@ -606,7 +615,7 @@ def _read_records_or_reduce_window( ) for record in records: emitted_any = True - yield record + yield self._reassociate_with_original_slice(record, original_slice) except RequestWindowReductionRequiredException as exception: if self.request_window_reduction is None or self.window_reducer is None: raise RequestWindowReductionNotSupportedException( @@ -631,6 +640,20 @@ def _read_records_or_reduce_window( failure_type=FailureType.config_error, ) from exception + if depth >= self.request_window_reduction.max_split_depth: + raise AirbyteTracedException( + internal_message=f"Stream {self.name} exceeded the maximum request window split depth of {self.request_window_reduction.max_split_depth} while reducing {original_slice}", + message=( + f"Stream {self.name} could not reduce its request window to a size the API accepts " + f"within {self.request_window_reduction.max_split_depth} splits. This usually means the " + f"stream's cursor is not actually shrinking the window on each split; if it uses a " + f"custom cursor, check its `reduce_window` implementation. Raise " + f"`request_window_reduction.max_split_depth` only if this stream's cursor granularity " + f"genuinely requires more splits than that." + ), + failure_type=FailureType.config_error, + ) from exception + children = self.window_reducer.reduce_window(stream_slice) if children is None: failure_message = ( @@ -648,7 +671,33 @@ def _read_records_or_reduce_window( ) from exception for child in children: - yield from self._read_records_or_reduce_window(records_schema, child) + yield from self._read_records_or_reduce_window( + records_schema, child, original_slice, depth + 1 + ) + + @staticmethod + def _reassociate_with_original_slice( + record: StreamData, original_slice: StreamSlice + ) -> StreamData: + """ + Records read while recursing into a reduced child window are parsed against that child's `StreamSlice`, + so `RecordSelector` stamps them with the child as `associated_slice`. `DeclarativePartition.read()` + passes an already-built `Record` through unchanged rather than re-wrapping it, so left uncorrected the + child slice - not the partition's own slice - is what `ConcurrentCursor.observe()` would key its + per-partition bookkeeping by. `close_partition()` always looks that bookkeeping up by the partition's own + slice, so it would never find it, silently losing the precise most-recently-observed cursor value for a + split partition (recoverable in practice today only because callers of that lookup fall back to the + slice's end boundary when it is missing). Re-stamping every record with `original_slice` here restores + the same tracking a non-split read already gets, rather than relying on that fallback. + """ + if isinstance(record, Record) and record.associated_slice is not original_slice: + return Record( + data=record.data, + stream_name=record.stream_name, + associated_slice=original_slice, + file_reference=record.file_reference, + ) + return record def _parse_records( self, diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index d33ec65d76..c0d248e950 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -52,3 +52,13 @@ class RequestWindowReduction: on_partial_response: OnPartialResponse = OnPartialResponse.FAIL failure_message: Optional[str] = None + # A defense-in-depth bound independent of any specific WindowReducible's own no-progress guard: the retriever + # enforces this itself so a misbehaving custom cursor (returning children that do not actually shrink) fails + # deterministically rather than recursing indefinitely. Each reduction bisects a single already-generated + # slice - bounded by the cursor's own `step`, not the whole sync range - and real-world APIs that reject + # oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step + # bisected down to a 12-hour floor needs ~10 halvings (log2(hours in a year / 12) ~= 9.5), which already + # covers a wider window than the request-window failures this feature targets in practice tend to involve + # (on the order of days to a few months). A connector whose cursor genuinely needs finer-than-half-day + # granularity, or windows spanning multiple years, should raise this explicitly. + max_split_depth: int = 10 diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 42d82f6e0c..01acb79d90 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -7629,7 +7629,8 @@ def test_given_request_window_reduction_values_then_create_retriever_with_those_ "request_window_reduction:\n" " type: RequestWindowReduction\n" " on_partial_response: ALLOW_REPLAY\n" - " failure_message: Lower time_window so that each request covers less data." + " failure_message: Lower time_window so that each request covers less data.\n" + " max_split_depth: 20" ) ) ) @@ -7637,6 +7638,7 @@ def test_given_request_window_reduction_values_then_create_retriever_with_those_ assert retriever.request_window_reduction == RequestWindowReduction( on_partial_response=OnPartialResponse.ALLOW_REPLAY, failure_message="Lower time_window so that each request covers less data.", + max_split_depth=20, ) diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 9baa176c5c..d585b9d25d 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -2376,3 +2376,237 @@ def test_given_request_window_reduction_when_read_records_then_each_child_gets_f {"next_page_token": "initial token"}, {"next_page_token": "initial token"}, ] + + +def test_reassociate_with_original_slice_rewraps_a_record_stamped_with_a_different_slice(): + child_slice = StreamSlice(cursor_slice={"start_time": "child"}, partition={}) + record = Record( + data={"id": 1}, + stream_name=A_STREAM_NAME, + associated_slice=child_slice, + ) + + result = SimpleRetriever._reassociate_with_original_slice(record, A_WINDOW_SLICE) + + assert result.associated_slice is A_WINDOW_SLICE + assert result.data == {"id": 1} + assert result.stream_name == A_STREAM_NAME + + +def test_reassociate_with_original_slice_is_a_no_op_when_already_associated(): + record = Record( + data={"id": 1}, + stream_name=A_STREAM_NAME, + associated_slice=A_WINDOW_SLICE, + ) + + result = SimpleRetriever._reassociate_with_original_slice(record, A_WINDOW_SLICE) + + assert result is record + + +def test_reassociate_with_original_slice_passes_through_non_record_stream_data(): + message = {"raw": "not a Record instance"} + + result = SimpleRetriever._reassociate_with_original_slice(message, A_WINDOW_SLICE) + + assert result is message + + +def test_given_request_window_reduction_when_split_then_records_are_associated_with_original_slice(): + """ + Regression test: `ConcurrentCursor.observe()` keys its per-partition bookkeeping by `Record.associated_slice`, + but `ConcurrentCursor.close_partition()` always looks that bookkeeping up by the partition's own (original) + slice. A record read from a reduced child window - via a real `RecordSelector`, not the mocks the other + tests in this module use - would otherwise carry the child as its `associated_slice`, a key + `close_partition()` can never find, silently degrading the most-recently-observed cursor value tracked for a + split partition. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [ + RequestWindowReductionRequiredException(), + Mock(), + Mock(), + ] + + def select_records(*, response, stream_slice, **kwargs): + return [ + Record( + data={"start": stream_slice.cursor_slice["start_time"]}, + stream_name=A_STREAM_NAME, + associated_slice=stream_slice, + ) + ] + + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = select_records + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert len(records) == 2 + assert all(record.associated_slice is A_WINDOW_SLICE for record in records) + # the underlying data still reflects which child the record actually came from + assert {record.data["start"] for record in records} == { + A_FIRST_HALF_SLICE.cursor_slice["start_time"], + A_SECOND_HALF_SLICE.cursor_slice["start_time"], + } + + +def test_given_exception_raised_during_record_extraction_when_read_then_still_reduce_window(): + """ + Google Ads/Iterable-style exception-driven cases fail during response streaming/decoding, not at the point + the request is sent - unlike PayPal, where the API rejects the request outright before any body is read. + The catch boundary must cover the entire read, not only the requester call, so custom code raising + `RequestWindowReductionRequiredException` from anywhere in the pipeline is still honored. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = [Mock(), Mock(), Mock()] + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = [ + RequestWindowReductionRequiredException(), + [{"id": 1}], + [{"id": 2}], + ] + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert records == [{"id": 1}, {"id": 2}] + + +def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_error(): + """ + Defense-in-depth: even if a (misbehaving, e.g. custom) `WindowReducible` never itself signals "cannot split + any further", the retriever's own depth counter must still terminate the read deterministically rather than + recursing without bound. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = RequestWindowReductionRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + window_reducer = Mock(spec=WindowReducible) + # never signals "no further progress possible" - always splits into two children identical to the parent + window_reducer.reduce_window.side_effect = lambda stream_slice: [stream_slice, stream_slice] + + retriever = _request_window_reduction_retriever( + requester, + paginator, + record_selector, + window_reducer, + request_window_reduction=RequestWindowReduction(max_split_depth=3), + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert exception.value.failure_type == FailureType.config_error + assert "maximum request window split depth" in exception.value.internal_message + assert "within 3 splits" in exception.value.message + # the single leftmost recursion path is explored to depth 3 (three reduce_window calls, at depth 0, 1, and + # 2) before the cap stops a fourth call at depth 3 - the exception then propagates immediately, so sibling + # branches at shallower depths are never explored + assert window_reducer.reduce_window.call_count == 3 + + +def test_given_partitions_read_concurrently_then_window_reduction_isolates_between_partitions(): + """ + One retriever instance - and one shared `window_reducer` (the stream's cursor) - is used for every partition + of a stream, so a reduction triggered while reading one partition must not affect a concurrently-running + read of another partition at all: the other partition must complete normally, and the reducer must only ever + be asked to split the slice that actually needed it. + """ + slice_a = StreamSlice( + cursor_slice={"start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T23:59:59Z"}, + partition={"id": "a"}, + ) + slice_b = StreamSlice( + cursor_slice={"start_time": "2024-02-01T00:00:00Z", "end_time": "2024-02-01T23:59:59Z"}, + partition={"id": "b"}, + ) + b_requested = threading.Event() + + def send_request(*, stream_slice, **kwargs): + # only the original top-level slice_a is rejected (its children must succeed, or the recursion would + # never terminate); identity, not partition id, distinguishes it from the children reduce_window builds + if stream_slice is slice_a: + # only raise once b's own, unrelated request is known to be in flight, so the two reads + # genuinely overlap rather than running one after the other + assert b_requested.wait(timeout=10) + raise RequestWindowReductionRequiredException() + if stream_slice is slice_b: + b_requested.set() + return Mock() + + requester = Mock(spec=Requester) + requester.send_request.side_effect = send_request + + def select_records(*, response, stream_slice, **kwargs): + return [ + { + "partition": stream_slice.partition["id"], + "start": stream_slice.cursor_slice["start_time"], + } + ] + + record_selector = Mock(spec=HttpSelector) + record_selector.select_records.side_effect = select_records + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + paginator.next_page_token.return_value = None + + def reduce_window(stream_slice): + # preserve the parent's own partition, matching what a real WindowReducible would do + return [ + StreamSlice( + cursor_slice={ + "start_time": stream_slice.cursor_slice["start_time"], + "end_time": "midpoint", + }, + partition=stream_slice.partition, + ), + StreamSlice( + cursor_slice={ + "start_time": "midpoint", + "end_time": stream_slice.cursor_slice["end_time"], + }, + partition=stream_slice.partition, + ), + ] + + window_reducer = Mock(spec=WindowReducible) + window_reducer.reduce_window.side_effect = reduce_window + + retriever = _request_window_reduction_retriever( + requester, paginator, record_selector, window_reducer + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + future_a = executor.submit(lambda: list(retriever.read_records(A_RECORD_SCHEMA, slice_a))) + future_b = executor.submit(lambda: list(retriever.read_records(A_RECORD_SCHEMA, slice_b))) + records_a = future_a.result(timeout=10) + records_b = future_b.result(timeout=10) + + assert records_b == [{"partition": "b", "start": "2024-02-01T00:00:00Z"}] + assert len(records_a) == 2 + window_reducer.reduce_window.assert_called_once_with(slice_a) diff --git a/unit_tests/sources/declarative/test_concurrent_declarative_source.py b/unit_tests/sources/declarative/test_concurrent_declarative_source.py index 4966dfb8f1..250a5b6452 100644 --- a/unit_tests/sources/declarative/test_concurrent_declarative_source.py +++ b/unit_tests/sources/declarative/test_concurrent_declarative_source.py @@ -5279,6 +5279,59 @@ def test_given_reduce_request_window_action_when_read_then_split_and_read_both_h ) == [1, 2] +def test_given_reduce_request_window_action_when_read_then_final_state_reflects_full_original_window(): + """ + Regression coverage for a bug where records read from a split child window carried the child's own slice + as `Record.associated_slice`, which `ConcurrentCursor.observe()` keys its bookkeeping by - a key + `close_partition()` (which always looks up by the *original* partition's slice) could never find. Left + unfixed, `_get_latest_complete_time`'s `first_interval.get("most_recent_cursor_value") or + first_interval[START_KEY]` fallback would silently commit the window's *start* boundary as the emitted + state instead of the actual highest cursor value observed across the two split children - reverting the + cursor all the way back to the beginning of the (already fully synced) day on the very next sync, rather + than a merely imprecise-but-safe value. + """ + full_window_request = HttpRequest( + "https://example.org/test", + query_params={"start": "2024-01-01T00:00:00Z", "end": "2024-01-01T23:59:59Z"}, + ) + first_half_request = HttpRequest( + "https://example.org/test", + query_params={"start": "2024-01-01T00:00:00Z", "end": "2024-01-01T11:59:59Z"}, + ) + second_half_request = HttpRequest( + "https://example.org/test", + query_params={"start": "2024-01-01T12:00:00Z", "end": "2024-01-01T23:59:59Z"}, + ) + + with HttpMocker() as http_mocker: + http_mocker.get(full_window_request, HttpResponse("", 400)) + http_mocker.get( + first_half_request, + HttpResponse( + json.dumps({"items": [{"id": 1, "updated_at": "2024-01-01T05:00:00Z"}]}), 200 + ), + ) + http_mocker.get( + second_half_request, + # the higher cursor value lives in the *second* half, so a fallback to the slice's end boundary + # instead of the true observed maximum would be easy to miss if it happened to match by accident + HttpResponse( + json.dumps({"items": [{"id": 2, "updated_at": "2024-01-01T20:00:00Z"}]}), 200 + ), + ) + + messages = list(_read_request_window_reduction_source(_request_window_reduction_manifest())) + + states = get_states_for_stream(stream_name="Test", messages=messages) + assert states + # the declarative DatetimeBasedCursor emits sequential-format state (a flat {cursor_field: value} dict) by + # default; every emitted state must show the true observed maximum, never the window's start boundary + assert all( + state.stream.stream_state.__dict__ == {"updated_at": "2024-01-01T20:00:00Z"} + for state in states + ) + + def test_given_pagination_limit_reached_when_read_then_reset_pagination(): input_config = {} manifest = { diff --git a/unit_tests/sources/streams/concurrent/test_cursor.py b/unit_tests/sources/streams/concurrent/test_cursor.py index 64580ebdd7..db4625bea5 100644 --- a/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/unit_tests/sources/streams/concurrent/test_cursor.py @@ -1614,3 +1614,17 @@ def test_given_multiple_datetime_formats_when_reduce_then_round_trip_format_corr self._slice("2024-01-01 00:00:00", "2024-01-01 11:59:59"), self._slice("2024-01-01 12:00:00", "2024-01-01 23:59:59"), ] + + def test_given_non_utc_timezone_offset_when_reduce_then_preserve_offset_in_round_trip( + self, + ) -> None: + cursor = self._cursor(datetime_format="%Y-%m-%dT%H:%M:%S%z") + + children = cursor.reduce_window( + self._slice("2024-01-01T00:00:00+0530", "2024-01-01T23:59:59+0530") + ) + + assert children == [ + self._slice("2024-01-01T00:00:00+0530", "2024-01-01T11:59:59+0530"), + self._slice("2024-01-01T12:00:00+0530", "2024-01-01T23:59:59+0530"), + ] From 82cba07a01e9a980fe72d3f341aa2286ef7324e1 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Tue, 22 Sep 2026 14:19:06 -0700 Subject: [PATCH 05/21] Apply batched suggestions from code review Co-authored-by: Copilot Autofix powered by AI <223894421+github-code-quality[bot]@users.noreply.github.com> --- .../declarative/parsers/model_to_component_factory.py | 6 ------ .../sources/declarative/retrievers/window_reducible.py | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 5783251af0..d619a0f514 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -385,9 +385,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( OffsetIncrement as OffsetIncrementModel, ) -from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( - OnPartialResponse as OnPartialResponseModel, -) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( PageIncrement as PageIncrementModel, ) @@ -445,9 +442,6 @@ from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( RequestPath as RequestPathModel, ) -from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( - RequestWindowReduction as RequestWindowReductionModel, -) from airbyte_cdk.sources.declarative.models.declarative_component_schema import ( ResponseToFileExtractor as ResponseToFileExtractorModel, ) diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index c0d248e950..a3400b8103 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -36,7 +36,7 @@ def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice] smaller than the parent (a no-progress guard independent of the granularity check). Callers should treat `None` as a terminal condition, not retry with the same slice. """ - ... + raise NotImplementedError("WindowReducible.reduce_window must be implemented by protocol implementers") @dataclass(frozen=True) From d02ebbf78d4dff8f1c1fd10d4917f0a3c882341f Mon Sep 17 00:00:00 2001 From: kaizerch Date: Tue, 22 Sep 2026 14:34:34 -0700 Subject: [PATCH 06/21] style: fix ruff format after applying Copilot autofix suggestions The autofix commit (82cba07a) changed the WindowReducible.reduce_window body from `...` to an explicit raise, which pushed the line over ruff's line-length limit. ruff format --check/ruff check are clean across the repo now. --- .../sources/declarative/retrievers/window_reducible.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index a3400b8103..9b424cd7b7 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -36,7 +36,9 @@ def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice] smaller than the parent (a no-progress guard independent of the granularity check). Callers should treat `None` as a terminal condition, not retry with the same slice. """ - raise NotImplementedError("WindowReducible.reduce_window must be implemented by protocol implementers") + raise NotImplementedError( + "WindowReducible.reduce_window must be implemented by protocol implementers" + ) @dataclass(frozen=True) From 51e5e61ee5f568e4907d24c2495f896b4bf0ed2f Mon Sep 17 00:00:00 2001 From: "kaizer.charania" Date: Wed, 23 Sep 2026 21:21:17 +0000 Subject: [PATCH 07/21] feat(declarative): log request window reductions in SimpleRetriever Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../sources/declarative/retrievers/simple_retriever.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 4391998886..47b7545f56 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -670,6 +670,10 @@ def _read_records_or_reduce_window( failure_type=exception.failure_type or FailureType.config_error, ) from exception + LOGGER.info( + f"Stream {self.name}: the API rejected request window {stream_slice} (split depth {depth}); " + f"reducing it to {children} and reading each in turn." + ) for child in children: yield from self._read_records_or_reduce_window( records_schema, child, original_slice, depth + 1 From 1f5912e91a4ed2985c05e191669bf1f2220daa6e Mon Sep 17 00:00:00 2001 From: kaizerch Date: Thu, 24 Sep 2026 11:20:46 -0700 Subject: [PATCH 08/21] refactor(cdk): rename request window reduction to splitting (Phase 1) Renames REDUCE_REQUEST_WINDOW/request_window_reduction/reduce_window and related identifiers to SPLIT_REQUEST_WINDOW/request_window_splitting/ split_request_window across the manifest schema, factory, retriever, cursor, and exceptions, per Patrick's review on PR #1171. Also drops the unused `model` parameter from `_validate_request_window_splitting_is_supported`, per the same review. --- .../declarative_component_schema.yaml | 28 +-- .../models/declarative_component_schema.py | 16 +- .../parsers/model_to_component_factory.py | 101 +++++---- .../error_handlers/composite_error_handler.py | 2 +- .../retrievers/simple_retriever.py | 84 +++---- .../retrievers/window_reducible.py | 20 +- .../sources/streams/concurrent/cursor.py | 2 +- .../http/error_handlers/response_models.py | 2 +- .../sources/streams/http/http_client.py | 12 +- ...n.py => request_window_split_exception.py} | 28 +-- .../test_model_to_component_factory.py | 104 ++++----- .../test_composite_error_handler.py | 8 +- .../retrievers/test_simple_retriever.py | 206 +++++++++--------- .../test_concurrent_declarative_source.py | 16 +- .../sources/streams/concurrent/test_cursor.py | 32 +-- .../sources/streams/http/test_http_client.py | 28 +-- 16 files changed, 344 insertions(+), 345 deletions(-) rename airbyte_cdk/sources/streams/http/{request_window_reduction_exception.py => request_window_split_exception.py} (54%) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 7a1e0e652e..8bf337d8be 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2686,7 +2686,7 @@ definitions: - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY - REDUCE_PAGE_SIZE - - REDUCE_REQUEST_WINDOW + - SPLIT_REQUEST_WINDOW examples: - SUCCESS - FAIL @@ -2696,7 +2696,7 @@ definitions: - RATE_LIMITED - REFRESH_TOKEN_THEN_RETRY - REDUCE_PAGE_SIZE - - REDUCE_REQUEST_WINDOW + - SPLIT_REQUEST_WINDOW failure_type: title: Failure Type description: Failure type of traced exception if a response matches the filter. @@ -4263,17 +4263,17 @@ definitions: re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size. "$ref": "#/definitions/PageSizeReduction" - request_window_reduction: + request_window_splitting: description: >- - Describes how the request window is reduced when an error handler resolves to the - REDUCE_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, + Describes how the request window is split when an error handler resolves to the + SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, since that is both the smallest window the connector will request and what keeps two child windows from overlapping at their shared edge. Distinct from page_size_reduction: an API can reject an entire date/time range as too large even though every individual page of it would be within the API's page size limit, and no page can be returned to page through in the first place. On such a response the connector replaces the failing window with two smaller, ordered windows covering the same range and reads each one fully, with its own pagination state, before checkpointing the original partition. - "$ref": "#/definitions/RequestWindowReduction" + "$ref": "#/definitions/RequestWindowSplitting" ignore_stream_slicer_parameters_on_paginated_requests: description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored. type: boolean @@ -4454,10 +4454,10 @@ definitions: - NEVER - AFTER_SUCCESSFUL_PAGE default: NEVER - RequestWindowReduction: - title: Request Window Reduction + RequestWindowSplitting: + title: Request Window Splitting description: >- - Describes how the request window is reduced when an error handler resolves to the REDUCE_REQUEST_WINDOW + Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. On such a response, the connector replaces the failing window with two smaller, ordered windows that together cover the same range, and reads each one fully - with its own pagination state - instead of retrying the identical request. Only supported on a stream whose incremental_sync cursor defines @@ -4471,15 +4471,15 @@ definitions: properties: type: type: string - enum: [RequestWindowReduction] + enum: [RequestWindowSplitting] on_partial_response: title: On Partial Response description: >- - What to do when a REDUCE_REQUEST_WINDOW response arrives after the current window has already - emitted a page or a record. FAIL stops the sync with a configuration error, since reducing and + What to do when a SPLIT_REQUEST_WINDOW response arrives after the current window has already + emitted a page or a record. FAIL stops the sync with a configuration error, since splitting and re-reading the window from scratch would emit those records a second time - this is correct for an API, such as PayPal's, that always rejects a window before returning its first page. ALLOW_REPLAY - reduces and re-reads the window anyway, accepting at-least-once delivery of the records already + splits and re-reads the window anyway, accepting at-least-once delivery of the records already emitted; use it only when the destination or a downstream deduplication step can tolerate duplicate records, for instance when adopting this action for a transport or streamed-decoding failure that can occur after some records of the window were already read. @@ -4505,7 +4505,7 @@ definitions: description: >- Maximum number of times in a row a single request window may be split before the sync fails with a configuration error. This is a safety bound independent of the cursor's own granularity floor: it - protects against a custom cursor whose reduce_window implementation returns children that do not + protects against a custom cursor whose split_request_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index d906630682..27c40028e7 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -716,7 +716,7 @@ class Action(Enum): RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" - REDUCE_REQUEST_WINDOW = "REDUCE_REQUEST_WINDOW" + SPLIT_REQUEST_WINDOW = "SPLIT_REQUEST_WINDOW" class FailureType(Enum): @@ -739,7 +739,7 @@ class HttpResponseFilter(BaseModel): "RATE_LIMITED", "REFRESH_TOKEN_THEN_RETRY", "REDUCE_PAGE_SIZE", - "REDUCE_REQUEST_WINDOW", + "SPLIT_REQUEST_WINDOW", ], title="Action", ) @@ -1478,11 +1478,11 @@ class OnPartialResponse(Enum): ALLOW_REPLAY = "ALLOW_REPLAY" -class RequestWindowReduction(BaseModel): - type: Literal["RequestWindowReduction"] +class RequestWindowSplitting(BaseModel): + type: Literal["RequestWindowSplitting"] on_partial_response: Optional[OnPartialResponse] = Field( OnPartialResponse.FAIL, - description="What to do when a REDUCE_REQUEST_WINDOW response arrives after the current window has already emitted a page or a record. FAIL stops the sync with a configuration error, since reducing and re-reading the window from scratch would emit those records a second time - this is correct for an API, such as PayPal's, that always rejects a window before returning its first page. ALLOW_REPLAY reduces and re-reads the window anyway, accepting at-least-once delivery of the records already emitted; use it only when the destination or a downstream deduplication step can tolerate duplicate records, for instance when adopting this action for a transport or streamed-decoding failure that can occur after some records of the window were already read.", + description="What to do when a SPLIT_REQUEST_WINDOW response arrives after the current window has already emitted a page or a record. FAIL stops the sync with a configuration error, since splitting and re-reading the window from scratch would emit those records a second time - this is correct for an API, such as PayPal's, that always rejects a window before returning its first page. ALLOW_REPLAY splits and re-reads the window anyway, accepting at-least-once delivery of the records already emitted; use it only when the destination or a downstream deduplication step can tolerate duplicate records, for instance when adopting this action for a transport or streamed-decoding failure that can occur after some records of the window were already read.", title="On Partial Response", ) failure_message: Optional[str] = Field( @@ -1496,7 +1496,7 @@ class RequestWindowReduction(BaseModel): ) max_split_depth: Optional[int] = Field( 10, - description="Maximum number of times in a row a single request window may be split before the sync fails with a configuration error. This is a safety bound independent of the cursor's own granularity floor: it protects against a custom cursor whose reduce_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a wider window than the request-window failures this feature targets in practice tend to involve (days to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning multiple years, should raise this explicitly.", + description="Maximum number of times in a row a single request window may be split before the sync fails with a configuration error. This is a safety bound independent of the cursor's own granularity floor: it protects against a custom cursor whose split_request_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a wider window than the request-window failures this feature targets in practice tend to involve (days to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning multiple years, should raise this explicitly.", examples=[10, 5], ge=1, title="Maximum Split Depth", @@ -3330,9 +3330,9 @@ class SimpleRetriever(BaseModel): None, description="Describes how the page size is reduced when an error handler resolves to the REDUCE_PAGE_SIZE action. Requires a DefaultPaginator that defines both page_size_option and a pagination strategy with a page_size. Cannot be combined with query properties, a file uploader, or a parent stream read lazily through lazy_read_pointer, because in those cases records of the failing page have already been emitted and re-issuing the page would emit them twice. A page_token_option of type RequestPath is rejected as well, because the next page is then a URL built by the API which already carries the page size.", ) - request_window_reduction: Optional[RequestWindowReduction] = Field( + request_window_splitting: Optional[RequestWindowSplitting] = Field( None, - description="Describes how the request window is reduced when an error handler resolves to the REDUCE_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, since that is both the smallest window the connector will request and what keeps two child windows from overlapping at their shared edge. Distinct from page_size_reduction: an API can reject an entire date/time range as too large even though every individual page of it would be within the API's page size limit, and no page can be returned to page through in the first place. On such a response the connector replaces the failing window with two smaller, ordered windows covering the same range and reads each one fully, with its own pagination state, before checkpointing the original partition.", + description="Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, since that is both the smallest window the connector will request and what keeps two child windows from overlapping at their shared edge. Distinct from page_size_reduction: an API can reject an entire date/time range as too large even though every individual page of it would be within the API's page size limit, and no page can be returned to page through in the first place. On such a response the connector replaces the failing window with two smaller, ordered windows covering the same range and reads each one fully, with its own pagination state, before checkpointing the original partition.", ) ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( False, diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index d619a0f514..5cfeccf3e8 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -597,7 +597,7 @@ from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, - RequestWindowReduction, + RequestWindowSplitting, WindowReducible, ) from airbyte_cdk.sources.declarative.schema import ( @@ -1220,7 +1220,7 @@ def create_session_token_authenticator( self._reject_reduce_page_size_action( model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" ) - self._reject_reduce_request_window_action( + self._reject_split_request_window_action( model.login_requester, f"`login_requester` of the SessionTokenAuthenticator of {name}" ) decoder = ( @@ -3701,18 +3701,18 @@ def _get_url(req: Requester) -> str: ) if reads_parent_stream_lazily and ( - model.request_window_reduction - or self._uses_reduce_request_window_action( + model.request_window_splitting + or self._uses_split_request_window_action( getattr(model.requester, "error_handler", None) ) ): # Same reasoning as the page_size_reduction check above: LazySimpleRetriever reads records # embedded in the parent's own pages rather than requesting a window of its own, so there is - # nothing here for a reduced child window to be read from. + # nothing here for a split child window to be read from. raise ValueError( - f"`request_window_reduction` and the REDUCE_REQUEST_WINDOW response action are not " + f"`request_window_splitting` and the SPLIT_REQUEST_WINDOW response action are not " f"supported when reading a parent stream lazily. Remove either the request window " - f"reduction or the parent stream's `lazy_read_pointer` for stream {name}." + f"splitting or the parent stream's `lazy_read_pointer` for stream {name}." ) if reads_parent_stream_lazily and not bool( @@ -3755,7 +3755,7 @@ def _get_url(req: Requester) -> str: ): raise ValueError("PaginationResetLimits are not supported while having record filter.") - request_window_reduction = self._create_request_window_reduction( + request_window_splitting = self._create_request_window_splitting( model, name, cursor, incremental_sync, query_properties, file_uploader ) return SimpleRetriever( @@ -3776,9 +3776,9 @@ def _get_url(req: Requester) -> str: page_size_reduction=self._create_page_size_reduction( model, name, query_properties, file_uploader ), - request_window_reduction=request_window_reduction, - window_reducer=( - cursor if request_window_reduction and isinstance(cursor, WindowReducible) else None + request_window_splitting=request_window_splitting, + request_window_splitter=( + cursor if request_window_splitting and isinstance(cursor, WindowReducible) else None ), post_pagination_filter=post_pagination_filter, parameters=model.parameters or {}, @@ -4085,7 +4085,7 @@ def _uses_reduce_page_size_action(self, error_handler: Any) -> bool: # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. return False - def _create_request_window_reduction( + def _create_request_window_splitting( self, model: SimpleRetrieverModel, name: str, @@ -4095,50 +4095,49 @@ def _create_request_window_reduction( ] = None, query_properties: Optional[QueryProperties] = None, file_uploader: Optional[DefaultFileUploader] = None, - ) -> Optional[RequestWindowReduction]: + ) -> Optional[RequestWindowSplitting]: # A CustomRequester does not necessarily define an error handler. A CustomRequester that does define # one keeps it as a raw dict rather than a typed model, so this returns False for it as well. error_handler = getattr(model.requester, "error_handler", None) - uses_action = self._uses_reduce_request_window_action(error_handler) - if uses_action and not model.request_window_reduction: + uses_action = self._uses_split_request_window_action(error_handler) + if uses_action and not model.request_window_splitting: raise ValueError( - f"Stream {name} has a response filter with the REDUCE_REQUEST_WINDOW action but the " - f"retriever does not define `request_window_reduction`. Add a `request_window_reduction` " + f"Stream {name} has a response filter with the SPLIT_REQUEST_WINDOW action but the " + f"retriever does not define `request_window_splitting`. Add a `request_window_splitting` " f"block to the retriever." ) - if not model.request_window_reduction: + if not model.request_window_splitting: return None if not uses_action: - # Not raised: a CustomErrorHandler can resolve to REDUCE_REQUEST_WINDOW without us being able to - # see it, and custom code can raise RequestWindowReductionRequiredException directly without + # Not raised: a CustomErrorHandler can resolve to SPLIT_REQUEST_WINDOW without us being able to + # see it, and custom code can raise RequestWindowSplitRequiredException directly without # going through any error handler at all - so the only safe reaction to a block we cannot tie to # a known trigger is a warning. LOGGER.warning( - f"Stream {name} defines `request_window_reduction` but no response filter with the " - f"REDUCE_REQUEST_WINDOW action was found on its requester. The request window will never " - f"be reduced unless a custom error handler or custom code resolves to that action." + f"Stream {name} defines `request_window_splitting` but no response filter with the " + f"SPLIT_REQUEST_WINDOW action was found on its requester. The request window will never " + f"be split unless a custom error handler or custom code resolves to that action." ) - self._validate_request_window_reduction_is_supported( - model, name, cursor, incremental_sync, query_properties, file_uploader + self._validate_request_window_splitting_is_supported( + name, cursor, incremental_sync, query_properties, file_uploader ) - on_partial_response = model.request_window_reduction.on_partial_response - return RequestWindowReduction( + on_partial_response = model.request_window_splitting.on_partial_response + return RequestWindowSplitting( on_partial_response=OnPartialResponse(on_partial_response.value) if on_partial_response is not None else OnPartialResponse.FAIL, - failure_message=model.request_window_reduction.failure_message, - max_split_depth=model.request_window_reduction.max_split_depth # type: ignore[arg-type] # the schema defines a default - if model.request_window_reduction.max_split_depth is not None + failure_message=model.request_window_splitting.failure_message, + max_split_depth=model.request_window_splitting.max_split_depth # type: ignore[arg-type] # the schema defines a default + if model.request_window_splitting.max_split_depth is not None else 10, ) - def _validate_request_window_reduction_is_supported( + def _validate_request_window_splitting_is_supported( self, - model: SimpleRetrieverModel, name: str, cursor: Optional[Cursor], incremental_sync: Optional[ @@ -4148,16 +4147,16 @@ def _validate_request_window_reduction_is_supported( file_uploader: Optional[DefaultFileUploader] = None, ) -> None: """ - Request window reduction replaces a failing slice with smaller children derived from the stream's own + Request window splitting replaces a failing slice with smaller children derived from the stream's own cursor, so it only makes sense on a stream whose cursor can do that (see `WindowReducible`) and it is rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and a `file_uploader` both mean records of the failing window may already have been emitted by the time the - reduction is requested, from inside the record generator rather than from a page boundary this retriever + split is requested, from inside the record generator rather than from a page boundary this retriever controls. """ if not isinstance(cursor, WindowReducible): raise ValueError( - f"`request_window_reduction` requires an incremental cursor that supports window reduction on " + f"`request_window_splitting` requires an incremental cursor that supports window splitting on " f"stream {name}. Found {type(cursor).__name__ if cursor is not None else 'no cursor'}: this is " f"only supported today for a `DatetimeBasedCursor` with `cursor_granularity` set and no " f"partition router combining multiple cursors." @@ -4168,54 +4167,54 @@ def _validate_request_window_reduction_is_supported( and incremental_sync.cursor_granularity ): # `cursor_granularity` is both the smallest window the connector will ever request and what keeps - # two child windows from overlapping at their shared edge, so `WindowReducible.reduce_window` + # two child windows from overlapping at their shared edge, so `WindowReducible.split_request_window` # cannot split at all without it - checked here, on the manifest model, rather than by reaching # into the cursor's private state from outside the class it belongs to. raise ValueError( - f"`request_window_reduction` requires `cursor_granularity` on the `DatetimeBasedCursor` of " + f"`request_window_splitting` requires `cursor_granularity` on the `DatetimeBasedCursor` of " f"stream {name}: without it there is no smallest window to stop splitting at, and no way to " f"keep two child windows from overlapping at their shared edge." ) if query_properties: raise ValueError( - f"`request_window_reduction` cannot be used together with query properties on stream {name}. " + f"`request_window_splitting` cannot be used together with query properties on stream {name}. " f"Records from the earlier property chunks may have already been emitted when a chunk asks to " - f"reduce the window, so reducing and re-reading it could duplicate them." + f"split the window, so splitting and re-reading it could duplicate them." ) if file_uploader: raise ValueError( - f"`request_window_reduction` cannot be used together with a `file_uploader` on stream {name}. " + f"`request_window_splitting` cannot be used together with a `file_uploader` on stream {name}. " f"The file uploader sends one request per record from inside the window's record generator, so " - f"a reduction requested partway through could re-emit records already yielded by it." + f"a split requested partway through could re-emit records already yielded by it." ) - def _reject_reduce_request_window_action(self, requester: Any, description: str) -> None: + def _reject_split_request_window_action(self, requester: Any, description: str) -> None: """ `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no window of - their own to reduce, so REDUCE_REQUEST_WINDOW is schema-legal in places where nothing can honor it. Only + their own to split, so SPLIT_REQUEST_WINDOW is schema-legal in places where nothing can honor it. Only the main requester of a `SimpleRetriever` reads a cursor-sliced window, so every other requester is rejected here rather than surfacing the exception mid-sync as a generic failure. """ if requester is None: return - if self._uses_reduce_request_window_action(getattr(requester, "error_handler", None)): + if self._uses_split_request_window_action(getattr(requester, "error_handler", None)): raise ValueError( - f"The REDUCE_REQUEST_WINDOW response action is not supported on the {description}: only the " - f"main requester of a SimpleRetriever reads a window that can be reduced. Use a different " + f"The SPLIT_REQUEST_WINDOW response action is not supported on the {description}: only the " + f"main requester of a SimpleRetriever reads a window that can be split. Use a different " f"action on that error handler." ) - def _uses_reduce_request_window_action(self, error_handler: Any) -> bool: + def _uses_split_request_window_action(self, error_handler: Any) -> bool: if isinstance(error_handler, CompositeErrorHandlerModel): return any( - self._uses_reduce_request_window_action(nested) + self._uses_split_request_window_action(nested) for nested in error_handler.error_handlers ) if isinstance(error_handler, DefaultErrorHandlerModel): return any( - response_filter.action == HttpResponseFilterActionModel.REDUCE_REQUEST_WINDOW + response_filter.action == HttpResponseFilterActionModel.SPLIT_REQUEST_WINDOW for response_filter in error_handler.response_filters or [] ) # A CustomErrorHandler can return any action and we cannot inspect it, so we do not validate it. @@ -4500,7 +4499,7 @@ def create_async_retriever( getattr(model, requester_field, None), f"`{requester_field}` of the AsyncRetriever of stream {name}", ) - self._reject_reduce_request_window_action( + self._reject_split_request_window_action( getattr(model, requester_field, None), f"`{requester_field}` of the AsyncRetriever of stream {name}", ) @@ -5119,7 +5118,7 @@ def create_file_uploader( self, model: FileUploaderModel, config: Config, **kwargs: Any ) -> FileUploader: self._reject_reduce_page_size_action(model.requester, "requester of a `file_uploader`") - self._reject_reduce_request_window_action(model.requester, "requester of a `file_uploader`") + self._reject_split_request_window_action(model.requester, "requester of a `file_uploader`") name = "File Uploader" requester = self._create_component_from_model( model=model.requester, diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py index 708189bbbb..2ef35a06ff 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/composite_error_handler.py @@ -72,7 +72,7 @@ def interpret_response( ResponseAction.IGNORE, ResponseAction.RESET_PAGINATION, ResponseAction.REDUCE_PAGE_SIZE, - ResponseAction.REDUCE_REQUEST_WINDOW, + ResponseAction.SPLIT_REQUEST_WINDOW, ]: return matched_error_resolution diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 4391998886..ba01e9ebca 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -51,7 +51,7 @@ from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, - RequestWindowReduction, + RequestWindowSplitting, WindowReducible, ) from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer @@ -64,9 +64,9 @@ from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) -from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( - RequestWindowReductionNotSupportedException, - RequestWindowReductionRequiredException, +from airbyte_cdk.sources.streams.http.request_window_split_exception import ( + RequestWindowSplitNotSupportedException, + RequestWindowSplitRequiredException, ) from airbyte_cdk.sources.types import Config, Record, StreamSlice from airbyte_cdk.utils.mapping_helpers import combine_mappings @@ -105,15 +105,15 @@ class SimpleRetriever(Retriever): `_read_pages` creates per call, so the retriever and its paginator - both shared by every partition of the stream, read concurrently - stay stateless. When `page_size_reduction` is `None` no reducer is created and `_read_pages` keeps its previous behaviour - request_window_reduction (Optional[RequestWindowReduction]): Policy applied when an error handler - resolves to `ResponseAction.REDUCE_REQUEST_WINDOW`, or when `RequestWindowReductionRequiredException` - is raised directly by custom code. `None` disables request-window reduction entirely: `read_records` - converts the exception into `RequestWindowReductionNotSupportedException` instead of attempting a - reduction it cannot perform. - window_reducer (Optional[WindowReducible]): The component - normally the stream's own cursor - asked to + request_window_splitting (Optional[RequestWindowSplitting]): Policy applied when an error handler + resolves to `ResponseAction.SPLIT_REQUEST_WINDOW`, or when `RequestWindowSplitRequiredException` + is raised directly by custom code. `None` disables request-window splitting entirely: `read_records` + converts the exception into `RequestWindowSplitNotSupportedException` instead of attempting a + split it cannot perform. + request_window_splitter (Optional[WindowReducible]): The component - normally the stream's own cursor - asked to replace a failing `StreamSlice` with smaller children. `None` has the same effect as - `request_window_reduction` being `None`: reduction is not attempted, and a - `RequestWindowReductionRequiredException` is re-raised as `RequestWindowReductionNotSupportedException`. + `request_window_splitting` being `None`: splitting is not attempted, and a + `RequestWindowSplitRequiredException` is re-raised as `RequestWindowSplitNotSupportedException`. """ requester: Requester @@ -139,8 +139,8 @@ class SimpleRetriever(Retriever): ) post_pagination_filter: Optional[ClientSideIncrementalRecordFilterDecorator] = None page_size_reduction: Optional[PageSizeReduction] = None - request_window_reduction: Optional[RequestWindowReduction] = None - window_reducer: Optional[WindowReducible] = None + request_window_splitting: Optional[RequestWindowSplitting] = None + request_window_splitter: Optional[WindowReducible] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._paginator = self.paginator or NoPagination(parameters=parameters) @@ -563,9 +563,9 @@ def read_records( :return: The records read from the API source """ _slice = stream_slice or StreamSlice(partition={}, cursor_slice={}) # None-check - yield from self._read_records_or_reduce_window(records_schema, _slice, _slice, depth=0) + yield from self._read_records_or_split_request_window(records_schema, _slice, _slice, depth=0) - def _read_records_or_reduce_window( + def _read_records_or_split_request_window( self, records_schema: Mapping[str, Any], stream_slice: StreamSlice, @@ -574,7 +574,7 @@ def _read_records_or_reduce_window( ) -> Iterable[StreamData]: """ Read `stream_slice` to completion, replacing it with smaller children and recursing into each of them in - turn when a `RequestWindowReductionRequiredException` is raised while reading it. + turn when a `RequestWindowSplitRequiredException` is raised while reading it. Each recursive call re-enters this method - and, through it, `_read_pages` - from scratch for the child slice it is given, so every child gets its own paginator token, `PaginationTracker`, and @@ -582,12 +582,12 @@ def _read_records_or_reduce_window( recursion lives inside the single generator `DeclarativePartition.read()` consumes, `PartitionReader.process_partition()` only calls `cursor.close_partition()` once, after this generator is fully exhausted - so a failure anywhere in the recursion (a child that cannot be read, or a window that - cannot be reduced any further) propagates out without ever checkpointing the original partition. + cannot be split any further) propagates out without ever checkpointing the original partition. `original_slice` is threaded through the recursion unchanged so every record yielded, however deep the recursion went to produce it, is re-stamped with the partition's own slice rather than the child slice it was actually read against - see `_reassociate_with_original_slice`. `depth` bounds the recursion - independently of `WindowReducible.reduce_window`'s own no-progress guard: it is enforced here so a + independently of `WindowReducible.split_request_window`'s own no-progress guard: it is enforced here so a misbehaving custom cursor cannot recurse indefinitely regardless of what that guard does or does not catch. """ @@ -616,62 +616,62 @@ def _read_records_or_reduce_window( for record in records: emitted_any = True yield self._reassociate_with_original_slice(record, original_slice) - except RequestWindowReductionRequiredException as exception: - if self.request_window_reduction is None or self.window_reducer is None: - raise RequestWindowReductionNotSupportedException( + except RequestWindowSplitRequiredException as exception: + if self.request_window_splitting is None or self.request_window_splitter is None: + raise RequestWindowSplitNotSupportedException( stream_name=self.name ) from exception if ( emitted_any - and self.request_window_reduction.on_partial_response == OnPartialResponse.FAIL + and self.request_window_splitting.on_partial_response == OnPartialResponse.FAIL ): - # The reduction is safe only because it re-reads a window whose records were not emitted yet. + # The split is safe only because it re-reads a window whose records were not emitted yet. # Re-reading it as smaller children here would duplicate the records already yielded above. raise AirbyteTracedException( - internal_message=f"Stream {self.name} requested a request window reduction after records of the current window had already been emitted", + internal_message=f"Stream {self.name} requested a request window split after records of the current window had already been emitted", message=( - f"Stream {self.name} asked to reduce its request window after some records of it were " - f"already read. Reducing and re-reading the window would duplicate those records. Set " - f"`on_partial_response: ALLOW_REPLAY` on `request_window_reduction` if duplicate records " - f"are acceptable for this stream, or move the REDUCE_REQUEST_WINDOW action so it only " + f"Stream {self.name} asked to split its request window after some records of it were " + f"already read. Splitting and re-reading the window would duplicate those records. Set " + f"`on_partial_response: ALLOW_REPLAY` on `request_window_splitting` if duplicate records " + f"are acceptable for this stream, or move the SPLIT_REQUEST_WINDOW action so it only " f"triggers before any record of the window has been read." ), failure_type=FailureType.config_error, ) from exception - if depth >= self.request_window_reduction.max_split_depth: + if depth >= self.request_window_splitting.max_split_depth: raise AirbyteTracedException( - internal_message=f"Stream {self.name} exceeded the maximum request window split depth of {self.request_window_reduction.max_split_depth} while reducing {original_slice}", + internal_message=f"Stream {self.name} exceeded the maximum request window split depth of {self.request_window_splitting.max_split_depth} while splitting {original_slice}", message=( - f"Stream {self.name} could not reduce its request window to a size the API accepts " - f"within {self.request_window_reduction.max_split_depth} splits. This usually means the " + f"Stream {self.name} could not split its request window to a size the API accepts " + f"within {self.request_window_splitting.max_split_depth} splits. This usually means the " f"stream's cursor is not actually shrinking the window on each split; if it uses a " - f"custom cursor, check its `reduce_window` implementation. Raise " - f"`request_window_reduction.max_split_depth` only if this stream's cursor granularity " + f"custom cursor, check its `split_request_window` implementation. Raise " + f"`request_window_splitting.max_split_depth` only if this stream's cursor granularity " f"genuinely requires more splits than that." ), failure_type=FailureType.config_error, ) from exception - children = self.window_reducer.reduce_window(stream_slice) + children = self.request_window_splitter.split_request_window(stream_slice) if children is None: failure_message = ( f"The API kept rejecting stream {self.name}'s request window even at the smallest window " - f"its cursor allows, or reducing the window further would not make progress." + f"its cursor allows, or splitting the window further would not make progress." ) - if self.request_window_reduction.failure_message: + if self.request_window_splitting.failure_message: failure_message = ( - f"{failure_message} {self.request_window_reduction.failure_message}" + f"{failure_message} {self.request_window_splitting.failure_message}" ) raise AirbyteTracedException( - internal_message=f"Stream {self.name} could not reduce its request window {stream_slice} any further", + internal_message=f"Stream {self.name} could not split its request window {stream_slice} any further", message=failure_message, failure_type=exception.failure_type or FailureType.config_error, ) from exception for child in children: - yield from self._read_records_or_reduce_window( + yield from self._read_records_or_split_request_window( records_schema, child, original_slice, depth + 1 ) @@ -680,7 +680,7 @@ def _reassociate_with_original_slice( record: StreamData, original_slice: StreamSlice ) -> StreamData: """ - Records read while recursing into a reduced child window are parsed against that child's `StreamSlice`, + Records read while recursing into a split child window are parsed against that child's `StreamSlice`, so `RecordSelector` stamps them with the child as `associated_slice`. `DeclarativePartition.read()` passes an already-built `Record` through unchanged rather than re-wrapping it, so left uncorrected the child slice - not the partition's own slice - is what `ConcurrentCursor.observe()` would key its diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index 9b424cd7b7..5f4bb820b4 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -21,12 +21,12 @@ class WindowReducible(Protocol): covering the same range. The component responsible for slicing owns the boundary field names, parsing, output formatting, and - interval semantics (granularity, clamping, comparison), so `reduce_window` lives on that component - not on + interval semantics (granularity, clamping, comparison), so `split_request_window` lives on that component - not on a standalone retriever-side parser - and is checked with `isinstance(slicer, WindowReducible)` wherever a - retriever needs to know whether the stream it is reading supports request-window reduction at all. + retriever needs to know whether the stream it is reading supports request-window splitting at all. """ - def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: + def split_request_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: """ Split `stream_slice` into two or more smaller, non-overlapping child slices that together cover exactly the same range, preserving `partition` and `extra_fields` unchanged. @@ -37,26 +37,26 @@ def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice] `None` as a terminal condition, not retry with the same slice. """ raise NotImplementedError( - "WindowReducible.reduce_window must be implemented by protocol implementers" + "WindowReducible.split_request_window must be implemented by protocol implementers" ) @dataclass(frozen=True) -class RequestWindowReduction: +class RequestWindowSplitting: """ - Configuration for the `request_window_reduction` retriever field - created once per stream and shared, + Configuration for the `request_window_splitting` retriever field - created once per stream and shared, mirroring `PageSizeReduction`. There is no per-partition mutable counterpart to instantiate the way - `PageSizeReducer` is: reducing a window does not accumulate attempts across sibling requests the way - reducing a page size does, since a reduced window is never retried at the same size twice - it is either + `PageSizeReducer` is: splitting a window does not accumulate attempts across sibling requests the way + reducing a page size does, since a split window is never retried at the same size twice - it is either split again (recursion) or the sync fails. All state needed to decide that lives in the `StreamSlice` being - read and the `WindowReducible.reduce_window` result for it, so a plain config object is enough. + read and the `WindowReducible.split_request_window` result for it, so a plain config object is enough. """ on_partial_response: OnPartialResponse = OnPartialResponse.FAIL failure_message: Optional[str] = None # A defense-in-depth bound independent of any specific WindowReducible's own no-progress guard: the retriever # enforces this itself so a misbehaving custom cursor (returning children that do not actually shrink) fails - # deterministically rather than recursing indefinitely. Each reduction bisects a single already-generated + # deterministically rather than recursing indefinitely. Each split bisects a single already-generated # slice - bounded by the cursor's own `step`, not the whole sync range - and real-world APIs that reject # oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step # bisected down to a 12-hour floor needs ~10 halvings (log2(hours in a year / 12) ~= 9.5), which already diff --git a/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte_cdk/sources/streams/concurrent/cursor.py index 77c614fc3c..b4b767ccf8 100644 --- a/airbyte_cdk/sources/streams/concurrent/cursor.py +++ b/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -607,7 +607,7 @@ def reduce_slice_range(self, stream_slice: StreamSlice) -> StreamSlice: else: return stream_slice - def reduce_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: + def split_request_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: """ Split `stream_slice` in half along this cursor's own boundary fields, granularity, and output format. diff --git a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py index cec05f6ce8..4cda91ffb8 100644 --- a/airbyte_cdk/sources/streams/http/error_handlers/response_models.py +++ b/airbyte_cdk/sources/streams/http/error_handlers/response_models.py @@ -20,7 +20,7 @@ class ResponseAction(Enum): RATE_LIMITED = "RATE_LIMITED" REFRESH_TOKEN_THEN_RETRY = "REFRESH_TOKEN_THEN_RETRY" REDUCE_PAGE_SIZE = "REDUCE_PAGE_SIZE" - REDUCE_REQUEST_WINDOW = "REDUCE_REQUEST_WINDOW" + SPLIT_REQUEST_WINDOW = "SPLIT_REQUEST_WINDOW" @dataclass diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 58bc3eef20..7268ae6976 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -53,8 +53,8 @@ rate_limit_default_backoff_handler, user_defined_backoff_handler, ) -from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( - RequestWindowReductionRequiredException, +from airbyte_cdk.sources.streams.http.request_window_split_exception import ( + RequestWindowSplitRequiredException, ) # Imported from the leaf module rather than the package: `protocols` pulls in nothing from the @@ -477,14 +477,14 @@ def _send( and self._message_repository is not None ): formatter = log_formatter - # A response resolving to REDUCE_PAGE_SIZE or REDUCE_REQUEST_WINDOW is not a page of the stream: + # A response resolving to REDUCE_PAGE_SIZE or SPLIT_REQUEST_WINDOW is not a page of the stream: # the retriever discards it and re-issues the request with a smaller page size or a narrower # window. Logging it as an auxiliary request keeps it visible in the Connector Builder while # keeping it out of the per-slice page count, which would otherwise report "limit reached" on a # read that only retried. log_as_auxiliary = error_resolution.response_action in ( ResponseAction.REDUCE_PAGE_SIZE, - ResponseAction.REDUCE_REQUEST_WINDOW, + ResponseAction.SPLIT_REQUEST_WINDOW, ) self._message_repository.log_message( Level.DEBUG, @@ -576,8 +576,8 @@ def _handle_error_resolution( stream_name=self._name, error_message=error_resolution.error_message ) - if error_resolution.response_action == ResponseAction.REDUCE_REQUEST_WINDOW: - raise RequestWindowReductionRequiredException( + if error_resolution.response_action == ResponseAction.SPLIT_REQUEST_WINDOW: + raise RequestWindowSplitRequiredException( stream_name=self._name, error_message=error_resolution.error_message, failure_type=error_resolution.failure_type, diff --git a/airbyte_cdk/sources/streams/http/request_window_reduction_exception.py b/airbyte_cdk/sources/streams/http/request_window_split_exception.py similarity index 54% rename from airbyte_cdk/sources/streams/http/request_window_reduction_exception.py rename to airbyte_cdk/sources/streams/http/request_window_split_exception.py index 961b3d3986..4d88ba78f6 100644 --- a/airbyte_cdk/sources/streams/http/request_window_reduction_exception.py +++ b/airbyte_cdk/sources/streams/http/request_window_split_exception.py @@ -8,22 +8,22 @@ from airbyte_cdk.utils.traced_exception import AirbyteTracedException -class RequestWindowReductionRequiredException(AirbyteTracedException): +class RequestWindowSplitRequiredException(AirbyteTracedException): """ - Signals that the request window backing the current read must be reduced and retried as smaller children. + Signals that the request window backing the current read must be split and retried as smaller children. This is control flow rather than a failure: `SimpleRetriever.read_records` catches it and replaces the failing window with ordered, non-overlapping child windows produced by the stream's cursor. Unlike `PageSizeReductionRequiredException`, which is only ever raised by `HttpClient`, this exception is public and is meant to be raised directly by custom code - a `CustomRetriever`, a `CustomExtractor`, or code - handling a transport or streamed-decoding failure - so a connector can opt into window reduction for a - failure the built-in `HttpResponseFilter`/`ResponseAction.REDUCE_REQUEST_WINDOW` path cannot see, without + handling a transport or streamed-decoding failure - so a connector can opt into window splitting for a + failure the built-in `HttpResponseFilter`/`ResponseAction.SPLIT_REQUEST_WINDOW` path cannot see, without the CDK needing to know about that connector's specific exception types. The classifying error's `failure_type` and `error_message` are preserved on this exception so the eventual - terminal failure - if the window cannot be reduced any further - reports the original cause rather than a - generic "please reduce" message. Callers with an underlying exception should chain it the normal Python way, - `raise RequestWindowReductionRequiredException(...) from original_exception`, so the original traceback is + terminal failure - if the window cannot be split any further - reports the original cause rather than a + generic "please split" message. Callers with an underlying exception should chain it the normal Python way, + `raise RequestWindowSplitRequiredException(...) from original_exception`, so the original traceback is not lost. """ @@ -36,26 +36,26 @@ def __init__( stream = f" of stream {stream_name}" if stream_name else "" detail = f": {error_message}" if error_message else "" super().__init__( - internal_message=f"A request window reduction was requested{stream}{detail}", - message=f"The API rejected the current request window{stream} and requires a smaller one. If this message ends a sync, the stream is not set up to reduce its window: add `request_window_reduction` to its retriever, or remove the `REDUCE_REQUEST_WINDOW` action from its error handler.", + internal_message=f"A request window split was requested{stream}{detail}", + message=f"The API rejected the current request window{stream} and requires a smaller one. If this message ends a sync, the stream is not set up to split its window: add `request_window_splitting` to its retriever, or remove the `SPLIT_REQUEST_WINDOW` action from its error handler.", failure_type=failure_type or FailureType.config_error, ) -class RequestWindowReductionNotSupportedException(AirbyteTracedException): +class RequestWindowSplitNotSupportedException(AirbyteTracedException): """ - Raised when a window reduction is requested but the retriever cannot honor it. + Raised when a window split is requested but the retriever cannot honor it. The factory rejects this at config time wherever it can see the error handler, so reaching this means the signal came from somewhere it cannot inspect - a custom error handler, a custom requester/retriever raising - `RequestWindowReductionRequiredException` directly, or a stream whose cursor does not implement + `RequestWindowSplitRequiredException` directly, or a stream whose cursor does not implement `WindowReducible` (for example, a non-datetime or non-incremental slicer). """ def __init__(self, stream_name: Optional[str] = None) -> None: stream = f"Stream {stream_name}" if stream_name else "The stream" super().__init__( - internal_message=f"A request window reduction was requested for stream {stream_name} but its retriever defines no `request_window_reduction`, or its cursor does not support window reduction. The action is only supported on a SimpleRetriever whose cursor implements WindowReducible and that defines `request_window_reduction`.", - message=f"{stream} resolves to a request window reduction but is not set up to reduce its window. Add `request_window_reduction` to the stream's retriever (this requires a datetime-based incremental cursor with `cursor_granularity` set), or remove the `REDUCE_REQUEST_WINDOW` action from its error handler.", + internal_message=f"A request window split was requested for stream {stream_name} but its retriever defines no `request_window_splitting`, or its cursor does not support window splitting. The action is only supported on a SimpleRetriever whose cursor implements WindowReducible and that defines `request_window_splitting`.", + message=f"{stream} resolves to a request window split but is not set up to split its window. Add `request_window_splitting` to the stream's retriever (this requires a datetime-based incremental cursor with `cursor_granularity` set), or remove the `SPLIT_REQUEST_WINDOW` action from its error handler.", failure_type=FailureType.config_error, ) diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 01acb79d90..de0caaf598 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -207,7 +207,7 @@ ) from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, - RequestWindowReduction, + RequestWindowSplitting, WindowReducible, ) from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader, JsonFileSchemaLoader @@ -7565,7 +7565,7 @@ def test_given_file_uploader_and_page_size_reduction_then_raise(): {incremental_sync} retriever: type: SimpleRetriever - {request_window_reduction} + {request_window_splitting} requester: type: HttpRequester url_base: "https://airbyte.io" @@ -7594,14 +7594,14 @@ def test_given_file_uploader_and_page_size_reduction_then_raise(): datetime_format: "%Y-%m-%dT%H:%M:%SZ\"""" -def _request_window_reduction_stream( - request_window_reduction="request_window_reduction:\n type: RequestWindowReduction", - action="REDUCE_REQUEST_WINDOW", +def _request_window_splitting_stream( + request_window_splitting="request_window_splitting:\n type: RequestWindowSplitting", + action="SPLIT_REQUEST_WINDOW", incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, ): content = _REQUEST_WINDOW_REDUCTION_STREAM.format( incremental_sync=incremental_sync, - request_window_reduction=request_window_reduction, + request_window_splitting=request_window_splitting, action=action, ) stream_manifest = transformer.propagate_types_and_parameters( @@ -7612,22 +7612,22 @@ def _request_window_reduction_stream( ) -def test_given_request_window_reduction_then_create_retriever_with_defaults(): - retriever = get_retriever(_request_window_reduction_stream()) +def test_given_request_window_splitting_then_create_retriever_with_defaults(): + retriever = get_retriever(_request_window_splitting_stream()) - assert retriever.request_window_reduction == RequestWindowReduction( + assert retriever.request_window_splitting == RequestWindowSplitting( on_partial_response=OnPartialResponse.FAIL, failure_message=None, ) - assert isinstance(retriever.window_reducer, WindowReducible) + assert isinstance(retriever.request_window_splitter, WindowReducible) -def test_given_request_window_reduction_values_then_create_retriever_with_those_values(): +def test_given_request_window_splitting_values_then_create_retriever_with_those_values(): retriever = get_retriever( - _request_window_reduction_stream( - request_window_reduction=( - "request_window_reduction:\n" - " type: RequestWindowReduction\n" + _request_window_splitting_stream( + request_window_splitting=( + "request_window_splitting:\n" + " type: RequestWindowSplitting\n" " on_partial_response: ALLOW_REPLAY\n" " failure_message: Lower time_window so that each request covers less data.\n" " max_split_depth: 20" @@ -7635,44 +7635,44 @@ def test_given_request_window_reduction_values_then_create_retriever_with_those_ ) ) - assert retriever.request_window_reduction == RequestWindowReduction( + assert retriever.request_window_splitting == RequestWindowSplitting( on_partial_response=OnPartialResponse.ALLOW_REPLAY, failure_message="Lower time_window so that each request covers less data.", max_split_depth=20, ) -def test_given_no_request_window_reduction_then_retriever_has_none(): +def test_given_no_request_window_splitting_then_retriever_has_none(): retriever = get_retriever( - _request_window_reduction_stream(request_window_reduction="", action="RETRY") + _request_window_splitting_stream(request_window_splitting="", action="RETRY") ) - assert retriever.request_window_reduction is None - assert retriever.window_reducer is None + assert retriever.request_window_splitting is None + assert retriever.request_window_splitter is None -def test_given_reduce_request_window_action_without_request_window_reduction_then_raise(): +def test_given_split_request_window_action_without_request_window_splitting_then_raise(): with pytest.raises(ValueError) as exception: - _request_window_reduction_stream(request_window_reduction="") + _request_window_splitting_stream(request_window_splitting="") - assert "REDUCE_REQUEST_WINDOW" in str(exception.value) + assert "SPLIT_REQUEST_WINDOW" in str(exception.value) -def test_given_request_window_reduction_without_reduce_request_window_action_then_warn(caplog): +def test_given_request_window_splitting_without_split_request_window_action_then_warn(caplog): """ A `CustomErrorHandler` can resolve to the action without being inspectable, and custom code can raise - `RequestWindowReductionRequiredException` directly without going through any error handler at all - so this + `RequestWindowSplitRequiredException` directly without going through any error handler at all - so this cannot raise. It must not stay silent either: the feature would be dead on a stream that only defines the block because it expects to need it. """ with caplog.at_level(logging.WARNING, logger="airbyte.model_to_component_factory"): - retriever = get_retriever(_request_window_reduction_stream(action="RETRY")) + retriever = get_retriever(_request_window_splitting_stream(action="RETRY")) - assert retriever.request_window_reduction == RequestWindowReduction() - assert "REDUCE_REQUEST_WINDOW" in caplog.text + assert retriever.request_window_splitting == RequestWindowSplitting() + assert "SPLIT_REQUEST_WINDOW" in caplog.text -def test_given_composite_error_handler_with_reduce_request_window_action_then_require_request_window_reduction(): +def test_given_composite_error_handler_with_split_request_window_action_then_require_request_window_splitting(): """The action can be nested in a CompositeErrorHandler, which the guard has to recurse into.""" stream_definition = { "type": "DeclarativeStream", @@ -7714,7 +7714,7 @@ def test_given_composite_error_handler_with_reduce_request_window_action_then_re { "type": "HttpResponseFilter", "http_codes": [400], - "action": "REDUCE_REQUEST_WINDOW", + "action": "SPLIT_REQUEST_WINDOW", } ], }, @@ -7735,10 +7735,10 @@ def test_given_composite_error_handler_with_reduce_request_window_action_then_re config={}, ) - assert "REDUCE_REQUEST_WINDOW" in str(exception.value) + assert "SPLIT_REQUEST_WINDOW" in str(exception.value) # and the same manifest with the block present builds - stream_definition["retriever"]["request_window_reduction"] = {"type": "RequestWindowReduction"} + stream_definition["retriever"]["request_window_splitting"] = {"type": "RequestWindowSplitting"} retriever = get_retriever( factory.create_component( model_type=DeclarativeStreamModel, @@ -7746,19 +7746,19 @@ def test_given_composite_error_handler_with_reduce_request_window_action_then_re config={}, ) ) - assert retriever.request_window_reduction == RequestWindowReduction() + assert retriever.request_window_splitting == RequestWindowSplitting() -def test_given_no_incremental_sync_and_request_window_reduction_then_raise(): +def test_given_no_incremental_sync_and_request_window_splitting_then_raise(): with pytest.raises(ValueError) as exception: - _request_window_reduction_stream(incremental_sync="") + _request_window_splitting_stream(incremental_sync="") assert "cursor_granularity" in str(exception.value) or "WindowReducible" in str(exception.value) def test_given_datetime_based_cursor_without_cursor_granularity_then_raise(): with pytest.raises(ValueError) as exception: - _request_window_reduction_stream( + _request_window_splitting_stream( incremental_sync=( 'incremental_sync:\n type: DatetimeBasedCursor\n start_datetime: "2024-01-01T00:00:00Z"\n' ' end_datetime: "2024-01-31T00:00:00Z"\n step: "P7D"\n cursor_field: "updated_at"\n' @@ -7769,14 +7769,14 @@ def test_given_datetime_based_cursor_without_cursor_granularity_then_raise(): assert "cursor_granularity" in str(exception.value) -def test_given_incrementing_count_cursor_and_request_window_reduction_then_raise(): +def test_given_incrementing_count_cursor_and_request_window_splitting_then_raise(): """ `IncrementingCountCursor` builds a `ConcurrentCursor` too, but not one carrying the datetime boundary - fields, granularity, or formatting `WindowReducible.reduce_window` needs, so it must be rejected the same + fields, granularity, or formatting `WindowReducible.split_request_window` needs, so it must be rejected the same way a stream with no incremental_sync at all is. """ with pytest.raises(ValueError) as exception: - _request_window_reduction_stream( + _request_window_splitting_stream( incremental_sync=( "incremental_sync:\n type: IncrementingCountCursor\n cursor_field: id" ) @@ -7785,11 +7785,11 @@ def test_given_incrementing_count_cursor_and_request_window_reduction_then_raise assert "cursor_granularity" in str(exception.value) -def test_given_query_properties_and_request_window_reduction_then_raise(): +def test_given_query_properties_and_request_window_splitting_then_raise(): content = _REQUEST_WINDOW_REDUCTION_STREAM.format( incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, - request_window_reduction="request_window_reduction:\n type: RequestWindowReduction", - action="REDUCE_REQUEST_WINDOW", + request_window_splitting="request_window_splitting:\n type: RequestWindowSplitting", + action="SPLIT_REQUEST_WINDOW", ).replace( " http_method: GET\n", " http_method: GET\n" @@ -7809,12 +7809,12 @@ def test_given_query_properties_and_request_window_reduction_then_raise(): assert "query properties" in str(exception.value) -def test_given_file_uploader_and_request_window_reduction_then_raise(): +def test_given_file_uploader_and_request_window_splitting_then_raise(): content = ( _REQUEST_WINDOW_REDUCTION_STREAM.format( incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, - request_window_reduction="request_window_reduction:\n type: RequestWindowReduction", - action="REDUCE_REQUEST_WINDOW", + request_window_splitting="request_window_splitting:\n type: RequestWindowSplitting", + action="SPLIT_REQUEST_WINDOW", ) + """file_uploader: type: FileUploader @@ -7839,7 +7839,7 @@ def test_given_file_uploader_and_request_window_reduction_then_raise(): assert "file_uploader" in str(exception.value) -def test_given_lazy_read_pointer_and_request_window_reduction_then_raise(): +def test_given_lazy_read_pointer_and_request_window_splitting_then_raise(): """ A LazySimpleRetriever reads records embedded in the parent's own pages rather than requesting a window of its own, so there is nothing for a reduced child window to be read from. @@ -7854,7 +7854,7 @@ def test_given_lazy_read_pointer_and_request_window_reduction_then_raise(): }, "retriever": { "type": "SimpleRetriever", - "request_window_reduction": {"type": "RequestWindowReduction"}, + "request_window_splitting": {"type": "RequestWindowSplitting"}, "requester": { "type": "HttpRequester", "url_base": "https://api.test.com", @@ -7866,7 +7866,7 @@ def test_given_lazy_read_pointer_and_request_window_reduction_then_raise(): { "type": "HttpResponseFilter", "http_codes": [400], - "action": "REDUCE_REQUEST_WINDOW", + "action": "SPLIT_REQUEST_WINDOW", } ], }, @@ -7920,11 +7920,11 @@ def test_given_lazy_read_pointer_and_request_window_reduction_then_raise(): config={}, ) - assert "request_window_reduction" in str(exception.value) + assert "request_window_splitting" in str(exception.value) assert "lazily" in str(exception.value) -def test_given_login_requester_with_reduce_request_window_action_then_raise(): +def test_given_login_requester_with_split_request_window_action_then_raise(): """ Only the main requester of a SimpleRetriever reads a cursor-sliced window, so the action is rejected on a SessionTokenAuthenticator's login_requester even though it is schema-legal there. @@ -7954,7 +7954,7 @@ def test_given_login_requester_with_reduce_request_window_action_then_raise(): { "type": "HttpResponseFilter", "http_codes": [400], - "action": "REDUCE_REQUEST_WINDOW", + "action": "SPLIT_REQUEST_WINDOW", } ], }, @@ -7985,7 +7985,7 @@ def test_given_login_requester_with_reduce_request_window_action_then_raise(): config={}, ) - assert "REDUCE_REQUEST_WINDOW" in str(exception.value) + assert "SPLIT_REQUEST_WINDOW" in str(exception.value) assert "login_requester" in str(exception.value) diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py index d5e901d670..78d3b7ee8a 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_composite_error_handler.py @@ -374,14 +374,14 @@ def test_given_reduce_page_size_when_interpret_response_then_stop_at_the_matchin assert failing_handler.interpret_response.call_count == 0 -def test_given_reduce_request_window_when_interpret_response_then_stop_at_the_matching_handler(): +def test_given_split_request_window_when_interpret_response_then_stop_at_the_matching_handler(): """ Mirrors test_given_reduce_page_size_when_interpret_response_then_stop_at_the_matching_handler: without the short circuit, a later handler resolving to FAIL would win over the request window reduction. """ reducing_handler = MagicMock() reducing_handler.interpret_response.return_value = ErrorResolution( - response_action=ResponseAction.REDUCE_REQUEST_WINDOW, + response_action=ResponseAction.SPLIT_REQUEST_WINDOW, failure_type=FailureType.config_error, ) failing_handler = MagicMock() @@ -394,13 +394,13 @@ def test_given_reduce_request_window_when_interpret_response_then_stop_at_the_ma error_resolution = error_handler.interpret_response(MagicMock()) - assert error_resolution.response_action == ResponseAction.REDUCE_REQUEST_WINDOW + assert error_resolution.response_action == ResponseAction.SPLIT_REQUEST_WINDOW assert failing_handler.interpret_response.call_count == 0 def test_given_unrelated_response_error_when_interpret_response_then_behavior_is_unaffected(): """ - Adding REDUCE_REQUEST_WINDOW to the short-circuit list must not change behavior for actions that were + Adding SPLIT_REQUEST_WINDOW to the short-circuit list must not change behavior for actions that were never in it: RATE_LIMITED still does not short-circuit, so a later handler's FAIL still wins over an earlier handler's RATE_LIMITED, exactly as before this action existed. """ diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index d585b9d25d..ac9ab62276 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -55,7 +55,7 @@ from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, - RequestWindowReduction, + RequestWindowSplitting, WindowReducible, ) from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( @@ -64,9 +64,9 @@ from airbyte_cdk.sources.streams.http.pagination_reset_exception import ( PaginationResetRequiredException, ) -from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( - RequestWindowReductionNotSupportedException, - RequestWindowReductionRequiredException, +from airbyte_cdk.sources.streams.http.request_window_split_exception import ( + RequestWindowSplitNotSupportedException, + RequestWindowSplitRequiredException, ) from airbyte_cdk.sources.types import Record, StreamSlice from airbyte_cdk.sources.utils.transform import TransformConfig, TypeTransformer @@ -2013,7 +2013,7 @@ def test_given_no_data_feed_when_read_records_then_emit_every_record(): assert actual_records == page -# --- request_window_reduction --- +# --- request_window_splitting --- A_WINDOW_SLICE = StreamSlice( cursor_slice={"start_time": "2024-01-01T00:00:00Z", "end_time": "2024-01-01T23:59:59Z"}, @@ -2029,12 +2029,12 @@ def test_given_no_data_feed_when_read_records_then_emit_every_record(): ) -def _request_window_reduction_retriever( +def _request_window_splitting_retriever( requester: Requester, paginator: Paginator, record_selector: HttpSelector, - window_reducer: WindowReducible, - request_window_reduction: Optional[RequestWindowReduction] = None, + request_window_splitter: WindowReducible, + request_window_splitting: Optional[RequestWindowSplitting] = None, ) -> SimpleRetriever: return SimpleRetriever( name=A_STREAM_NAME, @@ -2042,17 +2042,17 @@ def _request_window_reduction_retriever( requester=requester, record_selector=record_selector, paginator=paginator, - request_window_reduction=request_window_reduction or RequestWindowReduction(), - window_reducer=window_reducer, + request_window_splitting=request_window_splitting or RequestWindowSplitting(), + request_window_splitter=request_window_splitter, parameters={}, config={}, ) -def test_given_request_window_reduction_when_read_records_then_split_and_read_both_children(): +def test_given_request_window_splitting_when_read_records_then_split_and_read_both_children(): requester = Mock(spec=Requester) requester.send_request.side_effect = [ - RequestWindowReductionRequiredException(), + RequestWindowSplitRequiredException(), [{"id": 1}], [{"id": 2}], ] @@ -2062,17 +2062,17 @@ def test_given_request_window_reduction_when_read_records_then_split_and_read_bo paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [{"id": 1}, {"id": 2}] - window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) assert requester.send_request.call_count == 3 assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ A_WINDOW_SLICE, @@ -2088,8 +2088,8 @@ def test_given_only_one_child_needs_further_reduction_when_read_records_then_rec """ requester = Mock(spec=Requester) requester.send_request.side_effect = [ - RequestWindowReductionRequiredException(), # top-level window - RequestWindowReductionRequiredException(), # first half, rejected again + RequestWindowSplitRequiredException(), # top-level window + RequestWindowSplitRequiredException(), # first half, rejected again [{"id": 1}], # first quarter of the first half [{"id": 2}], # second quarter of the first half [{"id": 3}], # second half, succeeds directly @@ -2108,20 +2108,20 @@ def test_given_only_one_child_needs_further_reduction_when_read_records_then_rec cursor_slice={"start_time": "2024-01-01T06:00:00Z", "end_time": "2024-01-01T11:59:59Z"}, partition={}, ) - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.side_effect = [ + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.side_effect = [ [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE], [a_quarter_slice, another_quarter_slice], ] - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [{"id": 1}, {"id": 2}, {"id": 3}] - assert window_reducer.reduce_window.call_args_list == [ + assert request_window_splitter.split_request_window.call_args_list == [ ((A_WINDOW_SLICE,),), ((A_FIRST_HALF_SLICE,),), ] @@ -2134,9 +2134,9 @@ def test_given_only_one_child_needs_further_reduction_when_read_records_then_rec ] -def test_given_no_request_window_reduction_configured_when_reduction_required_then_raise_not_supported(): +def test_given_no_request_window_splitting_configured_when_reduction_required_then_raise_not_supported(): requester = Mock(spec=Requester) - requester.send_request.side_effect = RequestWindowReductionRequiredException() + requester.send_request.side_effect = RequestWindowSplitRequiredException() record_selector = Mock(spec=HttpSelector) paginator = _mock_paginator() paginator.get_initial_token.return_value = None @@ -2151,13 +2151,13 @@ def test_given_no_request_window_reduction_configured_when_reduction_required_th config={}, ) - with pytest.raises(RequestWindowReductionNotSupportedException): + with pytest.raises(RequestWindowSplitNotSupportedException): list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) -def test_given_no_window_reducer_when_reduction_required_then_raise_not_supported(): +def test_given_no_request_window_splitter_when_reduction_required_then_raise_not_supported(): requester = Mock(spec=Requester) - requester.send_request.side_effect = RequestWindowReductionRequiredException() + requester.send_request.side_effect = RequestWindowSplitRequiredException() record_selector = Mock(spec=HttpSelector) paginator = _mock_paginator() paginator.get_initial_token.return_value = None @@ -2168,13 +2168,13 @@ def test_given_no_window_reducer_when_reduction_required_then_raise_not_supporte requester=requester, record_selector=record_selector, paginator=paginator, - request_window_reduction=RequestWindowReduction(), - window_reducer=None, + request_window_splitting=RequestWindowSplitting(), + request_window_splitter=None, parameters={}, config={}, ) - with pytest.raises(RequestWindowReductionNotSupportedException): + with pytest.raises(RequestWindowSplitNotSupportedException): list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) @@ -2185,28 +2185,28 @@ def test_given_records_already_emitted_and_fail_policy_when_reduction_requested_ must raise a hard config_error instead of silently reducing. """ requester = Mock(spec=Requester) - requester.send_request.side_effect = [Mock(), RequestWindowReductionRequiredException()] + requester.send_request.side_effect = [Mock(), RequestWindowSplitRequiredException()] record_selector = Mock(spec=HttpSelector) record_selector.select_records.return_value = [{"id": 1}] paginator = _mock_paginator() paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = {"next_page_token": "page2"} - window_reducer = Mock(spec=WindowReducible) + request_window_splitter = Mock(spec=WindowReducible) - retriever = _request_window_reduction_retriever( + retriever = _request_window_splitting_retriever( requester, paginator, record_selector, - window_reducer, - request_window_reduction=RequestWindowReduction(on_partial_response=OnPartialResponse.FAIL), + request_window_splitter, + request_window_splitting=RequestWindowSplitting(on_partial_response=OnPartialResponse.FAIL), ) with pytest.raises(AirbyteTracedException) as exception: list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert exception.value.failure_type == FailureType.config_error - window_reducer.reduce_window.assert_not_called() + request_window_splitter.split_request_window.assert_not_called() def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_requested_then_reduce_anyway(): @@ -2217,7 +2217,7 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re requester = Mock(spec=Requester) requester.send_request.side_effect = [ Mock(), - RequestWindowReductionRequiredException(), + RequestWindowSplitRequiredException(), [{"id": 2}], [{"id": 3}], ] @@ -2227,15 +2227,15 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re paginator.get_initial_token.return_value = None paginator.next_page_token.side_effect = [{"next_page_token": "page2"}, None, None] - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] - retriever = _request_window_reduction_retriever( + retriever = _request_window_splitting_retriever( requester, paginator, record_selector, - window_reducer, - request_window_reduction=RequestWindowReduction( + request_window_splitter, + request_window_splitting=RequestWindowSplitting( on_partial_response=OnPartialResponse.ALLOW_REPLAY ), ) @@ -2245,32 +2245,32 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re # the record from the page emitted before the reduction signal is included once more: ALLOW_REPLAY # accepts this at-least-once duplication rather than silently dropping it or failing the sync. assert records == [{"id": 1}, {"id": 2}, {"id": 3}] - window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) -def test_given_window_reducer_returns_none_when_reduction_requested_then_raise_terminal_error(): +def test_given_request_window_splitter_returns_none_when_reduction_requested_then_raise_terminal_error(): """ - `WindowReducible.reduce_window` returning `None` means the window cannot be split any further - either + `WindowReducible.split_request_window` returning `None` means the window cannot be split any further - either the granularity floor was reached or reducing would not make progress. Either way this must terminate the sync deterministically rather than loop or silently drop the window. """ requester = Mock(spec=Requester) - requester.send_request.side_effect = RequestWindowReductionRequiredException( + requester.send_request.side_effect = RequestWindowSplitRequiredException( failure_type=FailureType.transient_error ) record_selector = Mock(spec=HttpSelector) paginator = _mock_paginator() paginator.get_initial_token.return_value = None - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.return_value = None + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = None - retriever = _request_window_reduction_retriever( + retriever = _request_window_splitting_retriever( requester, paginator, record_selector, - window_reducer, - request_window_reduction=RequestWindowReduction( + request_window_splitter, + request_window_splitting=RequestWindowSplitting( failure_message="Lower time_window so that each request covers less data." ), ) @@ -2281,27 +2281,27 @@ def test_given_window_reducer_returns_none_when_reduction_requested_then_raise_t # the classifying error's own failure_type is preserved rather than defaulting to config_error assert exception.value.failure_type == FailureType.transient_error assert "Lower time_window so that each request covers less data." in exception.value.message - window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) def test_given_empty_children_list_when_reduction_requested_then_emit_no_records_and_do_not_loop(): requester = Mock(spec=Requester) - requester.send_request.side_effect = RequestWindowReductionRequiredException() + requester.send_request.side_effect = RequestWindowSplitRequiredException() record_selector = Mock(spec=HttpSelector) paginator = _mock_paginator() paginator.get_initial_token.return_value = None - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.return_value = [] + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = [] - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [] - window_reducer.reduce_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates(): @@ -2311,9 +2311,9 @@ def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates """ requester = Mock(spec=Requester) requester.send_request.side_effect = [ - RequestWindowReductionRequiredException(), + RequestWindowSplitRequiredException(), [{"id": 1}], - RequestWindowReductionRequiredException(), + RequestWindowSplitRequiredException(), ] record_selector = Mock(spec=HttpSelector) record_selector.select_records.return_value = [{"id": 1}] @@ -2321,14 +2321,14 @@ def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.side_effect = [ + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.side_effect = [ [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE], None, ] - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) emitted = [] @@ -2338,13 +2338,13 @@ def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates # the first child's records were already emitted to the caller before the second child failed assert emitted == [{"id": 1}] - assert window_reducer.reduce_window.call_args_list == [ + assert request_window_splitter.split_request_window.call_args_list == [ ((A_WINDOW_SLICE,),), ((A_SECOND_HALF_SLICE,),), ] -def test_given_request_window_reduction_when_read_records_then_each_child_gets_fresh_pagination_state(): +def test_given_request_window_splitting_when_read_records_then_each_child_gets_fresh_pagination_state(): """ Each recursive call re-enters read_records/_read_pages from scratch for its child slice, so the paginator's initial token is requested again for every child rather than resuming from wherever the @@ -2352,7 +2352,7 @@ def test_given_request_window_reduction_when_read_records_then_each_child_gets_f """ requester = Mock(spec=Requester) requester.send_request.side_effect = [ - RequestWindowReductionRequiredException(), + RequestWindowSplitRequiredException(), [{"id": 1}], [{"id": 2}], ] @@ -2362,11 +2362,11 @@ def test_given_request_window_reduction_when_read_records_then_each_child_gets_f paginator.get_initial_token.return_value = "initial token" paginator.next_page_token.return_value = None - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) @@ -2413,7 +2413,7 @@ def test_reassociate_with_original_slice_passes_through_non_record_stream_data() assert result is message -def test_given_request_window_reduction_when_split_then_records_are_associated_with_original_slice(): +def test_given_request_window_splitting_when_split_then_records_are_associated_with_original_slice(): """ Regression test: `ConcurrentCursor.observe()` keys its per-partition bookkeeping by `Record.associated_slice`, but `ConcurrentCursor.close_partition()` always looks that bookkeeping up by the partition's own (original) @@ -2424,7 +2424,7 @@ def test_given_request_window_reduction_when_split_then_records_are_associated_w """ requester = Mock(spec=Requester) requester.send_request.side_effect = [ - RequestWindowReductionRequiredException(), + RequestWindowSplitRequiredException(), Mock(), Mock(), ] @@ -2444,11 +2444,11 @@ def select_records(*, response, stream_slice, **kwargs): paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) @@ -2462,18 +2462,18 @@ def select_records(*, response, stream_slice, **kwargs): } -def test_given_exception_raised_during_record_extraction_when_read_then_still_reduce_window(): +def test_given_exception_raised_during_record_extraction_when_read_then_still_split_request_window(): """ Google Ads/Iterable-style exception-driven cases fail during response streaming/decoding, not at the point the request is sent - unlike PayPal, where the API rejects the request outright before any body is read. The catch boundary must cover the entire read, not only the requester call, so custom code raising - `RequestWindowReductionRequiredException` from anywhere in the pipeline is still honored. + `RequestWindowSplitRequiredException` from anywhere in the pipeline is still honored. """ requester = Mock(spec=Requester) requester.send_request.side_effect = [Mock(), Mock(), Mock()] record_selector = Mock(spec=HttpSelector) record_selector.select_records.side_effect = [ - RequestWindowReductionRequiredException(), + RequestWindowSplitRequiredException(), [{"id": 1}], [{"id": 2}], ] @@ -2481,11 +2481,11 @@ def test_given_exception_raised_during_record_extraction_when_read_then_still_re paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) @@ -2500,21 +2500,21 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er recursing without bound. """ requester = Mock(spec=Requester) - requester.send_request.side_effect = RequestWindowReductionRequiredException() + requester.send_request.side_effect = RequestWindowSplitRequiredException() record_selector = Mock(spec=HttpSelector) paginator = _mock_paginator() paginator.get_initial_token.return_value = None - window_reducer = Mock(spec=WindowReducible) + request_window_splitter = Mock(spec=WindowReducible) # never signals "no further progress possible" - always splits into two children identical to the parent - window_reducer.reduce_window.side_effect = lambda stream_slice: [stream_slice, stream_slice] + request_window_splitter.split_request_window.side_effect = lambda stream_slice: [stream_slice, stream_slice] - retriever = _request_window_reduction_retriever( + retriever = _request_window_splitting_retriever( requester, paginator, record_selector, - window_reducer, - request_window_reduction=RequestWindowReduction(max_split_depth=3), + request_window_splitter, + request_window_splitting=RequestWindowSplitting(max_split_depth=3), ) with pytest.raises(AirbyteTracedException) as exception: @@ -2523,15 +2523,15 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er assert exception.value.failure_type == FailureType.config_error assert "maximum request window split depth" in exception.value.internal_message assert "within 3 splits" in exception.value.message - # the single leftmost recursion path is explored to depth 3 (three reduce_window calls, at depth 0, 1, and + # the single leftmost recursion path is explored to depth 3 (three split_request_window calls, at depth 0, 1, and # 2) before the cap stops a fourth call at depth 3 - the exception then propagates immediately, so sibling # branches at shallower depths are never explored - assert window_reducer.reduce_window.call_count == 3 + assert request_window_splitter.split_request_window.call_count == 3 def test_given_partitions_read_concurrently_then_window_reduction_isolates_between_partitions(): """ - One retriever instance - and one shared `window_reducer` (the stream's cursor) - is used for every partition + One retriever instance - and one shared `request_window_splitter` (the stream's cursor) - is used for every partition of a stream, so a reduction triggered while reading one partition must not affect a concurrently-running read of another partition at all: the other partition must complete normally, and the reducer must only ever be asked to split the slice that actually needed it. @@ -2548,12 +2548,12 @@ def test_given_partitions_read_concurrently_then_window_reduction_isolates_betwe def send_request(*, stream_slice, **kwargs): # only the original top-level slice_a is rejected (its children must succeed, or the recursion would - # never terminate); identity, not partition id, distinguishes it from the children reduce_window builds + # never terminate); identity, not partition id, distinguishes it from the children split_request_window builds if stream_slice is slice_a: # only raise once b's own, unrelated request is known to be in flight, so the two reads # genuinely overlap rather than running one after the other assert b_requested.wait(timeout=10) - raise RequestWindowReductionRequiredException() + raise RequestWindowSplitRequiredException() if stream_slice is slice_b: b_requested.set() return Mock() @@ -2575,7 +2575,7 @@ def select_records(*, response, stream_slice, **kwargs): paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - def reduce_window(stream_slice): + def split_request_window(stream_slice): # preserve the parent's own partition, matching what a real WindowReducible would do return [ StreamSlice( @@ -2594,11 +2594,11 @@ def reduce_window(stream_slice): ), ] - window_reducer = Mock(spec=WindowReducible) - window_reducer.reduce_window.side_effect = reduce_window + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.side_effect = split_request_window - retriever = _request_window_reduction_retriever( - requester, paginator, record_selector, window_reducer + retriever = _request_window_splitting_retriever( + requester, paginator, record_selector, request_window_splitter ) with ThreadPoolExecutor(max_workers=2) as executor: @@ -2609,4 +2609,4 @@ def reduce_window(stream_slice): assert records_b == [{"partition": "b", "start": "2024-02-01T00:00:00Z"}] assert len(records_a) == 2 - window_reducer.reduce_window.assert_called_once_with(slice_a) + request_window_splitter.split_request_window.assert_called_once_with(slice_a) diff --git a/unit_tests/sources/declarative/test_concurrent_declarative_source.py b/unit_tests/sources/declarative/test_concurrent_declarative_source.py index 250a5b6452..fd6393762a 100644 --- a/unit_tests/sources/declarative/test_concurrent_declarative_source.py +++ b/unit_tests/sources/declarative/test_concurrent_declarative_source.py @@ -5167,7 +5167,7 @@ def test_given_reductions_exhausted_when_read_then_emit_a_transient_error(): ) -def _request_window_reduction_manifest(): +def _request_window_splitting_manifest(): return { "version": "0.34.2", "type": "DeclarativeSource", @@ -5201,7 +5201,7 @@ def _request_window_reduction_manifest(): }, "retriever": { "type": "SimpleRetriever", - "request_window_reduction": {"type": "RequestWindowReduction"}, + "request_window_splitting": {"type": "RequestWindowSplitting"}, "requester": { "type": "HttpRequester", "url_base": "https://example.org", @@ -5213,7 +5213,7 @@ def _request_window_reduction_manifest(): { "type": "HttpResponseFilter", "http_codes": [400], - "action": "REDUCE_REQUEST_WINDOW", + "action": "SPLIT_REQUEST_WINDOW", }, ], }, @@ -5233,7 +5233,7 @@ def _request_window_reduction_manifest(): } -def _read_request_window_reduction_source(manifest): +def _read_request_window_splitting_source(manifest): catalog = create_catalog("Test") source = ConcurrentDeclarativeSource( source_config=manifest, @@ -5244,7 +5244,7 @@ def _read_request_window_reduction_source(manifest): yield from source.read(logger=source.logger, config={}, catalog=catalog, state=[]) -def test_given_reduce_request_window_action_when_read_then_split_and_read_both_halves(): +def test_given_split_request_window_action_when_read_then_split_and_read_both_halves(): """ Mirrors the real PayPal `RESULTSET_TOO_LARGE` incident this feature exists to generalize: a request for the full day is rejected outright, and the connector reads it back as two half-day requests instead, with @@ -5268,7 +5268,7 @@ def test_given_reduce_request_window_action_when_read_then_split_and_read_both_h http_mocker.get(first_half_request, HttpResponse(json.dumps({"items": [{"id": 1}]}), 200)) http_mocker.get(second_half_request, HttpResponse(json.dumps({"items": [{"id": 2}]}), 200)) - messages = list(_read_request_window_reduction_source(_request_window_reduction_manifest())) + messages = list(_read_request_window_splitting_source(_request_window_splitting_manifest())) http_mocker.assert_number_of_calls(full_window_request, 1) http_mocker.assert_number_of_calls(first_half_request, 1) @@ -5279,7 +5279,7 @@ def test_given_reduce_request_window_action_when_read_then_split_and_read_both_h ) == [1, 2] -def test_given_reduce_request_window_action_when_read_then_final_state_reflects_full_original_window(): +def test_given_split_request_window_action_when_read_then_final_state_reflects_full_original_window(): """ Regression coverage for a bug where records read from a split child window carried the child's own slice as `Record.associated_slice`, which `ConcurrentCursor.observe()` keys its bookkeeping by - a key @@ -5320,7 +5320,7 @@ def test_given_reduce_request_window_action_when_read_then_final_state_reflects_ ), ) - messages = list(_read_request_window_reduction_source(_request_window_reduction_manifest())) + messages = list(_read_request_window_splitting_source(_request_window_splitting_manifest())) states = get_states_for_stream(stream_name="Test", messages=messages) assert states diff --git a/unit_tests/sources/streams/concurrent/test_cursor.py b/unit_tests/sources/streams/concurrent/test_cursor.py index db4625bea5..1992fe29e9 100644 --- a/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/unit_tests/sources/streams/concurrent/test_cursor.py @@ -1415,8 +1415,8 @@ def test_final_state_cursor_get_cursor_datetime_from_state_returns_now_for_no_cu class ConcurrentCursorReduceWindowTest(TestCase): """ - Covers ConcurrentCursor.reduce_window(), the WindowReducible implementation backing declarative - `request_window_reduction`. See airbyte_cdk.sources.declarative.retrievers.window_reducible. + Covers ConcurrentCursor.split_request_window(), the WindowReducible implementation backing declarative + `request_window_splitting`. See airbyte_cdk.sources.declarative.retrievers.window_reducible. """ def setUp(self) -> None: @@ -1456,7 +1456,7 @@ def test_given_window_when_reduce_then_split_in_half_at_cursor_granularity_bound ) -> None: cursor = self._cursor() - children = cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z")) + children = cursor.split_request_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z")) assert children == [ self._slice("2024-01-01T00:00:00Z", "2024-01-01T11:59:59Z"), @@ -1469,7 +1469,7 @@ def test_given_odd_number_of_granularity_units_when_reduce_then_children_still_c cursor = self._cursor() # 3 whole seconds: [00:00:00, 00:00:02] - children = cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:02Z")) + children = cursor.split_request_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:02Z")) assert children == [ self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z"), @@ -1480,7 +1480,7 @@ def test_given_single_granularity_unit_window_when_reduce_then_return_none(self) cursor = self._cursor() assert ( - cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z")) + cursor.split_request_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z")) is None ) @@ -1492,7 +1492,7 @@ def test_given_repeated_reduction_when_reduce_then_eventually_reaches_granularit depth = 0 while True: - children = cursor.reduce_window(current) + children = cursor.split_request_window(current) if children is None: break # exactly at the boundary @@ -1502,7 +1502,7 @@ def test_given_repeated_reduction_when_reduce_then_eventually_reaches_granularit ) current = children[0] depth += 1 - assert depth < 30, "reduce_window did not converge to the granularity floor" + assert depth < 30, "split_request_window did not converge to the granularity floor" assert ( current.cursor_slice[_LOWER_SLICE_BOUNDARY_FIELD] @@ -1514,7 +1514,7 @@ def test_given_reversed_boundaries_when_reduce_then_return_none(self) -> None: cursor = self._cursor() assert ( - cursor.reduce_window(self._slice("2024-01-01T00:00:05Z", "2024-01-01T00:00:00Z")) + cursor.split_request_window(self._slice("2024-01-01T00:00:05Z", "2024-01-01T00:00:00Z")) is None ) @@ -1522,7 +1522,7 @@ def test_given_no_cursor_granularity_when_reduce_then_return_none(self) -> None: cursor = self._cursor(granularity=None) assert ( - cursor.reduce_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z")) + cursor.split_request_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z")) is None ) @@ -1530,7 +1530,7 @@ def test_given_malformed_boundary_when_reduce_then_return_none(self) -> None: cursor = self._cursor() assert ( - cursor.reduce_window( + cursor.split_request_window( StreamSlice( partition={}, cursor_slice={ @@ -1546,11 +1546,11 @@ def test_given_missing_boundary_field_when_reduce_then_return_none(self) -> None cursor = self._cursor() assert ( - cursor.reduce_window(StreamSlice(partition={}, cursor_slice={"only_one_field": "x"})) + cursor.split_request_window(StreamSlice(partition={}, cursor_slice={"only_one_field": "x"})) is None ) - def test_reduce_window_preserves_partition_and_extra_fields(self) -> None: + def test_split_request_window_preserves_partition_and_extra_fields(self) -> None: cursor = self._cursor() original = StreamSlice( partition={"parent_id": "123"}, @@ -1561,7 +1561,7 @@ def test_reduce_window_preserves_partition_and_extra_fields(self) -> None: extra_fields={"some_extra": "value"}, ) - children = cursor.reduce_window(original) + children = cursor.split_request_window(original) for child in children: assert child.partition == {"parent_id": "123"} @@ -1582,7 +1582,7 @@ def test_given_epoch_cursor_when_reduce_then_split_using_epoch_output_format(sel cursor_granularity=timedelta(seconds=1), ) - children = cursor.reduce_window( + children = cursor.split_request_window( StreamSlice( partition={}, cursor_slice={ @@ -1608,7 +1608,7 @@ def test_given_multiple_datetime_formats_when_reduce_then_round_trip_format_corr ) -> None: cursor = self._cursor(datetime_format="%Y-%m-%d %H:%M:%S") - children = cursor.reduce_window(self._slice("2024-01-01 00:00:00", "2024-01-01 23:59:59")) + children = cursor.split_request_window(self._slice("2024-01-01 00:00:00", "2024-01-01 23:59:59")) assert children == [ self._slice("2024-01-01 00:00:00", "2024-01-01 11:59:59"), @@ -1620,7 +1620,7 @@ def test_given_non_utc_timezone_offset_when_reduce_then_preserve_offset_in_round ) -> None: cursor = self._cursor(datetime_format="%Y-%m-%dT%H:%M:%S%z") - children = cursor.reduce_window( + children = cursor.split_request_window( self._slice("2024-01-01T00:00:00+0530", "2024-01-01T23:59:59+0530") ) diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 8d50dd79b9..9a87e2dc7a 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -33,8 +33,8 @@ from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( PageSizeReductionRequiredException, ) -from airbyte_cdk.sources.streams.http.request_window_reduction_exception import ( - RequestWindowReductionRequiredException, +from airbyte_cdk.sources.streams.http.request_window_split_exception import ( + RequestWindowSplitRequiredException, ) from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator from airbyte_cdk.utils.traced_exception import AirbyteTracedException @@ -1570,7 +1570,7 @@ def test_given_no_reduce_page_size_action_then_log_the_response_as_a_page(): assert [entry["http"].get("is_auxiliary") for entry in logged] == [None] -def test_send_raises_request_window_reduction_required_exception_with_reduce_request_window_response_action(): +def test_send_raises_request_window_splitting_required_exception_with_split_request_window_response_action(): mocked_session = MagicMock(spec=requests.Session) http_client = HttpClient( name="test", @@ -1579,7 +1579,7 @@ def test_send_raises_request_window_reduction_required_exception_with_reduce_req logger=MagicMock(), error_mapping={ 400: ErrorResolution( - ResponseAction.REDUCE_REQUEST_WINDOW, + ResponseAction.SPLIT_REQUEST_WINDOW, FailureType.config_error, "test reduce request window message", ) @@ -1592,28 +1592,28 @@ def test_send_raises_request_window_reduction_required_exception_with_reduce_req mocked_session.send.return_value = mocked_response # the retriever is responsible for reducing and re-reading the window, so the backoff handlers must not retry - with pytest.raises(RequestWindowReductionRequiredException) as exception: + with pytest.raises(RequestWindowSplitRequiredException) as exception: http_client.send_request(http_method="get", url="https://airbyte.io", request_kwargs={}) assert http_client._session.send.call_count == 1 assert "test reduce request window message" in exception.value.internal_message assert exception.value.message == ( "The API rejected the current request window of stream test and requires a smaller one. If this " - "message ends a sync, the stream is not set up to reduce its window: add `request_window_reduction` " - "to its retriever, or remove the `REDUCE_REQUEST_WINDOW` action from its error handler." + "message ends a sync, the stream is not set up to split its window: add `request_window_splitting` " + "to its retriever, or remove the `SPLIT_REQUEST_WINDOW` action from its error handler." ) # the classifying error's own failure_type is preserved rather than defaulting to config_error assert exception.value.failure_type == FailureType.config_error -def test_given_no_classified_failure_type_when_reduce_request_window_then_default_to_config_error(): +def test_given_no_classified_failure_type_when_split_request_window_then_default_to_config_error(): mocked_session = MagicMock(spec=requests.Session) http_client = HttpClient( name="test", logger=MagicMock(), error_handler=HttpStatusErrorHandler( logger=MagicMock(), - error_mapping={400: ErrorResolution(ResponseAction.REDUCE_REQUEST_WINDOW, None, None)}, + error_mapping={400: ErrorResolution(ResponseAction.SPLIT_REQUEST_WINDOW, None, None)}, ), session=mocked_session, ) @@ -1621,16 +1621,16 @@ def test_given_no_classified_failure_type_when_reduce_request_window_then_defaul mocked_response.status_code = 400 mocked_session.send.return_value = mocked_response - with pytest.raises(RequestWindowReductionRequiredException) as exception: + with pytest.raises(RequestWindowSplitRequiredException) as exception: http_client.send_request(http_method="get", url="https://airbyte.io", request_kwargs={}) assert exception.value.failure_type == FailureType.config_error -def test_given_reduce_request_window_action_then_log_the_response_as_an_auxiliary_request(): +def test_given_split_request_window_action_then_log_the_response_as_an_auxiliary_request(): """ Mirrors test_given_reduce_page_size_action_then_log_the_response_as_an_auxiliary_request: a response - resolving to REDUCE_REQUEST_WINDOW never becomes a page either - the retriever splits the window and + resolving to SPLIT_REQUEST_WINDOW never becomes a page either - the retriever splits the window and re-reads it - so it must not count against the Connector Builder's per-slice page limit. """ message_repository = InMemoryMessageRepository(Level.DEBUG) @@ -1642,7 +1642,7 @@ def test_given_reduce_request_window_action_then_log_the_response_as_an_auxiliar logger=MagicMock(), error_mapping={ 400: ErrorResolution( - ResponseAction.REDUCE_REQUEST_WINDOW, + ResponseAction.SPLIT_REQUEST_WINDOW, FailureType.config_error, "test reduce request window message", ) @@ -1656,7 +1656,7 @@ def test_given_reduce_request_window_action_then_log_the_response_as_an_auxiliar mocked_response.request = requests.Request(method="GET", url="https://airbyte.io").prepare() mocked_session.send.return_value = mocked_response - with pytest.raises(RequestWindowReductionRequiredException): + with pytest.raises(RequestWindowSplitRequiredException): http_client.send_request( http_method="get", url="https://airbyte.io", From ab74c3fcd70a775379bc44df9cb65d3f52bf0906 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Thu, 24 Sep 2026 11:48:42 -0700 Subject: [PATCH 09/21] fix(cdk): classify request window split exhaustion as transient_error (Phase 2) Genuine exhaustion of REQUEST_WINDOW splitting - max_split_depth exceeded, or the cursor's split_request_window reporting no further progress - is not the user's fault, so it now raises transient_error instead of config_error, matching PageSizeReducer's precedent for the same situation. The "records already emitted, on_partial_response: FAIL" path is unaffected and stays config_error, since that one is user-fixable. Also: - Threads the configured `failure_message` through both exhaustion raises via a new `_with_request_window_failure_message` helper (mirroring `PageSizeReducer._with_failure_message`); previously it was dropped on the max_split_depth path. - Fixes RequestWindowSplitRequiredException always baking a concrete failure_type into itself (defaulting to config_error), which made `exception.failure_type or FailureType.transient_error` in the retriever dead code - it could never actually see an unclassified exception. Adds `classified_failure_type` to distinguish "not classified by whatever raised this" from an explicit classification. Per Patrick's review on PR #1171. --- .../declarative_component_schema.yaml | 2 +- .../models/declarative_component_schema.py | 2 +- .../retrievers/simple_retriever.py | 32 +++++++++------ .../http/request_window_split_exception.py | 1 + .../retrievers/test_simple_retriever.py | 41 +++++++++++++++++-- 5 files changed, 61 insertions(+), 17 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 8bf337d8be..571556cfdf 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4504,7 +4504,7 @@ definitions: title: Maximum Split Depth description: >- Maximum number of times in a row a single request window may be split before the sync fails with a - configuration error. This is a safety bound independent of the cursor's own granularity floor: it + transient error. This is a safety bound independent of the cursor's own granularity floor: it protects against a custom cursor whose split_request_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 27c40028e7..aee2bdb4e4 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1496,7 +1496,7 @@ class RequestWindowSplitting(BaseModel): ) max_split_depth: Optional[int] = Field( 10, - description="Maximum number of times in a row a single request window may be split before the sync fails with a configuration error. This is a safety bound independent of the cursor's own granularity floor: it protects against a custom cursor whose split_request_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a wider window than the request-window failures this feature targets in practice tend to involve (days to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning multiple years, should raise this explicitly.", + description="Maximum number of times in a row a single request window may be split before the sync fails with a transient error. This is a safety bound independent of the cursor's own granularity floor: it protects against a custom cursor whose split_request_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a wider window than the request-window failures this feature targets in practice tend to involve (days to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning multiple years, should raise this explicitly.", examples=[10, 5], ge=1, title="Maximum Split Depth", diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index ba01e9ebca..acd6f12621 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -643,7 +643,8 @@ def _read_records_or_split_request_window( if depth >= self.request_window_splitting.max_split_depth: raise AirbyteTracedException( internal_message=f"Stream {self.name} exceeded the maximum request window split depth of {self.request_window_splitting.max_split_depth} while splitting {original_slice}", - message=( + # `transient_error`, so the only remediation is the connector's own, if it defined one. + message=self._with_request_window_failure_message( f"Stream {self.name} could not split its request window to a size the API accepts " f"within {self.request_window_splitting.max_split_depth} splits. This usually means the " f"stream's cursor is not actually shrinking the window on each split; if it uses a " @@ -651,23 +652,19 @@ def _read_records_or_split_request_window( f"`request_window_splitting.max_split_depth` only if this stream's cursor granularity " f"genuinely requires more splits than that." ), - failure_type=FailureType.config_error, + failure_type=FailureType.transient_error, ) from exception children = self.request_window_splitter.split_request_window(stream_slice) if children is None: - failure_message = ( - f"The API kept rejecting stream {self.name}'s request window even at the smallest window " - f"its cursor allows, or splitting the window further would not make progress." - ) - if self.request_window_splitting.failure_message: - failure_message = ( - f"{failure_message} {self.request_window_splitting.failure_message}" - ) raise AirbyteTracedException( internal_message=f"Stream {self.name} could not split its request window {stream_slice} any further", - message=failure_message, - failure_type=exception.failure_type or FailureType.config_error, + # `transient_error`, so the only remediation is the connector's own, if it defined one. + message=self._with_request_window_failure_message( + f"The API kept rejecting stream {self.name}'s request window even at the smallest window " + f"its cursor allows, or splitting the window further would not make progress." + ), + failure_type=exception.classified_failure_type or FailureType.transient_error, ) from exception for child in children: @@ -675,6 +672,17 @@ def _read_records_or_split_request_window( records_schema, child, original_slice, depth + 1 ) + def _with_request_window_failure_message(self, message: str) -> str: + """ + :return: the message followed by `request_window_splitting`'s configured `failure_message`, when it + defined one - mirroring `PageSizeReducer._with_failure_message` + """ + assert self.request_window_splitting is not None + failure_message = (self.request_window_splitting.failure_message or "").strip() + if not failure_message: + return message + return f"{message} {failure_message}" + @staticmethod def _reassociate_with_original_slice( record: StreamData, original_slice: StreamSlice diff --git a/airbyte_cdk/sources/streams/http/request_window_split_exception.py b/airbyte_cdk/sources/streams/http/request_window_split_exception.py index 4d88ba78f6..e57e154570 100644 --- a/airbyte_cdk/sources/streams/http/request_window_split_exception.py +++ b/airbyte_cdk/sources/streams/http/request_window_split_exception.py @@ -35,6 +35,7 @@ def __init__( ) -> None: stream = f" of stream {stream_name}" if stream_name else "" detail = f": {error_message}" if error_message else "" + self.classified_failure_type = failure_type super().__init__( internal_message=f"A request window split was requested{stream}{detail}", message=f"The API rejected the current request window{stream} and requires a smaller one. If this message ends a sync, the stream is not set up to split its window: add `request_window_splitting` to its retriever, or remove the `SPLIT_REQUEST_WINDOW` action from its error handler.", diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index ac9ab62276..6a2df58964 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -2278,12 +2278,41 @@ def test_given_request_window_splitter_returns_none_when_reduction_requested_the with pytest.raises(AirbyteTracedException) as exception: list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) - # the classifying error's own failure_type is preserved rather than defaulting to config_error + # the classifying error's own failure_type is preserved rather than falling back to this branch's default assert exception.value.failure_type == FailureType.transient_error assert "Lower time_window so that each request covers less data." in exception.value.message request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) +def test_given_request_window_splitter_returns_none_and_no_classified_failure_type_when_reduction_requested_then_raise_transient_error(): + """ + When whatever raised `RequestWindowSplitRequiredException` didn't classify a `failure_type` (the common + case: a manifest response filter with no explicit `failure_type`, or custom code that omits it), exhausting + the split floor must default to `transient_error` - the API kept rejecting every window size tried, which is + not the user's fault - matching `PageSizeReducer`'s equivalent exhaustion branch, not `config_error`. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = RequestWindowSplitRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = None + + retriever = _request_window_splitting_retriever( + requester, + paginator, + record_selector, + request_window_splitter, + ) + + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + assert exception.value.failure_type == FailureType.transient_error + + def test_given_empty_children_list_when_reduction_requested_then_emit_no_records_and_do_not_loop(): requester = Mock(spec=Requester) requester.send_request.side_effect = RequestWindowSplitRequiredException() @@ -2514,15 +2543,21 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er paginator, record_selector, request_window_splitter, - request_window_splitting=RequestWindowSplitting(max_split_depth=3), + request_window_splitting=RequestWindowSplitting( + max_split_depth=3, + failure_message="Lower time_window so that each request covers less data.", + ), ) with pytest.raises(AirbyteTracedException) as exception: list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) - assert exception.value.failure_type == FailureType.config_error + # exhausting the split depth is not the user's fault - the API kept rejecting every window size tried - + # so it is transient_error, matching PageSizeReducer's equivalent exhaustion branch, not config_error + assert exception.value.failure_type == FailureType.transient_error assert "maximum request window split depth" in exception.value.internal_message assert "within 3 splits" in exception.value.message + assert "Lower time_window so that each request covers less data." in exception.value.message # the single leftmost recursion path is explored to depth 3 (three split_request_window calls, at depth 0, 1, and # 2) before the cap stops a fourth call at depth 3 - the exception then propagates immediately, so sibling # branches at shallower depths are never explored From f962c3677a258e873df29c65b1866c55a9103527 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Thu, 24 Sep 2026 12:18:35 -0700 Subject: [PATCH 10/21] feat(cdk): tighten request window splitting validation, replace max_split_depth with min_split_window (Phases 3-4) Phase 3 - validation tightening: - Reject a `cursor_granularity` that parses to a zero-length duration (PT0S/P0D), which passes today's truthy check but can never produce a smaller child window. - Reject when `datetime_format` is coarser than `cursor_granularity` (e.g. %Y-%m-%d with PT1S), via a new `_smallest_datetime_format_unit` helper that maps format directives to the finest duration they can represent. Phase 4 - max_split_depth -> internal constant + min_split_window: - `max_split_depth` is no longer a per-connector manifest field; it's now the internal constant `_MAX_REQUEST_WINDOW_SPLIT_DEPTH = 10` in SimpleRetriever, with a warning logged when it's hit (a rare safety net, not something legitimate splitting is expected to reach). - Adds `min_split_window` (ISO-8601 duration) so connector authors can express a splitting floor in domain terms instead of a raw split count. Threaded through RequestWindowSplitting -> SimpleRetriever -> WindowReducible.split_request_window -> ConcurrentCursor's stopping condition, independent of (and typically looser than) the existing cursor_granularity floor. Per Patrick's review on PR #1171. --- .../declarative_component_schema.yaml | 31 +++--- .../models/declarative_component_schema.py | 11 +-- .../parsers/model_to_component_factory.py | 69 ++++++++++++- .../retrievers/simple_retriever.py | 45 ++++++--- .../retrievers/window_reducible.py | 28 +++--- .../sources/streams/concurrent/cursor.py | 19 ++-- .../test_model_to_component_factory.py | 37 ++++++- .../retrievers/test_simple_retriever.py | 98 ++++++++++++++----- .../sources/streams/concurrent/test_cursor.py | 47 ++++++++- 9 files changed, 297 insertions(+), 88 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 571556cfdf..703d04183e 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4500,26 +4500,21 @@ definitions: examples: - Narrow the sync down by selecting fewer fields on this stream. - Lower time_window so that each request covers less data. - max_split_depth: - title: Maximum Split Depth + min_split_window: + title: Minimum Split Window description: >- - Maximum number of times in a row a single request window may be split before the sync fails with a - transient error. This is a safety bound independent of the cursor's own granularity floor: it - protects against a custom cursor whose split_request_window implementation returns children that do not - actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single - already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world - APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a - one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a - wider window than the request-window failures this feature targets in practice tend to involve (days - to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to - approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning - multiple years, should raise this explicitly. - type: integer - default: 10 - minimum: 1 + Smallest window (ISO 8601 duration) the connector will ever request, expressed in the same domain + terms as `cursor_granularity` rather than a raw split count. Independent of - and typically looser + than - the cursor's own `cursor_granularity` floor: a connector whose cursor could technically split + down to the second may still want to stop earlier, for instance because the API's rate limit makes + many small requests worse than a few large ones. A window already at or below this size is not split + any further, the same way one at the `cursor_granularity` floor is not. Splitting is also bounded by + an internal safety net independent of this field, which protects against a custom cursor whose + `split_request_window` implementation returns children that do not actually shrink the window. + type: string examples: - - 10 - - 5 + - "P1D" + - "PT1H" GzipDecoder: title: gzip description: Select 'gzip' for response data that is compressed with gzip. Requires specifying an inner data type/decoder to parse the decompressed data. diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index aee2bdb4e4..983078d489 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1494,12 +1494,11 @@ class RequestWindowSplitting(BaseModel): ], title="Failure Message", ) - max_split_depth: Optional[int] = Field( - 10, - description="Maximum number of times in a row a single request window may be split before the sync fails with a transient error. This is a safety bound independent of the cursor's own granularity floor: it protects against a custom cursor whose split_request_window implementation returns children that do not actually shrink the window, which would otherwise recurse indefinitely. Each reduction bisects a single already-generated slice - bounded by the cursor's own step, not the whole sync range - and real-world APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step bisected down to a twelve-hour floor needs about 10 halvings, which already covers a wider window than the request-window failures this feature targets in practice tend to involve (days to a few months). The default of 10 is a safety net, not a limit legitimate reductions are expected to approach. A stream whose cursor genuinely needs finer-than-half-day granularity, or windows spanning multiple years, should raise this explicitly.", - examples=[10, 5], - ge=1, - title="Maximum Split Depth", + min_split_window: Optional[str] = Field( + None, + description="Smallest window (ISO 8601 duration) the connector will ever request, expressed in the same domain terms as `cursor_granularity` rather than a raw split count. Independent of - and typically looser than - the cursor's own `cursor_granularity` floor: a connector whose cursor could technically split down to the second may still want to stop earlier, for instance because the API's rate limit makes many small requests worse than a few large ones. A window already at or below this size is not split any further, the same way one at the `cursor_granularity` floor is not. Splitting is also bounded by an internal safety net independent of this field, which protects against a custom cursor whose `split_request_window` implementation returns children that do not actually shrink the window.", + examples=["P1D", "PT1H"], + title="Minimum Split Window", ) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 5cfeccf3e8..957cbdcbcd 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -720,6 +720,24 @@ # this would be a circular import MAX_SLICES = 5 +# The smallest duration each `DatetimeParser`/`strftime` directive can distinguish between two values, finest +# first so ties are irrelevant - used by `_smallest_datetime_format_unit` to check that `datetime_format` can +# actually represent whatever `cursor_granularity` claims. Directives this table has no entry for (a bare +# `%Y`/`%m`, or any other formatting scheme) are treated as "cannot determine" rather than rejected. +_DATETIME_FORMAT_DIRECTIVE_GRANULARITY: List[Tuple[str, datetime.timedelta]] = [ + ("%epoch_microseconds", datetime.timedelta(microseconds=1)), + ("%s_as_float", datetime.timedelta(microseconds=1)), + ("%f", datetime.timedelta(microseconds=1)), + ("%_ms", datetime.timedelta(milliseconds=1)), + ("%ms", datetime.timedelta(milliseconds=1)), + ("%S", datetime.timedelta(seconds=1)), + ("%s", datetime.timedelta(seconds=1)), + ("%M", datetime.timedelta(minutes=1)), + ("%H", datetime.timedelta(hours=1)), + ("%I", datetime.timedelta(hours=1)), + ("%d", datetime.timedelta(days=1)), +] + LOGGER = logging.getLogger(f"airbyte.model_to_component_factory") @@ -4131,9 +4149,9 @@ def _create_request_window_splitting( if on_partial_response is not None else OnPartialResponse.FAIL, failure_message=model.request_window_splitting.failure_message, - max_split_depth=model.request_window_splitting.max_split_depth # type: ignore[arg-type] # the schema defines a default - if model.request_window_splitting.max_split_depth is not None - else 10, + min_split_window=parse_duration(model.request_window_splitting.min_split_window) + if model.request_window_splitting.min_split_window + else None, ) def _validate_request_window_splitting_is_supported( @@ -4176,6 +4194,37 @@ def _validate_request_window_splitting_is_supported( f"keep two child windows from overlapping at their shared edge." ) + parsed_cursor_granularity = parse_duration(incremental_sync.cursor_granularity) + if isinstance( + parsed_cursor_granularity, datetime.timedelta + ) and parsed_cursor_granularity <= datetime.timedelta(0): + # A zero-length (or, via a leading `-`, negative) granularity passes the truthy check above - the + # string is non-empty - but `ConcurrentCursor.split_request_window` divides the window's span by it + # to find the split point, so it can never produce a child strictly smaller than the parent. + raise ValueError( + f"`request_window_splitting` requires a `cursor_granularity` greater than zero on stream " + f"{name}: `{incremental_sync.cursor_granularity}` parses to a zero-length duration, which can " + f"never produce a smaller child window." + ) + + smallest_format_unit = self._smallest_datetime_format_unit(incremental_sync.datetime_format) + if ( + smallest_format_unit is not None + and isinstance(parsed_cursor_granularity, datetime.timedelta) + and parsed_cursor_granularity < smallest_format_unit + ): + # `cursor_granularity` claims a finer boundary than `datetime_format` can render: two children + # split that close together would format to the same boundary value, so the retriever would ask + # for the same window twice - or, once formatted, request a reversed one - instead of splitting. + raise ValueError( + f"`request_window_splitting` requires `cursor_granularity` to be no finer than what " + f"`datetime_format` can represent on stream {name}: `datetime_format` " + f"{incremental_sync.datetime_format!r} cannot represent a value finer than " + f"{smallest_format_unit}, but `cursor_granularity` {incremental_sync.cursor_granularity!r} is " + f"finer than that. Use a `datetime_format` precise enough to represent `cursor_granularity`, " + f"or a coarser `cursor_granularity`." + ) + if query_properties: raise ValueError( f"`request_window_splitting` cannot be used together with query properties on stream {name}. " @@ -4190,6 +4239,20 @@ def _validate_request_window_splitting_is_supported( f"a split requested partway through could re-emit records already yielded by it." ) + @staticmethod + def _smallest_datetime_format_unit(datetime_format: str) -> Optional[datetime.timedelta]: + """ + :return: the smallest duration `datetime_format` can distinguish between two values - the finest + directive it contains, per `_DATETIME_FORMAT_DIRECTIVE_GRANULARITY` - or `None` if it contains none + of the directives that table recognizes, in which case the caller cannot validate against it. + """ + matches = [ + granularity + for directive, granularity in _DATETIME_FORMAT_DIRECTIVE_GRANULARITY + if directive in datetime_format + ] + return min(matches) if matches else None + def _reject_split_request_window_action(self, requester: Any, description: str) -> None: """ `error_handler` is defined on `HttpRequester`, which is referenced by requesters that have no window of diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index acd6f12621..5be12fbcfd 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -75,6 +75,18 @@ FULL_REFRESH_SYNC_COMPLETE_KEY = "__ab_full_refresh_sync_complete" LOGGER = logging.getLogger("airbyte") +# A defense-in-depth bound independent of any specific WindowReducible's own no-progress guard (cursor_granularity +# or min_split_window): the retriever enforces this itself so a misbehaving custom cursor (returning children +# that do not actually shrink) fails deterministically rather than recursing indefinitely. Each split bisects a +# single already-generated slice - bounded by the cursor's own `step`, not the whole sync range - and real-world +# APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a +# one-year step bisected down to a 12-hour floor needs ~10 halvings (log2(hours in a year / 12) ~= 9.5), which +# already covers a wider window than the request-window failures this feature targets in practice tend to +# involve (on the order of days to a few months). Not user-configurable: a connector whose cursor genuinely +# needs finer-than-half-day granularity, or windows spanning multiple years, should express that through +# `min_split_window` instead of raising this safety net. +_MAX_REQUEST_WINDOW_SPLIT_DEPTH = 10 + @dataclass class SimpleRetriever(Retriever): @@ -563,7 +575,9 @@ def read_records( :return: The records read from the API source """ _slice = stream_slice or StreamSlice(partition={}, cursor_slice={}) # None-check - yield from self._read_records_or_split_request_window(records_schema, _slice, _slice, depth=0) + yield from self._read_records_or_split_request_window( + records_schema, _slice, _slice, depth=0 + ) def _read_records_or_split_request_window( self, @@ -618,9 +632,7 @@ def _read_records_or_split_request_window( yield self._reassociate_with_original_slice(record, original_slice) except RequestWindowSplitRequiredException as exception: if self.request_window_splitting is None or self.request_window_splitter is None: - raise RequestWindowSplitNotSupportedException( - stream_name=self.name - ) from exception + raise RequestWindowSplitNotSupportedException(stream_name=self.name) from exception if ( emitted_any @@ -640,22 +652,31 @@ def _read_records_or_split_request_window( failure_type=FailureType.config_error, ) from exception - if depth >= self.request_window_splitting.max_split_depth: + if depth >= _MAX_REQUEST_WINDOW_SPLIT_DEPTH: + # A safety net, expected to be rare: logged separately from the exception below so it is + # visible even if whatever catches the trace message does not surface its internal_message. + LOGGER.warning( + f"Stream {self.name} hit the maximum request window split depth " + f"({_MAX_REQUEST_WINDOW_SPLIT_DEPTH}) while splitting {original_slice}. This is a safety " + f"net against a cursor whose `split_request_window` returns children that do not actually " + f"shrink the window; a stream needing more splits than that for a legitimate reason should " + f"raise `min_split_window` instead of relying on more splits." + ) raise AirbyteTracedException( - internal_message=f"Stream {self.name} exceeded the maximum request window split depth of {self.request_window_splitting.max_split_depth} while splitting {original_slice}", + internal_message=f"Stream {self.name} exceeded the maximum request window split depth of {_MAX_REQUEST_WINDOW_SPLIT_DEPTH} while splitting {original_slice}", # `transient_error`, so the only remediation is the connector's own, if it defined one. message=self._with_request_window_failure_message( f"Stream {self.name} could not split its request window to a size the API accepts " - f"within {self.request_window_splitting.max_split_depth} splits. This usually means the " - f"stream's cursor is not actually shrinking the window on each split; if it uses a " - f"custom cursor, check its `split_request_window` implementation. Raise " - f"`request_window_splitting.max_split_depth` only if this stream's cursor granularity " - f"genuinely requires more splits than that." + f"within {_MAX_REQUEST_WINDOW_SPLIT_DEPTH} splits. This usually means the stream's " + f"cursor is not actually shrinking the window on each split; if it uses a custom " + f"cursor, check its `split_request_window` implementation." ), failure_type=FailureType.transient_error, ) from exception - children = self.request_window_splitter.split_request_window(stream_slice) + children = self.request_window_splitter.split_request_window( + stream_slice, self.request_window_splitting.min_split_window + ) if children is None: raise AirbyteTracedException( internal_message=f"Stream {self.name} could not split its request window {stream_slice} any further", diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index 5f4bb820b4..0109272556 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -2,6 +2,7 @@ # Copyright (c) 2026 Airbyte, Inc., all rights reserved. # +import datetime from dataclasses import dataclass from enum import Enum from typing import List, Optional, Protocol, runtime_checkable @@ -26,15 +27,18 @@ class WindowReducible(Protocol): retriever needs to know whether the stream it is reading supports request-window splitting at all. """ - def split_request_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: + def split_request_window( + self, stream_slice: StreamSlice, min_split_window: Optional[datetime.timedelta] = None + ) -> Optional[List[StreamSlice]]: """ Split `stream_slice` into two or more smaller, non-overlapping child slices that together cover exactly the same range, preserving `partition` and `extra_fields` unchanged. Returns `None` when the slice cannot be split any further - either because it is already at (or below) - the minimum granularity the cursor supports, or because splitting it would not produce children strictly - smaller than the parent (a no-progress guard independent of the granularity check). Callers should treat - `None` as a terminal condition, not retry with the same slice. + the minimum granularity the cursor supports, because it is already at or below `min_split_window` (a + connector-configured floor expressed in domain terms rather than raw split count), or because splitting + it would not produce children strictly smaller than the parent (a no-progress guard independent of + either floor). Callers should treat `None` as a terminal condition, not retry with the same slice. """ raise NotImplementedError( "WindowReducible.split_request_window must be implemented by protocol implementers" @@ -54,13 +58,9 @@ class RequestWindowSplitting: on_partial_response: OnPartialResponse = OnPartialResponse.FAIL failure_message: Optional[str] = None - # A defense-in-depth bound independent of any specific WindowReducible's own no-progress guard: the retriever - # enforces this itself so a misbehaving custom cursor (returning children that do not actually shrink) fails - # deterministically rather than recursing indefinitely. Each split bisects a single already-generated - # slice - bounded by the cursor's own `step`, not the whole sync range - and real-world APIs that reject - # oversized windows are typically satisfied well before reaching sub-day granularity: a one-year step - # bisected down to a 12-hour floor needs ~10 halvings (log2(hours in a year / 12) ~= 9.5), which already - # covers a wider window than the request-window failures this feature targets in practice tend to involve - # (on the order of days to a few months). A connector whose cursor genuinely needs finer-than-half-day - # granularity, or windows spanning multiple years, should raise this explicitly. - max_split_depth: int = 10 + # An upper bound on how small a window `split_request_window` is asked to produce, expressed in the same + # domain terms as `cursor_granularity` (a duration) rather than a raw split count. Independent of - and + # typically looser than - the cursor's own granularity floor: a connector whose cursor could technically + # split down to the second may still want to stop earlier, e.g. because the API's rate limit makes many + # small requests worse than a few large ones. + min_split_window: Optional[datetime.timedelta] = None diff --git a/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte_cdk/sources/streams/concurrent/cursor.py index b4b767ccf8..7b7699ee8f 100644 --- a/airbyte_cdk/sources/streams/concurrent/cursor.py +++ b/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -607,16 +607,20 @@ def reduce_slice_range(self, stream_slice: StreamSlice) -> StreamSlice: else: return stream_slice - def split_request_window(self, stream_slice: StreamSlice) -> Optional[List[StreamSlice]]: + def split_request_window( + self, stream_slice: StreamSlice, min_split_window: Optional[datetime.timedelta] = None + ) -> Optional[List[StreamSlice]]: """ Split `stream_slice` in half along this cursor's own boundary fields, granularity, and output format. Returns `None` when the slice cannot be split any further: either it spans at most one - `cursor_granularity` unit already (the floor), the boundaries are missing/malformed/reversed, or no - `cursor_granularity` was configured at all (there is then no way to know the smallest addressable unit, - or how to keep two children from overlapping at their shared edge). This is a generic implementation of - `WindowReducible` (see `airbyte_cdk.sources.declarative.retrievers.window_reducible`) that reuses the - same parsing/formatting/comparison operations `_split_per_slice_range` already relies on, so it works for + `cursor_granularity` unit already (the floor), it is already at or below `min_split_window` (a + connector-configured floor independent of and typically looser than `cursor_granularity`), the + boundaries are missing/malformed/reversed, or no `cursor_granularity` was configured at all (there is + then no way to know the smallest addressable unit, or how to keep two children from overlapping at + their shared edge). This is a generic implementation of `WindowReducible` (see + `airbyte_cdk.sources.declarative.retrievers.window_reducible`) that reuses the same + parsing/formatting/comparison operations `_split_per_slice_range` already relies on, so it works for any `CursorValueType`/`GapType` pair this cursor was built with (datetime/timedelta, or int/int), without a second, duplicate implementation of that logic living outside the cursor. """ @@ -639,6 +643,9 @@ def split_request_window(self, stream_slice: StreamSlice) -> Optional[List[Strea return None span = end_value - start_value # type: ignore[operator] # concretely a GapType (timedelta/int) at runtime + if min_split_window is not None and span <= min_split_window: # type: ignore[operator] + return None + total_units = span // self._cursor_granularity + 1 # type: ignore[operator] # inclusive count of granularity units if total_units <= 1: # Already at the minimum granularity and still rejected; nothing left to split. diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index de0caaf598..435a585e7e 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -7630,7 +7630,7 @@ def test_given_request_window_splitting_values_then_create_retriever_with_those_ " type: RequestWindowSplitting\n" " on_partial_response: ALLOW_REPLAY\n" " failure_message: Lower time_window so that each request covers less data.\n" - " max_split_depth: 20" + " min_split_window: P1D" ) ) ) @@ -7638,7 +7638,7 @@ def test_given_request_window_splitting_values_then_create_retriever_with_those_ assert retriever.request_window_splitting == RequestWindowSplitting( on_partial_response=OnPartialResponse.ALLOW_REPLAY, failure_message="Lower time_window so that each request covers less data.", - max_split_depth=20, + min_split_window=timedelta(days=1), ) @@ -7769,6 +7769,39 @@ def test_given_datetime_based_cursor_without_cursor_granularity_then_raise(): assert "cursor_granularity" in str(exception.value) +def test_given_zero_length_cursor_granularity_and_request_window_splitting_then_raise(): + with pytest.raises(ValueError) as exception: + _request_window_splitting_stream( + incremental_sync=( + 'incremental_sync:\n type: DatetimeBasedCursor\n start_datetime: "2024-01-01T00:00:00Z"\n' + ' end_datetime: "2024-01-31T00:00:00Z"\n step: "P7D"\n cursor_field: "updated_at"\n' + ' cursor_granularity: "PT0S"\n datetime_format: "%Y-%m-%dT%H:%M:%SZ"' + ) + ) + + assert "cursor_granularity" in str(exception.value) + assert "zero-length" in str(exception.value) + + +def test_given_datetime_format_coarser_than_cursor_granularity_and_request_window_splitting_then_raise(): + """ + `cursor_granularity: PT1S` claims second-level precision, but `%Y-%m-%d` can only render day boundaries: two + children split less than a day apart would format to the same date and the retriever would re-request the + same (or a reversed) window instead of a smaller one. + """ + with pytest.raises(ValueError) as exception: + _request_window_splitting_stream( + incremental_sync=( + 'incremental_sync:\n type: DatetimeBasedCursor\n start_datetime: "2024-01-01"\n' + ' end_datetime: "2024-01-31"\n step: "P7D"\n cursor_field: "updated_at"\n' + ' cursor_granularity: "PT1S"\n datetime_format: "%Y-%m-%d"' + ) + ) + + assert "cursor_granularity" in str(exception.value) + assert "datetime_format" in str(exception.value) + + def test_given_incrementing_count_cursor_and_request_window_splitting_then_raise(): """ `IncrementingCountCursor` builds a `ConcurrentCursor` too, but not one carrying the datetime boundary diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 6a2df58964..09d6bd76b3 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -6,6 +6,7 @@ import threading from collections import defaultdict from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta from functools import partial from typing import Any, Iterable, Mapping, Optional from unittest.mock import MagicMock, Mock, patch @@ -52,7 +53,10 @@ PageSizeResetPolicy, ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker -from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever +from airbyte_cdk.sources.declarative.retrievers.simple_retriever import ( + _MAX_REQUEST_WINDOW_SPLIT_DEPTH, + SimpleRetriever, +) from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, RequestWindowSplitting, @@ -2063,7 +2067,10 @@ def test_given_request_window_splitting_when_read_records_then_split_and_read_bo paginator.next_page_token.return_value = None request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter.split_request_window.return_value = [ + A_FIRST_HALF_SLICE, + A_SECOND_HALF_SLICE, + ] retriever = _request_window_splitting_retriever( requester, paginator, record_selector, request_window_splitter @@ -2072,7 +2079,7 @@ def test_given_request_window_splitting_when_read_records_then_split_and_read_bo records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [{"id": 1}, {"id": 2}] - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) assert requester.send_request.call_count == 3 assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ A_WINDOW_SLICE, @@ -2081,6 +2088,36 @@ def test_given_request_window_splitting_when_read_records_then_split_and_read_bo ] +def test_given_min_split_window_configured_when_read_records_then_pass_it_to_the_splitter(): + """ + `min_split_window` lives on `request_window_splitting`, not on the splitter itself, so the retriever must + thread it through on every call rather than the splitter reading it from somewhere else. + """ + requester = Mock(spec=Requester) + requester.send_request.side_effect = RequestWindowSplitRequiredException() + record_selector = Mock(spec=HttpSelector) + paginator = _mock_paginator() + paginator.get_initial_token.return_value = None + + request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter.split_request_window.return_value = None + + retriever = _request_window_splitting_retriever( + requester, + paginator, + record_selector, + request_window_splitter, + request_window_splitting=RequestWindowSplitting(min_split_window=timedelta(days=1)), + ) + + with pytest.raises(AirbyteTracedException): + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + + request_window_splitter.split_request_window.assert_called_once_with( + A_WINDOW_SLICE, timedelta(days=1) + ) + + def test_given_only_one_child_needs_further_reduction_when_read_records_then_recurse_asymmetrically(): """ Nested/asymmetric reduction: the first child is itself rejected and split again, while the second @@ -2122,8 +2159,8 @@ def test_given_only_one_child_needs_further_reduction_when_read_records_then_rec assert records == [{"id": 1}, {"id": 2}, {"id": 3}] assert request_window_splitter.split_request_window.call_args_list == [ - ((A_WINDOW_SLICE,),), - ((A_FIRST_HALF_SLICE,),), + ((A_WINDOW_SLICE, None),), + ((A_FIRST_HALF_SLICE, None),), ] assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ A_WINDOW_SLICE, @@ -2228,7 +2265,10 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re paginator.next_page_token.side_effect = [{"next_page_token": "page2"}, None, None] request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter.split_request_window.return_value = [ + A_FIRST_HALF_SLICE, + A_SECOND_HALF_SLICE, + ] retriever = _request_window_splitting_retriever( requester, @@ -2245,7 +2285,7 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re # the record from the page emitted before the reduction signal is included once more: ALLOW_REPLAY # accepts this at-least-once duplication rather than silently dropping it or failing the sync. assert records == [{"id": 1}, {"id": 2}, {"id": 3}] - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) def test_given_request_window_splitter_returns_none_when_reduction_requested_then_raise_terminal_error(): @@ -2281,7 +2321,7 @@ def test_given_request_window_splitter_returns_none_when_reduction_requested_the # the classifying error's own failure_type is preserved rather than falling back to this branch's default assert exception.value.failure_type == FailureType.transient_error assert "Lower time_window so that each request covers less data." in exception.value.message - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) def test_given_request_window_splitter_returns_none_and_no_classified_failure_type_when_reduction_requested_then_raise_transient_error(): @@ -2330,7 +2370,7 @@ def test_given_empty_children_list_when_reduction_requested_then_emit_no_records records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [] - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE) + request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates(): @@ -2368,8 +2408,8 @@ def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates # the first child's records were already emitted to the caller before the second child failed assert emitted == [{"id": 1}] assert request_window_splitter.split_request_window.call_args_list == [ - ((A_WINDOW_SLICE,),), - ((A_SECOND_HALF_SLICE,),), + ((A_WINDOW_SLICE, None),), + ((A_SECOND_HALF_SLICE, None),), ] @@ -2392,7 +2432,10 @@ def test_given_request_window_splitting_when_read_records_then_each_child_gets_f paginator.next_page_token.return_value = None request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter.split_request_window.return_value = [ + A_FIRST_HALF_SLICE, + A_SECOND_HALF_SLICE, + ] retriever = _request_window_splitting_retriever( requester, paginator, record_selector, request_window_splitter @@ -2474,7 +2517,10 @@ def select_records(*, response, stream_slice, **kwargs): paginator.next_page_token.return_value = None request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter.split_request_window.return_value = [ + A_FIRST_HALF_SLICE, + A_SECOND_HALF_SLICE, + ] retriever = _request_window_splitting_retriever( requester, paginator, record_selector, request_window_splitter @@ -2511,7 +2557,10 @@ def test_given_exception_raised_during_record_extraction_when_read_then_still_sp paginator.next_page_token.return_value = None request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE] + request_window_splitter.split_request_window.return_value = [ + A_FIRST_HALF_SLICE, + A_SECOND_HALF_SLICE, + ] retriever = _request_window_splitting_retriever( requester, paginator, record_selector, request_window_splitter @@ -2536,7 +2585,9 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er request_window_splitter = Mock(spec=WindowReducible) # never signals "no further progress possible" - always splits into two children identical to the parent - request_window_splitter.split_request_window.side_effect = lambda stream_slice: [stream_slice, stream_slice] + request_window_splitter.split_request_window.side_effect = ( + lambda stream_slice, min_split_window=None: [stream_slice, stream_slice] + ) retriever = _request_window_splitting_retriever( requester, @@ -2544,7 +2595,6 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er record_selector, request_window_splitter, request_window_splitting=RequestWindowSplitting( - max_split_depth=3, failure_message="Lower time_window so that each request covers less data.", ), ) @@ -2556,12 +2606,14 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er # so it is transient_error, matching PageSizeReducer's equivalent exhaustion branch, not config_error assert exception.value.failure_type == FailureType.transient_error assert "maximum request window split depth" in exception.value.internal_message - assert "within 3 splits" in exception.value.message + assert f"within {_MAX_REQUEST_WINDOW_SPLIT_DEPTH} splits" in exception.value.message assert "Lower time_window so that each request covers less data." in exception.value.message - # the single leftmost recursion path is explored to depth 3 (three split_request_window calls, at depth 0, 1, and - # 2) before the cap stops a fourth call at depth 3 - the exception then propagates immediately, so sibling - # branches at shallower depths are never explored - assert request_window_splitter.split_request_window.call_count == 3 + # the single leftmost recursion path is explored to the full depth (one split_request_window call per + # depth, from 0 up to the cap) before the cap stops the next call - the exception then propagates + # immediately, so sibling branches at shallower depths are never explored + assert ( + request_window_splitter.split_request_window.call_count == _MAX_REQUEST_WINDOW_SPLIT_DEPTH + ) def test_given_partitions_read_concurrently_then_window_reduction_isolates_between_partitions(): @@ -2610,7 +2662,7 @@ def select_records(*, response, stream_slice, **kwargs): paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - def split_request_window(stream_slice): + def split_request_window(stream_slice, min_split_window=None): # preserve the parent's own partition, matching what a real WindowReducible would do return [ StreamSlice( @@ -2644,4 +2696,4 @@ def split_request_window(stream_slice): assert records_b == [{"partition": "b", "start": "2024-02-01T00:00:00Z"}] assert len(records_a) == 2 - request_window_splitter.split_request_window.assert_called_once_with(slice_a) + request_window_splitter.split_request_window.assert_called_once_with(slice_a, None) diff --git a/unit_tests/sources/streams/concurrent/test_cursor.py b/unit_tests/sources/streams/concurrent/test_cursor.py index 1992fe29e9..3bac4be4ad 100644 --- a/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/unit_tests/sources/streams/concurrent/test_cursor.py @@ -1456,7 +1456,9 @@ def test_given_window_when_reduce_then_split_in_half_at_cursor_granularity_bound ) -> None: cursor = self._cursor() - children = cursor.split_request_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z")) + children = cursor.split_request_window( + self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z") + ) assert children == [ self._slice("2024-01-01T00:00:00Z", "2024-01-01T11:59:59Z"), @@ -1469,7 +1471,9 @@ def test_given_odd_number_of_granularity_units_when_reduce_then_children_still_c cursor = self._cursor() # 3 whole seconds: [00:00:00, 00:00:02] - children = cursor.split_request_window(self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:02Z")) + children = cursor.split_request_window( + self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:02Z") + ) assert children == [ self._slice("2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z"), @@ -1484,6 +1488,37 @@ def test_given_single_granularity_unit_window_when_reduce_then_return_none(self) is None ) + def test_given_window_at_or_below_min_split_window_when_reduce_then_return_none(self) -> None: + """ + `min_split_window` is a connector-configured floor independent of, and typically looser than, + `cursor_granularity`: a window already at or below it must not be split further even though + `cursor_granularity` alone would still allow it. + """ + cursor = self._cursor() + + # spans exactly 1 day - at the floor when min_split_window is 1 day, even though cursor_granularity + # (1 second) would otherwise allow splitting this down much further + assert ( + cursor.split_request_window( + self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z"), + min_split_window=timedelta(days=1), + ) + is None + ) + + def test_given_window_above_min_split_window_when_reduce_then_split_normally(self) -> None: + cursor = self._cursor() + + children = cursor.split_request_window( + self._slice("2024-01-01T00:00:00Z", "2024-01-02T23:59:59Z"), + min_split_window=timedelta(days=1), + ) + + assert children == [ + self._slice("2024-01-01T00:00:00Z", "2024-01-01T23:59:59Z"), + self._slice("2024-01-02T00:00:00Z", "2024-01-02T23:59:59Z"), + ] + def test_given_repeated_reduction_when_reduce_then_eventually_reaches_granularity_floor( self, ) -> None: @@ -1546,7 +1581,9 @@ def test_given_missing_boundary_field_when_reduce_then_return_none(self) -> None cursor = self._cursor() assert ( - cursor.split_request_window(StreamSlice(partition={}, cursor_slice={"only_one_field": "x"})) + cursor.split_request_window( + StreamSlice(partition={}, cursor_slice={"only_one_field": "x"}) + ) is None ) @@ -1608,7 +1645,9 @@ def test_given_multiple_datetime_formats_when_reduce_then_round_trip_format_corr ) -> None: cursor = self._cursor(datetime_format="%Y-%m-%d %H:%M:%S") - children = cursor.split_request_window(self._slice("2024-01-01 00:00:00", "2024-01-01 23:59:59")) + children = cursor.split_request_window( + self._slice("2024-01-01 00:00:00", "2024-01-01 23:59:59") + ) assert children == [ self._slice("2024-01-01 00:00:00", "2024-01-01 11:59:59"), From 1df603f1dc0a6c73f347178b46383567fad221e3 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Thu, 24 Sep 2026 13:01:19 -0700 Subject: [PATCH 11/21] refactor(cdk): drop the WindowReducible protocol (Phase 5) Only one thing ever implemented WindowReducible (ConcurrentCursor), so the protocol wasn't earning its keep. Replaces the isinstance(cursor, WindowReducible) capability check with hasattr(cursor, "split_request_window"), and SimpleRetriever.request_window_splitter is now a plain callable (the cursor's bound split_request_window method) instead of a protocol-typed object, matching how log_formatter is already passed on the same class. Per Patrick's review on PR #1171. --- .../parsers/model_to_component_factory.py | 19 ++-- .../retrievers/simple_retriever.py | 40 ++++---- .../retrievers/window_reducible.py | 38 +------- .../sources/streams/concurrent/cursor.py | 4 +- .../http/request_window_split_exception.py | 6 +- .../test_model_to_component_factory.py | 10 +- .../retrievers/test_simple_retriever.py | 96 +++++++++---------- .../sources/streams/concurrent/test_cursor.py | 5 +- 8 files changed, 95 insertions(+), 123 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 957cbdcbcd..3b29b62612 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -598,7 +598,6 @@ from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, RequestWindowSplitting, - WindowReducible, ) from airbyte_cdk.sources.declarative.schema import ( ComplexFieldType, @@ -3796,7 +3795,9 @@ def _get_url(req: Requester) -> str: ), request_window_splitting=request_window_splitting, request_window_splitter=( - cursor if request_window_splitting and isinstance(cursor, WindowReducible) else None + cursor.split_request_window + if request_window_splitting and hasattr(cursor, "split_request_window") + else None ), post_pagination_filter=post_pagination_filter, parameters=model.parameters or {}, @@ -4166,13 +4167,13 @@ def _validate_request_window_splitting_is_supported( ) -> None: """ Request window splitting replaces a failing slice with smaller children derived from the stream's own - cursor, so it only makes sense on a stream whose cursor can do that (see `WindowReducible`) and it is - rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and a - `file_uploader` both mean records of the failing window may already have been emitted by the time the + cursor, so it only makes sense on a stream whose cursor exposes a `split_request_window` method and it + is rejected for the same structural reasons `page_size_reduction` is: `additional_query_properties` and + a `file_uploader` both mean records of the failing window may already have been emitted by the time the split is requested, from inside the record generator rather than from a page boundary this retriever controls. """ - if not isinstance(cursor, WindowReducible): + if not hasattr(cursor, "split_request_window"): raise ValueError( f"`request_window_splitting` requires an incremental cursor that supports window splitting on " f"stream {name}. Found {type(cursor).__name__ if cursor is not None else 'no cursor'}: this is " @@ -4185,9 +4186,9 @@ def _validate_request_window_splitting_is_supported( and incremental_sync.cursor_granularity ): # `cursor_granularity` is both the smallest window the connector will ever request and what keeps - # two child windows from overlapping at their shared edge, so `WindowReducible.split_request_window` - # cannot split at all without it - checked here, on the manifest model, rather than by reaching - # into the cursor's private state from outside the class it belongs to. + # two child windows from overlapping at their shared edge, so `split_request_window` cannot split + # at all without it - checked here, on the manifest model, rather than by reaching into the + # cursor's private state from outside the class it belongs to. raise ValueError( f"`request_window_splitting` requires `cursor_granularity` on the `DatetimeBasedCursor` of " f"stream {name}: without it there is no smallest window to stop splitting at, and no way to " diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index fe2aab848d..26730934f8 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -2,6 +2,7 @@ # Copyright (c) 2025 Airbyte, Inc., all rights reserved. # +import datetime import json import logging from collections import defaultdict @@ -52,7 +53,6 @@ from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, RequestWindowSplitting, - WindowReducible, ) from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.source import ExperimentalClassWarning @@ -75,16 +75,16 @@ FULL_REFRESH_SYNC_COMPLETE_KEY = "__ab_full_refresh_sync_complete" LOGGER = logging.getLogger("airbyte") -# A defense-in-depth bound independent of any specific WindowReducible's own no-progress guard (cursor_granularity -# or min_split_window): the retriever enforces this itself so a misbehaving custom cursor (returning children -# that do not actually shrink) fails deterministically rather than recursing indefinitely. Each split bisects a -# single already-generated slice - bounded by the cursor's own `step`, not the whole sync range - and real-world -# APIs that reject oversized windows are typically satisfied well before reaching sub-day granularity: a -# one-year step bisected down to a 12-hour floor needs ~10 halvings (log2(hours in a year / 12) ~= 9.5), which -# already covers a wider window than the request-window failures this feature targets in practice tend to -# involve (on the order of days to a few months). Not user-configurable: a connector whose cursor genuinely -# needs finer-than-half-day granularity, or windows spanning multiple years, should express that through -# `min_split_window` instead of raising this safety net. +# A defense-in-depth bound independent of any specific `request_window_splitter`'s own no-progress guard +# (cursor_granularity or min_split_window): the retriever enforces this itself so a misbehaving custom cursor +# (returning children that do not actually shrink) fails deterministically rather than recursing indefinitely. +# Each split bisects a single already-generated slice - bounded by the cursor's own `step`, not the whole sync +# range - and real-world APIs that reject oversized windows are typically satisfied well before reaching +# sub-day granularity: a one-year step bisected down to a 12-hour floor needs ~10 halvings +# (log2(hours in a year / 12) ~= 9.5), which already covers a wider window than the request-window failures +# this feature targets in practice tend to involve (on the order of days to a few months). Not +# user-configurable: a connector whose cursor genuinely needs finer-than-half-day granularity, or windows +# spanning multiple years, should express that through `min_split_window` instead of raising this safety net. _MAX_REQUEST_WINDOW_SPLIT_DEPTH = 10 @@ -122,10 +122,12 @@ class SimpleRetriever(Retriever): is raised directly by custom code. `None` disables request-window splitting entirely: `read_records` converts the exception into `RequestWindowSplitNotSupportedException` instead of attempting a split it cannot perform. - request_window_splitter (Optional[WindowReducible]): The component - normally the stream's own cursor - asked to - replace a failing `StreamSlice` with smaller children. `None` has the same effect as - `request_window_splitting` being `None`: splitting is not attempted, and a - `RequestWindowSplitRequiredException` is re-raised as `RequestWindowSplitNotSupportedException`. + request_window_splitter (Optional[Callable[[StreamSlice, Optional[datetime.timedelta]], Optional[List[StreamSlice]]]]): + Bound method - normally the stream's own cursor's `split_request_window` - asked to replace a + failing `StreamSlice` with smaller children, given the slice and `request_window_splitting`'s + `min_split_window`. `None` has the same effect as `request_window_splitting` being `None`: + splitting is not attempted, and a `RequestWindowSplitRequiredException` is re-raised as + `RequestWindowSplitNotSupportedException`. """ requester: Requester @@ -152,7 +154,9 @@ class SimpleRetriever(Retriever): post_pagination_filter: Optional[ClientSideIncrementalRecordFilterDecorator] = None page_size_reduction: Optional[PageSizeReduction] = None request_window_splitting: Optional[RequestWindowSplitting] = None - request_window_splitter: Optional[WindowReducible] = None + request_window_splitter: Optional[ + Callable[[StreamSlice, Optional[datetime.timedelta]], Optional[List[StreamSlice]]] + ] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._paginator = self.paginator or NoPagination(parameters=parameters) @@ -601,7 +605,7 @@ def _read_records_or_split_request_window( `original_slice` is threaded through the recursion unchanged so every record yielded, however deep the recursion went to produce it, is re-stamped with the partition's own slice rather than the child slice it was actually read against - see `_reassociate_with_original_slice`. `depth` bounds the recursion - independently of `WindowReducible.split_request_window`'s own no-progress guard: it is enforced here so a + independently of `request_window_splitter`'s own no-progress guard: it is enforced here so a misbehaving custom cursor cannot recurse indefinitely regardless of what that guard does or does not catch. """ @@ -674,7 +678,7 @@ def _read_records_or_split_request_window( failure_type=FailureType.transient_error, ) from exception - children = self.request_window_splitter.split_request_window( + children = self.request_window_splitter( stream_slice, self.request_window_splitting.min_split_window ) if children is None: diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index 0109272556..994f9d1bd8 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -5,9 +5,7 @@ import datetime from dataclasses import dataclass from enum import Enum -from typing import List, Optional, Protocol, runtime_checkable - -from airbyte_cdk.sources.types import StreamSlice +from typing import Optional class OnPartialResponse(Enum): @@ -15,36 +13,6 @@ class OnPartialResponse(Enum): ALLOW_REPLAY = "ALLOW_REPLAY" -@runtime_checkable -class WindowReducible(Protocol): - """ - Implemented by a stream slicer/cursor that can replace a failing `StreamSlice` with smaller child slices - covering the same range. - - The component responsible for slicing owns the boundary field names, parsing, output formatting, and - interval semantics (granularity, clamping, comparison), so `split_request_window` lives on that component - not on - a standalone retriever-side parser - and is checked with `isinstance(slicer, WindowReducible)` wherever a - retriever needs to know whether the stream it is reading supports request-window splitting at all. - """ - - def split_request_window( - self, stream_slice: StreamSlice, min_split_window: Optional[datetime.timedelta] = None - ) -> Optional[List[StreamSlice]]: - """ - Split `stream_slice` into two or more smaller, non-overlapping child slices that together cover exactly - the same range, preserving `partition` and `extra_fields` unchanged. - - Returns `None` when the slice cannot be split any further - either because it is already at (or below) - the minimum granularity the cursor supports, because it is already at or below `min_split_window` (a - connector-configured floor expressed in domain terms rather than raw split count), or because splitting - it would not produce children strictly smaller than the parent (a no-progress guard independent of - either floor). Callers should treat `None` as a terminal condition, not retry with the same slice. - """ - raise NotImplementedError( - "WindowReducible.split_request_window must be implemented by protocol implementers" - ) - - @dataclass(frozen=True) class RequestWindowSplitting: """ @@ -53,12 +21,12 @@ class RequestWindowSplitting: `PageSizeReducer` is: splitting a window does not accumulate attempts across sibling requests the way reducing a page size does, since a split window is never retried at the same size twice - it is either split again (recursion) or the sync fails. All state needed to decide that lives in the `StreamSlice` being - read and the `WindowReducible.split_request_window` result for it, so a plain config object is enough. + read and the `request_window_splitter` result for it, so a plain config object is enough. """ on_partial_response: OnPartialResponse = OnPartialResponse.FAIL failure_message: Optional[str] = None - # An upper bound on how small a window `split_request_window` is asked to produce, expressed in the same + # An upper bound on how small a window `request_window_splitter` is asked to produce, expressed in the same # domain terms as `cursor_granularity` (a duration) rather than a raw split count. Independent of - and # typically looser than - the cursor's own granularity floor: a connector whose cursor could technically # split down to the second may still want to stop earlier, e.g. because the API's rate limit makes many diff --git a/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte_cdk/sources/streams/concurrent/cursor.py index 7b7699ee8f..c808075489 100644 --- a/airbyte_cdk/sources/streams/concurrent/cursor.py +++ b/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -618,8 +618,8 @@ def split_request_window( connector-configured floor independent of and typically looser than `cursor_granularity`), the boundaries are missing/malformed/reversed, or no `cursor_granularity` was configured at all (there is then no way to know the smallest addressable unit, or how to keep two children from overlapping at - their shared edge). This is a generic implementation of `WindowReducible` (see - `airbyte_cdk.sources.declarative.retrievers.window_reducible`) that reuses the same + their shared edge). This is the implementation `SimpleRetriever.request_window_splitter` binds to (see + `airbyte_cdk.sources.declarative.retrievers.simple_retriever`) that reuses the same parsing/formatting/comparison operations `_split_per_slice_range` already relies on, so it works for any `CursorValueType`/`GapType` pair this cursor was built with (datetime/timedelta, or int/int), without a second, duplicate implementation of that logic living outside the cursor. diff --git a/airbyte_cdk/sources/streams/http/request_window_split_exception.py b/airbyte_cdk/sources/streams/http/request_window_split_exception.py index e57e154570..c722b3151c 100644 --- a/airbyte_cdk/sources/streams/http/request_window_split_exception.py +++ b/airbyte_cdk/sources/streams/http/request_window_split_exception.py @@ -49,14 +49,14 @@ class RequestWindowSplitNotSupportedException(AirbyteTracedException): The factory rejects this at config time wherever it can see the error handler, so reaching this means the signal came from somewhere it cannot inspect - a custom error handler, a custom requester/retriever raising - `RequestWindowSplitRequiredException` directly, or a stream whose cursor does not implement - `WindowReducible` (for example, a non-datetime or non-incremental slicer). + `RequestWindowSplitRequiredException` directly, or a stream whose cursor has no `split_request_window` + method (for example, a non-datetime or non-incremental slicer). """ def __init__(self, stream_name: Optional[str] = None) -> None: stream = f"Stream {stream_name}" if stream_name else "The stream" super().__init__( - internal_message=f"A request window split was requested for stream {stream_name} but its retriever defines no `request_window_splitting`, or its cursor does not support window splitting. The action is only supported on a SimpleRetriever whose cursor implements WindowReducible and that defines `request_window_splitting`.", + internal_message=f"A request window split was requested for stream {stream_name} but its retriever defines no `request_window_splitting`, or its cursor does not support window splitting. The action is only supported on a SimpleRetriever whose cursor has a `split_request_window` method and that defines `request_window_splitting`.", message=f"{stream} resolves to a request window split but is not set up to split its window. Add `request_window_splitting` to the stream's retriever (this requires a datetime-based incremental cursor with `cursor_granularity` set), or remove the `SPLIT_REQUEST_WINDOW` action from its error handler.", failure_type=FailureType.config_error, ) diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 435a585e7e..ecd0b7e5d5 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -208,7 +208,6 @@ from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, RequestWindowSplitting, - WindowReducible, ) from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( @@ -7619,7 +7618,8 @@ def test_given_request_window_splitting_then_create_retriever_with_defaults(): on_partial_response=OnPartialResponse.FAIL, failure_message=None, ) - assert isinstance(retriever.request_window_splitter, WindowReducible) + assert callable(retriever.request_window_splitter) + assert retriever.request_window_splitter.__name__ == "split_request_window" def test_given_request_window_splitting_values_then_create_retriever_with_those_values(): @@ -7753,7 +7753,7 @@ def test_given_no_incremental_sync_and_request_window_splitting_then_raise(): with pytest.raises(ValueError) as exception: _request_window_splitting_stream(incremental_sync="") - assert "cursor_granularity" in str(exception.value) or "WindowReducible" in str(exception.value) + assert "cursor_granularity" in str(exception.value) def test_given_datetime_based_cursor_without_cursor_granularity_then_raise(): @@ -7805,8 +7805,8 @@ def test_given_datetime_format_coarser_than_cursor_granularity_and_request_windo def test_given_incrementing_count_cursor_and_request_window_splitting_then_raise(): """ `IncrementingCountCursor` builds a `ConcurrentCursor` too, but not one carrying the datetime boundary - fields, granularity, or formatting `WindowReducible.split_request_window` needs, so it must be rejected the same - way a stream with no incremental_sync at all is. + fields, granularity, or formatting `split_request_window` needs, so it must be rejected the same way a + stream with no incremental_sync at all is. """ with pytest.raises(ValueError) as exception: _request_window_splitting_stream( diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 09d6bd76b3..07cc7fc532 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -8,7 +8,7 @@ from concurrent.futures import ThreadPoolExecutor from datetime import timedelta from functools import partial -from typing import Any, Iterable, Mapping, Optional +from typing import Any, Callable, Iterable, List, Mapping, Optional from unittest.mock import MagicMock, Mock, patch import pytest @@ -60,7 +60,6 @@ from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( OnPartialResponse, RequestWindowSplitting, - WindowReducible, ) from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( PageSizeReductionRequiredException, @@ -2037,7 +2036,9 @@ def _request_window_splitting_retriever( requester: Requester, paginator: Paginator, record_selector: HttpSelector, - request_window_splitter: WindowReducible, + request_window_splitter: Callable[ + [StreamSlice, Optional[timedelta]], Optional[List[StreamSlice]] + ], request_window_splitting: Optional[RequestWindowSplitting] = None, ) -> SimpleRetriever: return SimpleRetriever( @@ -2066,8 +2067,8 @@ def test_given_request_window_splitting_when_read_records_then_split_and_read_bo paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [ + request_window_splitter = Mock() + request_window_splitter.return_value = [ A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE, ] @@ -2079,7 +2080,7 @@ def test_given_request_window_splitting_when_read_records_then_split_and_read_bo records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [{"id": 1}, {"id": 2}] - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) + request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, None) assert requester.send_request.call_count == 3 assert [call.kwargs["stream_slice"] for call in requester.send_request.call_args_list] == [ A_WINDOW_SLICE, @@ -2099,8 +2100,8 @@ def test_given_min_split_window_configured_when_read_records_then_pass_it_to_the paginator = _mock_paginator() paginator.get_initial_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = None + request_window_splitter = Mock() + request_window_splitter.return_value = None retriever = _request_window_splitting_retriever( requester, @@ -2113,9 +2114,7 @@ def test_given_min_split_window_configured_when_read_records_then_pass_it_to_the with pytest.raises(AirbyteTracedException): list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) - request_window_splitter.split_request_window.assert_called_once_with( - A_WINDOW_SLICE, timedelta(days=1) - ) + request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, timedelta(days=1)) def test_given_only_one_child_needs_further_reduction_when_read_records_then_recurse_asymmetrically(): @@ -2145,8 +2144,8 @@ def test_given_only_one_child_needs_further_reduction_when_read_records_then_rec cursor_slice={"start_time": "2024-01-01T06:00:00Z", "end_time": "2024-01-01T11:59:59Z"}, partition={}, ) - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.side_effect = [ + request_window_splitter = Mock() + request_window_splitter.side_effect = [ [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE], [a_quarter_slice, another_quarter_slice], ] @@ -2158,7 +2157,7 @@ def test_given_only_one_child_needs_further_reduction_when_read_records_then_rec records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [{"id": 1}, {"id": 2}, {"id": 3}] - assert request_window_splitter.split_request_window.call_args_list == [ + assert request_window_splitter.call_args_list == [ ((A_WINDOW_SLICE, None),), ((A_FIRST_HALF_SLICE, None),), ] @@ -2229,7 +2228,7 @@ def test_given_records_already_emitted_and_fail_policy_when_reduction_requested_ paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = {"next_page_token": "page2"} - request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter = Mock() retriever = _request_window_splitting_retriever( requester, @@ -2243,7 +2242,7 @@ def test_given_records_already_emitted_and_fail_policy_when_reduction_requested_ list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert exception.value.failure_type == FailureType.config_error - request_window_splitter.split_request_window.assert_not_called() + request_window_splitter.assert_not_called() def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_requested_then_reduce_anyway(): @@ -2264,8 +2263,8 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re paginator.get_initial_token.return_value = None paginator.next_page_token.side_effect = [{"next_page_token": "page2"}, None, None] - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [ + request_window_splitter = Mock() + request_window_splitter.return_value = [ A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE, ] @@ -2285,12 +2284,12 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re # the record from the page emitted before the reduction signal is included once more: ALLOW_REPLAY # accepts this at-least-once duplication rather than silently dropping it or failing the sync. assert records == [{"id": 1}, {"id": 2}, {"id": 3}] - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) + request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, None) def test_given_request_window_splitter_returns_none_when_reduction_requested_then_raise_terminal_error(): """ - `WindowReducible.split_request_window` returning `None` means the window cannot be split any further - either + `request_window_splitter` returning `None` means the window cannot be split any further - either the granularity floor was reached or reducing would not make progress. Either way this must terminate the sync deterministically rather than loop or silently drop the window. """ @@ -2302,8 +2301,8 @@ def test_given_request_window_splitter_returns_none_when_reduction_requested_the paginator = _mock_paginator() paginator.get_initial_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = None + request_window_splitter = Mock() + request_window_splitter.return_value = None retriever = _request_window_splitting_retriever( requester, @@ -2321,7 +2320,7 @@ def test_given_request_window_splitter_returns_none_when_reduction_requested_the # the classifying error's own failure_type is preserved rather than falling back to this branch's default assert exception.value.failure_type == FailureType.transient_error assert "Lower time_window so that each request covers less data." in exception.value.message - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) + request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, None) def test_given_request_window_splitter_returns_none_and_no_classified_failure_type_when_reduction_requested_then_raise_transient_error(): @@ -2337,8 +2336,8 @@ def test_given_request_window_splitter_returns_none_and_no_classified_failure_ty paginator = _mock_paginator() paginator.get_initial_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = None + request_window_splitter = Mock() + request_window_splitter.return_value = None retriever = _request_window_splitting_retriever( requester, @@ -2360,8 +2359,8 @@ def test_given_empty_children_list_when_reduction_requested_then_emit_no_records paginator = _mock_paginator() paginator.get_initial_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [] + request_window_splitter = Mock() + request_window_splitter.return_value = [] retriever = _request_window_splitting_retriever( requester, paginator, record_selector, request_window_splitter @@ -2370,7 +2369,7 @@ def test_given_empty_children_list_when_reduction_requested_then_emit_no_records records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) assert records == [] - request_window_splitter.split_request_window.assert_called_once_with(A_WINDOW_SLICE, None) + request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, None) def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates(): @@ -2390,8 +2389,8 @@ def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.side_effect = [ + request_window_splitter = Mock() + request_window_splitter.side_effect = [ [A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE], None, ] @@ -2407,7 +2406,7 @@ def test_given_a_later_child_fails_when_read_records_then_the_failure_propagates # the first child's records were already emitted to the caller before the second child failed assert emitted == [{"id": 1}] - assert request_window_splitter.split_request_window.call_args_list == [ + assert request_window_splitter.call_args_list == [ ((A_WINDOW_SLICE, None),), ((A_SECOND_HALF_SLICE, None),), ] @@ -2431,8 +2430,8 @@ def test_given_request_window_splitting_when_read_records_then_each_child_gets_f paginator.get_initial_token.return_value = "initial token" paginator.next_page_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [ + request_window_splitter = Mock() + request_window_splitter.return_value = [ A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE, ] @@ -2516,8 +2515,8 @@ def select_records(*, response, stream_slice, **kwargs): paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [ + request_window_splitter = Mock() + request_window_splitter.return_value = [ A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE, ] @@ -2556,8 +2555,8 @@ def test_given_exception_raised_during_record_extraction_when_read_then_still_sp paginator.get_initial_token.return_value = None paginator.next_page_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.return_value = [ + request_window_splitter = Mock() + request_window_splitter.return_value = [ A_FIRST_HALF_SLICE, A_SECOND_HALF_SLICE, ] @@ -2573,7 +2572,7 @@ def test_given_exception_raised_during_record_extraction_when_read_then_still_sp def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_error(): """ - Defense-in-depth: even if a (misbehaving, e.g. custom) `WindowReducible` never itself signals "cannot split + Defense-in-depth: even if a (misbehaving, e.g. custom) `request_window_splitter` never itself signals "cannot split any further", the retriever's own depth counter must still terminate the read deterministically rather than recursing without bound. """ @@ -2583,11 +2582,12 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er paginator = _mock_paginator() paginator.get_initial_token.return_value = None - request_window_splitter = Mock(spec=WindowReducible) + request_window_splitter = Mock() # never signals "no further progress possible" - always splits into two children identical to the parent - request_window_splitter.split_request_window.side_effect = ( - lambda stream_slice, min_split_window=None: [stream_slice, stream_slice] - ) + request_window_splitter.side_effect = lambda stream_slice, min_split_window=None: [ + stream_slice, + stream_slice, + ] retriever = _request_window_splitting_retriever( requester, @@ -2611,9 +2611,7 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er # the single leftmost recursion path is explored to the full depth (one split_request_window call per # depth, from 0 up to the cap) before the cap stops the next call - the exception then propagates # immediately, so sibling branches at shallower depths are never explored - assert ( - request_window_splitter.split_request_window.call_count == _MAX_REQUEST_WINDOW_SPLIT_DEPTH - ) + assert request_window_splitter.call_count == _MAX_REQUEST_WINDOW_SPLIT_DEPTH def test_given_partitions_read_concurrently_then_window_reduction_isolates_between_partitions(): @@ -2663,7 +2661,7 @@ def select_records(*, response, stream_slice, **kwargs): paginator.next_page_token.return_value = None def split_request_window(stream_slice, min_split_window=None): - # preserve the parent's own partition, matching what a real WindowReducible would do + # preserve the parent's own partition, matching what a real request_window_splitter would do return [ StreamSlice( cursor_slice={ @@ -2681,8 +2679,8 @@ def split_request_window(stream_slice, min_split_window=None): ), ] - request_window_splitter = Mock(spec=WindowReducible) - request_window_splitter.split_request_window.side_effect = split_request_window + request_window_splitter = Mock() + request_window_splitter.side_effect = split_request_window retriever = _request_window_splitting_retriever( requester, paginator, record_selector, request_window_splitter @@ -2696,4 +2694,4 @@ def split_request_window(stream_slice, min_split_window=None): assert records_b == [{"partition": "b", "start": "2024-02-01T00:00:00Z"}] assert len(records_a) == 2 - request_window_splitter.split_request_window.assert_called_once_with(slice_a, None) + request_window_splitter.assert_called_once_with(slice_a, None) diff --git a/unit_tests/sources/streams/concurrent/test_cursor.py b/unit_tests/sources/streams/concurrent/test_cursor.py index 3bac4be4ad..64bbb2726c 100644 --- a/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/unit_tests/sources/streams/concurrent/test_cursor.py @@ -1415,8 +1415,9 @@ def test_final_state_cursor_get_cursor_datetime_from_state_returns_now_for_no_cu class ConcurrentCursorReduceWindowTest(TestCase): """ - Covers ConcurrentCursor.split_request_window(), the WindowReducible implementation backing declarative - `request_window_splitting`. See airbyte_cdk.sources.declarative.retrievers.window_reducible. + Covers ConcurrentCursor.split_request_window(), the implementation `SimpleRetriever.request_window_splitter` + binds to for declarative `request_window_splitting`. See + airbyte_cdk.sources.declarative.retrievers.simple_retriever. """ def setUp(self) -> None: From d2e977c6d9c5c5d4379a48dfabd50297b5ab5c73 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Thu, 24 Sep 2026 13:08:36 -0700 Subject: [PATCH 12/21] style(cdk): shorten request window splitting descriptions and comments No behavior change: trims overly long manifest schema descriptions (RequestWindowSplitting and its fields) and several verbose internal comments/log messages left over from Phases 1-5, keeping the same information more concisely. --- .../declarative_component_schema.yaml | 51 ++++++------------- .../models/declarative_component_schema.py | 8 +-- .../parsers/model_to_component_factory.py | 22 ++++---- .../retrievers/simple_retriever.py | 31 ++++------- .../retrievers/window_reducible.py | 14 ++--- 5 files changed, 42 insertions(+), 84 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 703d04183e..4237c1a2cb 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4266,13 +4266,9 @@ definitions: request_window_splitting: description: >- Describes how the request window is split when an error handler resolves to the - SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, - since that is both the smallest window the connector will request and what keeps two child windows - from overlapping at their shared edge. Distinct from page_size_reduction: an API can reject an entire - date/time range as too large even though every individual page of it would be within the API's page - size limit, and no page can be returned to page through in the first place. On such a response the - connector replaces the failing window with two smaller, ordered windows covering the same range and - reads each one fully, with its own pagination state, before checkpointing the original partition. + SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity. + Distinct from page_size_reduction: this splits the cursor's window rather than a paginator's page + size, and applies even when the API rejects the request before returning a single page. "$ref": "#/definitions/RequestWindowSplitting" ignore_stream_slicer_parameters_on_paginated_requests: description: If true, the partition router and incremental request options will be ignored when paginating requests. Request options set directly on the requester will not be ignored. @@ -4458,13 +4454,10 @@ definitions: title: Request Window Splitting description: >- Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW - action. On such a response, the connector replaces the failing window with two smaller, ordered windows - that together cover the same range, and reads each one fully - with its own pagination state - instead of - retrying the identical request. Only supported on a stream whose incremental_sync cursor defines - cursor_granularity, since that is both the smallest window the connector will ever request and what keeps - the two child windows from overlapping at their shared edge. Distinct from page_size_reduction: this - operates on the window a DatetimeBasedCursor slices by, not on a paginator's page size, and applies even - when the API rejects the request before returning a single page. + action. The connector replaces the failing window with two smaller, ordered windows covering the same + range and reads each one fully before checkpointing. Only supported on a stream whose incremental_sync + cursor defines cursor_granularity. Distinct from page_size_reduction, which shrinks a paginator's page + size rather than the cursor's window. type: object required: - type @@ -4475,14 +4468,10 @@ definitions: on_partial_response: title: On Partial Response description: >- - What to do when a SPLIT_REQUEST_WINDOW response arrives after the current window has already - emitted a page or a record. FAIL stops the sync with a configuration error, since splitting and - re-reading the window from scratch would emit those records a second time - this is correct for an - API, such as PayPal's, that always rejects a window before returning its first page. ALLOW_REPLAY - splits and re-reads the window anyway, accepting at-least-once delivery of the records already - emitted; use it only when the destination or a downstream deduplication step can tolerate duplicate - records, for instance when adopting this action for a transport or streamed-decoding failure that can - occur after some records of the window were already read. + What to do when SPLIT_REQUEST_WINDOW arrives after the current window already emitted a record. + FAIL stops the sync with a configuration error, since re-reading the window would duplicate those + records. ALLOW_REPLAY splits and re-reads anyway, accepting duplicate records; use it only when + duplicates are tolerated downstream. type: string enum: - FAIL @@ -4491,11 +4480,8 @@ definitions: failure_message: title: Failure Message description: >- - Sentence appended to the error message shown to the user when the connector runs out of reductions, - either because the window is already at the cursor's minimum granularity or because reducing it would - not make progress. Use it to tell the user what they can do about it in terms of this specific API, - for instance which filter narrows the query down. Without it the message only states that the API - kept rejecting every window size the connector asked for. + Sentence appended to the error message when the connector can no longer split the window further. + Use it to tell the user what to do for this API, for instance which filter narrows the query down. type: string examples: - Narrow the sync down by selecting fewer fields on this stream. @@ -4503,14 +4489,9 @@ definitions: min_split_window: title: Minimum Split Window description: >- - Smallest window (ISO 8601 duration) the connector will ever request, expressed in the same domain - terms as `cursor_granularity` rather than a raw split count. Independent of - and typically looser - than - the cursor's own `cursor_granularity` floor: a connector whose cursor could technically split - down to the second may still want to stop earlier, for instance because the API's rate limit makes - many small requests worse than a few large ones. A window already at or below this size is not split - any further, the same way one at the `cursor_granularity` floor is not. Splitting is also bounded by - an internal safety net independent of this field, which protects against a custom cursor whose - `split_request_window` implementation returns children that do not actually shrink the window. + Smallest window (ISO 8601 duration) the connector will request. Independent of, and usually looser + than, cursor_granularity - useful when a cursor could split further but many small requests are + worse for this API than a few large ones. type: string examples: - "P1D" diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 983078d489..de1d15d733 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1482,12 +1482,12 @@ class RequestWindowSplitting(BaseModel): type: Literal["RequestWindowSplitting"] on_partial_response: Optional[OnPartialResponse] = Field( OnPartialResponse.FAIL, - description="What to do when a SPLIT_REQUEST_WINDOW response arrives after the current window has already emitted a page or a record. FAIL stops the sync with a configuration error, since splitting and re-reading the window from scratch would emit those records a second time - this is correct for an API, such as PayPal's, that always rejects a window before returning its first page. ALLOW_REPLAY splits and re-reads the window anyway, accepting at-least-once delivery of the records already emitted; use it only when the destination or a downstream deduplication step can tolerate duplicate records, for instance when adopting this action for a transport or streamed-decoding failure that can occur after some records of the window were already read.", + description="What to do when SPLIT_REQUEST_WINDOW arrives after the current window already emitted a record. FAIL stops the sync with a configuration error, since re-reading the window would duplicate those records. ALLOW_REPLAY splits and re-reads anyway, accepting duplicate records; use it only when duplicates are tolerated downstream.", title="On Partial Response", ) failure_message: Optional[str] = Field( None, - description="Sentence appended to the error message shown to the user when the connector runs out of reductions, either because the window is already at the cursor's minimum granularity or because reducing it would not make progress. Use it to tell the user what they can do about it in terms of this specific API, for instance which filter narrows the query down. Without it the message only states that the API kept rejecting every window size the connector asked for.", + description="Sentence appended to the error message when the connector can no longer split the window further. Use it to tell the user what to do for this API, for instance which filter narrows the query down.", examples=[ "Narrow the sync down by selecting fewer fields on this stream.", "Lower time_window so that each request covers less data.", @@ -1496,7 +1496,7 @@ class RequestWindowSplitting(BaseModel): ) min_split_window: Optional[str] = Field( None, - description="Smallest window (ISO 8601 duration) the connector will ever request, expressed in the same domain terms as `cursor_granularity` rather than a raw split count. Independent of - and typically looser than - the cursor's own `cursor_granularity` floor: a connector whose cursor could technically split down to the second may still want to stop earlier, for instance because the API's rate limit makes many small requests worse than a few large ones. A window already at or below this size is not split any further, the same way one at the `cursor_granularity` floor is not. Splitting is also bounded by an internal safety net independent of this field, which protects against a custom cursor whose `split_request_window` implementation returns children that do not actually shrink the window.", + description="Smallest window (ISO 8601 duration) the connector will request. Independent of, and usually looser than, cursor_granularity - useful when a cursor could split further but many small requests are worse for this API than a few large ones.", examples=["P1D", "PT1H"], title="Minimum Split Window", ) @@ -3331,7 +3331,7 @@ class SimpleRetriever(BaseModel): ) request_window_splitting: Optional[RequestWindowSplitting] = Field( None, - description="Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity, since that is both the smallest window the connector will request and what keeps two child windows from overlapping at their shared edge. Distinct from page_size_reduction: an API can reject an entire date/time range as too large even though every individual page of it would be within the API's page size limit, and no page can be returned to page through in the first place. On such a response the connector replaces the failing window with two smaller, ordered windows covering the same range and reads each one fully, with its own pagination state, before checkpointing the original partition.", + description="Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. Requires an incremental_sync cursor that defines cursor_granularity. Distinct from page_size_reduction: this splits the cursor's window rather than a paginator's page size, and applies even when the API rejects the request before returning a single page.", ) ignore_stream_slicer_parameters_on_paginated_requests: Optional[bool] = Field( False, diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 3b29b62612..4cda71f17a 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -719,10 +719,9 @@ # this would be a circular import MAX_SLICES = 5 -# The smallest duration each `DatetimeParser`/`strftime` directive can distinguish between two values, finest -# first so ties are irrelevant - used by `_smallest_datetime_format_unit` to check that `datetime_format` can -# actually represent whatever `cursor_granularity` claims. Directives this table has no entry for (a bare -# `%Y`/`%m`, or any other formatting scheme) are treated as "cannot determine" rather than rejected. +# Smallest duration each `DatetimeParser`/`strftime` directive can distinguish, used by +# `_smallest_datetime_format_unit` to check `datetime_format` can represent `cursor_granularity`. A format with +# no entry here (e.g. a bare `%Y`/`%m`) is treated as "cannot determine" rather than rejected. _DATETIME_FORMAT_DIRECTIVE_GRANULARITY: List[Tuple[str, datetime.timedelta]] = [ ("%epoch_microseconds", datetime.timedelta(microseconds=1)), ("%s_as_float", datetime.timedelta(microseconds=1)), @@ -4199,9 +4198,8 @@ def _validate_request_window_splitting_is_supported( if isinstance( parsed_cursor_granularity, datetime.timedelta ) and parsed_cursor_granularity <= datetime.timedelta(0): - # A zero-length (or, via a leading `-`, negative) granularity passes the truthy check above - the - # string is non-empty - but `ConcurrentCursor.split_request_window` divides the window's span by it - # to find the split point, so it can never produce a child strictly smaller than the parent. + # Passes the truthy check above (the string is non-empty), but a zero-or-negative granularity can + # never produce a child strictly smaller than the parent. raise ValueError( f"`request_window_splitting` requires a `cursor_granularity` greater than zero on stream " f"{name}: `{incremental_sync.cursor_granularity}` parses to a zero-length duration, which can " @@ -4214,9 +4212,8 @@ def _validate_request_window_splitting_is_supported( and isinstance(parsed_cursor_granularity, datetime.timedelta) and parsed_cursor_granularity < smallest_format_unit ): - # `cursor_granularity` claims a finer boundary than `datetime_format` can render: two children - # split that close together would format to the same boundary value, so the retriever would ask - # for the same window twice - or, once formatted, request a reversed one - instead of splitting. + # Two children split closer together than `datetime_format` can render would format to the same + # boundary value instead of splitting. raise ValueError( f"`request_window_splitting` requires `cursor_granularity` to be no finer than what " f"`datetime_format` can represent on stream {name}: `datetime_format` " @@ -4243,9 +4240,8 @@ def _validate_request_window_splitting_is_supported( @staticmethod def _smallest_datetime_format_unit(datetime_format: str) -> Optional[datetime.timedelta]: """ - :return: the smallest duration `datetime_format` can distinguish between two values - the finest - directive it contains, per `_DATETIME_FORMAT_DIRECTIVE_GRANULARITY` - or `None` if it contains none - of the directives that table recognizes, in which case the caller cannot validate against it. + :return: the finest duration `datetime_format` can distinguish, per + `_DATETIME_FORMAT_DIRECTIVE_GRANULARITY`, or `None` if it contains none of those directives. """ matches = [ granularity diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index 26730934f8..a3f264422e 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -75,16 +75,9 @@ FULL_REFRESH_SYNC_COMPLETE_KEY = "__ab_full_refresh_sync_complete" LOGGER = logging.getLogger("airbyte") -# A defense-in-depth bound independent of any specific `request_window_splitter`'s own no-progress guard -# (cursor_granularity or min_split_window): the retriever enforces this itself so a misbehaving custom cursor -# (returning children that do not actually shrink) fails deterministically rather than recursing indefinitely. -# Each split bisects a single already-generated slice - bounded by the cursor's own `step`, not the whole sync -# range - and real-world APIs that reject oversized windows are typically satisfied well before reaching -# sub-day granularity: a one-year step bisected down to a 12-hour floor needs ~10 halvings -# (log2(hours in a year / 12) ~= 9.5), which already covers a wider window than the request-window failures -# this feature targets in practice tend to involve (on the order of days to a few months). Not -# user-configurable: a connector whose cursor genuinely needs finer-than-half-day granularity, or windows -# spanning multiple years, should express that through `min_split_window` instead of raising this safety net. +# A defense-in-depth bound independent of any `request_window_splitter`'s own no-progress guard: protects +# against a misbehaving custom cursor whose children don't actually shrink the window. Not user-configurable; +# use `min_split_window` for a legitimate need to split more than this. _MAX_REQUEST_WINDOW_SPLIT_DEPTH = 10 @@ -122,12 +115,9 @@ class SimpleRetriever(Retriever): is raised directly by custom code. `None` disables request-window splitting entirely: `read_records` converts the exception into `RequestWindowSplitNotSupportedException` instead of attempting a split it cannot perform. - request_window_splitter (Optional[Callable[[StreamSlice, Optional[datetime.timedelta]], Optional[List[StreamSlice]]]]): - Bound method - normally the stream's own cursor's `split_request_window` - asked to replace a - failing `StreamSlice` with smaller children, given the slice and `request_window_splitting`'s - `min_split_window`. `None` has the same effect as `request_window_splitting` being `None`: - splitting is not attempted, and a `RequestWindowSplitRequiredException` is re-raised as - `RequestWindowSplitNotSupportedException`. + request_window_splitter: Bound method - normally the stream's own cursor's `split_request_window` - + asked to replace a failing `StreamSlice` with smaller children. `None` has the same effect as + `request_window_splitting` being `None`. """ requester: Requester @@ -657,14 +647,11 @@ def _read_records_or_split_request_window( ) from exception if depth >= _MAX_REQUEST_WINDOW_SPLIT_DEPTH: - # A safety net, expected to be rare: logged separately from the exception below so it is - # visible even if whatever catches the trace message does not surface its internal_message. + # Logged separately from the exception below so it stays visible even if the trace message's + # internal_message isn't surfaced by whatever catches it. LOGGER.warning( f"Stream {self.name} hit the maximum request window split depth " - f"({_MAX_REQUEST_WINDOW_SPLIT_DEPTH}) while splitting {original_slice}. This is a safety " - f"net against a cursor whose `split_request_window` returns children that do not actually " - f"shrink the window; a stream needing more splits than that for a legitimate reason should " - f"raise `min_split_window` instead of relying on more splits." + f"({_MAX_REQUEST_WINDOW_SPLIT_DEPTH}) while splitting {original_slice}." ) raise AirbyteTracedException( internal_message=f"Stream {self.name} exceeded the maximum request window split depth of {_MAX_REQUEST_WINDOW_SPLIT_DEPTH} while splitting {original_slice}", diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index 994f9d1bd8..9786d3c108 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -17,18 +17,12 @@ class OnPartialResponse(Enum): class RequestWindowSplitting: """ Configuration for the `request_window_splitting` retriever field - created once per stream and shared, - mirroring `PageSizeReduction`. There is no per-partition mutable counterpart to instantiate the way - `PageSizeReducer` is: splitting a window does not accumulate attempts across sibling requests the way - reducing a page size does, since a split window is never retried at the same size twice - it is either - split again (recursion) or the sync fails. All state needed to decide that lives in the `StreamSlice` being - read and the `request_window_splitter` result for it, so a plain config object is enough. + mirroring `PageSizeReduction`. Unlike `PageSizeReducer`, there is no per-partition mutable counterpart: a + split window is either split again (recursion) or the sync fails, so a plain config object is enough. """ on_partial_response: OnPartialResponse = OnPartialResponse.FAIL failure_message: Optional[str] = None - # An upper bound on how small a window `request_window_splitter` is asked to produce, expressed in the same - # domain terms as `cursor_granularity` (a duration) rather than a raw split count. Independent of - and - # typically looser than - the cursor's own granularity floor: a connector whose cursor could technically - # split down to the second may still want to stop earlier, e.g. because the API's rate limit makes many - # small requests worse than a few large ones. + # Independent of, and typically looser than, the cursor's own granularity floor - useful when a cursor + # could split further but the API's rate limit makes many small requests worse than a few large ones. min_split_window: Optional[datetime.timedelta] = None From 92ec49592498875241aa439bafca9cbba1ce5ae6 Mon Sep 17 00:00:00 2001 From: "octavia-bot[bot]" <108746235+octavia-bot[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 20:23:05 +0000 Subject: [PATCH 13/21] Auto-committed changes from Poe command `build` --- .../models/declarative_component_schema.py | 571 +++++++++--------- 1 file changed, 282 insertions(+), 289 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index de1d15d733..1f9a3b2171 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -6,7 +6,7 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic.v1 import BaseModel, Extra, Field +from pydantic.v1 import BaseModel, Extra, Field, confloat, conint from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( BaseModelWithDeprecations, @@ -18,12 +18,6 @@ class AuthFlowType(Enum): oauth1_0 = "oauth1.0" -class ScopesJoinStrategy(Enum): - space = "space" - comma = "comma" - plus = "plus" - - class BasicHttpAuthenticator(BaseModel): type: Literal["BasicHttpAuthenticator"] username: str = Field( @@ -52,15 +46,50 @@ class BearerAuthenticator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class HttpMethod(Enum): + GET = "GET" + POST = "POST" + + +class QuotaStatusSource(BaseModel): + type: Literal["QuotaStatusSource"] + url: str = Field( + ..., + description="The full URL of the quota status endpoint.", + examples=[ + "https://api.github.com/rate_limit", + "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", + ], + title="URL", + ) + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description="The HTTP method used to fetch the quota status.", + title="HTTP Method", + ) + request_headers: Optional[Dict[str, str]] = Field( + None, + description="Additional headers to send with the quota status request.", + title="Request Headers", + ) + unavailable_status_codes: Optional[List[int]] = Field( + None, + description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", + examples=[[404]], + title="Unavailable Status Codes", + unique_items=True, + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class DynamicStreamCheckConfig(BaseModel): type: Literal["DynamicStreamCheckConfig"] dynamic_stream_name: str = Field( ..., description="The dynamic stream name.", title="Dynamic Stream Name" ) - stream_count: Optional[int] = Field( + stream_count: Optional[conint(ge=1)] = Field( None, description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", - ge=1, title="Stream Count", ) @@ -104,17 +133,16 @@ class ConcurrencyLevel(BaseModel): class ConstantBackoffStrategy(BaseModel): type: Literal["ConstantBackoffStrategy"] - backoff_time_in_seconds: Union[float, str] = Field( + backoff_time_in_seconds: Union[confloat(ge=0.0), str] = Field( ..., description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], title="Backoff Time", ) - jitter_range_in_seconds: Optional[float] = Field( + jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], - ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -503,7 +531,7 @@ class Config: ) weight: Optional[Union[int, str]] = Field( None, - description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", + description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", title="Weight", ) @@ -531,11 +559,10 @@ class ExponentialBackoffStrategy(BaseModel): examples=[5, 5.5, "10"], title="Factor", ) - jitter_range_in_seconds: Optional[float] = Field( + jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], - ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -556,157 +583,6 @@ class SessionTokenRequestBearerAuthenticator(BaseModel): type: Literal["Bearer"] -class HttpMethod(Enum): - GET = "GET" - POST = "POST" - - -class QuotaStatusSource(BaseModel): - type: Literal["QuotaStatusSource"] - url: str = Field( - ..., - description="The full URL of the quota status endpoint.", - examples=[ - "https://api.github.com/rate_limit", - "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", - ], - title="URL", - ) - http_method: Optional[HttpMethod] = Field( - HttpMethod.GET, - description="The HTTP method used to fetch the quota status.", - title="HTTP Method", - ) - request_headers: Optional[Dict[str, str]] = Field( - None, - description="Additional headers to send with the quota status request.", - title="Request Headers", - ) - unavailable_status_codes: Optional[List[int]] = Field( - None, - description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", - examples=[[404]], - title="Unavailable Status Codes", - unique_items=True, - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class TokenQuota(BaseModel): - type: Literal["TokenQuota"] - name: str = Field( - ..., - description="Name of the quota pool.", - examples=["rest", "graphql"], - title="Name", - ) - remaining_path: List[str] = Field( - ..., - description="Path to the remaining call count for this pool in the quota status response.", - examples=[["resources", "core", "remaining"]], - title="Remaining Path", - ) - reset_path: List[str] = Field( - ..., - description="Path to the quota reset timestamp for this pool in the quota status response.", - examples=[["resources", "core", "reset"]], - title="Reset Path", - ) - limit_path: Optional[List[str]] = Field( - None, - description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", - examples=[["resources", "core", "limit"]], - title="Limit Path", - ) - matchers: Optional[List[HttpRequestRegexMatcher]] = Field( - None, - description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", - title="Matchers", - ) - remaining_header: Optional[str] = Field( - None, - description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", - examples=["X-RateLimit-Remaining"], - title="Remaining Header", - ) - reset_header: Optional[str] = Field( - None, - description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", - examples=["X-RateLimit-Reset"], - title="Reset Header", - ) - limit_header: Optional[str] = Field( - None, - description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", - examples=["X-RateLimit-Limit"], - title="Limit Header", - ) - exhaustion_status_codes: Optional[List[int]] = Field( - None, - description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", - examples=[[429]], - title="Exhaustion Status Codes", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class RateLimitedMultipleTokenAuthenticator(BaseModel): - type: Literal["RateLimitedMultipleTokenAuthenticator"] - tokens: Union[str, List[str]] = Field( - ..., - description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", - examples=[ - "{{ config['credentials']['personal_access_token'] }}", - ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], - ], - title="Tokens", - ) - token_delimiter: Optional[str] = Field( - ",", - description="Delimiter used to split a single token string into multiple tokens.", - title="Token Delimiter", - ) - auth_method: Optional[str] = Field( - "Bearer", - description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", - examples=["Bearer", "token"], - title="Auth Method", - ) - header: Optional[str] = Field( - "Authorization", - description="The name of the HTTP header in which to inject the token.", - title="Header Name", - ) - quota_status_source: QuotaStatusSource = Field( - ..., - description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", - title="Quota Status Source", - ) - quotas: List[TokenQuota] = Field( - ..., - description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", - min_items=1, - title="Quota Pools", - ) - max_wait_time: Optional[str] = Field( - "PT2H", - description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", - examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], - title="Maximum Wait Time", - ) - budget_reserve_fraction: Optional[float] = Field( - 0.1, - description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", - title="Budget Reserve Fraction", - ) - budget_min_reserve: Optional[int] = Field( - 50, - description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", - title="Budget Minimum Reserve", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class Action(Enum): SUCCESS = "SUCCESS" FAIL = "FAIL" @@ -840,12 +716,13 @@ class JsonItemsDecoder(BaseModel): type: Literal["JsonItemsDecoder"] items_path: str = Field( ..., - description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", + description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", + examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], title="Items Path", ) encoding: Optional[str] = Field( "utf-8", - description="The character encoding of the JSON data. Defaults to UTF-8.", + description="Text encoding used to decode the streamed bytes before JSON parsing.", title="Encoding", ) @@ -1008,22 +885,32 @@ class NoPagination(BaseModel): type: Literal["NoPagination"] -class State(BaseModel): +class Scope(BaseModel): class Config: extra = Extra.allow - min: int - max: int + scope: str = Field(..., description="The OAuth scope string to request from the provider.") -class OAuthScope(BaseModel): +class OptionalScope(BaseModel): class Config: extra = Extra.allow - scope: str = Field( - ..., - description="The OAuth scope string to request from the provider.", - ) + scope: str = Field(..., description="The OAuth scope string to request from the provider.") + + +class ScopesJoinStrategy(Enum): + space = "space" + comma = "comma" + plus = "plus" + + +class State(BaseModel): + class Config: + extra = Extra.allow + + min: int + max: int class OauthConnectorInputSpecification(BaseModel): @@ -1045,17 +932,13 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) - # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the - # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. - # The CDK schema defines the manifest contract; the platform reads these fields - # during the OAuth consent flow to build the authorization URL. - scopes: Optional[List[OAuthScope]] = Field( + scopes: Optional[List[Scope]] = Field( None, description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], title="Scopes", ) - optional_scopes: Optional[List[OAuthScope]] = Field( + optional_scopes: Optional[List[OptionalScope]] = Field( None, description="Optional OAuth scope objects that may or may not be granted.", examples=[[{"scope": "admin:read"}]], @@ -1422,39 +1305,34 @@ class ResetPolicy(Enum): class PageSizeReduction(BaseModel): type: Literal["PageSizeReduction"] - reduction_factor: Optional[float] = Field( + reduction_factor: Optional[confloat(gt=1.0)] = Field( 2, description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", examples=[2, 4], - gt=1.0, title="Reduction Factor", ) - minimum_page_size: Optional[int] = Field( + minimum_page_size: Optional[conint(ge=1)] = Field( 1, description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", examples=[1, 10], - ge=1, title="Minimum Page Size", ) - max_attempts: Optional[int] = Field( + max_attempts: Optional[conint(ge=1)] = Field( 5, description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", examples=[5, 10], - ge=1, title="Maximum Reduction Attempts", ) - backoff_seconds: Optional[float] = Field( + backoff_seconds: Optional[confloat(ge=0.0)] = Field( 0.5, description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', examples=[0.5, 5], - ge=0.0, title="Backoff Seconds", ) - retries_at_minimum_page_size: Optional[int] = Field( + retries_at_minimum_page_size: Optional[conint(ge=0)] = Field( 0, description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", examples=[0, 3], - ge=0, title="Retries At Minimum Page Size", ) failure_message: Optional[str] = Field( @@ -1515,7 +1393,14 @@ class AsyncJobStatusMap(BaseModel): completed: List[str] failed: List[str] timeout: List[str] - skipped: Optional[List[str]] = None + skipped: Optional[List[str]] = Field( + None, + description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", + ) + + +class BlockSimultaneousSyncsAction(BaseModel): + type: Literal["BlockSimultaneousSyncsAction"] class ValueType(Enum): @@ -1883,6 +1768,64 @@ class AuthFlow(BaseModel): oauth_config_specification: Optional[OAuthConfigSpecification] = None +class TokenQuota(BaseModel): + type: Literal["TokenQuota"] + name: str = Field( + ..., + description="Name of the quota pool.", + examples=["rest", "graphql"], + title="Name", + ) + remaining_path: List[str] = Field( + ..., + description="Path to the remaining call count for this pool in the quota status response.", + examples=[["resources", "core", "remaining"]], + title="Remaining Path", + ) + reset_path: List[str] = Field( + ..., + description="Path to the quota reset timestamp for this pool in the quota status response.", + examples=[["resources", "core", "reset"]], + title="Reset Path", + ) + limit_path: Optional[List[str]] = Field( + None, + description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", + examples=[["resources", "core", "limit"]], + title="Limit Path", + ) + matchers: Optional[List[HttpRequestRegexMatcher]] = Field( + None, + description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", + title="Matchers", + ) + remaining_header: Optional[str] = Field( + None, + description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", + examples=["X-RateLimit-Remaining"], + title="Remaining Header", + ) + reset_header: Optional[str] = Field( + None, + description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", + examples=["X-RateLimit-Reset"], + title="Reset Header", + ) + limit_header: Optional[str] = Field( + None, + description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", + examples=["X-RateLimit-Limit"], + title="Limit Header", + ) + exhaustion_status_codes: Optional[List[int]] = Field( + None, + description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", + examples=[[429]], + title="Exhaustion Status Codes", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class CheckStream(BaseModel): type: Literal["CheckStream"] stream_names: Optional[List[str]] = Field( @@ -2374,43 +2317,6 @@ class DefaultPaginator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class RecordExpander(BaseModel): - type: Literal["RecordExpander"] - expand_records_from_field: List[str] = Field( - ..., - description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", - examples=[ - ["lines", "data"], - ["items"], - ["nested", "array"], - ["sections", "*", "items"], - ], - title="Expand Records From Field", - ) - remain_original_record: Optional[bool] = Field( - False, - description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', - title="Remain Original Record", - ) - on_no_records: Optional[OnNoRecords] = Field( - OnNoRecords.skip, - description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', - title="On No Records", - ) - truncation_indicator_path: Optional[List[str]] = Field( - None, - description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", - examples=[["data", "object", "lines", "has_more"]], - title="Truncation Indicator Path", - ) - truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( - None, - description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", - title="Truncated List Retriever", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class SessionTokenRequestApiKeyAuthenticator(BaseModel): type: Literal["ApiKey"] inject_into: RequestOption = Field( @@ -2554,6 +2460,63 @@ class ConfigAddFields(BaseModel): ) +class RateLimitedMultipleTokenAuthenticator(BaseModel): + type: Literal["RateLimitedMultipleTokenAuthenticator"] + tokens: Union[str, List[str]] = Field( + ..., + description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", + examples=[ + "{{ config['credentials']['personal_access_token'] }}", + ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], + ], + title="Tokens", + ) + token_delimiter: Optional[str] = Field( + ",", + description="Delimiter used to split a single token string into multiple tokens.", + title="Token Delimiter", + ) + auth_method: Optional[str] = Field( + "Bearer", + description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", + examples=["Bearer", "token"], + title="Auth Method", + ) + header: Optional[str] = Field( + "Authorization", + description="The name of the HTTP header in which to inject the token.", + title="Header Name", + ) + quota_status_source: QuotaStatusSource = Field( + ..., + description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", + title="Quota Status Source", + ) + quotas: List[TokenQuota] = Field( + ..., + description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", + min_items=1, + title="Quota Pools", + ) + max_wait_time: Optional[str] = Field( + "PT2H", + description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", + examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], + title="Maximum Wait Time", + ) + budget_reserve_fraction: Optional[float] = Field( + 0.1, + description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", + title="Budget Reserve Fraction", + ) + budget_min_reserve: Optional[int] = Field( + 50, + description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", + title="Budget Minimum Reserve", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( @@ -2599,27 +2562,6 @@ class Config: ) -class DpathExtractor(BaseModel): - type: Literal["DpathExtractor"] - field_path: List[str] = Field( - ..., - description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', - examples=[ - ["data"], - ["data", "records"], - ["data", "{{ parameters.name }}"], - ["data", "*", "record"], - ], - title="Field Path", - ) - record_expander: Optional[RecordExpander] = Field( - None, - description="Optional component to expand records by extracting items from nested array fields.", - title="Record Expander", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ZipfileDecoder(BaseModel): class Config: extra = Extra.allow @@ -2632,27 +2574,6 @@ class Config: ) -class RecordSelector(BaseModel): - type: Literal["RecordSelector"] - extractor: Union[DpathExtractor, CustomRecordExtractor] - record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( - None, - description="Responsible for filtering records to be emitted by the Source.", - title="Record Filter", - ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( - None, - description="Responsible for normalization according to the schema.", - title="Schema Normalization", - ) - transform_before_filtering: Optional[bool] = Field( - None, - description="If true, transformation will be applied before record filtering.", - title="Transform Before Filtering", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ConfigMigration(BaseModel): type: Literal["ConfigMigration"] description: Optional[str] = Field( @@ -2745,7 +2666,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2785,7 +2706,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2968,6 +2889,64 @@ class Config: parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class DpathExtractor(BaseModel): + type: Literal["DpathExtractor"] + field_path: List[str] = Field( + ..., + description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', + examples=[ + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], + ], + title="Field Path", + ) + record_expander: Optional[RecordExpander] = Field( + None, + description="Optional component to expand records by extracting items from nested array fields.", + title="Record Expander", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class RecordExpander(BaseModel): + type: Literal["RecordExpander"] + expand_records_from_field: List[str] = Field( + ..., + description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", + examples=[ + ["lines", "data"], + ["items"], + ["nested", "array"], + ["sections", "*", "items"], + ], + title="Expand Records From Field", + ) + remain_original_record: Optional[bool] = Field( + False, + description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', + title="Remain Original Record", + ) + on_no_records: Optional[OnNoRecords] = Field( + OnNoRecords.skip, + description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', + title="On No Records", + ) + truncation_indicator_path: Optional[List[str]] = Field( + None, + description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", + examples=[["data", "object", "lines", "has_more"]], + title="Truncation Indicator Path", + ) + truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( + None, + description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", + title="Truncated List Retriever", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class SessionTokenAuthenticator(BaseModel): type: Literal["SessionTokenAuthenticator"] login_requester: HttpRequester = Field( @@ -3268,6 +3247,27 @@ class QueryProperties(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class RecordSelector(BaseModel): + type: Literal["RecordSelector"] + extractor: Union[DpathExtractor, CustomRecordExtractor] + record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( + None, + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", + ) + schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + None, + description="Responsible for normalization according to the schema.", + title="Schema Normalization", + ) + transform_before_filtering: Optional[bool] = Field( + None, + description="If true, transformation will be applied before record filtering.", + title="Transform Before Filtering", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] name: str = Field(..., description="The stream name.", example=["Users"], title="Name") @@ -3283,7 +3283,7 @@ class StateDelegatingStream(BaseModel): ) api_retention_period: Optional[str] = Field( None, - description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", + description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", examples=["P30D", "P90D", "P1Y"], title="API Retention Period", ) @@ -3393,10 +3393,9 @@ class AsyncRetriever(BaseModel): None, description="The time in minutes after which the single Async Job should be considered as Timed Out.", ) - failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( + failed_retry_wait_time_in_seconds: Optional[Union[conint(ge=1), str]] = Field( None, description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", - ge=1, ) download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( None, @@ -3477,20 +3476,14 @@ class AsyncRetriever(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class BlockSimultaneousSyncsAction(BaseModel): - type: Literal["BlockSimultaneousSyncsAction"] - - class StreamGroup(BaseModel): - streams: List[str] = Field( + streams: List[DeclarativeStream] = Field( ..., - description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', + description="List of references to streams that belong to this group.\n", title="Streams", ) action: BlockSimultaneousSyncsAction = Field( - ..., - description="The action to apply to streams in this group.", - title="Action", + ..., description="The action to apply to streams in this group.", title="Action" ) @@ -3515,7 +3508,7 @@ class GroupingPartitionRouter(BaseModel): underlying_partition_router: Union[ ListPartitionRouter, SubstreamPartitionRouter, - "UnionPartitionRouter", + UnionPartitionRouter, CustomPartitionRouter, ] = Field( ..., @@ -3595,8 +3588,9 @@ class DynamicDeclarativeStream(BaseModel): ConditionalStreams.update_forward_refs() FileUploader.update_forward_refs() DeclarativeStream.update_forward_refs() -SessionTokenAuthenticator.update_forward_refs() +DpathExtractor.update_forward_refs() RecordExpander.update_forward_refs() +SessionTokenAuthenticator.update_forward_refs() HttpRequester.update_forward_refs() DynamicSchemaLoader.update_forward_refs() ParentStreamConfig.update_forward_refs() @@ -3604,4 +3598,3 @@ class DynamicDeclarativeStream(BaseModel): SimpleRetriever.update_forward_refs() AsyncRetriever.update_forward_refs() GroupingPartitionRouter.update_forward_refs() -UnionPartitionRouter.update_forward_refs() From 37697f66824f2de087c823d3bf566bb6010bca3a Mon Sep 17 00:00:00 2001 From: kaizerch Date: Fri, 25 Sep 2026 09:08:16 -0700 Subject: [PATCH 14/21] fix(cdk): always replay after a partial request window split, drop on_partial_response Per review feedback (Anatolii Yatsuk): FAIL never actually prevented duplicates. A failed partition is never checkpointed, so the next sync attempt just re-reads the same window and re-emits the same records anyway - and for a deterministic "window too large" condition, FAIL retries the identical oversized window forever, so the sync can never succeed. A connector that doesn't want splitting at all can already opt out by not mapping to SPLIT_REQUEST_WINDOW in its response filter, so on_partial_response added nothing beyond that. Removes OnPartialResponse, the on_partial_response manifest field, and the FAIL raise entirely. Splitting after a partial response now always proceeds, logging a warning with how many records were already emitted so the replay is visible. --- .../declarative_component_schema.yaml | 19 +---- .../models/declarative_component_schema.py | 10 --- .../parsers/model_to_component_factory.py | 9 +-- .../retrievers/simple_retriever.py | 35 +++------ .../retrievers/window_reducible.py | 7 -- .../test_model_to_component_factory.py | 8 +- .../retrievers/test_simple_retriever.py | 73 ++++++------------- 7 files changed, 40 insertions(+), 121 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 4237c1a2cb..d7e907118f 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4455,9 +4455,10 @@ definitions: description: >- Describes how the request window is split when an error handler resolves to the SPLIT_REQUEST_WINDOW action. The connector replaces the failing window with two smaller, ordered windows covering the same - range and reads each one fully before checkpointing. Only supported on a stream whose incremental_sync - cursor defines cursor_granularity. Distinct from page_size_reduction, which shrinks a paginator's page - size rather than the cursor's window. + range and reads each one fully before checkpointing. If the failing window had already emitted records, + those may be re-emitted; downstream primary-key deduplication is expected to handle that. Only supported + on a stream whose incremental_sync cursor defines cursor_granularity. Distinct from page_size_reduction, + which shrinks a paginator's page size rather than the cursor's window. type: object required: - type @@ -4465,18 +4466,6 @@ definitions: type: type: string enum: [RequestWindowSplitting] - on_partial_response: - title: On Partial Response - description: >- - What to do when SPLIT_REQUEST_WINDOW arrives after the current window already emitted a record. - FAIL stops the sync with a configuration error, since re-reading the window would duplicate those - records. ALLOW_REPLAY splits and re-reads anyway, accepting duplicate records; use it only when - duplicates are tolerated downstream. - type: string - enum: - - FAIL - - ALLOW_REPLAY - default: FAIL failure_message: title: Failure Message description: >- diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index de1d15d733..8eddbaa079 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1473,18 +1473,8 @@ class PageSizeReduction(BaseModel): ) -class OnPartialResponse(Enum): - FAIL = "FAIL" - ALLOW_REPLAY = "ALLOW_REPLAY" - - class RequestWindowSplitting(BaseModel): type: Literal["RequestWindowSplitting"] - on_partial_response: Optional[OnPartialResponse] = Field( - OnPartialResponse.FAIL, - description="What to do when SPLIT_REQUEST_WINDOW arrives after the current window already emitted a record. FAIL stops the sync with a configuration error, since re-reading the window would duplicate those records. ALLOW_REPLAY splits and re-reads anyway, accepting duplicate records; use it only when duplicates are tolerated downstream.", - title="On Partial Response", - ) failure_message: Optional[str] = Field( None, description="Sentence appended to the error message when the connector can no longer split the window further. Use it to tell the user what to do for this API, for instance which filter narrows the query down.", diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 4cda71f17a..8c33989536 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -595,10 +595,7 @@ PageSizeResetPolicy, ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker -from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( - OnPartialResponse, - RequestWindowSplitting, -) +from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting from airbyte_cdk.sources.declarative.schema import ( ComplexFieldType, DefaultSchemaLoader, @@ -4143,11 +4140,7 @@ def _create_request_window_splitting( name, cursor, incremental_sync, query_properties, file_uploader ) - on_partial_response = model.request_window_splitting.on_partial_response return RequestWindowSplitting( - on_partial_response=OnPartialResponse(on_partial_response.value) - if on_partial_response is not None - else OnPartialResponse.FAIL, failure_message=model.request_window_splitting.failure_message, min_split_window=parse_duration(model.request_window_splitting.min_split_window) if model.request_window_splitting.min_split_window diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index a3f264422e..f24b2f1191 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -50,10 +50,7 @@ ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever -from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( - OnPartialResponse, - RequestWindowSplitting, -) +from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.source import ExperimentalClassWarning from airbyte_cdk.sources.streams.core import StreamData @@ -605,7 +602,7 @@ def _read_records_or_split_request_window( records_schema=records_schema, ) - emitted_any = False + emitted_count = 0 try: records: Iterable[Mapping[str, Any]] = self._read_pages(record_generator, stream_slice) if self.post_pagination_filter: @@ -622,29 +619,21 @@ def _read_records_or_split_request_window( stream_slice=stream_slice, ) for record in records: - emitted_any = True + emitted_count += 1 yield self._reassociate_with_original_slice(record, original_slice) except RequestWindowSplitRequiredException as exception: if self.request_window_splitting is None or self.request_window_splitter is None: raise RequestWindowSplitNotSupportedException(stream_name=self.name) from exception - if ( - emitted_any - and self.request_window_splitting.on_partial_response == OnPartialResponse.FAIL - ): - # The split is safe only because it re-reads a window whose records were not emitted yet. - # Re-reading it as smaller children here would duplicate the records already yielded above. - raise AirbyteTracedException( - internal_message=f"Stream {self.name} requested a request window split after records of the current window had already been emitted", - message=( - f"Stream {self.name} asked to split its request window after some records of it were " - f"already read. Splitting and re-reading the window would duplicate those records. Set " - f"`on_partial_response: ALLOW_REPLAY` on `request_window_splitting` if duplicate records " - f"are acceptable for this stream, or move the SPLIT_REQUEST_WINDOW action so it only " - f"triggers before any record of the window has been read." - ), - failure_type=FailureType.config_error, - ) from exception + if emitted_count: + # Splitting and re-reading the window re-emits these records: a failed partition is never + # checkpointed, so the next attempt would re-emit them anyway. Always replaying makes forward + # progress instead of retrying the same oversized window forever; primary-key dedup downstream + # handles the duplicates either way. + LOGGER.warning( + f"Stream {self.name} already emitted {emitted_count} record(s) from {stream_slice} before " + f"it was rejected; splitting and re-reading the window may re-emit them." + ) if depth >= _MAX_REQUEST_WINDOW_SPLIT_DEPTH: # Logged separately from the exception below so it stays visible even if the trace message's diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py index 9786d3c108..1431928b92 100644 --- a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py +++ b/airbyte_cdk/sources/declarative/retrievers/window_reducible.py @@ -4,15 +4,9 @@ import datetime from dataclasses import dataclass -from enum import Enum from typing import Optional -class OnPartialResponse(Enum): - FAIL = "FAIL" - ALLOW_REPLAY = "ALLOW_REPLAY" - - @dataclass(frozen=True) class RequestWindowSplitting: """ @@ -21,7 +15,6 @@ class RequestWindowSplitting: split window is either split again (recursion) or the sync fails, so a plain config object is enough. """ - on_partial_response: OnPartialResponse = OnPartialResponse.FAIL failure_message: Optional[str] = None # Independent of, and typically looser than, the cursor's own granularity floor - useful when a cursor # could split further but the API's rate limit makes many small requests worse than a few large ones. diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index ecd0b7e5d5..95ff17eea4 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -205,10 +205,7 @@ PageSizeReduction, PageSizeResetPolicy, ) -from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( - OnPartialResponse, - RequestWindowSplitting, -) +from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( CachingSchemaLoaderDecorator, @@ -7615,7 +7612,6 @@ def test_given_request_window_splitting_then_create_retriever_with_defaults(): retriever = get_retriever(_request_window_splitting_stream()) assert retriever.request_window_splitting == RequestWindowSplitting( - on_partial_response=OnPartialResponse.FAIL, failure_message=None, ) assert callable(retriever.request_window_splitter) @@ -7628,7 +7624,6 @@ def test_given_request_window_splitting_values_then_create_retriever_with_those_ request_window_splitting=( "request_window_splitting:\n" " type: RequestWindowSplitting\n" - " on_partial_response: ALLOW_REPLAY\n" " failure_message: Lower time_window so that each request covers less data.\n" " min_split_window: P1D" ) @@ -7636,7 +7631,6 @@ def test_given_request_window_splitting_values_then_create_retriever_with_those_ ) assert retriever.request_window_splitting == RequestWindowSplitting( - on_partial_response=OnPartialResponse.ALLOW_REPLAY, failure_message="Lower time_window so that each request covers less data.", min_split_window=timedelta(days=1), ) diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 07cc7fc532..7dda8a60ed 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -3,6 +3,7 @@ # import json +import logging import threading from collections import defaultdict from concurrent.futures import ThreadPoolExecutor @@ -57,10 +58,7 @@ _MAX_REQUEST_WINDOW_SPLIT_DEPTH, SimpleRetriever, ) -from airbyte_cdk.sources.declarative.retrievers.window_reducible import ( - OnPartialResponse, - RequestWindowSplitting, -) +from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( PageSizeReductionRequiredException, ) @@ -2214,41 +2212,14 @@ def test_given_no_request_window_splitter_when_reduction_required_then_raise_not list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) -def test_given_records_already_emitted_and_fail_policy_when_reduction_requested_then_raise_config_error(): - """ - A later page of the same window fails after an earlier page already emitted records: with the default - `FAIL` policy, reducing and re-reading the window from scratch would duplicate those records, so this - must raise a hard config_error instead of silently reducing. - """ - requester = Mock(spec=Requester) - requester.send_request.side_effect = [Mock(), RequestWindowSplitRequiredException()] - record_selector = Mock(spec=HttpSelector) - record_selector.select_records.return_value = [{"id": 1}] - paginator = _mock_paginator() - paginator.get_initial_token.return_value = None - paginator.next_page_token.return_value = {"next_page_token": "page2"} - - request_window_splitter = Mock() - - retriever = _request_window_splitting_retriever( - requester, - paginator, - record_selector, - request_window_splitter, - request_window_splitting=RequestWindowSplitting(on_partial_response=OnPartialResponse.FAIL), - ) - - with pytest.raises(AirbyteTracedException) as exception: - list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) - - assert exception.value.failure_type == FailureType.config_error - request_window_splitter.assert_not_called() - - -def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_requested_then_reduce_anyway(): +def test_given_records_already_emitted_when_reduction_requested_then_split_and_replay_anyway( + caplog, +): """ - The opt-in `ALLOW_REPLAY` policy accepts at-least-once delivery: unlike the FAIL case above, this must - proceed to split and re-read the window even though the first page already emitted a record. + A later page of the same window fails after an earlier page already emitted a record: since a failed + partition is never checkpointed, refusing to split here wouldn't avoid the duplicate anyway - the next + attempt would just re-read the same window and re-emit it. So this always splits and re-reads, logging a + warning with how many records were already emitted. """ requester = Mock(spec=Requester) requester.send_request.side_effect = [ @@ -2270,20 +2241,16 @@ def test_given_records_already_emitted_and_allow_replay_policy_when_reduction_re ] retriever = _request_window_splitting_retriever( - requester, - paginator, - record_selector, - request_window_splitter, - request_window_splitting=RequestWindowSplitting( - on_partial_response=OnPartialResponse.ALLOW_REPLAY - ), + requester, paginator, record_selector, request_window_splitter ) - records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + with caplog.at_level(logging.WARNING, logger="airbyte"): + records = list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) - # the record from the page emitted before the reduction signal is included once more: ALLOW_REPLAY - # accepts this at-least-once duplication rather than silently dropping it or failing the sync. + # the record from the page emitted before the split signal is included once more: splitting and + # re-reading accepts this at-least-once duplication rather than silently dropping it or failing the sync. assert records == [{"id": 1}, {"id": 2}, {"id": 3}] + assert "already emitted 1 record" in caplog.text request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, None) @@ -2570,7 +2537,7 @@ def test_given_exception_raised_during_record_extraction_when_read_then_still_sp assert records == [{"id": 1}, {"id": 2}] -def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_error(): +def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_error(caplog): """ Defense-in-depth: even if a (misbehaving, e.g. custom) `request_window_splitter` never itself signals "cannot split any further", the retriever's own depth counter must still terminate the read deterministically rather than @@ -2599,8 +2566,9 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er ), ) - with pytest.raises(AirbyteTracedException) as exception: - list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) + with caplog.at_level(logging.WARNING, logger="airbyte"): + with pytest.raises(AirbyteTracedException) as exception: + list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) # exhausting the split depth is not the user's fault - the API kept rejecting every window size tried - # so it is transient_error, matching PageSizeReducer's equivalent exhaustion branch, not config_error @@ -2608,6 +2576,9 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er assert "maximum request window split depth" in exception.value.internal_message assert f"within {_MAX_REQUEST_WINDOW_SPLIT_DEPTH} splits" in exception.value.message assert "Lower time_window so that each request covers less data." in exception.value.message + # the safety net is expected to be rare, so it's logged separately from the exception (per review feedback + # to make it visible for later review even if the trace message's internal_message isn't surfaced) + assert f"hit the maximum request window split depth ({_MAX_REQUEST_WINDOW_SPLIT_DEPTH})" in caplog.text # the single leftmost recursion path is explored to the full depth (one split_request_window call per # depth, from 0 up to the cap) before the cap stops the next call - the exception then propagates # immediately, so sibling branches at shallower depths are never explored From c0cf4a5974f64fa416c8cbb38b1303940d99c1ed Mon Sep 17 00:00:00 2001 From: "kaizer.charania" Date: Fri, 25 Sep 2026 16:43:37 +0000 Subject: [PATCH 15/21] fix(declarative): drop conint/confloat annotations from generated models and format test Regenerating declarative_component_schema.py without --field-constraints emitted conint(...)/confloat(...) annotations that mypy rejects; restore the plain int/float annotations used on main. Also wrap an over-long assert for ruff format. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../models/declarative_component_schema.py | 22 +++++++++---------- .../retrievers/test_simple_retriever.py | 5 ++++- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 491d81c66e..6f35a669ba 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -6,7 +6,7 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic.v1 import BaseModel, Extra, Field, confloat, conint +from pydantic.v1 import BaseModel, Extra, Field from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( BaseModelWithDeprecations, @@ -87,7 +87,7 @@ class DynamicStreamCheckConfig(BaseModel): dynamic_stream_name: str = Field( ..., description="The dynamic stream name.", title="Dynamic Stream Name" ) - stream_count: Optional[conint(ge=1)] = Field( + stream_count: Optional[int] = Field( None, description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", title="Stream Count", @@ -133,13 +133,13 @@ class ConcurrencyLevel(BaseModel): class ConstantBackoffStrategy(BaseModel): type: Literal["ConstantBackoffStrategy"] - backoff_time_in_seconds: Union[confloat(ge=0.0), str] = Field( + backoff_time_in_seconds: Union[float, str] = Field( ..., description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], title="Backoff Time", ) - jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( + jitter_range_in_seconds: Optional[float] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], @@ -559,7 +559,7 @@ class ExponentialBackoffStrategy(BaseModel): examples=[5, 5.5, "10"], title="Factor", ) - jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( + jitter_range_in_seconds: Optional[float] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], @@ -1305,31 +1305,31 @@ class ResetPolicy(Enum): class PageSizeReduction(BaseModel): type: Literal["PageSizeReduction"] - reduction_factor: Optional[confloat(gt=1.0)] = Field( + reduction_factor: Optional[float] = Field( 2, description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", examples=[2, 4], title="Reduction Factor", ) - minimum_page_size: Optional[conint(ge=1)] = Field( + minimum_page_size: Optional[int] = Field( 1, description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", examples=[1, 10], title="Minimum Page Size", ) - max_attempts: Optional[conint(ge=1)] = Field( + max_attempts: Optional[int] = Field( 5, description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", examples=[5, 10], title="Maximum Reduction Attempts", ) - backoff_seconds: Optional[confloat(ge=0.0)] = Field( + backoff_seconds: Optional[float] = Field( 0.5, description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', examples=[0.5, 5], title="Backoff Seconds", ) - retries_at_minimum_page_size: Optional[conint(ge=0)] = Field( + retries_at_minimum_page_size: Optional[int] = Field( 0, description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", examples=[0, 3], @@ -3383,7 +3383,7 @@ class AsyncRetriever(BaseModel): None, description="The time in minutes after which the single Async Job should be considered as Timed Out.", ) - failed_retry_wait_time_in_seconds: Optional[Union[conint(ge=1), str]] = Field( + failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( None, description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", ) diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index 7dda8a60ed..d317705f66 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -2578,7 +2578,10 @@ def test_given_max_split_depth_exceeded_when_read_records_then_raise_terminal_er assert "Lower time_window so that each request covers less data." in exception.value.message # the safety net is expected to be rare, so it's logged separately from the exception (per review feedback # to make it visible for later review even if the trace message's internal_message isn't surfaced) - assert f"hit the maximum request window split depth ({_MAX_REQUEST_WINDOW_SPLIT_DEPTH})" in caplog.text + assert ( + f"hit the maximum request window split depth ({_MAX_REQUEST_WINDOW_SPLIT_DEPTH})" + in caplog.text + ) # the single leftmost recursion path is explored to the full depth (one split_request_window call per # depth, from 0 up to the cap) before the cap stops the next call - the exception then propagates # immediately, so sibling branches at shallower depths are never explored From 541dab5e3268f34b3d4b3ebeea29dc1436138a7d Mon Sep 17 00:00:00 2001 From: "kaizer.charania" Date: Fri, 25 Sep 2026 17:05:11 +0000 Subject: [PATCH 16/21] fix(declarative): restore ge=0 constraint on jitter_range_in_seconds via Field Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../sources/declarative/models/declarative_component_schema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 6f35a669ba..dba816e20b 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -141,6 +141,7 @@ class ConstantBackoffStrategy(BaseModel): ) jitter_range_in_seconds: Optional[float] = Field( None, + ge=0.0, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], title="Jitter Range", @@ -561,6 +562,7 @@ class ExponentialBackoffStrategy(BaseModel): ) jitter_range_in_seconds: Optional[float] = Field( None, + ge=0.0, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], title="Jitter Range", From e67caed580106f39283e17fa7dae17c76657b1af Mon Sep 17 00:00:00 2001 From: kaizerch Date: Fri, 25 Sep 2026 10:18:02 -0700 Subject: [PATCH 17/21] fix(cdk): restore ge/gt Field constraints dropped when removing conint/confloat The prior fix (c0cf4a59) correctly replaced conint(...)/confloat(...) type annotations with plain int/float to satisfy mypy, but dropped the numeric bound entirely instead of moving it to a `ge`/`gt` Field() keyword - which pydantic v1 enforces independently of the type annotation and which mypy has no issue with. This silently broke pydantic-level validation for 9 fields (stream_count, backoff jitter, PageSizeReduction's reduction_factor/minimum_page_size/max_attempts/ backoff_seconds/retries_at_minimum_page_size, and AsyncRetriever's failed_retry_wait_time_in_seconds), caught by the existing test_backoff_jitter_schema_validation test. Restores each field's ge/gt to match its `minimum`/`exclusiveMinimum` in declarative_component_schema.yaml, verified against main's own already-correct values for the pre-existing fields. --- .../declarative/models/declarative_component_schema.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 6f35a669ba..3039e2e171 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -90,6 +90,7 @@ class DynamicStreamCheckConfig(BaseModel): stream_count: Optional[int] = Field( None, description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", + ge=1, title="Stream Count", ) @@ -143,6 +144,7 @@ class ConstantBackoffStrategy(BaseModel): None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], + ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -563,6 +565,7 @@ class ExponentialBackoffStrategy(BaseModel): None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], + ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -1309,30 +1312,35 @@ class PageSizeReduction(BaseModel): 2, description="Divisor applied to the page size on each reduction. The new page size is floor(current page size / reduction factor).", examples=[2, 4], + gt=1, title="Reduction Factor", ) minimum_page_size: Optional[int] = Field( 1, description="Page size below which the connector stops reducing and fails the sync. It must be smaller than the page size configured on the pagination strategy, otherwise no reduction could ever be applied. It is one of two bounds on the reduction and whichever is tighter wins: an unbroken run of failing pages divides the page size by reduction_factor at most max_attempts times, so reaching this floor in a single run needs max_attempts of at least log(page_size / minimum_page_size) / log(reduction_factor) - with the defaults, a page size of 1000 bottoms out at 31 records per page and a floor of 10 is never reached. Pages that succeed in between restart the max_attempts budget, so the floor is still reachable over a partition.", examples=[1, 10], + ge=1, title="Minimum Page Size", ) max_attempts: Optional[int] = Field( 5, description="Maximum number of page size reductions made in a row without a single page succeeding, before the sync fails with a transient error. Every reduction follows a request that failed, so at most max_attempts + 1 failing requests are issued before giving up. The budget restarts after every page that succeeds, under either reset_policy, so it bounds the reductions needed to get a single page through and not the number of pages a partition may have: a stream that needs a reduction every now and then reads to the end however long it is. Under NEVER the page size also strictly decreases, so minimum_page_size bounds the reductions of the whole partition on its own. The wait between reduction attempts is the CDK's own - it grows with each attempt - and does not consult the error handler's backoff_strategies or a Retry-After header.", examples=[5, 10], + ge=1, title="Maximum Reduction Attempts", ) backoff_seconds: Optional[float] = Field( 0.5, description='Base number of seconds to wait before the page is re-issued, multiplied by the number of attempts made in a row, so the second attempt waits twice as long as the first. A REDUCE_PAGE_SIZE response never reaches the error handler\'s retry budget, backoff_strategies or a Retry-After header, so this is the only thing spacing those requests out. Raise it on an API whose error also means "we are briefly unwell" rather than only "your page is too big", since the default spaces the whole run of attempts over a few seconds.', examples=[0.5, 5], + ge=0, title="Backoff Seconds", ) retries_at_minimum_page_size: Optional[int] = Field( 0, description="Number of times the same page is re-issued unchanged, each after the backoff wait, once the page size cannot be shrunk any further, before the sync fails with a transient error. The default of 0 fails on the first response received at minimum_page_size. Raise it when the API returns the same error for a page that is too big and for a server-side hiccup: at the floor, reducing is no longer an option but waiting still is, and without this budget those responses end the stream on the first one. This budget is separate from max_attempts, which only counts reductions, and it restarts on every page that succeeds. It applies however the page size arrived at the floor, whether by reduction or because page_size was already there; a page size that minimum_page_size blocks from ever being reduced is still reported as a configuration error, but only once this budget is spent.", examples=[0, 3], + ge=0, title="Retries At Minimum Page Size", ) failure_message: Optional[str] = Field( @@ -3386,6 +3394,7 @@ class AsyncRetriever(BaseModel): failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( None, description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", + ge=1, ) download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( None, From 418334e209a1787490c9430ff13e947906d65b20 Mon Sep 17 00:00:00 2001 From: "kaizer.charania" Date: Fri, 25 Sep 2026 17:24:33 +0000 Subject: [PATCH 18/21] fix(declarative): dedupe ge constraint on jitter_range_in_seconds after merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../sources/declarative/models/declarative_component_schema.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 2703878f7d..3039e2e171 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -142,7 +142,6 @@ class ConstantBackoffStrategy(BaseModel): ) jitter_range_in_seconds: Optional[float] = Field( None, - ge=0.0, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], ge=0, @@ -564,7 +563,6 @@ class ExponentialBackoffStrategy(BaseModel): ) jitter_range_in_seconds: Optional[float] = Field( None, - ge=0.0, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], ge=0, From e6fd13fe5ad9508ebe7eed282918dc275a294a95 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Fri, 25 Sep 2026 12:33:07 -0700 Subject: [PATCH 19/21] fix(cdk): address Anatolii's review - failure type, min_split_window validation, client-side incremental Per Anatolii Yatsuk's review on PR #1171: - Hardcode transient_error on the floor-exhaustion path instead of inheriting the triggering response's classification, which almost always resolves to system_error via DEFAULT_ERROR_MAPPING and never actually hit the intended transient_error fallback. Removes the now- unused classified_failure_type attribute/machinery this replaces. - Validate min_split_window at config time: reject anything that isn't a positive datetime.timedelta (a month/year duration like P1M parses to an isodate.Duration and crashed the sync mid-read comparing it to a timedelta; negative values were silently accepted and behaved as unset). - Reword min_split_window's description instead of changing the split boundary math, since a window can land a bit under the configured value by design. - Name min_split_window in the terminal "could not split further" message when it's configured, since it may be the actual reason splitting stopped rather than the cursor's own granularity floor. - Reject is_client_side_incremental for request_window_splitting: the window is filtered client-side rather than sent to the API, so every split child would send an identical request and never resolve the triggering response. - Rename window_reducible.py to request_window_splitting.py now that the WindowReducible protocol is gone and it only holds the config dataclass, matching page_size_reducer.py. - Raise a distinct system_error naming the missing/malformed boundary field, instead of silently returning None (reported as "the API kept rejecting...", misattributing an internal bug to the API). - Fix the comment above _MAX_REQUEST_WINDOW_SPLIT_DEPTH: it told users to raise min_split_window for more splits, which does the opposite (it can only stop splitting earlier), and mentioned a "custom cursor" a manifest can't define. - Add a docstring note on PaginatorTestReadDecorator: a split child resets its page count via get_initial_token(), so it can fetch up to maximum_number_of_pages on its own rather than sharing one budget with the rest of the partition. - Add the three missing tests he flagged: SPLIT_REQUEST_WINDOW on an AsyncRetriever requester and on a file_uploader.requester, and request_window_splitting with a partition_router. Proposed as a follow-up rather than in this PR: a warning when neither start_time_option nor end_time_option is set, since the window can still reach the request through interpolation and a heuristic here risks false positives. --- .../declarative_component_schema.yaml | 7 +- .../models/declarative_component_schema.py | 2 +- .../parsers/model_to_component_factory.py | 36 +++- .../paginators/default_paginator.py | 4 + ...ducible.py => request_window_splitting.py} | 0 .../retrievers/simple_retriever.py | 20 +- .../sources/streams/concurrent/cursor.py | 28 ++- .../http/request_window_split_exception.py | 10 +- .../test_model_to_component_factory.py | 190 +++++++++++++++++- .../retrievers/test_simple_retriever.py | 29 +-- .../sources/streams/concurrent/test_cursor.py | 22 +- 11 files changed, 304 insertions(+), 44 deletions(-) rename airbyte_cdk/sources/declarative/retrievers/{window_reducible.py => request_window_splitting.py} (100%) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index d7e907118f..4fbd89fb81 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -4478,9 +4478,10 @@ definitions: min_split_window: title: Minimum Split Window description: >- - Smallest window (ISO 8601 duration) the connector will request. Independent of, and usually looser - than, cursor_granularity - useful when a cursor could split further but many small requests are - worse for this API than a few large ones. + Approximate smallest window (ISO 8601 duration) the connector will split down to - a window at or + below this size is not split further, though the result can end up a bit smaller than this, not + larger. Independent of, and usually looser than, cursor_granularity - useful when a cursor could + split further but many small requests are worse for this API than a few large ones. type: string examples: - "P1D" diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 3039e2e171..93a7b0656a 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -1372,7 +1372,7 @@ class RequestWindowSplitting(BaseModel): ) min_split_window: Optional[str] = Field( None, - description="Smallest window (ISO 8601 duration) the connector will request. Independent of, and usually looser than, cursor_granularity - useful when a cursor could split further but many small requests are worse for this API than a few large ones.", + description="Approximate smallest window (ISO 8601 duration) the connector will split down to - a window at or below this size is not split further, though the result can end up a bit smaller than this, not larger. Independent of, and usually looser than, cursor_granularity - useful when a cursor could split further but many small requests are worse for this API than a few large ones.", examples=["P1D", "PT1H"], title="Minimum Split Window", ) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 8c33989536..27f086fe81 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -595,7 +595,9 @@ PageSizeResetPolicy, ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker -from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting +from airbyte_cdk.sources.declarative.retrievers.request_window_splitting import ( + RequestWindowSplitting, +) from airbyte_cdk.sources.declarative.schema import ( ComplexFieldType, DefaultSchemaLoader, @@ -4137,7 +4139,12 @@ def _create_request_window_splitting( ) self._validate_request_window_splitting_is_supported( - name, cursor, incremental_sync, query_properties, file_uploader + name, + cursor, + incremental_sync, + model.request_window_splitting.min_split_window, + query_properties, + file_uploader, ) return RequestWindowSplitting( @@ -4154,6 +4161,7 @@ def _validate_request_window_splitting_is_supported( incremental_sync: Optional[ Union[IncrementingCountCursorModel, DatetimeBasedCursorModel] ] = None, + min_split_window: Optional[str] = None, query_properties: Optional[QueryProperties] = None, file_uploader: Optional[DefaultFileUploader] = None, ) -> None: @@ -4187,6 +4195,17 @@ def _validate_request_window_splitting_is_supported( f"keep two child windows from overlapping at their shared edge." ) + if incremental_sync.is_client_side_incremental: + # A client-side-incremental cursor filters records after they are read rather than sending the + # window as request parameters, so every child window sends the exact same request as its parent: + # splitting changes nothing, and the sync would keep splitting all the way to `min_split_window` + # or the safety-net depth before failing. + raise ValueError( + f"`request_window_splitting` cannot be used together with `is_client_side_incremental` on " + f"stream {name}: the window is filtered client-side rather than sent to the API, so splitting " + f"it would not change the request and could not resolve a SPLIT_REQUEST_WINDOW response." + ) + parsed_cursor_granularity = parse_duration(incremental_sync.cursor_granularity) if isinstance( parsed_cursor_granularity, datetime.timedelta @@ -4216,6 +4235,19 @@ def _validate_request_window_splitting_is_supported( f"or a coarser `cursor_granularity`." ) + if min_split_window: + parsed_min_split_window = parse_duration(min_split_window) + if not isinstance( + parsed_min_split_window, datetime.timedelta + ) or parsed_min_split_window <= datetime.timedelta(0): + # A duration with years or months (e.g. `P1M`) parses to an `isodate.Duration`, which has no + # fixed length to compare a window's span against - `split_request_window` would raise + # mid-sync trying to. A zero-or-negative duration would never stop splitting. + raise ValueError( + f"`min_split_window` must be a positive duration expressible as a fixed number of " + f"days/hours/minutes/seconds on stream {name}: `{min_split_window}` is not." + ) + if query_properties: raise ValueError( f"`request_window_splitting` cannot be used together with query properties on stream {name}. " diff --git a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py index d00e75ad29..7fdc593caa 100644 --- a/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py +++ b/airbyte_cdk/sources/declarative/requesters/paginators/default_paginator.py @@ -265,6 +265,10 @@ class PaginatorTestReadDecorator(Paginator): class (`connector_builder/test_reader/reader.py::_has_reached_limit`). The two counts agree because `HttpClient` logs a response resolving to `REDUCE_PAGE_SIZE` as an auxiliary request, which the Builder does not turn into a page. + + `get_initial_token()` also resets this count, and `SimpleRetriever._read_pages` calls it again from scratch + for every child window a `SPLIT_REQUEST_WINDOW` response produces. A split slice can therefore fetch up to + `maximum_number_of_pages` per child rather than sharing one budget across the whole original partition. """ _PAGE_COUNT_BEFORE_FIRST_NEXT_CALL = 1 diff --git a/airbyte_cdk/sources/declarative/retrievers/window_reducible.py b/airbyte_cdk/sources/declarative/retrievers/request_window_splitting.py similarity index 100% rename from airbyte_cdk/sources/declarative/retrievers/window_reducible.py rename to airbyte_cdk/sources/declarative/retrievers/request_window_splitting.py diff --git a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py index f24b2f1191..c7ea07be38 100644 --- a/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py +++ b/airbyte_cdk/sources/declarative/retrievers/simple_retriever.py @@ -49,8 +49,10 @@ PageSizeReduction, ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker +from airbyte_cdk.sources.declarative.retrievers.request_window_splitting import ( + RequestWindowSplitting, +) from airbyte_cdk.sources.declarative.retrievers.retriever import Retriever -from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting from airbyte_cdk.sources.declarative.stream_slicers.stream_slicer import StreamSlicer from airbyte_cdk.sources.source import ExperimentalClassWarning from airbyte_cdk.sources.streams.core import StreamData @@ -73,8 +75,7 @@ LOGGER = logging.getLogger("airbyte") # A defense-in-depth bound independent of any `request_window_splitter`'s own no-progress guard: protects -# against a misbehaving custom cursor whose children don't actually shrink the window. Not user-configurable; -# use `min_split_window` for a legitimate need to split more than this. +# against a misbehaving custom cursor whose children don't actually shrink the window. Not user-configurable. _MAX_REQUEST_WINDOW_SPLIT_DEPTH = 10 @@ -658,14 +659,21 @@ def _read_records_or_split_request_window( stream_slice, self.request_window_splitting.min_split_window ) if children is None: + min_split_window_note = ( + f", its configured `min_split_window` ({self.request_window_splitting.min_split_window})" + if self.request_window_splitting.min_split_window + else "" + ) raise AirbyteTracedException( internal_message=f"Stream {self.name} could not split its request window {stream_slice} any further", - # `transient_error`, so the only remediation is the connector's own, if it defined one. + # `transient_error` regardless of how the triggering response was classified: exhaustion is + # never the user's fault, matching `PageSizeReducer`'s equivalent exhaustion branch. message=self._with_request_window_failure_message( f"The API kept rejecting stream {self.name}'s request window even at the smallest window " - f"its cursor allows, or splitting the window further would not make progress." + f"its cursor allows{min_split_window_note}, or splitting the window further would not " + f"make progress." ), - failure_type=exception.classified_failure_type or FailureType.transient_error, + failure_type=FailureType.transient_error, ) from exception LOGGER.info( diff --git a/airbyte_cdk/sources/streams/concurrent/cursor.py b/airbyte_cdk/sources/streams/concurrent/cursor.py index c808075489..ce4e3f644e 100644 --- a/airbyte_cdk/sources/streams/concurrent/cursor.py +++ b/airbyte_cdk/sources/streams/concurrent/cursor.py @@ -19,6 +19,7 @@ Union, ) +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.message import MessageRepository, NoopMessageRepository from airbyte_cdk.sources.streams import NO_CURSOR_STATE_KEY @@ -30,6 +31,7 @@ AbstractStreamStateConverter, ) from airbyte_cdk.sources.types import Record, StreamSlice +from airbyte_cdk.utils.traced_exception import AirbyteTracedException LOGGER = logging.getLogger("airbyte") @@ -616,9 +618,11 @@ def split_request_window( Returns `None` when the slice cannot be split any further: either it spans at most one `cursor_granularity` unit already (the floor), it is already at or below `min_split_window` (a connector-configured floor independent of and typically looser than `cursor_granularity`), the - boundaries are missing/malformed/reversed, or no `cursor_granularity` was configured at all (there is - then no way to know the smallest addressable unit, or how to keep two children from overlapping at - their shared edge). This is the implementation `SimpleRetriever.request_window_splitter` binds to (see + boundaries are reversed, or no `cursor_granularity` was configured at all (there is then no way to know + the smallest addressable unit, or how to keep two children from overlapping at their shared edge). + Raises `AirbyteTracedException` instead if a boundary is missing or unparseable: that is a bug in how + the slice was produced, not the API rejecting a window, so it must not be reported as one. This is the + implementation `SimpleRetriever.request_window_splitter` binds to (see `airbyte_cdk.sources.declarative.retrievers.simple_retriever`) that reuses the same parsing/formatting/comparison operations `_split_per_slice_range` already relies on, so it works for any `CursorValueType`/`GapType` pair this cursor was built with (datetime/timedelta, or int/int), @@ -635,8 +639,22 @@ def split_request_window( end_value = self._connector_state_converter.parse_value( stream_slice.cursor_slice[end_field] ) - except (KeyError, ValueError, TypeError): - return None + except (KeyError, ValueError, TypeError) as exception: + # A missing or unparseable boundary means this slice was produced (or passed in) wrong - a bug in + # slice generation, not something the API did. Raising here, rather than returning `None` (which + # `SimpleRetriever` reports as "the API kept rejecting..."), keeps that distinction visible instead + # of blaming the API for our own state. + raise AirbyteTracedException( + internal_message=( + f"Cannot split {stream_slice}: boundary field {start_field!r} or {end_field!r} is missing " + f"or not parseable: {exception}" + ), + message=( + f"Stream {self._stream_name} produced a request window with a missing or invalid " + f"{start_field!r}/{end_field!r} boundary and could not split it further." + ), + failure_type=FailureType.system_error, + ) from exception if start_value >= end_value: # Reversed or degenerate boundaries: nothing safe to split. diff --git a/airbyte_cdk/sources/streams/http/request_window_split_exception.py b/airbyte_cdk/sources/streams/http/request_window_split_exception.py index c722b3151c..0ee937fa00 100644 --- a/airbyte_cdk/sources/streams/http/request_window_split_exception.py +++ b/airbyte_cdk/sources/streams/http/request_window_split_exception.py @@ -20,10 +20,11 @@ class RequestWindowSplitRequiredException(AirbyteTracedException): failure the built-in `HttpResponseFilter`/`ResponseAction.SPLIT_REQUEST_WINDOW` path cannot see, without the CDK needing to know about that connector's specific exception types. - The classifying error's `failure_type` and `error_message` are preserved on this exception so the eventual - terminal failure - if the window cannot be split any further - reports the original cause rather than a - generic "please split" message. Callers with an underlying exception should chain it the normal Python way, - `raise RequestWindowSplitRequiredException(...) from original_exception`, so the original traceback is + The classifying error's `error_message` is preserved on this exception so it can be surfaced if the window + is never split any further. `failure_type` is not: exhausting every window size the API will accept is + never the user's fault, so `SimpleRetriever` always reports that as `transient_error` regardless of how the + triggering response was classified. Callers with an underlying exception should chain it the normal Python + way, `raise RequestWindowSplitRequiredException(...) from original_exception`, so the original traceback is not lost. """ @@ -35,7 +36,6 @@ def __init__( ) -> None: stream = f" of stream {stream_name}" if stream_name else "" detail = f": {error_message}" if error_message else "" - self.classified_failure_type = failure_type super().__init__( internal_message=f"A request window split was requested{stream}{detail}", message=f"The API rejected the current request window{stream} and requires a smaller one. If this message ends a sync, the stream is not set up to split its window: add `request_window_splitting` to its retriever, or remove the `SPLIT_REQUEST_WINDOW` action from its error handler.", diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 95ff17eea4..b54ddf83f9 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -205,7 +205,9 @@ PageSizeReduction, PageSizeResetPolicy, ) -from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting +from airbyte_cdk.sources.declarative.retrievers.request_window_splitting import ( + RequestWindowSplitting, +) from airbyte_cdk.sources.declarative.schema import InlineSchemaLoader, JsonFileSchemaLoader from airbyte_cdk.sources.declarative.schema.caching_schema_loader_decorator import ( CachingSchemaLoaderDecorator, @@ -7796,6 +7798,52 @@ def test_given_datetime_format_coarser_than_cursor_granularity_and_request_windo assert "datetime_format" in str(exception.value) +def test_given_client_side_incremental_and_request_window_splitting_then_raise(): + """ + A client-side-incremental cursor filters records after they're read instead of sending the window as + request parameters, so every split child would send the exact same request as its parent - splitting would + never resolve a SPLIT_REQUEST_WINDOW response. + """ + with pytest.raises(ValueError) as exception: + _request_window_splitting_stream( + incremental_sync=( + 'incremental_sync:\n type: DatetimeBasedCursor\n start_datetime: "2024-01-01T00:00:00Z"\n' + ' end_datetime: "2024-01-31T00:00:00Z"\n step: "P7D"\n cursor_field: "updated_at"\n' + ' cursor_granularity: "PT1S"\n datetime_format: "%Y-%m-%dT%H:%M:%SZ"\n' + " is_client_side_incremental: true" + ) + ) + + assert "is_client_side_incremental" in str(exception.value) + + +def test_given_month_or_year_min_split_window_and_request_window_splitting_then_raise(): + """ + `P1M` parses to an `isodate.Duration`, which has no fixed length to compare a window's span against - + `ConcurrentCursor.split_request_window` would raise a `TypeError` mid-sync trying to. This must be rejected + at config time instead. + """ + with pytest.raises(ValueError) as exception: + _request_window_splitting_stream( + request_window_splitting=( + "request_window_splitting:\n type: RequestWindowSplitting\n min_split_window: P1M" + ) + ) + + assert "min_split_window" in str(exception.value) + + +def test_given_negative_min_split_window_and_request_window_splitting_then_raise(): + with pytest.raises(ValueError) as exception: + _request_window_splitting_stream( + request_window_splitting=( + "request_window_splitting:\n type: RequestWindowSplitting\n min_split_window: -PT1H" + ) + ) + + assert "min_split_window" in str(exception.value) + + def test_given_incrementing_count_cursor_and_request_window_splitting_then_raise(): """ `IncrementingCountCursor` builds a `ConcurrentCursor` too, but not one carrying the datetime boundary @@ -7812,6 +7860,38 @@ def test_given_incrementing_count_cursor_and_request_window_splitting_then_raise assert "cursor_granularity" in str(exception.value) +def test_given_partition_router_and_request_window_splitting_then_raise(): + """ + A partition router combines multiple cursors (one per partition) behind a `ConcurrentPerPartitionCursor`, + which does not implement `split_request_window` itself - there is no longer one cursor whose own + granularity/boundary fields splitting could rely on. Rejected the same way a stream with no incremental + cursor at all is. + """ + content = _REQUEST_WINDOW_REDUCTION_STREAM.format( + incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, + request_window_splitting="request_window_splitting:\n type: RequestWindowSplitting", + action="SPLIT_REQUEST_WINDOW", + ).replace( + " requester:\n", + " partition_router:\n" + " type: ListPartitionRouter\n" + ' values: ["a", "b"]\n' + " cursor_field: partition_field\n" + " requester:\n", + ) + stream_manifest = transformer.propagate_types_and_parameters( + "", resolver.preprocess_manifest(YamlDeclarativeSource._parse(content)), {} + ) + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=DeclarativeStreamModel, component_definition=stream_manifest, config={} + ) + + assert "request_window_splitting" in str(exception.value) + assert "partition router" in str(exception.value) + + def test_given_query_properties_and_request_window_splitting_then_raise(): content = _REQUEST_WINDOW_REDUCTION_STREAM.format( incremental_sync=_DATETIME_BASED_CURSOR_WITH_GRANULARITY, @@ -8152,6 +8232,114 @@ def test_given_reduce_page_size_action_on_an_async_retriever_requester_then_rais assert "REDUCE_PAGE_SIZE" in str(exception.value) +_SPLIT_REQUEST_WINDOW_ERROR_HANDLER = { + "type": "DefaultErrorHandler", + "response_filters": [ + {"type": "HttpResponseFilter", "http_codes": [400], "action": "SPLIT_REQUEST_WINDOW"} + ], +} + + +def test_given_split_request_window_action_on_the_file_uploader_requester_then_raise(): + """ + `error_handler` is defined on `HttpRequester`, so the action is manifest-legal on requesters that have no + window of their own. Nothing there can honor it, so it is rejected rather than raised mid-sync. + """ + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=FileUploaderModel, + component_definition={ + "type": "FileUploader", + "requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/download", + "error_handler": _SPLIT_REQUEST_WINDOW_ERROR_HANDLER, + }, + "download_target_extractor": { + "type": "DpathExtractor", + "field_path": ["url"], + }, + }, + config={}, + ) + + assert "SPLIT_REQUEST_WINDOW" in str(exception.value) + + +@pytest.mark.parametrize( + "requester_field", + [ + "creation_requester", + "polling_requester", + "download_requester", + "download_target_requester", + "abort_requester", + "delete_requester", + ], +) +def test_given_split_request_window_action_on_an_async_retriever_requester_then_raise( + requester_field, +): + definition = { + "type": "AsyncRetriever", + "status_mapping": { + "type": "AsyncJobStatusMap", + "running": ["running"], + "completed": ["ready"], + "failed": ["failed"], + "timeout": ["timeout"], + }, + "status_extractor": {"type": "DpathExtractor", "field_path": ["status"]}, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["items"]}, + }, + "creation_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs", + "http_method": "POST", + }, + "polling_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs/{{ creation_response.id }}", + }, + "download_requester": { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs/{{ creation_response.id }}/download", + }, + } + definition[requester_field] = { + "type": "HttpRequester", + "url_base": "https://airbyte.io", + "path": "/jobs", + "error_handler": _SPLIT_REQUEST_WINDOW_ERROR_HANDLER, + } + if requester_field == "download_target_requester": + # Without it the `download_target_extractor` guard fires first and the assertion below would pass for + # the wrong reason. + definition["download_target_extractor"] = { + "type": "DpathExtractor", + "field_path": ["url"], + } + + with pytest.raises(ValueError) as exception: + factory.create_component( + model_type=AsyncRetrieverModel, + component_definition=definition, + config={}, + name="a_stream", + primary_key=None, + stream_slicer=None, + transformations=[], + ) + + assert "SPLIT_REQUEST_WINDOW" in str(exception.value) + + def get_retriever(stream: Union[DeclarativeStream, DefaultStream]): return ( stream.retriever diff --git a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py index d317705f66..b11d8507d2 100644 --- a/unit_tests/sources/declarative/retrievers/test_simple_retriever.py +++ b/unit_tests/sources/declarative/retrievers/test_simple_retriever.py @@ -54,11 +54,13 @@ PageSizeResetPolicy, ) from airbyte_cdk.sources.declarative.retrievers.pagination_tracker import PaginationTracker +from airbyte_cdk.sources.declarative.retrievers.request_window_splitting import ( + RequestWindowSplitting, +) from airbyte_cdk.sources.declarative.retrievers.simple_retriever import ( _MAX_REQUEST_WINDOW_SPLIT_DEPTH, SimpleRetriever, ) -from airbyte_cdk.sources.declarative.retrievers.window_reducible import RequestWindowSplitting from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( PageSizeReductionRequiredException, ) @@ -2254,15 +2256,17 @@ def test_given_records_already_emitted_when_reduction_requested_then_split_and_r request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, None) -def test_given_request_window_splitter_returns_none_when_reduction_requested_then_raise_terminal_error(): +def test_given_request_window_splitter_returns_none_when_reduction_requested_then_raise_transient_error(): """ - `request_window_splitter` returning `None` means the window cannot be split any further - either - the granularity floor was reached or reducing would not make progress. Either way this must terminate - the sync deterministically rather than loop or silently drop the window. + `request_window_splitter` returning `None` means the window cannot be split any further - either the + granularity floor or `min_split_window` was reached, or splitting would not make progress. This is never + the user's fault - the API kept rejecting every window size tried - so it is always `transient_error`, + matching `PageSizeReducer`'s equivalent exhaustion branch, regardless of how the triggering response was + classified (here, deliberately the opposite - `config_error` - to prove it is overridden). """ requester = Mock(spec=Requester) requester.send_request.side_effect = RequestWindowSplitRequiredException( - failure_type=FailureType.transient_error + failure_type=FailureType.config_error ) record_selector = Mock(spec=HttpSelector) paginator = _mock_paginator() @@ -2284,18 +2288,15 @@ def test_given_request_window_splitter_returns_none_when_reduction_requested_the with pytest.raises(AirbyteTracedException) as exception: list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) - # the classifying error's own failure_type is preserved rather than falling back to this branch's default assert exception.value.failure_type == FailureType.transient_error assert "Lower time_window so that each request covers less data." in exception.value.message request_window_splitter.assert_called_once_with(A_WINDOW_SLICE, None) -def test_given_request_window_splitter_returns_none_and_no_classified_failure_type_when_reduction_requested_then_raise_transient_error(): +def test_given_request_window_splitter_returns_none_and_min_split_window_configured_then_name_it_in_the_message(): """ - When whatever raised `RequestWindowSplitRequiredException` didn't classify a `failure_type` (the common - case: a manifest response filter with no explicit `failure_type`, or custom code that omits it), exhausting - the split floor must default to `transient_error` - the API kept rejecting every window size tried, which is - not the user's fault - matching `PageSizeReducer`'s equivalent exhaustion branch, not `config_error`. + When `min_split_window` is configured, it may be the actual reason splitting stopped rather than the + cursor's own granularity floor - the terminal message should name it so the user knows which knob to check. """ requester = Mock(spec=Requester) requester.send_request.side_effect = RequestWindowSplitRequiredException() @@ -2311,12 +2312,14 @@ def test_given_request_window_splitter_returns_none_and_no_classified_failure_ty paginator, record_selector, request_window_splitter, + request_window_splitting=RequestWindowSplitting(min_split_window=timedelta(days=1)), ) with pytest.raises(AirbyteTracedException) as exception: list(retriever.read_records(A_RECORD_SCHEMA, A_WINDOW_SLICE)) - assert exception.value.failure_type == FailureType.transient_error + assert "min_split_window" in exception.value.message + assert str(timedelta(days=1)) in exception.value.message def test_given_empty_children_list_when_reduction_requested_then_emit_no_records_and_do_not_loop(): diff --git a/unit_tests/sources/streams/concurrent/test_cursor.py b/unit_tests/sources/streams/concurrent/test_cursor.py index 64bbb2726c..fb86f175d3 100644 --- a/unit_tests/sources/streams/concurrent/test_cursor.py +++ b/unit_tests/sources/streams/concurrent/test_cursor.py @@ -11,6 +11,7 @@ import freezegun import pytest +from airbyte_cdk.models import FailureType from airbyte_cdk.sources.connector_state_manager import ConnectorStateManager from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.streams import NO_CURSOR_STATE_KEY @@ -36,6 +37,7 @@ EpochValueConcurrentStreamStateConverter, ) from airbyte_cdk.sources.types import Record, StreamSlice +from airbyte_cdk.utils.traced_exception import AirbyteTracedException _A_STREAM_NAME = "a stream name" _A_STREAM_NAMESPACE = "a stream namespace" @@ -1562,10 +1564,15 @@ def test_given_no_cursor_granularity_when_reduce_then_return_none(self) -> None: is None ) - def test_given_malformed_boundary_when_reduce_then_return_none(self) -> None: + def test_given_malformed_boundary_when_reduce_then_raise(self) -> None: + """ + A malformed boundary means the slice was produced wrong - a bug, not the API rejecting a window - so + this must not be silently swallowed as `None` (which `SimpleRetriever` reports as "the API kept + rejecting...", misattributing the cause). + """ cursor = self._cursor() - assert ( + with pytest.raises(AirbyteTracedException) as exception: cursor.split_request_window( StreamSlice( partition={}, @@ -1575,18 +1582,17 @@ def test_given_malformed_boundary_when_reduce_then_return_none(self) -> None: }, ) ) - is None - ) + assert exception.value.failure_type == FailureType.system_error + assert _LOWER_SLICE_BOUNDARY_FIELD in exception.value.internal_message - def test_given_missing_boundary_field_when_reduce_then_return_none(self) -> None: + def test_given_missing_boundary_field_when_reduce_then_raise(self) -> None: cursor = self._cursor() - assert ( + with pytest.raises(AirbyteTracedException) as exception: cursor.split_request_window( StreamSlice(partition={}, cursor_slice={"only_one_field": "x"}) ) - is None - ) + assert exception.value.failure_type == FailureType.system_error def test_split_request_window_preserves_partition_and_extra_fields(self) -> None: cursor = self._cursor() From 04431f3974dae9baa134af36b37a22bf756c440b Mon Sep 17 00:00:00 2001 From: kaizerch Date: Mon, 28 Sep 2026 09:55:30 -0700 Subject: [PATCH 20/21] fixes after merge conflicts --- .../models/declarative_component_schema.py | 42 ------------------- 1 file changed, 42 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 03f9addc7d..b9e449f18c 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2956,27 +2956,6 @@ class Config: parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class DpathExtractor(BaseModel): - type: Literal["DpathExtractor"] - field_path: List[str] = Field( - ..., - description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', - examples=[ - ["data"], - ["data", "records"], - ["data", "{{ parameters.name }}"], - ["data", "*", "record"], - ], - title="Field Path", - ) - record_expander: Optional[RecordExpander] = Field( - None, - description="Optional component to expand records by extracting items from nested array fields.", - title="Record Expander", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class RecordExpander(BaseModel): type: Literal["RecordExpander"] expand_records_from_field: List[str] = Field( @@ -3314,27 +3293,6 @@ class QueryProperties(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class RecordSelector(BaseModel): - type: Literal["RecordSelector"] - extractor: Union[DpathExtractor, CustomRecordExtractor] - record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( - None, - description="Responsible for filtering records to be emitted by the Source.", - title="Record Filter", - ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( - None, - description="Responsible for normalization according to the schema.", - title="Schema Normalization", - ) - transform_before_filtering: Optional[bool] = Field( - None, - description="If true, transformation will be applied before record filtering.", - title="Transform Before Filtering", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] name: str = Field(..., description="The stream name.", example=["Users"], title="Name") From 0c8d5039f464deeed0de6d32d2fda33cdaa52036 Mon Sep 17 00:00:00 2001 From: kaizerch Date: Mon, 28 Sep 2026 11:08:21 -0700 Subject: [PATCH 21/21] fix: restore missing RequestWindowSplitRequiredException import Dropped during merge-conflict resolution, causing a NameError in test_http_client.py's SPLIT_REQUEST_WINDOW tests. --- unit_tests/sources/streams/http/test_http_client.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index 424c73badd..ad3ee5acd1 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -34,6 +34,9 @@ from airbyte_cdk.sources.streams.http.page_size_reduction_exception import ( PageSizeReductionRequiredException, ) +from airbyte_cdk.sources.streams.http.request_window_split_exception import ( + RequestWindowSplitRequiredException, +) from airbyte_cdk.sources.streams.http.requests_native_auth import ( Oauth2Authenticator, TokenAuthenticator,