From 34937e3c04bdb3908b20588881d119f3da9cdda5 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Wed, 26 Aug 2026 22:12:06 +0000 Subject: [PATCH 01/15] feat(low-code): support fetching complete nested lists when RecordExpander detects truncation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative_component_schema.yaml | 16 + .../declarative/expanders/record_expander.py | 60 +- .../models/declarative_component_schema.py | 714 ++++++++++-------- .../parsers/model_to_component_factory.py | 12 + .../sources/declarative/expanders/__init__.py | 3 + .../expanders/test_record_expander.py | 133 ++++ .../test_model_to_component_factory.py | 48 ++ 7 files changed, 649 insertions(+), 337 deletions(-) create mode 100644 unit_tests/sources/declarative/expanders/__init__.py create mode 100644 unit_tests/sources/declarative/expanders/test_record_expander.py diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index aa3bbbc5e0..bae5885b0e 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2258,6 +2258,22 @@ definitions: - skip - emit_parent default: skip + truncation_indicator_path: + title: Truncation Indicator Path + description: Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. Wildcards (*) are not supported in this path or in `expand_records_from_field` when this option is used. + type: array + items: + type: string + interpolation_context: + - config + examples: + - ["data", "object", "lines", "has_more"] + truncated_list_retriever: + title: Truncated List Retriever + description: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. If the retriever returns no records, the embedded items are expanded as a fallback. Requires `truncation_indicator_path`. + anyOf: + - "$ref": "#/definitions/SimpleRetriever" + - "$ref": "#/definitions/CustomRetriever" $parameters: type: object additionalProperties: true diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index 6b489243d1..9a05638d55 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -5,12 +5,15 @@ import copy from dataclasses import InitVar, dataclass from enum import Enum -from typing import Any, Iterable, Mapping, MutableMapping, Sequence +from typing import TYPE_CHECKING, Any, Iterable, Mapping, MutableMapping, Optional, Sequence import dpath from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString -from airbyte_cdk.sources.types import Config +from airbyte_cdk.sources.types import Config, Record, StreamSlice + +if TYPE_CHECKING: + from airbyte_cdk.sources.declarative.retrievers import Retriever class OnNoRecords(Enum): @@ -62,6 +65,12 @@ class RecordExpander: parent record in an "original_record" field. Defaults to False. on_no_records: Behavior when expansion produces no records. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged. + truncation_indicator_path: Path within each record to a field indicating that the + embedded nested list is truncated (e.g. a `has_more` flag on the list object). + truncated_list_retriever: Retriever used to fetch the complete list of items when + the field at `truncation_indicator_path` is truthy. The record being expanded is + exposed to the retriever's interpolation context as `stream_slice['parent_record']`. + If the retriever returns no records, the embedded items are expanded as a fallback. config: The user-provided configuration as specified by the source's spec. """ @@ -70,12 +79,29 @@ class RecordExpander: parameters: InitVar[Mapping[str, Any]] remain_original_record: bool = False on_no_records: OnNoRecords = OnNoRecords.skip + truncation_indicator_path: Optional[Sequence[str]] = None + truncated_list_retriever: Optional["Retriever"] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._expand_path: list[InterpolatedString] = [ InterpolatedString.create(path, parameters=parameters) for path in self.expand_records_from_field ] + if self.truncated_list_retriever and not self.truncation_indicator_path: + raise ValueError( + "`truncation_indicator_path` is required when `truncated_list_retriever` is configured." + ) + if self.truncation_indicator_path and any( + "*" in path + for path in (*self.expand_records_from_field, *self.truncation_indicator_path) + ): + raise ValueError( + "The '*' wildcard is not supported in `expand_records_from_field` or `truncation_indicator_path` when truncation handling is configured." + ) + self._truncation_indicator_path: list[InterpolatedString] = [ + InterpolatedString.create(path, parameters=parameters) + for path in (self.truncation_indicator_path or []) + ] def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMapping[Any, Any]]: """Expand a record by extracting items from a nested array field.""" @@ -89,6 +115,13 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap return parent_record = record + + if self.truncated_list_retriever and self._is_truncated(parent_record): + fetched_records = list(self._fetch_complete_list(parent_record)) + if fetched_records: + yield from fetched_records + return + expand_path = [path.eval(self.config) for path in self._expand_path] expanded_any = False @@ -120,6 +153,29 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap if not expanded_any and self.on_no_records == OnNoRecords.emit_parent: yield parent_record + def _is_truncated(self, parent_record: MutableMapping[Any, Any]) -> bool: + indicator_path = [path.eval(self.config) for path in self._truncation_indicator_path] + try: + return bool(dpath.get(parent_record, indicator_path)) + except KeyError: + return False + + def _fetch_complete_list( + self, parent_record: Mapping[str, Any] + ) -> Iterable[MutableMapping[str, Any]]: + if not self.truncated_list_retriever: + return + stream_slice = StreamSlice(partition={"parent_record": parent_record}, cursor_slice={}) + for item in self.truncated_list_retriever.read_records( + records_schema={}, stream_slice=stream_slice + ): + data = item.data if isinstance(item, Record) else item + if not isinstance(data, Mapping): + continue + expanded_record = dict(data) + self._apply_parent_context(parent_record, expanded_record) + yield expanded_record + def _apply_parent_context( self, parent_record: Mapping[str, Any], child_record: MutableMapping[str, Any] ) -> None: diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 21a92cc3be..a1fbcab701 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -6,7 +6,7 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic.v1 import BaseModel, Extra, Field +from pydantic.v1 import BaseModel, Extra, Field, confloat, conint from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( BaseModelWithDeprecations, @@ -18,12 +18,6 @@ class AuthFlowType(Enum): oauth1_0 = "oauth1.0" -class ScopesJoinStrategy(Enum): - space = "space" - comma = "comma" - plus = "plus" - - class BasicHttpAuthenticator(BaseModel): type: Literal["BasicHttpAuthenticator"] username: str = Field( @@ -52,15 +46,50 @@ class BearerAuthenticator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class HttpMethod(Enum): + GET = "GET" + POST = "POST" + + +class QuotaStatusSource(BaseModel): + type: Literal["QuotaStatusSource"] + url: str = Field( + ..., + description="The full URL of the quota status endpoint.", + examples=[ + "https://api.github.com/rate_limit", + "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", + ], + title="URL", + ) + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description="The HTTP method used to fetch the quota status.", + title="HTTP Method", + ) + request_headers: Optional[Dict[str, str]] = Field( + None, + description="Additional headers to send with the quota status request.", + title="Request Headers", + ) + unavailable_status_codes: Optional[List[int]] = Field( + None, + description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", + examples=[[404]], + title="Unavailable Status Codes", + unique_items=True, + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class DynamicStreamCheckConfig(BaseModel): type: Literal["DynamicStreamCheckConfig"] dynamic_stream_name: str = Field( ..., description="The dynamic stream name.", title="Dynamic Stream Name" ) - stream_count: Optional[int] = Field( + stream_count: Optional[conint(ge=1)] = Field( None, description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", - ge=1, title="Stream Count", ) @@ -104,17 +133,16 @@ class ConcurrencyLevel(BaseModel): class ConstantBackoffStrategy(BaseModel): type: Literal["ConstantBackoffStrategy"] - backoff_time_in_seconds: Union[float, str] = Field( + backoff_time_in_seconds: Union[confloat(ge=0.0), str] = Field( ..., description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], title="Backoff Time", ) - jitter_range_in_seconds: Optional[float] = Field( + jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], - ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -502,7 +530,7 @@ class Config: ) weight: Optional[Union[int, str]] = Field( None, - description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", + description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", title="Weight", ) @@ -530,11 +558,10 @@ class ExponentialBackoffStrategy(BaseModel): examples=[5, 5.5, "10"], title="Factor", ) - jitter_range_in_seconds: Optional[float] = Field( + jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], - ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -555,157 +582,6 @@ class SessionTokenRequestBearerAuthenticator(BaseModel): type: Literal["Bearer"] -class HttpMethod(Enum): - GET = "GET" - POST = "POST" - - -class QuotaStatusSource(BaseModel): - type: Literal["QuotaStatusSource"] - url: str = Field( - ..., - description="The full URL of the quota status endpoint.", - examples=[ - "https://api.github.com/rate_limit", - "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", - ], - title="URL", - ) - http_method: Optional[HttpMethod] = Field( - HttpMethod.GET, - description="The HTTP method used to fetch the quota status.", - title="HTTP Method", - ) - request_headers: Optional[Dict[str, str]] = Field( - None, - description="Additional headers to send with the quota status request.", - title="Request Headers", - ) - unavailable_status_codes: Optional[List[int]] = Field( - None, - description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", - examples=[[404]], - title="Unavailable Status Codes", - unique_items=True, - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class TokenQuota(BaseModel): - type: Literal["TokenQuota"] - name: str = Field( - ..., - description="Name of the quota pool.", - examples=["rest", "graphql"], - title="Name", - ) - remaining_path: List[str] = Field( - ..., - description="Path to the remaining call count for this pool in the quota status response.", - examples=[["resources", "core", "remaining"]], - title="Remaining Path", - ) - reset_path: List[str] = Field( - ..., - description="Path to the quota reset timestamp for this pool in the quota status response.", - examples=[["resources", "core", "reset"]], - title="Reset Path", - ) - limit_path: Optional[List[str]] = Field( - None, - description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", - examples=[["resources", "core", "limit"]], - title="Limit Path", - ) - matchers: Optional[List[HttpRequestRegexMatcher]] = Field( - None, - description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", - title="Matchers", - ) - remaining_header: Optional[str] = Field( - None, - description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", - examples=["X-RateLimit-Remaining"], - title="Remaining Header", - ) - reset_header: Optional[str] = Field( - None, - description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", - examples=["X-RateLimit-Reset"], - title="Reset Header", - ) - limit_header: Optional[str] = Field( - None, - description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", - examples=["X-RateLimit-Limit"], - title="Limit Header", - ) - exhaustion_status_codes: Optional[List[int]] = Field( - None, - description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", - examples=[[429]], - title="Exhaustion Status Codes", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class RateLimitedMultipleTokenAuthenticator(BaseModel): - type: Literal["RateLimitedMultipleTokenAuthenticator"] - tokens: Union[str, List[str]] = Field( - ..., - description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", - examples=[ - "{{ config['credentials']['personal_access_token'] }}", - ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], - ], - title="Tokens", - ) - token_delimiter: Optional[str] = Field( - ",", - description="Delimiter used to split a single token string into multiple tokens.", - title="Token Delimiter", - ) - auth_method: Optional[str] = Field( - "Bearer", - description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", - examples=["Bearer", "token"], - title="Auth Method", - ) - header: Optional[str] = Field( - "Authorization", - description="The name of the HTTP header in which to inject the token.", - title="Header Name", - ) - quota_status_source: QuotaStatusSource = Field( - ..., - description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", - title="Quota Status Source", - ) - quotas: List[TokenQuota] = Field( - ..., - description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", - min_items=1, - title="Quota Pools", - ) - max_wait_time: Optional[str] = Field( - "PT2H", - description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", - examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], - title="Maximum Wait Time", - ) - budget_reserve_fraction: Optional[float] = Field( - 0.1, - description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", - title="Budget Reserve Fraction", - ) - budget_min_reserve: Optional[int] = Field( - 50, - description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", - title="Budget Minimum Reserve", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class Action(Enum): SUCCESS = "SUCCESS" FAIL = "FAIL" @@ -835,12 +711,13 @@ class JsonItemsDecoder(BaseModel): type: Literal["JsonItemsDecoder"] items_path: str = Field( ..., - description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", + description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", + examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], title="Items Path", ) encoding: Optional[str] = Field( "utf-8", - description="The character encoding of the JSON data. Defaults to UTF-8.", + description="Text encoding used to decode the streamed bytes before JSON parsing.", title="Encoding", ) @@ -1003,24 +880,38 @@ class NoPagination(BaseModel): type: Literal["NoPagination"] -class State(BaseModel): +class Scope(BaseModel): class Config: extra = Extra.allow - min: int - max: int + scope: str = Field( + ..., description="The OAuth scope string to request from the provider." + ) -class OAuthScope(BaseModel): +class OptionalScope(BaseModel): class Config: extra = Extra.allow scope: str = Field( - ..., - description="The OAuth scope string to request from the provider.", + ..., description="The OAuth scope string to request from the provider." ) +class ScopesJoinStrategy(Enum): + space = "space" + comma = "comma" + plus = "plus" + + +class State(BaseModel): + class Config: + extra = Extra.allow + + min: int + max: int + + class OauthConnectorInputSpecification(BaseModel): class Config: extra = Extra.allow @@ -1040,17 +931,13 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) - # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the - # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. - # The CDK schema defines the manifest contract; the platform reads these fields - # during the OAuth consent flow to build the authorization URL. - scopes: Optional[List[OAuthScope]] = Field( + scopes: Optional[List[Scope]] = Field( None, description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], title="Scopes", ) - optional_scopes: Optional[List[OAuthScope]] = Field( + optional_scopes: Optional[List[OptionalScope]] = Field( None, description="Optional OAuth scope objects that may or may not be granted.", examples=[[{"scope": "admin:read"}]], @@ -1145,24 +1032,28 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( + Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", + ) ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( + Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", + ) ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -1180,7 +1071,9 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], + examples=[ + {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} + ], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1423,7 +1316,14 @@ class AsyncJobStatusMap(BaseModel): completed: List[str] failed: List[str] timeout: List[str] - skipped: Optional[List[str]] = None + skipped: Optional[List[str]] = Field( + None, + description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", + ) + + +class BlockSimultaneousSyncsAction(BaseModel): + type: Literal["BlockSimultaneousSyncsAction"] class ValueType(Enum): @@ -1691,7 +1591,9 @@ class CustomConfigTransformation(BaseModel): class_name: str = Field( ..., description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_..`.", - examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], + examples=[ + "source_declarative_manifest.components.MyCustomConfigTransformation" + ], ) parameters: Optional[Dict[str, Any]] = Field( None, @@ -1791,19 +1693,77 @@ class AuthFlow(BaseModel): oauth_config_specification: Optional[OAuthConfigSpecification] = None -class CheckStream(BaseModel): - type: Literal["CheckStream"] - stream_names: Optional[List[str]] = Field( - None, - description="Names of the streams to try reading from when running a check operation.", - examples=[["users"], ["users", "contacts"]], - title="Stream Names", +class TokenQuota(BaseModel): + type: Literal["TokenQuota"] + name: str = Field( + ..., + description="Name of the quota pool.", + examples=["rest", "graphql"], + title="Name", ) - dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None - config_overrides: Optional[Dict[str, Any]] = Field( - None, - description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", - examples=[{"max_waiting_time": 0}, {"page_size": 1}], + remaining_path: List[str] = Field( + ..., + description="Path to the remaining call count for this pool in the quota status response.", + examples=[["resources", "core", "remaining"]], + title="Remaining Path", + ) + reset_path: List[str] = Field( + ..., + description="Path to the quota reset timestamp for this pool in the quota status response.", + examples=[["resources", "core", "reset"]], + title="Reset Path", + ) + limit_path: Optional[List[str]] = Field( + None, + description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", + examples=[["resources", "core", "limit"]], + title="Limit Path", + ) + matchers: Optional[List[HttpRequestRegexMatcher]] = Field( + None, + description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", + title="Matchers", + ) + remaining_header: Optional[str] = Field( + None, + description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", + examples=["X-RateLimit-Remaining"], + title="Remaining Header", + ) + reset_header: Optional[str] = Field( + None, + description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", + examples=["X-RateLimit-Reset"], + title="Reset Header", + ) + limit_header: Optional[str] = Field( + None, + description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", + examples=["X-RateLimit-Limit"], + title="Limit Header", + ) + exhaustion_status_codes: Optional[List[int]] = Field( + None, + description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", + examples=[[429]], + title="Exhaustion Status Codes", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class CheckStream(BaseModel): + type: Literal["CheckStream"] + stream_names: Optional[List[str]] = Field( + None, + description="Names of the streams to try reading from when running a check operation.", + examples=[["users"], ["users", "contacts"]], + title="Stream Names", + ) + dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None + config_overrides: Optional[Dict[str, Any]] = Field( + None, + description="Values overlaid onto the connector config for the duration of the check operation only. Use this when a check must behave differently from a sync - for example a shorter rate limit wait budget, so that check fails fast with a clear message instead of sleeping until the quota resets. Keys should be fields declared in the connector's spec. Values are used as-is. They are not interpolated, a `$ref` inside them is not resolved, they replace a nested object rather than deep-merging into it, and they are applied after config migrations and transformations have run, so a field derived from an overridden field is not recomputed. Keys must be strings, and two combinations are rejected outright. Keys prefixed with `__airbyte` belong to the platform rather than to the connector's spec. And a manifest that declares a `refresh_token_updater` cannot use this field at all, because a token refresh during check emits the whole config it was handed as a CONNECTOR_CONFIG control message, which the platform persists - so a check-only override would become the connection's saved config and apply to every later sync.", + examples=[{"max_waiting_time": 0}, {"page_size": 1}], title="Config Overrides", ) @@ -2131,7 +2091,9 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], + examples=[ + ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] + ], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -2282,32 +2244,6 @@ class DefaultPaginator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class RecordExpander(BaseModel): - type: Literal["RecordExpander"] - expand_records_from_field: List[str] = Field( - ..., - description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", - examples=[ - ["lines", "data"], - ["items"], - ["nested", "array"], - ["sections", "*", "items"], - ], - title="Expand Records From Field", - ) - remain_original_record: Optional[bool] = Field( - False, - description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', - title="Remain Original Record", - ) - on_no_records: Optional[OnNoRecords] = Field( - OnNoRecords.skip, - description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', - title="On No Records", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class SessionTokenRequestApiKeyAuthenticator(BaseModel): type: Literal["ApiKey"] inject_into: RequestOption = Field( @@ -2405,10 +2341,12 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", + ) ) @@ -2425,10 +2363,12 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( + Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", + ) ) @@ -2451,14 +2391,71 @@ class ConfigAddFields(BaseModel): ) +class RateLimitedMultipleTokenAuthenticator(BaseModel): + type: Literal["RateLimitedMultipleTokenAuthenticator"] + tokens: Union[str, List[str]] = Field( + ..., + description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", + examples=[ + "{{ config['credentials']['personal_access_token'] }}", + ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], + ], + title="Tokens", + ) + token_delimiter: Optional[str] = Field( + ",", + description="Delimiter used to split a single token string into multiple tokens.", + title="Token Delimiter", + ) + auth_method: Optional[str] = Field( + "Bearer", + description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", + examples=["Bearer", "token"], + title="Auth Method", + ) + header: Optional[str] = Field( + "Authorization", + description="The name of the HTTP header in which to inject the token.", + title="Header Name", + ) + quota_status_source: QuotaStatusSource = Field( + ..., + description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", + title="Quota Status Source", + ) + quotas: List[TokenQuota] = Field( + ..., + description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", + min_items=1, + title="Quota Pools", + ) + max_wait_time: Optional[str] = Field( + "PT2H", + description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", + examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], + title="Maximum Wait Time", + ) + budget_reserve_fraction: Optional[float] = Field( + 0.1, + description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", + title="Budget Reserve Fraction", + ) + budget_min_reserve: Optional[int] = Field( + 50, + description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", + title="Budget Minimum Reserve", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] - error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( - Field( - ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", - ) + error_handlers: List[ + Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler] + ] = Field( + ..., + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2496,60 +2493,20 @@ class Config: ) -class DpathExtractor(BaseModel): - type: Literal["DpathExtractor"] - field_path: List[str] = Field( - ..., - description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', - examples=[ - ["data"], - ["data", "records"], - ["data", "{{ parameters.name }}"], - ["data", "*", "record"], - ], - title="Field Path", - ) - record_expander: Optional[RecordExpander] = Field( - None, - description="Optional component to expand records by extracting items from nested array fields.", - title="Record Expander", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ZipfileDecoder(BaseModel): class Config: extra = Extra.allow type: Literal["ZipfileDecoder"] - decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( + decoder: Union[ + CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder + ] = Field( ..., description="Parser to parse the decompressed data from the zipfile(s).", title="Parser", ) -class RecordSelector(BaseModel): - type: Literal["RecordSelector"] - extractor: Union[DpathExtractor, CustomRecordExtractor] - record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( - None, - description="Responsible for filtering records to be emitted by the Source.", - title="Record Filter", - ) - schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( - None, - description="Responsible for normalization according to the schema.", - title="Schema Normalization", - ) - transform_before_filtering: Optional[bool] = Field( - None, - description="If true, transformation will be applied before record filtering.", - title="Transform Before Filtering", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class ConfigMigration(BaseModel): type: Literal["ConfigMigration"] description: Optional[str] = Field( @@ -2642,7 +2599,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2667,9 +2624,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( - None - ) + streams: Optional[ + List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] + ] = None dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2682,7 +2639,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2800,16 +2757,20 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") + name: Optional[str] = Field( + "", description="The stream name.", example=["Users"], title="Name" + ) retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", title="Retriever", ) - incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", + incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = ( + Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", + ) ) primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ @@ -2865,6 +2826,64 @@ class Config: parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class DpathExtractor(BaseModel): + type: Literal["DpathExtractor"] + field_path: List[str] = Field( + ..., + description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', + examples=[ + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], + ], + title="Field Path", + ) + record_expander: Optional[RecordExpander] = Field( + None, + description="Optional component to expand records by extracting items from nested array fields.", + title="Record Expander", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class RecordExpander(BaseModel): + type: Literal["RecordExpander"] + expand_records_from_field: List[str] = Field( + ..., + description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", + examples=[ + ["lines", "data"], + ["items"], + ["nested", "array"], + ["sections", "*", "items"], + ], + title="Expand Records From Field", + ) + remain_original_record: Optional[bool] = Field( + False, + description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', + title="Remain Original Record", + ) + on_no_records: Optional[OnNoRecords] = Field( + OnNoRecords.skip, + description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', + title="On No Records", + ) + truncation_indicator_path: Optional[List[str]] = Field( + None, + description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. Wildcards (*) are not supported in this path or in `expand_records_from_field` when this option is used.", + examples=[["data", "object", "lines", "has_more"]], + title="Truncation Indicator Path", + ) + truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( + None, + description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. If the retriever returns no records, the embedded items are expanded as a fallback. Requires `truncation_indicator_path`.", + title="Truncated List Retriever", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class SessionTokenAuthenticator(BaseModel): type: Literal["SessionTokenAuthenticator"] login_requester: HttpRequester = Field( @@ -2984,18 +3003,20 @@ class HttpRequester(BaseModelWithDeprecations): description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", title="Query Properties", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( + Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", + ) ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -3165,9 +3186,34 @@ class QueryProperties(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class RecordSelector(BaseModel): + type: Literal["RecordSelector"] + extractor: Union[DpathExtractor, CustomRecordExtractor] + record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( + None, + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", + ) + schema_normalization: Optional[ + Union[SchemaNormalization, CustomSchemaNormalization] + ] = Field( + None, + description="Responsible for normalization according to the schema.", + title="Schema Normalization", + ) + transform_before_filtering: Optional[bool] = Field( + None, + description="If true, transformation will be applied before record filtering.", + title="Transform Before Filtering", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field(..., description="The stream name.", example=["Users"], title="Name") + name: str = Field( + ..., description="The stream name.", example=["Users"], title="Name" + ) full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -3180,7 +3226,7 @@ class StateDelegatingStream(BaseModel): ) api_retention_period: Optional[str] = Field( None, - description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", + description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", examples=["P30D", "P90D", "P1Y"], title="API Retention Period", ) @@ -3263,13 +3309,17 @@ class AsyncRetriever(BaseModel): status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the actual status of the async job." ) - download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( + download_target_extractor: Optional[ + Union[DpathExtractor, CustomRecordExtractor] + ] = Field( None, description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field(None, description="Responsible for fetching the records from provided urls.") + ] = Field( + None, description="Responsible for fetching the records from provided urls." + ) creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", @@ -3282,10 +3332,9 @@ class AsyncRetriever(BaseModel): None, description="The time in minutes after which the single Async Job should be considered as Timed Out.", ) - failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( + failed_retry_wait_time_in_seconds: Optional[Union[conint(ge=1), str]] = Field( None, description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", - ge=1, ) download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( None, @@ -3366,20 +3415,14 @@ class AsyncRetriever(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class BlockSimultaneousSyncsAction(BaseModel): - type: Literal["BlockSimultaneousSyncsAction"] - - class StreamGroup(BaseModel): - streams: List[str] = Field( + streams: List[DeclarativeStream] = Field( ..., - description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', + description="List of references to streams that belong to this group.\n", title="Streams", ) action: BlockSimultaneousSyncsAction = Field( - ..., - description="The action to apply to streams in this group.", - title="Action", + ..., description="The action to apply to streams in this group.", title="Action" ) @@ -3404,7 +3447,7 @@ class GroupingPartitionRouter(BaseModel): underlying_partition_router: Union[ ListPartitionRouter, SubstreamPartitionRouter, - "UnionPartitionRouter", + UnionPartitionRouter, CustomPartitionRouter, ] = Field( ..., @@ -3484,6 +3527,8 @@ class DynamicDeclarativeStream(BaseModel): ConditionalStreams.update_forward_refs() FileUploader.update_forward_refs() DeclarativeStream.update_forward_refs() +DpathExtractor.update_forward_refs() +RecordExpander.update_forward_refs() SessionTokenAuthenticator.update_forward_refs() HttpRequester.update_forward_refs() DynamicSchemaLoader.update_forward_refs() @@ -3492,4 +3537,3 @@ class DynamicDeclarativeStream(BaseModel): SimpleRetriever.update_forward_refs() AsyncRetriever.update_forward_refs() GroupingPartitionRouter.update_forward_refs() -UnionPartitionRouter.update_forward_refs() diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index aa546c73ab..3689a5b1f1 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2506,6 +2506,16 @@ def create_record_expander( config: Config, **kwargs: Any, ) -> RecordExpander: + truncated_list_retriever = None + if model.truncated_list_retriever: + truncated_list_retriever = self._create_component_from_model( + model=model.truncated_list_retriever, + config=config, + name="record_expander_truncated_list", + primary_key=None, + stream_slicer=None, + transformations=[], + ) return RecordExpander( expand_records_from_field=model.expand_records_from_field, config=config, @@ -2514,6 +2524,8 @@ def create_record_expander( on_no_records=OnNoRecords(model.on_no_records.value) if model.on_no_records else OnNoRecords.skip, + truncation_indicator_path=model.truncation_indicator_path, + truncated_list_retriever=truncated_list_retriever, ) @staticmethod diff --git a/unit_tests/sources/declarative/expanders/__init__.py b/unit_tests/sources/declarative/expanders/__init__.py new file mode 100644 index 0000000000..89adeb6e68 --- /dev/null +++ b/unit_tests/sources/declarative/expanders/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py new file mode 100644 index 0000000000..aaee11c9e7 --- /dev/null +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pytest + +from airbyte_cdk.sources.declarative.expanders.record_expander import RecordExpander +from airbyte_cdk.sources.types import Record, StreamSlice + +config = {} +parameters = {} + + +def _make_retriever(records): + retriever = MagicMock() + retriever.read_records.return_value = iter(records) + return retriever + + +def _event(lines, has_more, total_count): + return { + "id": "evt_1", + "data": { + "object": { + "id": "in_1", + "lines": { + "data": lines, + "has_more": has_more, + "total_count": total_count, + "url": "/v1/invoices/in_1/lines", + }, + } + }, + } + + +def test_truncated_list_is_fetched_via_retriever(): + embedded = [{"id": f"il_{i}"} for i in range(10)] + complete = [{"id": f"il_{i}"} for i in range(15)] + retriever = _make_retriever([Record(data=item, stream_name="test") for item in complete]) + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + remain_original_record=True, + truncation_indicator_path=["data", "object", "lines", "has_more"], + truncated_list_retriever=retriever, + ) + + parent = _event(embedded, has_more=True, total_count=15) + records = list(expander.expand_record(parent)) + + assert len(records) == 15 + assert [record["id"] for record in records] == [f"il_{i}" for i in range(15)] + assert all(record["original_record"] == parent for record in records) + stream_slice = retriever.read_records.call_args.kwargs["stream_slice"] + assert stream_slice == StreamSlice(partition={"parent_record": parent}, cursor_slice={}) + + +def test_no_retriever_call_when_not_truncated(): + embedded = [{"id": f"il_{i}"} for i in range(10)] + retriever = _make_retriever([]) + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + truncated_list_retriever=retriever, + ) + + records = list(expander.expand_record(_event(embedded, has_more=False, total_count=10))) + + assert len(records) == 10 + retriever.read_records.assert_not_called() + + +def test_no_retriever_call_when_indicator_missing(): + embedded = [{"id": "il_0"}] + retriever = _make_retriever([]) + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + truncated_list_retriever=retriever, + ) + + parent = _event(embedded, has_more=False, total_count=1) + del parent["data"]["object"]["lines"]["has_more"] + records = list(expander.expand_record(parent)) + + assert len(records) == 1 + retriever.read_records.assert_not_called() + + +def test_falls_back_to_embedded_items_when_retriever_returns_nothing(): + embedded = [{"id": f"il_{i}"} for i in range(10)] + retriever = _make_retriever([]) + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + truncated_list_retriever=retriever, + ) + + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + assert len(records) == 10 + retriever.read_records.assert_called_once() + + +def test_retriever_requires_truncation_indicator_path(): + with pytest.raises(ValueError): + RecordExpander( + expand_records_from_field=["lines", "data"], + config=config, + parameters=parameters, + truncated_list_retriever=_make_retriever([]), + ) + + +def test_wildcard_rejected_with_truncation_handling(): + with pytest.raises(ValueError): + RecordExpander( + expand_records_from_field=["sections", "*", "items"], + config=config, + parameters=parameters, + truncation_indicator_path=["lines", "has_more"], + truncated_list_retriever=_make_retriever([]), + ) diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 21c99adc71..83201b1417 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -1986,6 +1986,54 @@ def test_client_side_incremental_with_partition_router(): ) +def test_create_record_expander_with_truncated_list_retriever(): + content = """ + selector: + type: RecordSelector + $parameters: + name: "lists" + extractor: + type: DpathExtractor + field_path: ["data"] + record_expander: + type: RecordExpander + expand_records_from_field: ["data", "object", "lines", "data"] + remain_original_record: true + truncation_indicator_path: ["data", "object", "lines", "has_more"] + truncated_list_retriever: + type: SimpleRetriever + requester: + type: HttpRequester + url_base: "https://api.stripe.com/v1/" + path: "invoices/{{ stream_slice['parent_record']['data']['object']['id'] }}/lines" + http_method: "GET" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + """ + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + selector_manifest = transformer.propagate_types_and_parameters( + "", resolved_manifest["selector"], {} + ) + + selector = factory.create_component( + model_type=RecordSelectorModel, + name="test_stream", + component_definition=selector_manifest, + decoder=None, + transformations=[], + config=input_config, + ) + + expander = selector.extractor.record_expander + assert expander is not None + assert expander.truncation_indicator_path == ["data", "object", "lines", "has_more"] + assert isinstance(expander.truncated_list_retriever, SimpleRetriever) + + @pytest.mark.parametrize( "test_name, record_selector, expected_runtime_selector", [ From 236d69658474f5d025720ed3a868e2345412d12f Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Wed, 26 Aug 2026 22:40:44 +0000 Subject: [PATCH 02/15] chore: hand-scope generated model changes to RecordExpander additions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../models/declarative_component_schema.py | 716 +++++++++--------- 1 file changed, 342 insertions(+), 374 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index a1fbcab701..19f379008b 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -6,7 +6,7 @@ from enum import Enum from typing import Any, Dict, List, Literal, Optional, Union -from pydantic.v1 import BaseModel, Extra, Field, confloat, conint +from pydantic.v1 import BaseModel, Extra, Field from airbyte_cdk.sources.declarative.models.base_model_with_deprecations import ( BaseModelWithDeprecations, @@ -18,6 +18,12 @@ class AuthFlowType(Enum): oauth1_0 = "oauth1.0" +class ScopesJoinStrategy(Enum): + space = "space" + comma = "comma" + plus = "plus" + + class BasicHttpAuthenticator(BaseModel): type: Literal["BasicHttpAuthenticator"] username: str = Field( @@ -46,50 +52,15 @@ class BearerAuthenticator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class HttpMethod(Enum): - GET = "GET" - POST = "POST" - - -class QuotaStatusSource(BaseModel): - type: Literal["QuotaStatusSource"] - url: str = Field( - ..., - description="The full URL of the quota status endpoint.", - examples=[ - "https://api.github.com/rate_limit", - "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", - ], - title="URL", - ) - http_method: Optional[HttpMethod] = Field( - HttpMethod.GET, - description="The HTTP method used to fetch the quota status.", - title="HTTP Method", - ) - request_headers: Optional[Dict[str, str]] = Field( - None, - description="Additional headers to send with the quota status request.", - title="Request Headers", - ) - unavailable_status_codes: Optional[List[int]] = Field( - None, - description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", - examples=[[404]], - title="Unavailable Status Codes", - unique_items=True, - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class DynamicStreamCheckConfig(BaseModel): type: Literal["DynamicStreamCheckConfig"] dynamic_stream_name: str = Field( ..., description="The dynamic stream name.", title="Dynamic Stream Name" ) - stream_count: Optional[conint(ge=1)] = Field( + stream_count: Optional[int] = Field( None, description="The number of streams to attempt reading from during a check operation. If unset, all generated streams are checked. Must be a positive integer; if it exceeds the total number of available streams, all streams are checked.", + ge=1, title="Stream Count", ) @@ -133,16 +104,17 @@ class ConcurrencyLevel(BaseModel): class ConstantBackoffStrategy(BaseModel): type: Literal["ConstantBackoffStrategy"] - backoff_time_in_seconds: Union[confloat(ge=0.0), str] = Field( + backoff_time_in_seconds: Union[float, str] = Field( ..., description="Backoff time in seconds.", examples=[30, 30.5, "{{ config['backoff_time'] }}"], title="Backoff Time", ) - jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( + jitter_range_in_seconds: Optional[float] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between backoff_time_in_seconds and backoff_time_in_seconds + (jitter_range_in_seconds * 2), so jitter only increases the base backoff.", examples=[15], + ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -530,7 +502,7 @@ class Config: ) weight: Optional[Union[int, str]] = Field( None, - description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.\n", + description="The weight of a request matching this matcher when acquiring a call from the rate limiter. Different endpoints can consume different amounts from a shared budget by specifying different weights. If not set, each request counts as 1.", title="Weight", ) @@ -558,10 +530,11 @@ class ExponentialBackoffStrategy(BaseModel): examples=[5, 5.5, "10"], title="Factor", ) - jitter_range_in_seconds: Optional[confloat(ge=0.0)] = Field( + jitter_range_in_seconds: Optional[float] = Field( None, description="Optional additive jitter range in seconds. When set, the backoff time is uniformly distributed between computed_backoff and computed_backoff + (jitter_range_in_seconds * 2), so jitter only increases the computed backoff.", examples=[2], + ge=0, title="Jitter Range", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -582,6 +555,157 @@ class SessionTokenRequestBearerAuthenticator(BaseModel): type: Literal["Bearer"] +class HttpMethod(Enum): + GET = "GET" + POST = "POST" + + +class QuotaStatusSource(BaseModel): + type: Literal["QuotaStatusSource"] + url: str = Field( + ..., + description="The full URL of the quota status endpoint.", + examples=[ + "https://api.github.com/rate_limit", + "{{ config.get('api_url', 'https://api.github.com') }}/rate_limit", + ], + title="URL", + ) + http_method: Optional[HttpMethod] = Field( + HttpMethod.GET, + description="The HTTP method used to fetch the quota status.", + title="HTTP Method", + ) + request_headers: Optional[Dict[str, str]] = Field( + None, + description="Additional headers to send with the quota status request.", + title="Request Headers", + ) + unavailable_status_codes: Optional[List[int]] = Field( + None, + description="Status codes from the quota status endpoint that mean quota tracking is unavailable rather than broken, such as a self-hosted deployment with rate limiting turned off. Every pool of the token whose request returned that status is then treated as untracked, so the authenticator stops waiting for quota resets, stops throttling proactively and stops rotating on exhaustion for it, while still signing requests. A token untracked this way stays untracked for the rest of the sync, because the endpoint is never consulted for it again, so a status the endpoint can also return transiently costs quota tracking for the whole run. If only some tokens return that status the others stay tracked, but they are no longer refreshed either, because the authenticator stops waiting for quota resets as soon as one token is untracked; once their counters are locally spent all traffic moves onto the untracked tokens. Rate limiting reported by ordinary responses is still handled by the stream's error handler, so one that retries 429 or 403 keeps working, and a retry rotates onto the next token; it pays the backoff the response asks for rather than the shortened one a tracked pool would get, since an untracked pool has no counters with which to argue the rejection was about that credential. Any status not listed still fails the connection, and this field never excuses a quota path missing from a response the endpoint did answer, so list only the codes the endpoint uses to report that rate limiting is not enabled. Do not list authentication or authorization statuses, since a 401 or 403 from a revoked credential would then be read as quota tracking being unavailable rather than as a credentials failure.", + examples=[[404]], + title="Unavailable Status Codes", + unique_items=True, + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class TokenQuota(BaseModel): + type: Literal["TokenQuota"] + name: str = Field( + ..., + description="Name of the quota pool.", + examples=["rest", "graphql"], + title="Name", + ) + remaining_path: List[str] = Field( + ..., + description="Path to the remaining call count for this pool in the quota status response.", + examples=[["resources", "core", "remaining"]], + title="Remaining Path", + ) + reset_path: List[str] = Field( + ..., + description="Path to the quota reset timestamp for this pool in the quota status response.", + examples=[["resources", "core", "reset"]], + title="Reset Path", + ) + limit_path: Optional[List[str]] = Field( + None, + description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", + examples=[["resources", "core", "limit"]], + title="Limit Path", + ) + matchers: Optional[List[HttpRequestRegexMatcher]] = Field( + None, + description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", + title="Matchers", + ) + remaining_header: Optional[str] = Field( + None, + description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", + examples=["X-RateLimit-Remaining"], + title="Remaining Header", + ) + reset_header: Optional[str] = Field( + None, + description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", + examples=["X-RateLimit-Reset"], + title="Reset Header", + ) + limit_header: Optional[str] = Field( + None, + description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", + examples=["X-RateLimit-Limit"], + title="Limit Header", + ) + exhaustion_status_codes: Optional[List[int]] = Field( + None, + description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", + examples=[[429]], + title="Exhaustion Status Codes", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + +class RateLimitedMultipleTokenAuthenticator(BaseModel): + type: Literal["RateLimitedMultipleTokenAuthenticator"] + tokens: Union[str, List[str]] = Field( + ..., + description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", + examples=[ + "{{ config['credentials']['personal_access_token'] }}", + ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], + ], + title="Tokens", + ) + token_delimiter: Optional[str] = Field( + ",", + description="Delimiter used to split a single token string into multiple tokens.", + title="Token Delimiter", + ) + auth_method: Optional[str] = Field( + "Bearer", + description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", + examples=["Bearer", "token"], + title="Auth Method", + ) + header: Optional[str] = Field( + "Authorization", + description="The name of the HTTP header in which to inject the token.", + title="Header Name", + ) + quota_status_source: QuotaStatusSource = Field( + ..., + description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", + title="Quota Status Source", + ) + quotas: List[TokenQuota] = Field( + ..., + description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", + min_items=1, + title="Quota Pools", + ) + max_wait_time: Optional[str] = Field( + "PT2H", + description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", + examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], + title="Maximum Wait Time", + ) + budget_reserve_fraction: Optional[float] = Field( + 0.1, + description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", + title="Budget Reserve Fraction", + ) + budget_min_reserve: Optional[int] = Field( + 50, + description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", + title="Budget Minimum Reserve", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class Action(Enum): SUCCESS = "SUCCESS" FAIL = "FAIL" @@ -711,13 +835,12 @@ class JsonItemsDecoder(BaseModel): type: Literal["JsonItemsDecoder"] items_path: str = Field( ..., - description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax — do not include leading `$.` or trailing `[*]`.", - examples=["dataByDepartmentAndSearchTerm", "dataByAsin", "data.users"], + description="Dot-separated path to the JSON array whose elements should be yielded as records. Uses `ijson` path syntax (e.g. `data.users`), not JSONPath syntax \u2014 do not include leading `$.` or trailing `[*]`.", title="Items Path", ) encoding: Optional[str] = Field( "utf-8", - description="Text encoding used to decode the streamed bytes before JSON parsing.", + description="The character encoding of the JSON data. Defaults to UTF-8.", title="Encoding", ) @@ -880,38 +1003,24 @@ class NoPagination(BaseModel): type: Literal["NoPagination"] -class Scope(BaseModel): +class State(BaseModel): class Config: extra = Extra.allow - scope: str = Field( - ..., description="The OAuth scope string to request from the provider." - ) + min: int + max: int -class OptionalScope(BaseModel): +class OAuthScope(BaseModel): class Config: extra = Extra.allow scope: str = Field( - ..., description="The OAuth scope string to request from the provider." + ..., + description="The OAuth scope string to request from the provider.", ) -class ScopesJoinStrategy(Enum): - space = "space" - comma = "comma" - plus = "plus" - - -class State(BaseModel): - class Config: - extra = Extra.allow - - min: int - max: int - - class OauthConnectorInputSpecification(BaseModel): class Config: extra = Extra.allow @@ -931,13 +1040,17 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) - scopes: Optional[List[Scope]] = Field( + # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the + # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. + # The CDK schema defines the manifest contract; the platform reads these fields + # during the OAuth consent flow to build the authorization URL. + scopes: Optional[List[OAuthScope]] = Field( None, description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], title="Scopes", ) - optional_scopes: Optional[List[OptionalScope]] = Field( + optional_scopes: Optional[List[OAuthScope]] = Field( None, description="Optional OAuth scope objects that may or may not be granted.", examples=[[{"scope": "admin:read"}]], @@ -1032,28 +1145,24 @@ class OAuthConfigSpecification(BaseModel): class Config: extra = Extra.allow - oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = ( - Field( - None, - description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", - examples=[ - {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, - { - "app_id": { - "type": "string", - "path_in_connector_config": ["info", "app_id"], - } - }, - ], - title="OAuth user input", - ) + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = Field( + None, + description="OAuth specific blob. This is a Json Schema used to validate Json configurations used as input to OAuth.\nMust be a valid non-nested JSON that refers to properties from ConnectorSpecification.connectionSpecification\nusing special annotation 'path_in_connector_config'.\nThese are input values the user is entering through the UI to authenticate to the connector, that might also shared\nas inputs for syncing data via the connector.\nExamples:\nif no connector values is shared during oauth flow, oauth_user_input_from_connector_config_specification=[]\nif connector values such as 'app_id' inside the top level are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['app_id']\n }\n }\nif connector values such as 'info.app_id' nested inside another object are used to generate the API url for the oauth flow,\n oauth_user_input_from_connector_config_specification={\n app_id: {\n type: string\n path_in_connector_config: ['info', 'app_id']\n }\n }", + examples=[ + {"app_id": {"type": "string", "path_in_connector_config": ["app_id"]}}, + { + "app_id": { + "type": "string", + "path_in_connector_config": ["info", "app_id"], + } + }, + ], + title="OAuth user input", ) - oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = ( - Field( - None, - description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', - title="DeclarativeOAuth Connector Specification", - ) + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = Field( + None, + description='The DeclarativeOAuth specific blob.\nPertains to the fields defined by the connector relating to the OAuth flow.\n\nInterpolation capabilities:\n- The variables placeholders are declared as `{{my_var}}`.\n- The nested resolution variables like `{{ {{my_nested_var}} }}` is allowed as well.\n\n- The allowed interpolation context is:\n + base64Encoder - encode to `base64`, {{ {{my_var_a}}:{{my_var_b}} | base64Encoder }}\n + base64Decorer - decode from `base64` encoded string, {{ {{my_string_variable_or_string_value}} | base64Decoder }}\n + urlEncoder - encode the input string to URL-like format, {{ https://test.host.com/endpoint | urlEncoder}}\n + urlDecorer - decode the input url-encoded string into text format, {{ urlDecoder:https%3A%2F%2Fairbyte.io | urlDecoder}}\n + codeChallengeS256 - get the `codeChallenge` encoded value to provide additional data-provider specific authorisation values, {{ {{state_value}} | codeChallengeS256 }}\n\nExamples:\n - The TikTok Marketing DeclarativeOAuth spec:\n {\n "oauth_connector_input_specification": {\n "type": "object",\n "additionalProperties": false,\n "properties": {\n "consent_url": "https://ads.tiktok.com/marketing_api/auth?{{client_id_key}}={{client_id_value}}&{{redirect_uri_key}}={{ {{redirect_uri_value}} | urlEncoder}}&{{state_key}}={{state_value}}",\n "access_token_url": "https://business-api.tiktok.com/open_api/v1.3/oauth2/access_token/",\n "access_token_params": {\n "{{ auth_code_key }}": "{{ auth_code_value }}",\n "{{ client_id_key }}": "{{ client_id_value }}",\n "{{ client_secret_key }}": "{{ client_secret_value }}"\n },\n "access_token_headers": {\n "Content-Type": "application/json",\n "Accept": "application/json"\n },\n "extract_output": ["data.access_token"],\n "client_id_key": "app_id",\n "client_secret_key": "secret",\n "auth_code_key": "auth_code"\n }\n }\n }', + title="DeclarativeOAuth Connector Specification", ) complete_oauth_output_specification: Optional[Dict[str, Any]] = Field( None, @@ -1071,9 +1180,7 @@ class Config: complete_oauth_server_input_specification: Optional[Dict[str, Any]] = Field( None, description="OAuth specific blob. This is a Json Schema used to validate Json configurations persisted as Airbyte Server configurations.\nMust be a valid non-nested JSON describing additional fields configured by the Airbyte Instance or Workspace Admins to be used by the\nserver when completing an OAuth flow (typically exchanging an auth code for refresh token).\nExamples:\n complete_oauth_server_input_specification={\n client_id: {\n type: string\n },\n client_secret: {\n type: string\n }\n }", - examples=[ - {"client_id": {"type": "string"}, "client_secret": {"type": "string"}} - ], + examples=[{"client_id": {"type": "string"}, "client_secret": {"type": "string"}}], title="OAuth input specification", ) complete_oauth_server_output_specification: Optional[Dict[str, Any]] = Field( @@ -1316,14 +1423,7 @@ class AsyncJobStatusMap(BaseModel): completed: List[str] failed: List[str] timeout: List[str] - skipped: Optional[List[str]] = Field( - None, - description="Statuses that indicate the job was skipped because there is no data to return. Jobs with these statuses will not be retried and no records will be fetched.", - ) - - -class BlockSimultaneousSyncsAction(BaseModel): - type: Literal["BlockSimultaneousSyncsAction"] + skipped: Optional[List[str]] = None class ValueType(Enum): @@ -1591,9 +1691,7 @@ class CustomConfigTransformation(BaseModel): class_name: str = Field( ..., description="Fully-qualified name of the class that will be implementing the custom config transformation. The format is `source_..`.", - examples=[ - "source_declarative_manifest.components.MyCustomConfigTransformation" - ], + examples=["source_declarative_manifest.components.MyCustomConfigTransformation"], ) parameters: Optional[Dict[str, Any]] = Field( None, @@ -1693,71 +1791,13 @@ class AuthFlow(BaseModel): oauth_config_specification: Optional[OAuthConfigSpecification] = None -class TokenQuota(BaseModel): - type: Literal["TokenQuota"] - name: str = Field( - ..., - description="Name of the quota pool.", - examples=["rest", "graphql"], - title="Name", - ) - remaining_path: List[str] = Field( - ..., - description="Path to the remaining call count for this pool in the quota status response.", - examples=[["resources", "core", "remaining"]], - title="Remaining Path", - ) - reset_path: List[str] = Field( - ..., - description="Path to the quota reset timestamp for this pool in the quota status response.", - examples=[["resources", "core", "reset"]], - title="Reset Path", - ) - limit_path: Optional[List[str]] = Field( - None, - description="Optional path to the total call limit for this pool in the quota status response. Used to compute the proactive throttling reserve; falls back to the initially observed remaining count when not set. Setting it on every pool is recommended so the reserve does not shrink when a sync starts with the pool already partially consumed.", - examples=[["resources", "core", "limit"]], - title="Limit Path", - ) - matchers: Optional[List[HttpRequestRegexMatcher]] = Field( - None, - description="List of matchers that classify outgoing requests into this quota pool. The first pool whose matcher matches a request is used. A pool with no matchers acts as the default pool.", - title="Matchers", - ) - remaining_header: Optional[str] = Field( - None, - description="Optional response header carrying the remaining call count for this pool. When set, the pool's counter is reconciled against this header on every response, which corrects drift caused by sharing the token with other clients, by requests in flight concurrently, or by a sync running long enough for the initial quota status read to go stale. Without it the pool is only ever seeded from the quota status endpoint.", - examples=["X-RateLimit-Remaining"], - title="Remaining Header", - ) - reset_header: Optional[str] = Field( - None, - description="Optional response header carrying the quota reset timestamp for this pool. Parsed with the same rules as `reset_path`, so epoch seconds and ISO 8601 both work. Used to tell a rolled-over quota window from the current one; a response proving the window has rolled over restores the pool to its limit. Most useful alongside `remaining_header`.", - examples=["X-RateLimit-Reset"], - title="Reset Header", - ) - limit_header: Optional[str] = Field( - None, - description="Optional response header carrying the total call limit for this pool, used to keep the proactive throttling reserve accurate as the limit changes.", - examples=["X-RateLimit-Limit"], - title="Limit Header", - ) - exhaustion_status_codes: Optional[List[int]] = Field( - None, - description="Response status codes that mean this token's pool is spent. These have two effects. A response carrying one of them but no remaining count sets the pool to zero, so the next request rotates to another token instead of waiting out the reset window. They also mark which responses may report a zero for a quota window that has already elapsed, so a rate limit whose reset header trails the value being held still stops the token being used; a zero on any other response is treated as the last call of a finished window and ignored. Leaving this empty means such trailing rejections are ignored unless their reset is within the skew tolerance of the current window. Only list codes the API uses exclusively for rate limiting -- a code that also signals other failures would park a healthy token.", - examples=[[429]], - title="Exhaustion Status Codes", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class CheckStream(BaseModel): - type: Literal["CheckStream"] - stream_names: Optional[List[str]] = Field( - None, - description="Names of the streams to try reading from when running a check operation.", - examples=[["users"], ["users", "contacts"]], - title="Stream Names", +class CheckStream(BaseModel): + type: Literal["CheckStream"] + stream_names: Optional[List[str]] = Field( + None, + description="Names of the streams to try reading from when running a check operation.", + examples=[["users"], ["users", "contacts"]], + title="Stream Names", ) dynamic_streams_check_configs: Optional[List[DynamicStreamCheckConfig]] = None config_overrides: Optional[Dict[str, Any]] = Field( @@ -2091,9 +2131,7 @@ class OAuthAuthenticator(BaseModel): scopes: Optional[List[str]] = Field( None, description="List of scopes that should be granted to the access token.", - examples=[ - ["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"] - ], + examples=[["crm.list.read", "crm.objects.contacts.read", "crm.schema.contacts.read"]], title="Scopes", ) token_expiry_date: Optional[str] = Field( @@ -2244,6 +2282,43 @@ class DefaultPaginator(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class RecordExpander(BaseModel): + type: Literal["RecordExpander"] + expand_records_from_field: List[str] = Field( + ..., + description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", + examples=[ + ["lines", "data"], + ["items"], + ["nested", "array"], + ["sections", "*", "items"], + ], + title="Expand Records From Field", + ) + remain_original_record: Optional[bool] = Field( + False, + description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', + title="Remain Original Record", + ) + on_no_records: Optional[OnNoRecords] = Field( + OnNoRecords.skip, + description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', + title="On No Records", + ) + truncation_indicator_path: Optional[List[str]] = Field( + None, + description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. Wildcards (*) are not supported in this path or in `expand_records_from_field` when this option is used.", + examples=[["data", "object", "lines", "has_more"]], + title="Truncation Indicator Path", + ) + truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( + None, + description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. If the retriever returns no records, the embedded items are expanded as a fallback. Requires `truncation_indicator_path`.", + title="Truncated List Retriever", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class SessionTokenRequestApiKeyAuthenticator(BaseModel): type: Literal["ApiKey"] inject_into: RequestOption = Field( @@ -2341,12 +2416,10 @@ class DpathValidator(BaseModel): ], title="Field Path", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The condition that the specified config value will be evaluated against", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The condition that the specified config value will be evaluated against", + title="Validation Strategy", ) @@ -2363,12 +2436,10 @@ class PredicateValidator(BaseModel): ], title="Value", ) - validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = ( - Field( - ..., - description="The validation strategy to apply to the value.", - title="Validation Strategy", - ) + validation_strategy: Union[ValidateAdheresToSchema, CustomValidationStrategy] = Field( + ..., + description="The validation strategy to apply to the value.", + title="Validation Strategy", ) @@ -2391,71 +2462,14 @@ class ConfigAddFields(BaseModel): ) -class RateLimitedMultipleTokenAuthenticator(BaseModel): - type: Literal["RateLimitedMultipleTokenAuthenticator"] - tokens: Union[str, List[str]] = Field( - ..., - description="The tokens to rotate between. Either an explicit list of tokens, or a single string containing multiple tokens separated by `token_delimiter`.", - examples=[ - "{{ config['credentials']['personal_access_token'] }}", - ["{{ config['token_1'] }}", "{{ config['token_2'] }}"], - ], - title="Tokens", - ) - token_delimiter: Optional[str] = Field( - ",", - description="Delimiter used to split a single token string into multiple tokens.", - title="Token Delimiter", - ) - auth_method: Optional[str] = Field( - "Bearer", - description="The prefix to prepend to the token in the auth header value (e.g. `Authorization: Bearer `).", - examples=["Bearer", "token"], - title="Auth Method", - ) - header: Optional[str] = Field( - "Authorization", - description="The name of the HTTP header in which to inject the token.", - title="Header Name", - ) - quota_status_source: QuotaStatusSource = Field( - ..., - description="Defines where to fetch each token's current quota status. Called once per token at startup and after an exhaustion wait, not per data request.", - title="Quota Status Source", - ) - quotas: List[TokenQuota] = Field( - ..., - description="Quota pools tracked per token. Each outgoing request is classified into the first pool whose matchers match the request; a pool with no matchers acts as the default. The `remaining_path` and `reset_path` locate each pool's values in the quota status response.\n", - min_items=1, - title="Quota Pools", - ) - max_wait_time: Optional[str] = Field( - "PT2H", - description="ISO 8601 duration. When all tokens are exhausted, the maximum time to wait for a quota reset before raising a transient error.", - examples=["PT2H", "PT30M", "PT{{ config.get('max_waiting_time', 120) }}M"], - title="Maximum Wait Time", - ) - budget_reserve_fraction: Optional[float] = Field( - 0.1, - description="Fraction of each token's quota to keep in reserve. When every token drops below its reserve, requests are proactively throttled to spread the remaining calls until the quota reset. Set to 0 (along with `budget_min_reserve`) to disable throttling.", - title="Budget Reserve Fraction", - ) - budget_min_reserve: Optional[int] = Field( - 50, - description="Minimum number of calls to keep in reserve per token before proactive throttling kicks in.", - title="Budget Minimum Reserve", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class CompositeErrorHandler(BaseModel): type: Literal["CompositeErrorHandler"] - error_handlers: List[ - Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler] - ] = Field( - ..., - description="List of error handlers to iterate on to determine how to handle a failed response.", - title="Error Handlers", + error_handlers: List[Union[CompositeErrorHandler, DefaultErrorHandler, CustomErrorHandler]] = ( + Field( + ..., + description="List of error handlers to iterate on to determine how to handle a failed response.", + title="Error Handlers", + ) ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") @@ -2493,20 +2507,60 @@ class Config: ) +class DpathExtractor(BaseModel): + type: Literal["DpathExtractor"] + field_path: List[str] = Field( + ..., + description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', + examples=[ + ["data"], + ["data", "records"], + ["data", "{{ parameters.name }}"], + ["data", "*", "record"], + ], + title="Field Path", + ) + record_expander: Optional[RecordExpander] = Field( + None, + description="Optional component to expand records by extracting items from nested array fields.", + title="Record Expander", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class ZipfileDecoder(BaseModel): class Config: extra = Extra.allow type: Literal["ZipfileDecoder"] - decoder: Union[ - CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder - ] = Field( + decoder: Union[CsvDecoder, GzipDecoder, JsonDecoder, JsonItemsDecoder, JsonlDecoder] = Field( ..., description="Parser to parse the decompressed data from the zipfile(s).", title="Parser", ) +class RecordSelector(BaseModel): + type: Literal["RecordSelector"] + extractor: Union[DpathExtractor, CustomRecordExtractor] + record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( + None, + description="Responsible for filtering records to be emitted by the Source.", + title="Record Filter", + ) + schema_normalization: Optional[Union[SchemaNormalization, CustomSchemaNormalization]] = Field( + None, + description="Responsible for normalization according to the schema.", + title="Schema Normalization", + ) + transform_before_filtering: Optional[bool] = Field( + None, + description="If true, transformation will be applied before record filtering.", + title="Transform Before Filtering", + ) + parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") + + class ConfigMigration(BaseModel): type: Literal["ConfigMigration"] description: Optional[str] = Field( @@ -2599,7 +2653,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2624,9 +2678,9 @@ class Config: type: Literal["DeclarativeSource"] check: Union[CheckStream, CheckDynamicStream] - streams: Optional[ - List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]] - ] = None + streams: Optional[List[Union[ConditionalStreams, DeclarativeStream, StateDelegatingStream]]] = ( + None + ) dynamic_streams: List[DynamicDeclarativeStream] version: str = Field( ..., @@ -2639,7 +2693,7 @@ class Config: api_budget: Optional[HTTPAPIBudget] = None stream_groups: Optional[Dict[str, StreamGroup]] = Field( None, - description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.\n", + description="Groups of streams that share a common resource and should not be read simultaneously. Each group defines a set of stream references and an action that controls how concurrent reads are managed. Only applies to ConcurrentDeclarativeSource.", title="Stream Groups", ) max_concurrent_async_job_count: Optional[Union[int, str]] = Field( @@ -2757,20 +2811,16 @@ class Config: extra = Extra.allow type: Literal["DeclarativeStream"] - name: Optional[str] = Field( - "", description="The stream name.", example=["Users"], title="Name" - ) + name: Optional[str] = Field("", description="The stream name.", example=["Users"], title="Name") retriever: Union[SimpleRetriever, AsyncRetriever, CustomRetriever] = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages.", title="Retriever", ) - incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = ( - Field( - None, - description="Component used to fetch data incrementally based on a time field in the data.", - title="Incremental Sync", - ) + incremental_sync: Optional[Union[DatetimeBasedCursor, IncrementingCountCursor]] = Field( + None, + description="Component used to fetch data incrementally based on a time field in the data.", + title="Incremental Sync", ) primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ @@ -2826,64 +2876,6 @@ class Config: parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class DpathExtractor(BaseModel): - type: Literal["DpathExtractor"] - field_path: List[str] = Field( - ..., - description='List of potentially nested fields describing the full path of the field to extract. Use "*" to extract all values from an array. See more info in the [docs](https://docs.airbyte.com/connector-development/config-based/understanding-the-yaml-file/record-selector).', - examples=[ - ["data"], - ["data", "records"], - ["data", "{{ parameters.name }}"], - ["data", "*", "record"], - ], - title="Field Path", - ) - record_expander: Optional[RecordExpander] = Field( - None, - description="Optional component to expand records by extracting items from nested array fields.", - title="Record Expander", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - -class RecordExpander(BaseModel): - type: Literal["RecordExpander"] - expand_records_from_field: List[str] = Field( - ..., - description="Path to a nested array field within each record. Items from this array will be extracted and emitted as separate records. Supports wildcards (*) for matching multiple arrays.", - examples=[ - ["lines", "data"], - ["items"], - ["nested", "array"], - ["sections", "*", "items"], - ], - title="Expand Records From Field", - ) - remain_original_record: Optional[bool] = Field( - False, - description='If true, each expanded record will include the original parent record in an "original_record" field. Defaults to false.', - title="Remain Original Record", - ) - on_no_records: Optional[OnNoRecords] = Field( - OnNoRecords.skip, - description='Behavior when the expansion path is missing, not a list, or an empty list. "skip" (default) emits nothing. "emit_parent" emits the original parent record unchanged.', - title="On No Records", - ) - truncation_indicator_path: Optional[List[str]] = Field( - None, - description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. Wildcards (*) are not supported in this path or in `expand_records_from_field` when this option is used.", - examples=[["data", "object", "lines", "has_more"]], - title="Truncation Indicator Path", - ) - truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( - None, - description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. If the retriever returns no records, the embedded items are expanded as a fallback. Requires `truncation_indicator_path`.", - title="Truncated List Retriever", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class SessionTokenAuthenticator(BaseModel): type: Literal["SessionTokenAuthenticator"] login_requester: HttpRequester = Field( @@ -3003,20 +2995,18 @@ class HttpRequester(BaseModelWithDeprecations): description="For APIs that require explicit specification of the properties to query for, this component will take a static or dynamic set of properties (which can be optionally split into chunks) and allow them to be injected into an outbound request by accessing stream_partition.extra_fields.", title="Query Properties", ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = ( - Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", - ) + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", ) request_headers: Optional[Union[Dict[str, str], str]] = Field( None, @@ -3186,34 +3176,9 @@ class QueryProperties(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") -class RecordSelector(BaseModel): - type: Literal["RecordSelector"] - extractor: Union[DpathExtractor, CustomRecordExtractor] - record_filter: Optional[Union[RecordFilter, CustomRecordFilter]] = Field( - None, - description="Responsible for filtering records to be emitted by the Source.", - title="Record Filter", - ) - schema_normalization: Optional[ - Union[SchemaNormalization, CustomSchemaNormalization] - ] = Field( - None, - description="Responsible for normalization according to the schema.", - title="Schema Normalization", - ) - transform_before_filtering: Optional[bool] = Field( - None, - description="If true, transformation will be applied before record filtering.", - title="Transform Before Filtering", - ) - parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") - - class StateDelegatingStream(BaseModel): type: Literal["StateDelegatingStream"] - name: str = Field( - ..., description="The stream name.", example=["Users"], title="Name" - ) + name: str = Field(..., description="The stream name.", example=["Users"], title="Name") full_refresh_stream: DeclarativeStream = Field( ..., description="Component used to coordinate how records are extracted across stream slices and request pages when the state is empty or not provided.", @@ -3226,7 +3191,7 @@ class StateDelegatingStream(BaseModel): ) api_retention_period: Optional[str] = Field( None, - description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n* **PT1H**: 1 hour\n* **P1D**: 1 day\n* **P1W**: 1 week\n* **P1M**: 1 month\n* **P1Y**: 1 year\n* **P30D**: 30 days\n", + description="The data retention period of the incremental API (ISO8601 duration). If the cursor value is older than this retention period, the connector will automatically fall back to a full refresh to avoid data loss.\nThis is useful for APIs like Stripe Events API which only retain data for 30 days.\n * **PT1H**: 1 hour\n * **P1D**: 1 day\n * **P1W**: 1 week\n * **P1M**: 1 month\n * **P1Y**: 1 year\n * **P30D**: 30 days\n", examples=["P30D", "P90D", "P1Y"], title="API Retention Period", ) @@ -3309,17 +3274,13 @@ class AsyncRetriever(BaseModel): status_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the actual status of the async job." ) - download_target_extractor: Optional[ - Union[DpathExtractor, CustomRecordExtractor] - ] = Field( + download_target_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( None, description="Responsible for fetching the final result `urls` provided by the completed / finished / ready async job.", ) download_extractor: Optional[ Union[DpathExtractor, CustomRecordExtractor, ResponseToFileExtractor] - ] = Field( - None, description="Responsible for fetching the records from provided urls." - ) + ] = Field(None, description="Responsible for fetching the records from provided urls.") creation_requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API to create the async server-side job.", @@ -3332,9 +3293,10 @@ class AsyncRetriever(BaseModel): None, description="The time in minutes after which the single Async Job should be considered as Timed Out.", ) - failed_retry_wait_time_in_seconds: Optional[Union[conint(ge=1), str]] = Field( + failed_retry_wait_time_in_seconds: Optional[Union[int, str]] = Field( None, description="Time in seconds to wait before retrying a failed async job. Only applies to jobs that ran on the API side and reported a FAILED status (e.g. report generation failed due to a cooldown). Creation failures (HTTP errors when starting a job, such as 429s) and TIMED_OUT jobs are retried immediately and are not affected by this setting. When set, the orchestrator defers retry of real failed jobs until the wait time has elapsed, without blocking other jobs.", + ge=1, ) download_target_requester: Optional[Union[HttpRequester, CustomRequester]] = Field( None, @@ -3415,14 +3377,20 @@ class AsyncRetriever(BaseModel): parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") +class BlockSimultaneousSyncsAction(BaseModel): + type: Literal["BlockSimultaneousSyncsAction"] + + class StreamGroup(BaseModel): - streams: List[DeclarativeStream] = Field( + streams: List[str] = Field( ..., - description="List of references to streams that belong to this group.\n", + description='List of references to streams that belong to this group. Use JSON references to stream definitions (e.g., "#/definitions/my_stream").', title="Streams", ) action: BlockSimultaneousSyncsAction = Field( - ..., description="The action to apply to streams in this group.", title="Action" + ..., + description="The action to apply to streams in this group.", + title="Action", ) @@ -3447,7 +3415,7 @@ class GroupingPartitionRouter(BaseModel): underlying_partition_router: Union[ ListPartitionRouter, SubstreamPartitionRouter, - UnionPartitionRouter, + "UnionPartitionRouter", CustomPartitionRouter, ] = Field( ..., @@ -3527,9 +3495,8 @@ class DynamicDeclarativeStream(BaseModel): ConditionalStreams.update_forward_refs() FileUploader.update_forward_refs() DeclarativeStream.update_forward_refs() -DpathExtractor.update_forward_refs() -RecordExpander.update_forward_refs() SessionTokenAuthenticator.update_forward_refs() +RecordExpander.update_forward_refs() HttpRequester.update_forward_refs() DynamicSchemaLoader.update_forward_refs() ParentStreamConfig.update_forward_refs() @@ -3537,3 +3504,4 @@ class DynamicDeclarativeStream(BaseModel): SimpleRetriever.update_forward_refs() AsyncRetriever.update_forward_refs() GroupingPartitionRouter.update_forward_refs() +UnionPartitionRouter.update_forward_refs() From 39d2e22d5334918084415133b803ead63f986f14 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Wed, 26 Aug 2026 22:48:06 +0000 Subject: [PATCH 03/15] fix: scope wildcard validation to configured retriever and stream fetched records Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../sources/declarative/expanders/record_expander.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index 9a05638d55..c811c7d66b 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -91,9 +91,9 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: raise ValueError( "`truncation_indicator_path` is required when `truncated_list_retriever` is configured." ) - if self.truncation_indicator_path and any( - "*" in path - for path in (*self.expand_records_from_field, *self.truncation_indicator_path) + if self.truncated_list_retriever and any( + path == "*" + for path in (*self.expand_records_from_field, *(self.truncation_indicator_path or [])) ): raise ValueError( "The '*' wildcard is not supported in `expand_records_from_field` or `truncation_indicator_path` when truncation handling is configured." @@ -117,8 +117,10 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap parent_record = record if self.truncated_list_retriever and self._is_truncated(parent_record): - fetched_records = list(self._fetch_complete_list(parent_record)) - if fetched_records: + fetched_records = iter(self._fetch_complete_list(parent_record)) + first_fetched = next(fetched_records, None) + if first_fetched is not None: + yield first_fetched yield from fetched_records return From 87ab63253769fa39e6df604c3fdde57b4403003e Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 27 Aug 2026 21:48:17 +0000 Subject: [PATCH 04/15] feat: warn when a truncated embedded list has no truncated_list_retriever configured Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative_component_schema.yaml | 2 +- .../declarative/expanders/record_expander.py | 62 +++++++++++- .../models/declarative_component_schema.py | 2 +- .../expanders/test_record_expander.py | 94 +++++++++++++++++++ 4 files changed, 154 insertions(+), 6 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index bae5885b0e..6d02590e78 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2260,7 +2260,7 @@ definitions: default: skip truncation_indicator_path: title: Truncation Indicator Path - description: Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. Wildcards (*) are not supported in this path or in `expand_records_from_field` when this option is used. + description: Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Wildcards (*) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured. type: array items: type: string diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index c811c7d66b..8804df9d53 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -3,6 +3,7 @@ # import copy +import logging from dataclasses import InitVar, dataclass from enum import Enum from typing import TYPE_CHECKING, Any, Iterable, Mapping, MutableMapping, Optional, Sequence @@ -15,6 +16,8 @@ if TYPE_CHECKING: from airbyte_cdk.sources.declarative.retrievers import Retriever +logger = logging.getLogger("airbyte") + class OnNoRecords(Enum): """ @@ -67,6 +70,10 @@ class RecordExpander: emits nothing. "emit_parent" emits the original parent record unchanged. truncation_indicator_path: Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). + When the indicator is truthy and no `truncated_list_retriever` is configured, the + embedded items are expanded as normal and a WARNING is logged (once per stream + instance) describing the expansion path and the embedded item count, so that the + data loss is visible instead of silent. truncated_list_retriever: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. @@ -91,17 +98,21 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: raise ValueError( "`truncation_indicator_path` is required when `truncated_list_retriever` is configured." ) + if any(path == "*" for path in (self.truncation_indicator_path or [])): + raise ValueError( + "The '*' wildcard is not supported in `truncation_indicator_path`: the indicator must identify a single field." + ) if self.truncated_list_retriever and any( - path == "*" - for path in (*self.expand_records_from_field, *(self.truncation_indicator_path or [])) + path == "*" for path in self.expand_records_from_field ): raise ValueError( - "The '*' wildcard is not supported in `expand_records_from_field` or `truncation_indicator_path` when truncation handling is configured." + "The '*' wildcard is not supported in `expand_records_from_field` when `truncated_list_retriever` is configured." ) self._truncation_indicator_path: list[InterpolatedString] = [ InterpolatedString.create(path, parameters=parameters) for path in (self.truncation_indicator_path or []) ] + self._warned_truncation_without_retriever = False def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMapping[Any, Any]]: """Expand a record by extracting items from a nested array field.""" @@ -116,7 +127,9 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap parent_record = record - if self.truncated_list_retriever and self._is_truncated(parent_record): + truncated = bool(self._truncation_indicator_path) and self._is_truncated(parent_record) + + if truncated and self.truncated_list_retriever: fetched_records = iter(self._fetch_complete_list(parent_record)) first_fetched = next(fetched_records, None) if first_fetched is not None: @@ -126,6 +139,7 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap expand_path = [path.eval(self.config) for path in self._expand_path] expanded_any = False + embedded_count = 0 try: extracted_values = dpath.values(parent_record, expand_path) @@ -142,6 +156,7 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap self._apply_parent_context(parent_record, expanded_record) yield expanded_record expanded_any = True + embedded_count += 1 else: if self.remain_original_record: yield { @@ -151,10 +166,49 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap else: yield item expanded_any = True + embedded_count += 1 + + if truncated and not self.truncated_list_retriever: + self._warn_truncated_without_retriever(parent_record, embedded_count) if not expanded_any and self.on_no_records == OnNoRecords.emit_parent: yield parent_record + def _warn_truncated_without_retriever( + self, parent_record: Mapping[str, Any], embedded_count: int + ) -> None: + if self._warned_truncation_without_retriever: + return + self._warned_truncation_without_retriever = True + + indicator_path = [path.eval(self.config) for path in self._truncation_indicator_path] + expand_path = [path.eval(self.config) for path in self._expand_path] + total_count = self._get_sibling_total_count(parent_record, indicator_path) + total_fragment = f" of {total_count} total" if total_count is not None else "" + logger.warning( + "The nested list at %s is marked as truncated (the field at %s is truthy) but no " + "`truncated_list_retriever` is configured, so only the %d embedded item(s)%s were " + "expanded and the remaining items are not emitted. Configure `truncated_list_retriever` " + "to fetch the complete list if the API provides an endpoint for it. This warning is " + "emitted once per stream; other records may be truncated as well.", + expand_path, + indicator_path, + embedded_count, + total_fragment, + ) + + def _get_sibling_total_count( + self, parent_record: Mapping[str, Any], indicator_path: list[str] + ) -> Optional[int]: + """Best-effort lookup of a `total_count` field next to the truncation indicator.""" + if not indicator_path: + return None + try: + total = dpath.get(dict(parent_record), [*indicator_path[:-1], "total_count"]) + except (KeyError, ValueError): + return None + return total if isinstance(total, int) and not isinstance(total, bool) else None + def _is_truncated(self, parent_record: MutableMapping[Any, Any]) -> bool: indicator_path = [path.eval(self.config) for path in self._truncation_indicator_path] try: diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 19f379008b..db0dbdd613 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2307,7 +2307,7 @@ class RecordExpander(BaseModel): ) truncation_indicator_path: Optional[List[str]] = Field( None, - description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. Wildcards (*) are not supported in this path or in `expand_records_from_field` when this option is used.", + description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Wildcards (*) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured.", examples=[["data", "object", "lines", "has_more"]], title="Truncation Indicator Path", ) diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index aaee11c9e7..891bd9069b 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -131,3 +131,97 @@ def test_wildcard_rejected_with_truncation_handling(): truncation_indicator_path=["lines", "has_more"], truncated_list_retriever=_make_retriever([]), ) + + +def test_wildcard_rejected_in_truncation_indicator_path(): + with pytest.raises(ValueError): + RecordExpander( + expand_records_from_field=["lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["*", "has_more"], + ) + + +def _indicator_only_expander(): + return RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + ) + + +def test_warns_when_truncated_and_no_retriever_configured(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + expander = _indicator_only_expander() + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + assert len(records) == 10 + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + message = warnings[0].getMessage() + assert "['data', 'object', 'lines', 'data']" in message + assert "['data', 'object', 'lines', 'has_more']" in message + assert "10 embedded item(s)" in message + assert "of 15 total" in message + + +def test_warning_emitted_once_per_stream_instance(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + expander = _indicator_only_expander() + + with caplog.at_level("WARNING", logger="airbyte"): + list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + list(expander.expand_record(_event(embedded, has_more=True, total_count=20))) + list(expander.expand_record(_event(embedded, has_more=True, total_count=25))) + + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + + +def test_warning_omits_total_when_no_total_count_sibling(caplog): + embedded = [{"id": "il_0"}] + expander = _indicator_only_expander() + parent = _event(embedded, has_more=True, total_count=5) + del parent["data"]["object"]["lines"]["total_count"] + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(parent)) + + assert len(records) == 1 + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "total" not in warnings[0].getMessage().split("embedded item(s)")[1].split(" were")[0] + + +def test_no_warning_when_indicator_falsy(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + expander = _indicator_only_expander() + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(_event(embedded, has_more=False, total_count=10))) + + assert len(records) == 10 + assert not [r for r in caplog.records if r.levelname == "WARNING"] + + +def test_no_warning_when_retriever_configured(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + complete = [{"id": f"il_{i}"} for i in range(15)] + retriever = _make_retriever([Record(data=item, stream_name="test") for item in complete]) + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + truncated_list_retriever=retriever, + ) + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + assert len(records) == 15 + assert not [r for r in caplog.records if r.levelname == "WARNING"] From 73d86c4fe0076e88b8673fd309cd39aef83be329 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 14:28:58 +0000 Subject: [PATCH 05/15] fix(low-code): harden RecordExpander truncated-list handling per review - warn once when a configured retriever returns fewer records than total_count - reject all dpath glob metacharacters on interpolated indicator/expand paths; treat dpath ValueError as not truncated - keep scalar fetched items instead of dropping them - emit truncation warnings via MessageRepository so Builder test reads see them - factory: mark nested requests as auxiliary, drop dead stream_slicer=None, preserve CustomRetriever manifest fields, reject partition_router/pagination_reset - document amplification/use_cache, paginator, $parameters propagation, failure semantics and CDK compatibility in docstring, schema and generated model - add HttpMocker end-to-end test and factory/expander regression tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative_component_schema.yaml | 4 +- .../declarative/expanders/record_expander.py | 141 +++++++---- .../models/declarative_component_schema.py | 4 +- .../parsers/model_to_component_factory.py | 40 +++- .../expanders/test_record_expander.py | 178 +++++++++++++- .../expanders/test_record_expander_http.py | 219 ++++++++++++++++++ .../test_model_to_component_factory.py | 108 +++++++++ 7 files changed, 637 insertions(+), 57 deletions(-) create mode 100644 unit_tests/sources/declarative/expanders/test_record_expander_http.py diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 6d02590e78..9cb604f2fb 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2260,7 +2260,7 @@ definitions: default: skip truncation_indicator_path: title: Truncation Indicator Path - description: Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Wildcards (*) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured. + description: Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it. type: array items: type: string @@ -2270,7 +2270,7 @@ definitions: - ["data", "object", "lines", "has_more"] truncated_list_retriever: title: Truncated List Retriever - description: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. If the retriever returns no records, the embedded items are expanded as a fallback. Requires `truncation_indicator_path`. + description: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it. anyOf: - "$ref": "#/definitions/SimpleRetriever" - "$ref": "#/definitions/CustomRetriever" diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index 8804df9d53..e7ecc901cc 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -10,7 +10,10 @@ import dpath +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.message import MessageRepository from airbyte_cdk.sources.types import Config, Record, StreamSlice if TYPE_CHECKING: @@ -18,6 +21,9 @@ logger = logging.getLogger("airbyte") +# dpath treats these characters as glob metacharacters (fnmatch semantics) inside a path segment. +_GLOB_METACHARACTERS = ("*", "?", "[") + class OnNoRecords(Enum): """ @@ -73,11 +79,21 @@ class RecordExpander: When the indicator is truthy and no `truncated_list_retriever` is configured, the embedded items are expanded as normal and a WARNING is logged (once per stream instance) describing the expansion path and the embedded item count, so that the - data loss is visible instead of silent. + data loss is visible instead of silent. Glob metacharacters (`*`, `?`, `[`) are + rejected in this path, and in `expand_records_from_field` when a retriever is + configured; the check runs on the interpolated values. truncated_list_retriever: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. - If the retriever returns no records, the embedded items are expanded as a fallback. + One fetch is issued per truncated parent record; enable `use_cache` on its requester + when the same list can be fetched repeatedly. Without a `paginator` only the first + page is read. If the retriever returns no records, the embedded items are expanded + as a fallback; if it returns fewer records than the `total_count` field next to the + indicator, a WARNING is logged once per stream instance. Request failures surface + through the retriever's error handler and fail the stream like any other request. + `$parameters` of the enclosing stream propagate into this retriever's components. + message_repository: Optional repository used to also emit the truncation warnings as + Airbyte LOG messages so they are visible in the Connector Builder. config: The user-provided configuration as specified by the source's spec. """ @@ -88,6 +104,7 @@ class RecordExpander: on_no_records: OnNoRecords = OnNoRecords.skip truncation_indicator_path: Optional[Sequence[str]] = None truncated_list_retriever: Optional["Retriever"] = None + message_repository: Optional[MessageRepository] = None def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._expand_path: list[InterpolatedString] = [ @@ -98,21 +115,33 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: raise ValueError( "`truncation_indicator_path` is required when `truncated_list_retriever` is configured." ) - if any(path == "*" for path in (self.truncation_indicator_path or [])): - raise ValueError( - "The '*' wildcard is not supported in `truncation_indicator_path`: the indicator must identify a single field." - ) - if self.truncated_list_retriever and any( - path == "*" for path in self.expand_records_from_field - ): - raise ValueError( - "The '*' wildcard is not supported in `expand_records_from_field` when `truncated_list_retriever` is configured." - ) self._truncation_indicator_path: list[InterpolatedString] = [ InterpolatedString.create(path, parameters=parameters) for path in (self.truncation_indicator_path or []) ] + # The paths only interpolate `config`, so their evaluated values are validated up front. + self._reject_globs(self._evaluated_indicator_path(), "truncation_indicator_path") + if self.truncated_list_retriever: + self._reject_globs(self._evaluated_expand_path(), "expand_records_from_field") self._warned_truncation_without_retriever = False + self._warned_incomplete_fetch = False + + @staticmethod + def _reject_globs(path: Sequence[Any], field_name: str) -> None: + if any( + isinstance(segment, str) and any(char in segment for char in _GLOB_METACHARACTERS) + for segment in path + ): + raise ValueError( + f"Glob characters {_GLOB_METACHARACTERS} are not supported in `{field_name}` when " + "truncation handling is configured: the path must identify a single field." + ) + + def _evaluated_indicator_path(self) -> list[Any]: + return [segment.eval(self.config) for segment in self._truncation_indicator_path] + + def _evaluated_expand_path(self) -> list[Any]: + return [segment.eval(self.config) for segment in self._expand_path] def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMapping[Any, Any]]: """Expand a record by extracting items from a nested array field.""" @@ -127,17 +156,18 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap parent_record = record + expand_path = self._evaluated_expand_path() truncated = bool(self._truncation_indicator_path) and self._is_truncated(parent_record) if truncated and self.truncated_list_retriever: - fetched_records = iter(self._fetch_complete_list(parent_record)) - first_fetched = next(fetched_records, None) - if first_fetched is not None: - yield first_fetched - yield from fetched_records + fetched_count = 0 + for fetched in self._fetch_complete_list(parent_record): + fetched_count += 1 + yield fetched + if fetched_count > 0: + self._warn_if_fetch_incomplete(parent_record, expand_path, fetched_count) return - expand_path = [path.eval(self.config) for path in self._expand_path] expanded_any = False embedded_count = 0 @@ -169,36 +199,60 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap embedded_count += 1 if truncated and not self.truncated_list_retriever: - self._warn_truncated_without_retriever(parent_record, embedded_count) + self._warn_truncated_without_retriever(parent_record, expand_path, embedded_count) if not expanded_any and self.on_no_records == OnNoRecords.emit_parent: yield parent_record def _warn_truncated_without_retriever( - self, parent_record: Mapping[str, Any], embedded_count: int + self, parent_record: Mapping[str, Any], expand_path: list[Any], embedded_count: int ) -> None: if self._warned_truncation_without_retriever: return self._warned_truncation_without_retriever = True - indicator_path = [path.eval(self.config) for path in self._truncation_indicator_path] - expand_path = [path.eval(self.config) for path in self._expand_path] + indicator_path = self._evaluated_indicator_path() total_count = self._get_sibling_total_count(parent_record, indicator_path) total_fragment = f" of {total_count} total" if total_count is not None else "" - logger.warning( - "The nested list at %s is marked as truncated (the field at %s is truthy) but no " - "`truncated_list_retriever` is configured, so only the %d embedded item(s)%s were " - "expanded and the remaining items are not emitted. Configure `truncated_list_retriever` " - "to fetch the complete list if the API provides an endpoint for it. This warning is " - "emitted once per stream; other records may be truncated as well.", - expand_path, - indicator_path, - embedded_count, - total_fragment, + self._emit_warning( + f"The nested list at {expand_path} is marked as truncated (the field at {indicator_path} " + f"is truthy) but no `truncated_list_retriever` is configured, so only the " + f"{embedded_count} embedded item(s){total_fragment} were expanded and the remaining " + "items are not emitted. Configure `truncated_list_retriever` to fetch the complete list " + "if the API provides an endpoint for it. This warning is emitted once per stream; other " + "records may be truncated as well." ) + def _warn_if_fetch_incomplete( + self, parent_record: Mapping[str, Any], expand_path: list[Any], fetched_count: int + ) -> None: + if self._warned_incomplete_fetch: + return + indicator_path = self._evaluated_indicator_path() + total_count = self._get_sibling_total_count(parent_record, indicator_path) + if total_count is None or fetched_count >= total_count: + return + self._warned_incomplete_fetch = True + self._emit_warning( + f"The `truncated_list_retriever` for the nested list at {expand_path} returned " + f"{fetched_count} record(s) but the `total_count` field next to {indicator_path} reports " + f"{total_count}, so the fetched list is still incomplete. Check that the retriever has a " + "`paginator` configured and that its request matches the complete-list endpoint. This " + "warning is emitted once per stream; other records may be affected as well." + ) + + def _emit_warning(self, message: str) -> None: + logger.warning(message) + if self.message_repository: + self.message_repository.emit_message( + AirbyteMessage( + type=MessageType.LOG, + log=AirbyteLogMessage(level=Level.WARN, message=message), + ) + ) + def _get_sibling_total_count( - self, parent_record: Mapping[str, Any], indicator_path: list[str] + self, parent_record: Mapping[str, Any], indicator_path: list[Any] ) -> Optional[int]: """Best-effort lookup of a `total_count` field next to the truncation indicator.""" if not indicator_path: @@ -210,15 +264,13 @@ def _get_sibling_total_count( return total if isinstance(total, int) and not isinstance(total, bool) else None def _is_truncated(self, parent_record: MutableMapping[Any, Any]) -> bool: - indicator_path = [path.eval(self.config) for path in self._truncation_indicator_path] + indicator_path = self._evaluated_indicator_path() try: return bool(dpath.get(parent_record, indicator_path)) - except KeyError: + except (KeyError, ValueError): return False - def _fetch_complete_list( - self, parent_record: Mapping[str, Any] - ) -> Iterable[MutableMapping[str, Any]]: + def _fetch_complete_list(self, parent_record: Mapping[str, Any]) -> Iterable[Any]: if not self.truncated_list_retriever: return stream_slice = StreamSlice(partition={"parent_record": parent_record}, cursor_slice={}) @@ -226,11 +278,14 @@ def _fetch_complete_list( records_schema={}, stream_slice=stream_slice ): data = item.data if isinstance(item, Record) else item - if not isinstance(data, Mapping): - continue - expanded_record = dict(data) - self._apply_parent_context(parent_record, expanded_record) - yield expanded_record + if isinstance(data, Mapping): + expanded_record = dict(data) + self._apply_parent_context(parent_record, expanded_record) + yield expanded_record + elif self.remain_original_record: + yield {"value": data, "original_record": copy.deepcopy(parent_record)} + else: + yield data def _apply_parent_context( self, parent_record: Mapping[str, Any], child_record: MutableMapping[str, Any] diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index db0dbdd613..0c65edb4a5 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2307,13 +2307,13 @@ class RecordExpander(BaseModel): ) truncation_indicator_path: Optional[List[str]] = Field( None, - description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Wildcards (*) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured.", + description="Path within each record to a field indicating that the embedded nested list is truncated (e.g. a `has_more` flag on the list object). When the field evaluates to a truthy value and `truncated_list_retriever` is configured, the retriever is used to fetch the complete list instead of expanding the embedded items. When the field is truthy and no retriever is configured, the embedded items are expanded as normal and a WARNING is logged once per stream so the truncation is visible instead of silent. Glob characters (`*`, `?`, `[`) are not supported in this path, nor in `expand_records_from_field` when a retriever is configured; this is enforced on the interpolated values. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", examples=[["data", "object", "lines", "has_more"]], title="Truncation Indicator Path", ) truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( None, - description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. If the retriever returns no records, the embedded items are expanded as a fallback. Requires `truncation_indicator_path`.", + description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", title="Truncated List Retriever", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 3689a5b1f1..1d72600c46 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2508,14 +2508,37 @@ def create_record_expander( ) -> RecordExpander: truncated_list_retriever = None if model.truncated_list_retriever: - truncated_list_retriever = self._create_component_from_model( - model=model.truncated_list_retriever, - config=config, - name="record_expander_truncated_list", - primary_key=None, - stream_slicer=None, - transformations=[], - ) + retriever_model = model.truncated_list_retriever + name = "record_expander_truncated_list" + if isinstance(retriever_model, SimpleRetrieverModel): + if retriever_model.partition_router: + raise ValueError( + "`partition_router` is not supported on `truncated_list_retriever`." + ) + if retriever_model.pagination_reset: + raise ValueError( + "`pagination_reset` is not supported on `truncated_list_retriever`." + ) + truncated_list_retriever = self._create_component_from_model( + model=retriever_model, + config=config, + name=name, + primary_key=None, + transformations=[], + log_formatter=( + lambda response: format_http_message( + response, + f"Record expander '{name}' request", + "Request performed in order to fetch the complete nested list of a truncated record.", + name, + is_auxiliary=True, + ) + ), + ) + else: + truncated_list_retriever = self._create_component_from_model( + model=retriever_model, config=config + ) return RecordExpander( expand_records_from_field=model.expand_records_from_field, config=config, @@ -2526,6 +2549,7 @@ def create_record_expander( else OnNoRecords.skip, truncation_indicator_path=model.truncation_indicator_path, truncated_list_retriever=truncated_list_retriever, + message_repository=self._message_repository, ) @staticmethod diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index 891bd9069b..a41df9223d 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -6,7 +6,9 @@ import pytest +from airbyte_cdk.models import Level, Type from airbyte_cdk.sources.declarative.expanders.record_expander import RecordExpander +from airbyte_cdk.sources.message import InMemoryMessageRepository from airbyte_cdk.sources.types import Record, StreamSlice config = {} @@ -133,16 +135,188 @@ def test_wildcard_rejected_with_truncation_handling(): ) -def test_wildcard_rejected_in_truncation_indicator_path(): +@pytest.mark.parametrize("segment", ["*", "has_*", "has_more?", "[hl]ines", "**"]) +def test_glob_metacharacters_rejected_in_truncation_indicator_path(segment): with pytest.raises(ValueError): RecordExpander( expand_records_from_field=["lines", "data"], config=config, parameters=parameters, - truncation_indicator_path=["*", "has_more"], + truncation_indicator_path=[segment, "has_more"], ) +@pytest.mark.parametrize("segment", ["sect?ons", "[s]ections", "sec*"]) +def test_glob_metacharacters_rejected_in_expand_path_with_retriever(segment): + with pytest.raises(ValueError): + RecordExpander( + expand_records_from_field=[segment, "items"], + config=config, + parameters=parameters, + truncation_indicator_path=["lines", "has_more"], + truncated_list_retriever=_make_retriever([]), + ) + + +def test_glob_metacharacters_allowed_in_expand_path_without_retriever(): + expander = RecordExpander( + expand_records_from_field=["sections", "*", "items"], + config=config, + parameters=parameters, + truncation_indicator_path=["has_more"], + ) + records = list( + expander.expand_record({"sections": [{"items": [{"id": 1}]}], "has_more": False}) + ) + assert records == [{"id": 1}] + + +def test_interpolated_glob_rejected(): + with pytest.raises(ValueError): + RecordExpander( + expand_records_from_field=["lines", "data"], + config={"indicator": "has_*"}, + parameters=parameters, + truncation_indicator_path=["lines", "{{ config['indicator'] }}"], + ) + + +def test_interpolated_indicator_path_is_used(): + expander = RecordExpander( + expand_records_from_field=["lines", "data"], + config={"indicator": "has_more"}, + parameters=parameters, + truncation_indicator_path=["lines", "{{ config['indicator'] }}"], + truncated_list_retriever=_make_retriever([{"id": 2}]), + ) + records = list(expander.expand_record({"lines": {"data": [{"id": 1}], "has_more": True}})) + assert records == [{"id": 2}] + + +def test_indicator_path_through_non_mapping_is_not_truncated(): + expander = RecordExpander( + expand_records_from_field=["lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["lines", "has_more", "nested"], + ) + records = list(expander.expand_record({"lines": {"data": [{"id": 1}], "has_more": True}})) + assert records == [{"id": 1}] + + +def test_fetched_scalar_items_are_yielded_like_embedded_items(): + retriever = _make_retriever(["a", "b"]) + expander = RecordExpander( + expand_records_from_field=["lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["lines", "has_more"], + truncated_list_retriever=retriever, + ) + records = list(expander.expand_record({"lines": {"data": ["a"], "has_more": True}})) + assert records == ["a", "b"] + + +def test_fetched_scalar_items_are_wrapped_when_remain_original_record(): + retriever = _make_retriever(["a", "b"]) + parent = {"lines": {"data": ["a"], "has_more": True}} + expander = RecordExpander( + expand_records_from_field=["lines", "data"], + config=config, + parameters=parameters, + remain_original_record=True, + truncation_indicator_path=["lines", "has_more"], + truncated_list_retriever=retriever, + ) + records = list(expander.expand_record(parent)) + assert records == [ + {"value": "a", "original_record": parent}, + {"value": "b", "original_record": parent}, + ] + + +def test_retriever_errors_propagate(): + retriever = MagicMock() + retriever.read_records.side_effect = RuntimeError("boom") + expander = RecordExpander( + expand_records_from_field=["lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["lines", "has_more"], + truncated_list_retriever=retriever, + ) + with pytest.raises(RuntimeError): + list(expander.expand_record({"lines": {"data": [{"id": 1}], "has_more": True}})) + + +def _retriever_expander(retriever, message_repository=None): + return RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + truncated_list_retriever=retriever, + message_repository=message_repository, + ) + + +def test_warns_once_when_retriever_fetches_fewer_than_total_count(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + first_page_only = [{"id": f"il_{i}"} for i in range(12)] + retriever = MagicMock() + retriever.read_records.side_effect = lambda **_: iter(first_page_only) + expander = _retriever_expander(retriever) + + with caplog.at_level("WARNING", logger="airbyte"): + first = list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + second = list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + assert len(first) == len(second) == 12 + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + message = warnings[0].getMessage() + assert "`truncated_list_retriever`" in message + assert "returned 12 record(s)" in message + assert "reports 15" in message + assert "['data', 'object', 'lines', 'data']" in message + assert "il_" not in message + + +@pytest.mark.parametrize("total_count", [15, None, True, "15"]) +def test_no_incomplete_fetch_warning_when_count_matches_or_total_unknown(caplog, total_count): + embedded = [{"id": f"il_{i}"} for i in range(10)] + complete = [{"id": f"il_{i}"} for i in range(15)] + expander = _retriever_expander(_make_retriever(complete)) + + with caplog.at_level("WARNING", logger="airbyte"): + records = list( + expander.expand_record(_event(embedded, has_more=True, total_count=total_count)) + ) + + assert len(records) == 15 + assert not [r for r in caplog.records if r.levelname == "WARNING"] + + +def test_warnings_are_emitted_through_message_repository(): + repository = InMemoryMessageRepository() + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + message_repository=repository, + ) + embedded = [{"id": f"il_{i}"} for i in range(10)] + + list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + messages = list(repository.consume_queue()) + assert len(messages) == 1 + assert messages[0].type == Type.LOG + assert messages[0].log.level == Level.WARN + assert "10 embedded item(s) of 15 total" in messages[0].log.message + + def _indicator_only_expander(): return RecordExpander( expand_records_from_field=["data", "object", "lines", "data"], diff --git a/unit_tests/sources/declarative/expanders/test_record_expander_http.py b/unit_tests/sources/declarative/expanders/test_record_expander_http.py new file mode 100644 index 0000000000..ee45f32193 --- /dev/null +++ b/unit_tests/sources/declarative/expanders/test_record_expander_http.py @@ -0,0 +1,219 @@ +# +# Copyright (c) 2026 Airbyte, Inc., all rights reserved. +# + +import json +from copy import deepcopy +from typing import Any, Dict, List +from unittest.mock import MagicMock + +import pytest + +from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Type +from airbyte_cdk.sources.declarative.concurrent_declarative_source import ( + ConcurrentDeclarativeSource, +) +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest, HttpResponse + +_CONFIG: Dict[str, Any] = {} +_URL_BASE = "https://api.test.com/v1" + +_MANIFEST: Dict[str, Any] = { + "version": "6.0.0", + "type": "DeclarativeSource", + "check": {"type": "CheckStream", "stream_names": ["invoice_line_items"]}, + "streams": [ + { + "type": "DeclarativeStream", + "name": "invoice_line_items", + "primary_key": ["id"], + "schema_loader": { + "type": "InlineSchemaLoader", + "schema": { + "$schema": "http://json-schema.org/schema#", + "type": "object", + "properties": {"id": {"type": "string"}}, + }, + }, + "retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": _URL_BASE, + "path": "events", + "http_method": "GET", + "request_parameters": {"types[]": "invoice.created"}, + }, + "record_selector": { + "type": "RecordSelector", + "extractor": { + "type": "DpathExtractor", + "field_path": ["data"], + "record_expander": { + "type": "RecordExpander", + "expand_records_from_field": ["data", "object", "lines", "data"], + "truncation_indicator_path": ["data", "object", "lines", "has_more"], + "truncated_list_retriever": { + "type": "SimpleRetriever", + "requester": { + "type": "HttpRequester", + "url_base": _URL_BASE, + "path": "invoices/{{ stream_slice['parent_record']['data']['object']['id'] }}/lines", + "http_method": "GET", + }, + "record_selector": { + "type": "RecordSelector", + "extractor": {"type": "DpathExtractor", "field_path": ["data"]}, + }, + "paginator": { + "type": "DefaultPaginator", + "page_token_option": { + "type": "RequestOption", + "inject_into": "request_parameter", + "field_name": "starting_after", + }, + "pagination_strategy": { + "type": "CursorPagination", + "cursor_value": '{{ response["data"][-1]["id"] }}', + "stop_condition": '{{ not response.get("has_more", False) }}', + }, + }, + }, + }, + }, + }, + }, + } + ], + "spec": { + "type": "Spec", + "connection_specification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {}, + "additionalProperties": True, + }, + }, +} + + +def _invoice_event(invoice_id: str, embedded: List[str], has_more: bool, total_count: int): + return { + "id": f"evt_{invoice_id}", + "type": "invoice.created", + "data": { + "object": { + "id": invoice_id, + "object": "invoice", + "lines": { + "object": "list", + "data": [{"id": line_id, "object": "line_item"} for line_id in embedded], + "has_more": has_more, + "total_count": total_count, + "url": f"/v1/invoices/{invoice_id}/lines", + }, + } + }, + } + + +def _lines_page(line_ids: List[str], has_more: bool): + return HttpResponse( + body=json.dumps( + { + "object": "list", + "data": [{"id": line_id, "object": "line_item"} for line_id in line_ids], + "has_more": has_more, + } + ) + ) + + +def _read_records(manifest: Dict[str, Any]) -> List[Dict[str, Any]]: + source = ConcurrentDeclarativeSource( + source_config=manifest, config=_CONFIG, catalog=None, state=None + ) + catalog = ConfiguredAirbyteCatalog( + streams=[ + ConfiguredAirbyteStream( + stream=stream, sync_mode="full_refresh", destination_sync_mode="overwrite" + ) + for stream in source.discover(logger=source.logger, config=_CONFIG).streams + ] + ) + return [ + message.record.data + for message in source.read(logger=MagicMock(), config=_CONFIG, catalog=catalog, state=None) + if message.type == Type.RECORD + ] + + +def test_truncated_nested_list_is_fetched_over_http_with_pagination(): + with HttpMocker() as http_mocker: + http_mocker.get( + HttpRequest(url=f"{_URL_BASE}/events", query_params={"types[]": "invoice.created"}), + HttpResponse( + body=json.dumps( + { + "object": "list", + "data": [ + _invoice_event("in_1", ["il_1", "il_2"], has_more=True, total_count=5), + _invoice_event("in_2", ["il_6"], has_more=False, total_count=1), + ], + "has_more": False, + } + ) + ), + ) + first_page_request = HttpRequest(url=f"{_URL_BASE}/invoices/in_1/lines") + second_page_request = HttpRequest( + url=f"{_URL_BASE}/invoices/in_1/lines", query_params={"starting_after": "il_3"} + ) + http_mocker.get(first_page_request, _lines_page(["il_1", "il_2", "il_3"], has_more=True)) + http_mocker.get(second_page_request, _lines_page(["il_4", "il_5"], has_more=False)) + + records = _read_records(_MANIFEST) + + assert [record["id"] for record in records] == [ + "il_1", + "il_2", + "il_3", + "il_4", + "il_5", + "il_6", + ] + http_mocker.assert_number_of_calls(first_page_request, 1) + http_mocker.assert_number_of_calls(second_page_request, 1) + + +def test_stream_parameters_propagate_into_truncated_list_requester(): + """Request-shaping options declared in the stream's `$parameters` also reach the nested requester, + which is why the docs recommend declaring them on the outer requester's `request_parameters`.""" + with HttpMocker() as http_mocker: + manifest = deepcopy(_MANIFEST) + stream = manifest["streams"][0] + del stream["retriever"]["requester"]["request_parameters"] + stream["$parameters"] = {"request_parameters": {"types[]": "invoice.created"}} + + http_mocker.get( + HttpRequest(url=f"{_URL_BASE}/events", query_params={"types[]": "invoice.created"}), + HttpResponse( + body=json.dumps( + { + "object": "list", + "data": [_invoice_event("in_1", ["il_1"], has_more=True, total_count=2)], + "has_more": False, + } + ) + ), + ) + http_mocker.get( + HttpRequest( + url=f"{_URL_BASE}/invoices/in_1/lines", query_params={"types[]": "invoice.created"} + ), + _lines_page(["il_1", "il_2"], has_more=False), + ) + + records = _read_records(manifest) + + assert [record["id"] for record in records] == ["il_1", "il_2"] diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 83201b1417..533f1f1dae 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -225,6 +225,7 @@ from airbyte_cdk.utils import AirbyteTracedException from airbyte_cdk.utils.datetime_helpers import AirbyteDateTime, ab_datetime_now, ab_datetime_parse from unit_tests.sources.declarative.parsers.testing_components import ( + TestingCustomRetriever, TestingCustomSubstreamPartitionRouter, TestingSomeComponent, ) @@ -2032,6 +2033,113 @@ def test_create_record_expander_with_truncated_list_retriever(): assert expander is not None assert expander.truncation_indicator_path == ["data", "object", "lines", "has_more"] assert isinstance(expander.truncated_list_retriever, SimpleRetriever) + assert expander.truncated_list_retriever.name == "record_expander_truncated_list" + assert expander.message_repository is factory._message_repository + + +def _record_expander_selector(retriever_yaml: str) -> str: + return f""" + selector: + type: RecordSelector + $parameters: + name: "lists" + extractor: + type: DpathExtractor + field_path: ["data"] + record_expander: + type: RecordExpander + expand_records_from_field: ["lines", "data"] + truncation_indicator_path: ["lines", "has_more"] + truncated_list_retriever: +{retriever_yaml} + """ + + +def _create_record_expander(content: str): + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + selector_manifest = transformer.propagate_types_and_parameters( + "", resolved_manifest["selector"], {} + ) + selector = factory.create_component( + model_type=RecordSelectorModel, + name="test_stream", + component_definition=selector_manifest, + decoder=None, + transformations=[], + config=input_config, + ) + return selector.extractor.record_expander + + +def test_create_record_expander_with_custom_truncated_list_retriever(): + expander = _create_record_expander( + _record_expander_selector( + """ + type: CustomRetriever + class_name: unit_tests.sources.declarative.parsers.testing_components.TestingCustomRetriever + name: "custom_lines" + primary_key: "id" + requester: + type: HttpRequester + url_base: "https://api.test.com/" + path: "invoices/{{ stream_slice['parent_record']['id'] }}/lines" + http_method: "GET" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + """ + ) + ) + retriever = expander.truncated_list_retriever + assert isinstance(retriever, TestingCustomRetriever) + assert retriever.name == "custom_lines" + assert retriever.primary_key == "id" + + +@pytest.mark.parametrize( + "unsupported_option", + [ + pytest.param( + """ + partition_router: + type: ListPartitionRouter + values: ["a", "b"] + cursor_field: "partition" + """, + id="partition_router", + ), + pytest.param( + """ + pagination_reset: + type: PaginationReset + action: RESET + """, + id="pagination_reset", + ), + ], +) +def test_create_record_expander_rejects_unsupported_retriever_options(unsupported_option): + content = _record_expander_selector( + """ + type: SimpleRetriever + requester: + type: HttpRequester + url_base: "https://api.test.com/" + path: "lines" + http_method: "GET" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + """ + + unsupported_option + ) + with pytest.raises(ValueError, match="not supported on `truncated_list_retriever`"): + _create_record_expander(content) @pytest.mark.parametrize( From 55e8692d35e715ba883de17a99f240af930ee884 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 14:32:25 +0000 Subject: [PATCH 06/15] test: drop unused pytest import in record expander http test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../sources/declarative/expanders/test_record_expander_http.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/unit_tests/sources/declarative/expanders/test_record_expander_http.py b/unit_tests/sources/declarative/expanders/test_record_expander_http.py index ee45f32193..99d468a1b2 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander_http.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander_http.py @@ -7,8 +7,6 @@ from typing import Any, Dict, List from unittest.mock import MagicMock -import pytest - from airbyte_cdk.models import ConfiguredAirbyteCatalog, ConfiguredAirbyteStream, Type from airbyte_cdk.sources.declarative.concurrent_declarative_source import ( ConcurrentDeclarativeSource, From ebab4e753473bedfc7f79b5c824377eb34def13c Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 14:47:26 +0000 Subject: [PATCH 07/15] fix(low-code): validate nested retriever options for CustomRetriever, warn on empty fetch, single warning channel - reject partition_router/pagination_reset on truncated_list_retriever for both SimpleRetriever and CustomRetriever models - emit the incomplete-fetch warning when the retriever returns zero records but total_count is positive - guard the once-per-stream warning flags with a lock for concurrent partitions - emit warnings through message_repository when present, otherwise the logger (no duplicates) Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative/expanders/record_expander.py | 36 ++++++---- .../parsers/model_to_component_factory.py | 13 ++-- .../expanders/test_record_expander.py | 65 +++++++++++++++++++ .../test_model_to_component_factory.py | 16 ++++- 4 files changed, 107 insertions(+), 23 deletions(-) diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index e7ecc901cc..b140ea62bb 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -4,6 +4,7 @@ import copy import logging +import threading from dataclasses import InitVar, dataclass from enum import Enum from typing import TYPE_CHECKING, Any, Iterable, Mapping, MutableMapping, Optional, Sequence @@ -92,8 +93,9 @@ class RecordExpander: indicator, a WARNING is logged once per stream instance. Request failures surface through the retriever's error handler and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components. - message_repository: Optional repository used to also emit the truncation warnings as - Airbyte LOG messages so they are visible in the Connector Builder. + message_repository: Optional repository through which the truncation warnings are emitted + as Airbyte LOG messages so they are visible in the Connector Builder. When it is not + set, the warnings go to the `airbyte` logger instead. config: The user-provided configuration as specified by the source's spec. """ @@ -123,6 +125,7 @@ def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._reject_globs(self._evaluated_indicator_path(), "truncation_indicator_path") if self.truncated_list_retriever: self._reject_globs(self._evaluated_expand_path(), "expand_records_from_field") + self._warning_lock = threading.Lock() self._warned_truncation_without_retriever = False self._warned_incomplete_fetch = False @@ -164,8 +167,8 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap for fetched in self._fetch_complete_list(parent_record): fetched_count += 1 yield fetched + self._warn_if_fetch_incomplete(parent_record, expand_path, fetched_count) if fetched_count > 0: - self._warn_if_fetch_incomplete(parent_record, expand_path, fetched_count) return expanded_any = False @@ -207,9 +210,10 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap def _warn_truncated_without_retriever( self, parent_record: Mapping[str, Any], expand_path: list[Any], embedded_count: int ) -> None: - if self._warned_truncation_without_retriever: - return - self._warned_truncation_without_retriever = True + with self._warning_lock: + if self._warned_truncation_without_retriever: + return + self._warned_truncation_without_retriever = True indicator_path = self._evaluated_indicator_path() total_count = self._get_sibling_total_count(parent_record, indicator_path) @@ -226,23 +230,27 @@ def _warn_truncated_without_retriever( def _warn_if_fetch_incomplete( self, parent_record: Mapping[str, Any], expand_path: list[Any], fetched_count: int ) -> None: - if self._warned_incomplete_fetch: - return indicator_path = self._evaluated_indicator_path() total_count = self._get_sibling_total_count(parent_record, indicator_path) if total_count is None or fetched_count >= total_count: return - self._warned_incomplete_fetch = True + with self._warning_lock: + if self._warned_incomplete_fetch: + return + self._warned_incomplete_fetch = True + fallback_fragment = ( + " The embedded items were expanded as a fallback." if fetched_count == 0 else "" + ) self._emit_warning( f"The `truncated_list_retriever` for the nested list at {expand_path} returned " f"{fetched_count} record(s) but the `total_count` field next to {indicator_path} reports " - f"{total_count}, so the fetched list is still incomplete. Check that the retriever has a " - "`paginator` configured and that its request matches the complete-list endpoint. This " - "warning is emitted once per stream; other records may be affected as well." + f"{total_count}, so the fetched list is still incomplete.{fallback_fragment} Check that " + "the retriever has a `paginator` configured and that its request matches the " + "complete-list endpoint. This warning is emitted once per stream; other records may be " + "affected as well." ) def _emit_warning(self, message: str) -> None: - logger.warning(message) if self.message_repository: self.message_repository.emit_message( AirbyteMessage( @@ -250,6 +258,8 @@ def _emit_warning(self, message: str) -> None: log=AirbyteLogMessage(level=Level.WARN, message=message), ) ) + else: + logger.warning(message) def _get_sibling_total_count( self, parent_record: Mapping[str, Any], indicator_path: list[Any] diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 1d72600c46..19a1a4e3d6 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2510,15 +2510,14 @@ def create_record_expander( if model.truncated_list_retriever: retriever_model = model.truncated_list_retriever name = "record_expander_truncated_list" - if isinstance(retriever_model, SimpleRetrieverModel): - if retriever_model.partition_router: - raise ValueError( - "`partition_router` is not supported on `truncated_list_retriever`." - ) - if retriever_model.pagination_reset: + # `CustomRetriever` allows extra fields, so read from the dumped model to cover both types. + retriever_fields = retriever_model.dict() + for unsupported_option in ("partition_router", "pagination_reset"): + if retriever_fields.get(unsupported_option): raise ValueError( - "`pagination_reset` is not supported on `truncated_list_retriever`." + f"`{unsupported_option}` is not supported on `truncated_list_retriever`." ) + if isinstance(retriever_model, SimpleRetrieverModel): truncated_list_retriever = self._create_component_from_model( model=retriever_model, config=config, diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index a41df9223d..6e22f43faf 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -2,6 +2,7 @@ # Copyright (c) 2026 Airbyte, Inc., all rights reserved. # +import threading from unittest.mock import MagicMock import pytest @@ -297,6 +298,70 @@ def test_no_incomplete_fetch_warning_when_count_matches_or_total_unknown(caplog, assert not [r for r in caplog.records if r.levelname == "WARNING"] +def test_warns_when_retriever_returns_nothing_but_total_count_is_positive(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + expander = _retriever_expander(_make_retriever([])) + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + assert [record["id"] for record in records] == [f"il_{i}" for i in range(10)] + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + message = warnings[0].getMessage() + assert "returned 0 record(s)" in message + assert "reports 15" in message + assert "embedded items were expanded as a fallback" in message + + +def test_no_warning_when_retriever_returns_nothing_and_total_count_unknown(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + expander = _retriever_expander(_make_retriever([])) + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=None))) + + assert len(records) == 10 + assert not [r for r in caplog.records if r.levelname == "WARNING"] + + +def test_warning_is_emitted_once_across_concurrent_expansions(): + repository = InMemoryMessageRepository() + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + message_repository=repository, + ) + embedded = [{"id": f"il_{i}"} for i in range(10)] + barrier = threading.Barrier(8) + + def expand(): + barrier.wait() + list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + threads = [threading.Thread(target=expand) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + assert len(list(repository.consume_queue())) == 1 + + +def test_warnings_go_to_message_repository_instead_of_logger_when_configured(caplog): + repository = InMemoryMessageRepository() + expander = _retriever_expander(_make_retriever([]), message_repository=repository) + embedded = [{"id": f"il_{i}"} for i in range(10)] + + with caplog.at_level("WARNING", logger="airbyte"): + list(expander.expand_record(_event(embedded, has_more=True, total_count=15))) + + assert len(list(repository.consume_queue())) == 1 + assert not [r for r in caplog.records if r.levelname == "WARNING"] + + def test_warnings_are_emitted_through_message_repository(): repository = InMemoryMessageRepository() expander = RecordExpander( diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 533f1f1dae..08adb82643 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -2121,10 +2121,20 @@ def test_create_record_expander_with_custom_truncated_list_retriever(): ), ], ) -def test_create_record_expander_rejects_unsupported_retriever_options(unsupported_option): +@pytest.mark.parametrize( + "retriever_type", + [ + "type: SimpleRetriever", + "type: CustomRetriever\n class_name: unit_tests.sources.declarative.parsers.testing_components.TestingCustomRetriever", + ], + ids=["simple_retriever", "custom_retriever"], +) +def test_create_record_expander_rejects_unsupported_retriever_options( + unsupported_option, retriever_type +): content = _record_expander_selector( - """ - type: SimpleRetriever + f""" + {retriever_type} requester: type: HttpRequester url_base: "https://api.test.com/" From 692839c982496c60117b9c49916d4a7830360e59 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 15:02:28 +0000 Subject: [PATCH 08/15] fix(low-code): skip non-record protocol messages yielded by the nested truncated-list retriever Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative/expanders/record_expander.py | 9 +++++++- .../expanders/test_record_expander.py | 21 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index b140ea62bb..f79ee28e46 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -287,7 +287,14 @@ def _fetch_complete_list(self, parent_record: Mapping[str, Any]) -> Iterable[Any for item in self.truncated_list_retriever.read_records( records_schema={}, stream_slice=stream_slice ): - data = item.data if isinstance(item, Record) else item + if isinstance(item, AirbyteMessage): + if item.type != MessageType.RECORD or item.record is None: + continue + data: Any = item.record.data + elif isinstance(item, Record): + data = item.data + else: + data = item if isinstance(data, Mapping): expanded_record = dict(data) self._apply_parent_context(parent_record, expanded_record) diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index 6e22f43faf..8c67203695 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -7,7 +7,7 @@ import pytest -from airbyte_cdk.models import Level, Type +from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, AirbyteRecordMessage, Level, Type from airbyte_cdk.sources.declarative.expanders.record_expander import RecordExpander from airbyte_cdk.sources.message import InMemoryMessageRepository from airbyte_cdk.sources.types import Record, StreamSlice @@ -261,6 +261,25 @@ def _retriever_expander(retriever, message_repository=None): ) +def test_protocol_messages_from_retriever_are_not_treated_as_child_records(): + embedded = [{"id": f"il_{i}"} for i in range(10)] + log_message = AirbyteMessage( + type=Type.LOG, log=AirbyteLogMessage(level=Level.INFO, message="custom retriever log") + ) + record_message = AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage(stream="lines", data={"id": "il_1"}, emitted_at=0), + ) + retriever = _make_retriever( + [{"id": "il_0"}, log_message, record_message, Record(data={"id": "il_2"}, stream_name="t")] + ) + expander = _retriever_expander(retriever) + + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=3))) + + assert records == [{"id": "il_0"}, {"id": "il_1"}, {"id": "il_2"}] + + def test_warns_once_when_retriever_fetches_fewer_than_total_count(caplog): embedded = [{"id": f"il_{i}"} for i in range(10)] first_page_only = [{"id": f"il_{i}"} for i in range(12)] From 1dce8c7f0e311a4b1f663b9f43295cc873819a33 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 15:21:00 +0000 Subject: [PATCH 09/15] fix(low-code): pass the auxiliary log formatter to CustomRetriever nested retrievers too Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../parsers/model_to_component_factory.py | 19 +++++++++---------- .../test_model_to_component_factory.py | 9 +++++++++ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 19a1a4e3d6..9bce1c72bb 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2517,6 +2517,13 @@ def create_record_expander( raise ValueError( f"`{unsupported_option}` is not supported on `truncated_list_retriever`." ) + log_formatter = lambda response: format_http_message( + response, + f"Record expander '{name}' request", + "Request performed in order to fetch the complete nested list of a truncated record.", + name, + is_auxiliary=True, + ) if isinstance(retriever_model, SimpleRetrieverModel): truncated_list_retriever = self._create_component_from_model( model=retriever_model, @@ -2524,19 +2531,11 @@ def create_record_expander( name=name, primary_key=None, transformations=[], - log_formatter=( - lambda response: format_http_message( - response, - f"Record expander '{name}' request", - "Request performed in order to fetch the complete nested list of a truncated record.", - name, - is_auxiliary=True, - ) - ), + log_formatter=log_formatter, ) else: truncated_list_retriever = self._create_component_from_model( - model=retriever_model, config=config + model=retriever_model, config=config, log_formatter=log_formatter ) return RecordExpander( expand_records_from_field=model.expand_records_from_field, diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 08adb82643..a733a4e493 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -2098,6 +2098,15 @@ def test_create_record_expander_with_custom_truncated_list_retriever(): assert retriever.name == "custom_lines" assert retriever.primary_key == "id" + request = requests.PreparedRequest() + request.headers = {} + request.url = "https://api.test.com/invoices/in_1/lines" + response = requests.Response() + response.request = request + response.status_code = 200 + assert retriever.log_formatter is not None + assert retriever.log_formatter(response)["http"]["is_auxiliary"] is True + @pytest.mark.parametrize( "unsupported_option", From dc86078f5bf35ee6aea1119a92b537a1368e19e4 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 15:31:01 +0000 Subject: [PATCH 10/15] fix(low-code): skip bare protocol payloads yielded by the nested retriever Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative/expanders/record_expander.py | 19 ++++++++++++++++++- .../expanders/test_record_expander.py | 9 ++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index f79ee28e46..33bd1976d4 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -11,7 +11,14 @@ import dpath -from airbyte_cdk.models import AirbyteLogMessage, AirbyteMessage, Level +from airbyte_cdk.models import ( + AirbyteControlMessage, + AirbyteLogMessage, + AirbyteMessage, + AirbyteStateMessage, + AirbyteTraceMessage, + Level, +) from airbyte_cdk.models import Type as MessageType from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.message import MessageRepository @@ -25,6 +32,14 @@ # dpath treats these characters as glob metacharacters (fnmatch semantics) inside a path segment. _GLOB_METACHARACTERS = ("*", "?", "[") +# Protocol payloads a retriever may yield unwrapped (i.e. not inside an `AirbyteMessage` envelope). +_BARE_PROTOCOL_MESSAGES = ( + AirbyteControlMessage, + AirbyteLogMessage, + AirbyteStateMessage, + AirbyteTraceMessage, +) + class OnNoRecords(Enum): """ @@ -293,6 +308,8 @@ def _fetch_complete_list(self, parent_record: Mapping[str, Any]) -> Iterable[Any data: Any = item.record.data elif isinstance(item, Record): data = item.data + elif isinstance(item, _BARE_PROTOCOL_MESSAGES): + continue else: data = item if isinstance(data, Mapping): diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index 8c67203695..a502e726a7 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -270,8 +270,15 @@ def test_protocol_messages_from_retriever_are_not_treated_as_child_records(): type=Type.RECORD, record=AirbyteRecordMessage(stream="lines", data={"id": "il_1"}, emitted_at=0), ) + bare_log_message = AirbyteLogMessage(level=Level.DEBUG, message="request/response log") retriever = _make_retriever( - [{"id": "il_0"}, log_message, record_message, Record(data={"id": "il_2"}, stream_name="t")] + [ + {"id": "il_0"}, + log_message, + bare_log_message, + record_message, + Record(data={"id": "il_2"}, stream_name="t"), + ] ) expander = _retriever_expander(retriever) From afac261448d6b53ed127527e7849b2b2da8d9f52 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 15:48:29 +0000 Subject: [PATCH 11/15] fix(low-code): emit truncation warnings before yielding children so early-terminated consumers still see them Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative/expanders/record_expander.py | 50 ++++++++----------- .../expanders/test_record_expander.py | 22 ++++++++ 2 files changed, 44 insertions(+), 28 deletions(-) diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index 33bd1976d4..1234cdec94 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -177,49 +177,43 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap expand_path = self._evaluated_expand_path() truncated = bool(self._truncation_indicator_path) and self._is_truncated(parent_record) + # Warnings are emitted before any child is yielded: a consumer such as the Connector + # Builder record limit may stop iterating mid-list and never resume this generator. if truncated and self.truncated_list_retriever: - fetched_count = 0 - for fetched in self._fetch_complete_list(parent_record): - fetched_count += 1 - yield fetched - self._warn_if_fetch_incomplete(parent_record, expand_path, fetched_count) - if fetched_count > 0: + fetched = list(self._fetch_complete_list(parent_record)) + self._warn_if_fetch_incomplete(parent_record, expand_path, len(fetched)) + if fetched: + yield from fetched return - expanded_any = False - embedded_count = 0 - try: extracted_values = dpath.values(parent_record, expand_path) except KeyError: extracted_values = [] - for extracted in extracted_values: - if not isinstance(extracted, list): - continue - items = extracted + embedded_lists = [ + extracted for extracted in extracted_values if isinstance(extracted, list) + ] + embedded_count = sum(len(items) for items in embedded_lists) + + if truncated and not self.truncated_list_retriever: + self._warn_truncated_without_retriever(parent_record, expand_path, embedded_count) + + for items in embedded_lists: for item in items: if isinstance(item, dict): expanded_record = dict(item) self._apply_parent_context(parent_record, expanded_record) yield expanded_record - expanded_any = True - embedded_count += 1 + elif self.remain_original_record: + yield { + "value": item, + "original_record": copy.deepcopy(parent_record), + } else: - if self.remain_original_record: - yield { - "value": item, - "original_record": copy.deepcopy(parent_record), - } - else: - yield item - expanded_any = True - embedded_count += 1 - - if truncated and not self.truncated_list_retriever: - self._warn_truncated_without_retriever(parent_record, expand_path, embedded_count) + yield item - if not expanded_any and self.on_no_records == OnNoRecords.emit_parent: + if embedded_count == 0 and self.on_no_records == OnNoRecords.emit_parent: yield parent_record def _warn_truncated_without_retriever( diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index a502e726a7..3ecef88e6c 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -309,6 +309,28 @@ def test_warns_once_when_retriever_fetches_fewer_than_total_count(caplog): assert "il_" not in message +def test_warnings_are_emitted_even_when_consumer_stops_after_first_child(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + no_retriever = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + ) + short_fetch = _retriever_expander(_make_retriever([{"id": f"il_{i}"} for i in range(12)])) + + with caplog.at_level("WARNING", logger="airbyte"): + first_embedded = next(iter(no_retriever.expand_record(_event(embedded, True, 15)))) + first_fetched = next(iter(short_fetch.expand_record(_event(embedded, True, 15)))) + + assert first_embedded == first_fetched == {"id": "il_0"} + messages = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert len(messages) == 2 + assert "no `truncated_list_retriever` is configured" in messages[0] + assert "10 embedded item(s) of 15 total" in messages[0] + assert "returned 12 record(s)" in messages[1] + + @pytest.mark.parametrize("total_count", [15, None, True, "15"]) def test_no_incomplete_fetch_warning_when_count_matches_or_total_unknown(caplog, total_count): embedded = [{"id": f"il_{i}"} for i in range(10)] From eba756f7349a8a17be06a93847953c42fc7b002c Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Thu, 10 Sep 2026 16:02:36 +0000 Subject: [PATCH 12/15] fix(low-code): stream fetched children instead of buffering the complete nested list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative/expanders/record_expander.py | 15 +++++---- .../expanders/test_record_expander.py | 31 ++++++++++++++----- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index 1234cdec94..8e5ed91c75 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -177,13 +177,16 @@ def expand_record(self, record: MutableMapping[Any, Any]) -> Iterable[MutableMap expand_path = self._evaluated_expand_path() truncated = bool(self._truncation_indicator_path) and self._is_truncated(parent_record) - # Warnings are emitted before any child is yielded: a consumer such as the Connector - # Builder record limit may stop iterating mid-list and never resume this generator. if truncated and self.truncated_list_retriever: - fetched = list(self._fetch_complete_list(parent_record)) - self._warn_if_fetch_incomplete(parent_record, expand_path, len(fetched)) - if fetched: - yield from fetched + # Streamed, so the shortfall is only known once the retriever is exhausted. If the + # consumer stops early the fetch was cut short by it, not by the API, and no warning + # would be accurate anyway. + fetched_count = 0 + for fetched in self._fetch_complete_list(parent_record): + fetched_count += 1 + yield fetched + self._warn_if_fetch_incomplete(parent_record, expand_path, fetched_count) + if fetched_count > 0: return try: diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index 3ecef88e6c..7f1f42f408 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -309,26 +309,41 @@ def test_warns_once_when_retriever_fetches_fewer_than_total_count(caplog): assert "il_" not in message -def test_warnings_are_emitted_even_when_consumer_stops_after_first_child(caplog): +def test_no_retriever_warning_is_emitted_even_when_consumer_stops_after_first_child(caplog): embedded = [{"id": f"il_{i}"} for i in range(10)] - no_retriever = RecordExpander( + expander = RecordExpander( expand_records_from_field=["data", "object", "lines", "data"], config=config, parameters=parameters, truncation_indicator_path=["data", "object", "lines", "has_more"], ) - short_fetch = _retriever_expander(_make_retriever([{"id": f"il_{i}"} for i in range(12)])) with caplog.at_level("WARNING", logger="airbyte"): - first_embedded = next(iter(no_retriever.expand_record(_event(embedded, True, 15)))) - first_fetched = next(iter(short_fetch.expand_record(_event(embedded, True, 15)))) + first = next(iter(expander.expand_record(_event(embedded, True, 15)))) - assert first_embedded == first_fetched == {"id": "il_0"} + assert first == {"id": "il_0"} messages = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] - assert len(messages) == 2 + assert len(messages) == 1 assert "no `truncated_list_retriever` is configured" in messages[0] assert "10 embedded item(s) of 15 total" in messages[0] - assert "returned 12 record(s)" in messages[1] + + +def test_fetched_children_are_streamed_not_buffered(): + embedded = [{"id": f"il_{i}"} for i in range(10)] + consumed = [] + + def stream(): + for i in range(3): + consumed.append(i) + yield {"id": f"il_{i}"} + + retriever = MagicMock() + retriever.read_records.return_value = stream() + expander = _retriever_expander(retriever) + + records = expander.expand_record(_event(embedded, has_more=True, total_count=3)) + assert next(records) == {"id": "il_0"} + assert consumed == [0] @pytest.mark.parametrize("total_count", [15, None, True, "15"]) From ddb6adebd663730450baabed36051b61bfb6b583 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Mon, 14 Sep 2026 04:09:41 +0000 Subject: [PATCH 13/15] fix(low-code): skip the incomplete-fetch warning under the Connector Builder page cap In test reads, PaginatorTestReadDecorator caps the truncated_list_retriever's pagination like a stream's, so a shortfall against total_count is expected and indistinguishable from a real one. The factory passes suppress_incomplete_fetch_warning=bool(self._limit_pages_fetched_per_slice) so the warning is inert in a real sync. The truncated-without-retriever warning is unchanged. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative_component_schema.yaml | 2 +- .../declarative/expanders/record_expander.py | 10 +++++ .../models/declarative_component_schema.py | 2 +- .../parsers/model_to_component_factory.py | 1 + .../expanders/test_record_expander.py | 45 +++++++++++++++++++ .../test_model_to_component_factory.py | 36 +++++++++++++++ 6 files changed, 94 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 9cb604f2fb..428d6c5096 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2270,7 +2270,7 @@ definitions: - ["data", "object", "lines", "has_more"] truncated_list_retriever: title: Truncated List Retriever - description: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it. + description: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it. anyOf: - "$ref": "#/definitions/SimpleRetriever" - "$ref": "#/definitions/CustomRetriever" diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index 8e5ed91c75..196fbdd6f6 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -108,9 +108,16 @@ class RecordExpander: indicator, a WARNING is logged once per stream instance. Request failures surface through the retriever's error handler and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components. + In Connector Builder test reads the page limit applies to each fetch independently, + so the fetched list may be shorter than `total_count`; no incomplete-fetch warning + is emitted there. message_repository: Optional repository through which the truncation warnings are emitted as Airbyte LOG messages so they are visible in the Connector Builder. When it is not set, the warnings go to the `airbyte` logger instead. + suppress_incomplete_fetch_warning: Skip the incomplete-fetch WARNING. Set by the factory + for Connector Builder test reads, where the page limit caps the retriever's pagination + and a shortfall against `total_count` is expected. The truncated-without-retriever + warning is not affected. config: The user-provided configuration as specified by the source's spec. """ @@ -122,6 +129,7 @@ class RecordExpander: truncation_indicator_path: Optional[Sequence[str]] = None truncated_list_retriever: Optional["Retriever"] = None message_repository: Optional[MessageRepository] = None + suppress_incomplete_fetch_warning: bool = False def __post_init__(self, parameters: Mapping[str, Any]) -> None: self._expand_path: list[InterpolatedString] = [ @@ -242,6 +250,8 @@ def _warn_truncated_without_retriever( def _warn_if_fetch_incomplete( self, parent_record: Mapping[str, Any], expand_path: list[Any], fetched_count: int ) -> None: + if self.suppress_incomplete_fetch_warning: + return indicator_path = self._evaluated_indicator_path() total_count = self._get_sibling_total_count(parent_record, indicator_path) if total_count is None or fetched_count >= total_count: diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 0c65edb4a5..979c52915f 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2313,7 +2313,7 @@ class RecordExpander(BaseModel): ) truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( None, - description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", + description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", title="Truncated List Retriever", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 9bce1c72bb..75ba63e015 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2548,6 +2548,7 @@ def create_record_expander( truncation_indicator_path=model.truncation_indicator_path, truncated_list_retriever=truncated_list_retriever, message_repository=self._message_repository, + suppress_incomplete_fetch_warning=bool(self._limit_pages_fetched_per_slice), ) @staticmethod diff --git a/unit_tests/sources/declarative/expanders/test_record_expander.py b/unit_tests/sources/declarative/expanders/test_record_expander.py index 7f1f42f408..23731dda9e 100644 --- a/unit_tests/sources/declarative/expanders/test_record_expander.py +++ b/unit_tests/sources/declarative/expanders/test_record_expander.py @@ -309,6 +309,51 @@ def test_warns_once_when_retriever_fetches_fewer_than_total_count(caplog): assert "il_" not in message +@pytest.mark.parametrize( + "suppress_incomplete_fetch_warning, expected_warning_count", + [ + pytest.param(True, 0, id="test_read_page_cap_suppresses_warning"), + pytest.param(False, 1, id="no_page_cap_still_warns"), + ], +) +def test_incomplete_fetch_warning_under_test_read_page_cap( + caplog, suppress_incomplete_fetch_warning, expected_warning_count +): + embedded = [{"id": f"il_{i}"} for i in range(10)] + capped_pages = [{"id": f"il_{i}"} for i in range(200)] + expander = _retriever_expander(_make_retriever(capped_pages)) + expander.suppress_incomplete_fetch_warning = suppress_incomplete_fetch_warning + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=250))) + + assert len(records) == 200 + messages = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert len(messages) == expected_warning_count + for message in messages: + assert "returned 200 record(s)" in message + assert "reports 250" in message + + +def test_test_read_page_cap_flag_does_not_suppress_no_retriever_warning(caplog): + embedded = [{"id": f"il_{i}"} for i in range(10)] + expander = RecordExpander( + expand_records_from_field=["data", "object", "lines", "data"], + config=config, + parameters=parameters, + truncation_indicator_path=["data", "object", "lines", "has_more"], + suppress_incomplete_fetch_warning=True, + ) + + with caplog.at_level("WARNING", logger="airbyte"): + records = list(expander.expand_record(_event(embedded, has_more=True, total_count=250))) + + assert len(records) == 10 + messages = [r.getMessage() for r in caplog.records if r.levelname == "WARNING"] + assert len(messages) == 1 + assert "no `truncated_list_retriever` is configured" in messages[0] + + def test_no_retriever_warning_is_emitted_even_when_consumer_stops_after_first_child(caplog): embedded = [{"id": f"il_{i}"} for i in range(10)] expander = RecordExpander( diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index a733a4e493..156c84a114 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -2035,6 +2035,42 @@ def test_create_record_expander_with_truncated_list_retriever(): assert isinstance(expander.truncated_list_retriever, SimpleRetriever) assert expander.truncated_list_retriever.name == "record_expander_truncated_list" assert expander.message_repository is factory._message_repository + assert expander.suppress_incomplete_fetch_warning is False + + +def test_create_record_expander_suppresses_incomplete_fetch_warning_under_test_read_page_cap(): + content = _record_expander_selector( + """ + type: SimpleRetriever + requester: + type: HttpRequester + url_base: "https://api.test.com/" + path: "invoices/{{ stream_slice['parent_record']['id'] }}/lines" + http_method: "GET" + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data"] + """ + ) + parsed_manifest = YamlDeclarativeSource._parse(content) + resolved_manifest = resolver.preprocess_manifest(parsed_manifest) + selector_manifest = transformer.propagate_types_and_parameters( + "", resolved_manifest["selector"], {} + ) + test_read_factory = ModelToComponentFactory(limit_pages_fetched_per_slice=2) + + selector = test_read_factory.create_component( + model_type=RecordSelectorModel, + name="test_stream", + component_definition=selector_manifest, + decoder=None, + transformations=[], + config=input_config, + ) + + assert selector.extractor.record_expander.suppress_incomplete_fetch_warning is True def _record_expander_selector(retriever_yaml: str) -> str: From 95b7b2ac752d6703ce0aa0ad67360514cf2232bd Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Mon, 14 Sep 2026 04:20:01 +0000 Subject: [PATCH 14/15] fix(low-code): suppress incomplete-fetch warning only when the nested paginator is test-read capped A nested SimpleRetriever without a paginator gets NoPagination, which PaginatorTestReadDecorator never wraps, so its shortfall in a Builder test read is real and the warning must stay. Derive the flag from the constructed paginator instead of from the page limit alone; CustomRetriever keeps the page-limit heuristic since its paginator is opaque. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../declarative_component_schema.yaml | 2 +- .../declarative/expanders/record_expander.py | 4 +- .../models/declarative_component_schema.py | 2 +- .../parsers/model_to_component_factory.py | 7 ++- .../test_model_to_component_factory.py | 43 ++++++++++++++++--- 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 428d6c5096..071f5eeaf4 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2270,7 +2270,7 @@ definitions: - ["data", "object", "lines", "has_more"] truncated_list_retriever: title: Truncated List Retriever - description: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it. + description: Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it. anyOf: - "$ref": "#/definitions/SimpleRetriever" - "$ref": "#/definitions/CustomRetriever" diff --git a/airbyte_cdk/sources/declarative/expanders/record_expander.py b/airbyte_cdk/sources/declarative/expanders/record_expander.py index 196fbdd6f6..68c71af693 100644 --- a/airbyte_cdk/sources/declarative/expanders/record_expander.py +++ b/airbyte_cdk/sources/declarative/expanders/record_expander.py @@ -115,8 +115,8 @@ class RecordExpander: as Airbyte LOG messages so they are visible in the Connector Builder. When it is not set, the warnings go to the `airbyte` logger instead. suppress_incomplete_fetch_warning: Skip the incomplete-fetch WARNING. Set by the factory - for Connector Builder test reads, where the page limit caps the retriever's pagination - and a shortfall against `total_count` is expected. The truncated-without-retriever + for Connector Builder test reads when the page limit caps the retriever's paginator, + so a shortfall against `total_count` is expected. The truncated-without-retriever warning is not affected. config: The user-provided configuration as specified by the source's spec. """ diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 979c52915f..41265c9db9 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2313,7 +2313,7 @@ class RecordExpander(BaseModel): ) truncated_list_retriever: Optional[Union[SimpleRetriever, CustomRetriever]] = Field( None, - description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads. Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", + description="Retriever used to fetch the complete list of items when the field at `truncation_indicator_path` is truthy on a record. The record being expanded is exposed to the retriever's interpolation context as `stream_slice['parent_record']`. One fetch is issued per truncated record, so enable `use_cache` on the requester when the same list can be fetched repeatedly. Configure a `paginator`, since without one only the first page of the complete list is read. If the retriever returns no records, the embedded items are expanded as a fallback; if it returns fewer records than the `total_count` field next to the indicator, a WARNING is logged once per stream. Request failures surface through the retriever's `error_handler` and fail the stream like any other request. `$parameters` of the enclosing stream propagate into this retriever's components (including `request_parameters` on its requester); move request-shaping parameters to the outer requester's `request_parameters` when adopting this field. `partition_router` and `pagination_reset` are not supported. In Connector Builder test reads, its requests appear as auxiliary requests and the test-read page limit applies to each fetch independently, so the fetched list may be shorter than `total_count`; the incomplete-fetch warning is not emitted in test reads when a `paginator` is configured (without one the retriever is not capped, so the warning still applies). Requires `truncation_indicator_path`. This field is ignored by CDK versions that predate it, so pin the connector to a CDK version that supports it.", title="Truncated List Retriever", ) parameters: Optional[Dict[str, Any]] = Field(None, alias="$parameters") diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index 75ba63e015..a3cbd0b851 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2507,6 +2507,7 @@ def create_record_expander( **kwargs: Any, ) -> RecordExpander: truncated_list_retriever = None + suppress_incomplete_fetch_warning = bool(self._limit_pages_fetched_per_slice) if model.truncated_list_retriever: retriever_model = model.truncated_list_retriever name = "record_expander_truncated_list" @@ -2533,6 +2534,10 @@ def create_record_expander( transformations=[], log_formatter=log_formatter, ) + # Only a capped paginator makes a shortfall expected; `NoPagination` is never capped. + suppress_incomplete_fetch_warning = isinstance( + truncated_list_retriever.paginator, PaginatorTestReadDecorator + ) else: truncated_list_retriever = self._create_component_from_model( model=retriever_model, config=config, log_formatter=log_formatter @@ -2548,7 +2553,7 @@ def create_record_expander( truncation_indicator_path=model.truncation_indicator_path, truncated_list_retriever=truncated_list_retriever, message_repository=self._message_repository, - suppress_incomplete_fetch_warning=bool(self._limit_pages_fetched_per_slice), + suppress_incomplete_fetch_warning=suppress_incomplete_fetch_warning, ) @staticmethod diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 156c84a114..47db8b15d3 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -156,7 +156,10 @@ WaitUntilTimeFromHeaderBackoffStrategy, ) from airbyte_cdk.sources.declarative.requesters.http_job_repository import AsyncHttpJobRepository -from airbyte_cdk.sources.declarative.requesters.paginators import DefaultPaginator +from airbyte_cdk.sources.declarative.requesters.paginators import ( + DefaultPaginator, + PaginatorTestReadDecorator, +) from airbyte_cdk.sources.declarative.requesters.paginators.strategies import ( CursorPaginationStrategy, OffsetIncrement, @@ -2038,20 +2041,45 @@ def test_create_record_expander_with_truncated_list_retriever(): assert expander.suppress_incomplete_fetch_warning is False -def test_create_record_expander_suppresses_incomplete_fetch_warning_under_test_read_page_cap(): +@pytest.mark.parametrize( + "paginator_yaml, expected_suppressed", + [ + pytest.param( + """ + paginator: + type: DefaultPaginator + page_token_option: + type: RequestOption + inject_into: request_parameter + field_name: starting_after + pagination_strategy: + type: CursorPagination + cursor_value: "{{ last_record['id'] }}" + stop_condition: "{{ not response['has_more'] }}" + """, + True, + id="capped_default_paginator_suppresses_warning", + ), + pytest.param("", False, id="no_pagination_is_not_capped_so_warning_stays"), + ], +) +def test_create_record_expander_suppresses_incomplete_fetch_warning_only_for_capped_paginator( + paginator_yaml, expected_suppressed +): content = _record_expander_selector( - """ + f""" type: SimpleRetriever requester: type: HttpRequester url_base: "https://api.test.com/" - path: "invoices/{{ stream_slice['parent_record']['id'] }}/lines" + path: "invoices/{{{{ stream_slice['parent_record']['id'] }}}}/lines" http_method: "GET" record_selector: type: RecordSelector extractor: type: DpathExtractor field_path: ["data"] + {paginator_yaml} """ ) parsed_manifest = YamlDeclarativeSource._parse(content) @@ -2070,7 +2098,12 @@ def test_create_record_expander_suppresses_incomplete_fetch_warning_under_test_r config=input_config, ) - assert selector.extractor.record_expander.suppress_incomplete_fetch_warning is True + expander = selector.extractor.record_expander + assert expander.suppress_incomplete_fetch_warning is expected_suppressed + assert ( + isinstance(expander.truncated_list_retriever.paginator, PaginatorTestReadDecorator) + is expected_suppressed + ) def _record_expander_selector(retriever_yaml: str) -> str: From 4332e3c258afeb608ea32d6fd0027de08ef33436 Mon Sep 17 00:00:00 2001 From: "zane.hyatt" Date: Tue, 15 Sep 2026 11:05:17 +0000 Subject: [PATCH 15/15] fix(low-code): pass name/primary_key/transformations to CustomRetriever in create_record_expander Collapse the SimpleRetriever/CustomRetriever arms back into a single _create_component_from_model call so a CustomRetriever with nested requester/record_selector no longer fails with a missing $parameters.name. Derive suppress_incomplete_fetch_warning from the constructed retriever (SimpleRetriever whose paginator is a PaginatorTestReadDecorator) instead of the global page limit, so it defaults to False for retrievers without a capped paginator. Tests: drop the fixture's $parameters.name injection that masked the regression; parametrize the suppression test over both retriever types. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../parsers/model_to_component_factory.py | 34 +++++++------- .../test_model_to_component_factory.py | 44 +++++++++++-------- 2 files changed, 42 insertions(+), 36 deletions(-) diff --git a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py index a3cbd0b851..bec394f85e 100644 --- a/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py +++ b/airbyte_cdk/sources/declarative/parsers/model_to_component_factory.py @@ -2507,7 +2507,7 @@ def create_record_expander( **kwargs: Any, ) -> RecordExpander: truncated_list_retriever = None - suppress_incomplete_fetch_warning = bool(self._limit_pages_fetched_per_slice) + suppress_incomplete_fetch_warning = False if model.truncated_list_retriever: retriever_model = model.truncated_list_retriever name = "record_expander_truncated_list" @@ -2525,23 +2525,21 @@ def create_record_expander( name, is_auxiliary=True, ) - if isinstance(retriever_model, SimpleRetrieverModel): - truncated_list_retriever = self._create_component_from_model( - model=retriever_model, - config=config, - name=name, - primary_key=None, - transformations=[], - log_formatter=log_formatter, - ) - # Only a capped paginator makes a shortfall expected; `NoPagination` is never capped. - suppress_incomplete_fetch_warning = isinstance( - truncated_list_retriever.paginator, PaginatorTestReadDecorator - ) - else: - truncated_list_retriever = self._create_component_from_model( - model=retriever_model, config=config, log_formatter=log_formatter - ) + # `name`/`primary_key`/`transformations` are also what a `CustomRetriever` forwards to its + # nested `requester`/`record_selector`, so they are passed for both retriever types. + truncated_list_retriever = self._create_component_from_model( + model=retriever_model, + config=config, + name=name, + primary_key=None, + transformations=[], + log_formatter=log_formatter, + ) + # Only a capped paginator makes a shortfall expected; `NoPagination` and retrievers + # without a paginator are never capped. + suppress_incomplete_fetch_warning = isinstance( + truncated_list_retriever, SimpleRetriever + ) and isinstance(truncated_list_retriever.paginator, PaginatorTestReadDecorator) return RecordExpander( expand_records_from_field=model.expand_records_from_field, config=config, diff --git a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py index 47db8b15d3..39adbde700 100644 --- a/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py +++ b/unit_tests/sources/declarative/parsers/test_model_to_component_factory.py @@ -1994,8 +1994,6 @@ def test_create_record_expander_with_truncated_list_retriever(): content = """ selector: type: RecordSelector - $parameters: - name: "lists" extractor: type: DpathExtractor field_path: ["data"] @@ -2041,13 +2039,12 @@ def test_create_record_expander_with_truncated_list_retriever(): assert expander.suppress_incomplete_fetch_warning is False -@pytest.mark.parametrize( - "paginator_yaml, expected_suppressed", - [ - pytest.param( - """ +# `$parameters.url_base` is how a `CustomRetriever` shares the requester's `url_base` with a nested paginator. +_CURSOR_PAGINATOR_YAML = """ paginator: type: DefaultPaginator + $parameters: + url_base: "https://api.test.com/" page_token_option: type: RequestOption inject_into: request_parameter @@ -2056,19 +2053,32 @@ def test_create_record_expander_with_truncated_list_retriever(): type: CursorPagination cursor_value: "{{ last_record['id'] }}" stop_condition: "{{ not response['has_more'] }}" - """, - True, - id="capped_default_paginator_suppresses_warning", +""" + + +@pytest.mark.parametrize( + "paginator_yaml, expected_suppressed", + [ + pytest.param( + _CURSOR_PAGINATOR_YAML, True, id="capped_default_paginator_suppresses_warning" ), pytest.param("", False, id="no_pagination_is_not_capped_so_warning_stays"), ], ) +@pytest.mark.parametrize( + "retriever_type", + [ + "type: SimpleRetriever", + "type: CustomRetriever\n class_name: unit_tests.sources.declarative.parsers.testing_components.TestingCustomRetriever", + ], + ids=["simple_retriever", "custom_retriever"], +) def test_create_record_expander_suppresses_incomplete_fetch_warning_only_for_capped_paginator( - paginator_yaml, expected_suppressed + retriever_type, paginator_yaml, expected_suppressed ): content = _record_expander_selector( f""" - type: SimpleRetriever + {retriever_type} requester: type: HttpRequester url_base: "https://api.test.com/" @@ -2110,8 +2120,6 @@ def _record_expander_selector(retriever_yaml: str) -> str: return f""" selector: type: RecordSelector - $parameters: - name: "lists" extractor: type: DpathExtractor field_path: ["data"] @@ -2147,8 +2155,6 @@ def test_create_record_expander_with_custom_truncated_list_retriever(): """ type: CustomRetriever class_name: unit_tests.sources.declarative.parsers.testing_components.TestingCustomRetriever - name: "custom_lines" - primary_key: "id" requester: type: HttpRequester url_base: "https://api.test.com/" @@ -2164,8 +2170,10 @@ def test_create_record_expander_with_custom_truncated_list_retriever(): ) retriever = expander.truncated_list_retriever assert isinstance(retriever, TestingCustomRetriever) - assert retriever.name == "custom_lines" - assert retriever.primary_key == "id" + assert retriever.name == "record_expander_truncated_list" + assert isinstance(retriever.requester, HttpRequester) + assert isinstance(retriever.record_selector, RecordSelector) + assert expander.suppress_incomplete_fetch_warning is False request = requests.PreparedRequest() request.headers = {}