From f15dbd748c67764cd4e4b6dc8b9e025928ffe71e Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Thu, 5 Mar 2026 13:41:26 -0800 Subject: [PATCH 1/6] feat: add scopes array, optional_scopes, and scopes_join_strategy to declarative OAuth spec Adds structured scopes support to oauth_connector_input_specification: - scopes: array of strings, takes precedence over scope string - optional_scopes: array of optional scopes - scopes_join_strategy: enum (space/comma/plus), defaults to space per RFC 6749 Co-Authored-By: Claude Opus 4.6 --- .../declarative_component_schema.yaml | 33 +++++++++++++++++++ .../models/declarative_component_schema.py | 23 +++++++++++++ 2 files changed, 56 insertions(+) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 84aaa6c537..07a13da69a 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -3064,6 +3064,39 @@ definitions: The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user. examples: - user:read user:read_orders workspaces:read + scopes: + title: Scopes + type: array + items: + type: string + description: |- + The DeclarativeOAuth Specific list of scopes needed to be granted for the authenticated user. + When present, takes precedence over the `scope` string property. + The scopes are joined using the `scopes_join_strategy` (default: space) before being + sent to the OAuth provider. + examples: + - ["user:read", "user:write"] + optional_scopes: + title: Optional Scopes + type: array + items: + type: string + description: |- + The DeclarativeOAuth Specific list of optional scopes to request from the OAuth provider. + These scopes may or may not be granted depending on the provider and user consent. + examples: + - ["admin:read"] + scopes_join_strategy: + title: Scopes Join Strategy + type: string + enum: + - space + - comma + - plus + default: space + description: |- + The strategy used to join the `scopes` array into a single string for the OAuth request. + Defaults to `space` per RFC 6749. access_token_url: title: Access Token URL type: string diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 5d2f0521fa..54b3638b1f 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -20,6 +20,12 @@ class AuthFlowType(Enum): oauth1_0 = "oauth1.0" +class ScopesJoinStrategy(Enum): + space = "space" + comma = "comma" + plus = "plus" + + class BasicHttpAuthenticator(BaseModel): type: Literal["BasicHttpAuthenticator"] username: str = Field( @@ -846,6 +852,23 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) + scopes: Optional[List[str]] = Field( + None, + description="The DeclarativeOAuth Specific list of scopes needed to be granted for the authenticated user.\nWhen present, takes precedence over the `scope` string property.\nThe scopes are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", + examples=[["user:read", "user:write"]], + title="Scopes", + ) + optional_scopes: Optional[List[str]] = Field( + None, + description="The DeclarativeOAuth Specific list of optional scopes to request from the OAuth provider.\nThese scopes may or may not be granted depending on the provider and user consent.", + examples=[["admin:read"]], + title="Optional Scopes", + ) + scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( + ScopesJoinStrategy.space, + description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", + title="Scopes Join Strategy", + ) access_token_url: str = Field( ..., description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", From a577eed1381f5fa54b6be8a0a766194244a5f271 Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Thu, 5 Mar 2026 14:11:04 -0800 Subject: [PATCH 2/6] feat: change scopes from array of strings to array of objects Change scopes and optional_scopes from List[str] to List[OAuthScope] where each scope is an object with a required `scope` property. Uses additionalProperties: true for future extensibility. Co-Authored-By: Claude Opus 4.6 --- .../declarative_component_schema.yaml | 30 +++++++++++++------ .../models/declarative_component_schema.py | 22 ++++++++++---- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 07a13da69a..7a672e451b 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -3068,24 +3068,36 @@ definitions: title: Scopes type: array items: - type: string + type: object + required: + - scope + properties: + scope: + type: string + description: The OAuth scope string to request from the provider. + additionalProperties: true description: |- - The DeclarativeOAuth Specific list of scopes needed to be granted for the authenticated user. - When present, takes precedence over the `scope` string property. - The scopes are joined using the `scopes_join_strategy` (default: space) before being + List of OAuth scope objects. When present, takes precedence over the `scope` string property. + The scope values are joined using the `scopes_join_strategy` (default: space) before being sent to the OAuth provider. examples: - - ["user:read", "user:write"] + - [{"scope": "user:read"}, {"scope": "user:write"}] optional_scopes: title: Optional Scopes type: array items: - type: string + type: object + required: + - scope + properties: + scope: + type: string + description: The OAuth scope string to request from the provider. + additionalProperties: true description: |- - The DeclarativeOAuth Specific list of optional scopes to request from the OAuth provider. - These scopes may or may not be granted depending on the provider and user consent. + Optional OAuth scope objects that may or may not be granted. examples: - - ["admin:read"] + - [{"scope": "admin:read"}] scopes_join_strategy: title: Scopes Join Strategy type: string diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 54b3638b1f..a4789ee621 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -833,6 +833,16 @@ class Config: max: int +class OAuthScope(BaseModel): + class Config: + extra = Extra.allow + + scope: str = Field( + ..., + description="The OAuth scope string to request from the provider.", + ) + + class OauthConnectorInputSpecification(BaseModel): class Config: extra = Extra.allow @@ -852,16 +862,16 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) - scopes: Optional[List[str]] = Field( + scopes: Optional[List[OAuthScope]] = Field( None, - description="The DeclarativeOAuth Specific list of scopes needed to be granted for the authenticated user.\nWhen present, takes precedence over the `scope` string property.\nThe scopes are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", - examples=[["user:read", "user:write"]], + description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", + examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], title="Scopes", ) - optional_scopes: Optional[List[str]] = Field( + optional_scopes: Optional[List[OAuthScope]] = Field( None, - description="The DeclarativeOAuth Specific list of optional scopes to request from the OAuth provider.\nThese scopes may or may not be granted depending on the provider and user consent.", - examples=[["admin:read"]], + description="Optional OAuth scope objects that may or may not be granted.", + examples=[[{"scope": "admin:read"}]], title="Optional Scopes", ) scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( From cf3694407d8522b15da21ad4424654ed6afebe58 Mon Sep 17 00:00:00 2001 From: Aldo Gonzalez Date: Thu, 5 Mar 2026 14:24:23 -0800 Subject: [PATCH 3/6] docs: add inline comments clarifying platform-side processing of scopes fields Co-Authored-By: Claude Opus 4.6 --- .../sources/declarative/declarative_component_schema.yaml | 4 ++++ .../declarative/models/declarative_component_schema.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 7a672e451b..827a683f5a 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -3064,6 +3064,10 @@ definitions: The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user. examples: - user:read user:read_orders workspaces:read + # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the + # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. + # The CDK schema defines the manifest contract; the platform reads these fields + # during the OAuth consent flow to build the authorization URL. scopes: title: Scopes type: array diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index a4789ee621..a8db4d1221 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -862,6 +862,10 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) + # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the + # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. + # The CDK schema defines the manifest contract; the platform reads these fields + # during the OAuth consent flow to build the authorization URL. scopes: Optional[List[OAuthScope]] = Field( None, description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", From 8bb404acb806835b0870d9ce8fdf410aa01325ed Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 00:42:52 +0000 Subject: [PATCH 4/6] feat: override OauthConnectorInputSpecification in protocol model to preserve scopes fields Override OauthConnectorInputSpecification and cascade through OAuthConfigSpecification -> AdvancedAuth -> ConnectorSpecification -> AirbyteMessage to ensure serpyco_rs preserves scopes, optional_scopes, and scopes_join_strategy fields during deserialization. Also convert ScopesJoinStrategy enum to string value in spec.py before passing to ConnectorSpecificationSerializer.load(). Fixes: airbytehq/airbyte-internal-issues#15963 Co-Authored-By: aldo.gonzalez@airbyte.io --- airbyte_cdk/models/airbyte_protocol.py | 61 ++++++++++++++++++- airbyte_cdk/sources/declarative/spec/spec.py | 6 ++ .../sources/declarative/spec/test_spec.py | 1 + 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/models/airbyte_protocol.py b/airbyte_cdk/models/airbyte_protocol.py index 5c56244283..8702b83701 100644 --- a/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte_cdk/models/airbyte_protocol.py @@ -75,11 +75,70 @@ class AirbyteStateMessage: destinationStats: Optional[AirbyteStateStats] = None # type: ignore [name-defined] +# The following dataclasses have been redeclared to include scopes, optional_scopes, +# and scopes_join_strategy fields that are used by declarative OAuth connectors. +# The protocol model (OauthConnectorInputSpecification) does not include these fields, +# so serpyco_rs silently drops them during deserialization. By overriding the model here +# and cascading through OAuthConfigSpecification → AdvancedAuth → ConnectorSpecification, +# the fields are preserved in the connector's spec output. +# This follows the same override pattern used above for AirbyteStateBlob. +@dataclass +class OauthConnectorInputSpecification: + consent_url: str + access_token_url: str + scope: Optional[str] = None + scopes: Optional[List[Dict[str, Any]]] = None + optional_scopes: Optional[List[Dict[str, Any]]] = None + scopes_join_strategy: Optional[str] = None + access_token_headers: Optional[Dict[str, Any]] = None + access_token_params: Optional[Dict[str, Any]] = None + extract_output: Optional[List[str]] = None + state: Optional[State] = None # type: ignore [name-defined] + client_id_key: Optional[str] = None + client_secret_key: Optional[str] = None + scope_key: Optional[str] = None + state_key: Optional[str] = None + auth_code_key: Optional[str] = None + redirect_uri_key: Optional[str] = None + token_expiry_key: Optional[str] = None + + +@dataclass +class OAuthConfigSpecification: + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = None + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = None + complete_oauth_output_specification: Optional[Dict[str, Any]] = None + complete_oauth_server_input_specification: Optional[Dict[str, Any]] = None + complete_oauth_server_output_specification: Optional[Dict[str, Any]] = None + + +@dataclass +class AdvancedAuth: + auth_flow_type: Optional[AuthFlowType] = None # type: ignore [name-defined] + predicate_key: Optional[List[str]] = None + predicate_value: Optional[str] = None + oauth_config_specification: Optional[OAuthConfigSpecification] = None + + +@dataclass +class ConnectorSpecification: + connectionSpecification: Dict[str, Any] + documentationUrl: Optional[str] = None + changelogUrl: Optional[str] = None + supportsIncremental: Optional[bool] = None + supportsNormalization: Optional[bool] = False + supportsDBT: Optional[bool] = False + supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = None # type: ignore [name-defined] + authSpecification: Optional[AuthSpecification] = None # type: ignore [name-defined] + advanced_auth: Optional[AdvancedAuth] = None + protocol_version: Optional[str] = None + + @dataclass class AirbyteMessage: type: Type # type: ignore [name-defined] log: Optional[AirbyteLogMessage] = None # type: ignore [name-defined] - spec: Optional[ConnectorSpecification] = None # type: ignore [name-defined] + spec: Optional[ConnectorSpecification] = None connectionStatus: Optional[AirbyteConnectionStatus] = None # type: ignore [name-defined] catalog: Optional[AirbyteCatalog] = None # type: ignore [name-defined] record: Optional[AirbyteRecordMessage] = None # type: ignore [name-defined] diff --git a/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte_cdk/sources/declarative/spec/spec.py index 20fb3c5c9d..eebca8986a 100644 --- a/airbyte_cdk/sources/declarative/spec/spec.py +++ b/airbyte_cdk/sources/declarative/spec/spec.py @@ -55,6 +55,12 @@ def generate_spec(self) -> ConnectorSpecification: obj["documentationUrl"] = self.documentation_url if self.advanced_auth: self.advanced_auth.auth_flow_type = self.advanced_auth.auth_flow_type.value # type: ignore # We know this is always assigned to an AuthFlow which has the auth_flow_type field + # Convert scopes_join_strategy enum to its string value (same pattern as auth_flow_type above) + oauth_spec = getattr(self.advanced_auth, "oauth_config_specification", None) + if oauth_spec: + oauth_input = getattr(oauth_spec, "oauth_connector_input_specification", None) + if oauth_input and hasattr(oauth_input, "scopes_join_strategy") and oauth_input.scopes_join_strategy is not None: + oauth_input.scopes_join_strategy = oauth_input.scopes_join_strategy.value # type: ignore # Map CDK AuthFlow model to protocol AdvancedAuth model obj["advanced_auth"] = self.advanced_auth.dict() diff --git a/unit_tests/sources/declarative/spec/test_spec.py b/unit_tests/sources/declarative/spec/test_spec.py index 75287082c5..13c0c1c07a 100644 --- a/unit_tests/sources/declarative/spec/test_spec.py +++ b/unit_tests/sources/declarative/spec/test_spec.py @@ -131,6 +131,7 @@ oauth_connector_input_specification=model_declarative_oauth_connector_input_spec( consent_url="https://domain.host.com/endpoint/oauth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}", scope="reports:read campaigns:read", + scopes_join_strategy="space", # default from component schema, preserved through protocol override access_token_headers={"Content-Type": "application/json"}, access_token_params={"{auth_code_key}": "{{auth_code_key}}"}, access_token_url="https://domain.host.com/endpoint/v1/oauth2/access_token/", From 1e1f8f087c27f6097c4f55cb0061591fb6a4c45e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 10 Mar 2026 16:03:34 +0000 Subject: [PATCH 5/6] fix: ruff format for scopes_join_strategy condition Co-Authored-By: aldo.gonzalez@airbyte.io --- airbyte_cdk/sources/declarative/spec/spec.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte_cdk/sources/declarative/spec/spec.py index eebca8986a..9d328023e7 100644 --- a/airbyte_cdk/sources/declarative/spec/spec.py +++ b/airbyte_cdk/sources/declarative/spec/spec.py @@ -59,7 +59,11 @@ def generate_spec(self) -> ConnectorSpecification: oauth_spec = getattr(self.advanced_auth, "oauth_config_specification", None) if oauth_spec: oauth_input = getattr(oauth_spec, "oauth_connector_input_specification", None) - if oauth_input and hasattr(oauth_input, "scopes_join_strategy") and oauth_input.scopes_join_strategy is not None: + if ( + oauth_input + and hasattr(oauth_input, "scopes_join_strategy") + and oauth_input.scopes_join_strategy is not None + ): oauth_input.scopes_join_strategy = oauth_input.scopes_join_strategy.value # type: ignore # Map CDK AuthFlow model to protocol AdvancedAuth model obj["advanced_auth"] = self.advanced_auth.dict() From 6908084375fb34e30dbf15adc9ecbe4824579c2a Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 11 Mar 2026 00:12:49 +0000 Subject: [PATCH 6/6] fix: add protocol override sync-check test and clarifying comment for scopes_join_strategy Co-Authored-By: aldo.gonzalez@airbyte.io --- airbyte_cdk/models/airbyte_protocol.py | 2 + .../sources/declarative/spec/test_spec.py | 44 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/airbyte_cdk/models/airbyte_protocol.py b/airbyte_cdk/models/airbyte_protocol.py index 8702b83701..ec48e1ad53 100644 --- a/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte_cdk/models/airbyte_protocol.py @@ -89,6 +89,8 @@ class OauthConnectorInputSpecification: scope: Optional[str] = None scopes: Optional[List[Dict[str, Any]]] = None optional_scopes: Optional[List[Dict[str, Any]]] = None + # Stored as str (not ScopesJoinStrategy enum) because spec.py converts the enum + # to its .value before serialization. The protocol layer only sees plain strings. scopes_join_strategy: Optional[str] = None access_token_headers: Optional[Dict[str, Any]] = None access_token_params: Optional[Dict[str, Any]] = None diff --git a/unit_tests/sources/declarative/spec/test_spec.py b/unit_tests/sources/declarative/spec/test_spec.py index 13c0c1c07a..6516c146d1 100644 --- a/unit_tests/sources/declarative/spec/test_spec.py +++ b/unit_tests/sources/declarative/spec/test_spec.py @@ -2,6 +2,8 @@ # Copyright (c) 2025 Airbyte, Inc., all rights reserved. # +import dataclasses + import pytest from airbyte_cdk.models import ( @@ -271,3 +273,45 @@ def test_given_invalid_config_value_when_validating_then_exception_is_raised() - with pytest.raises(Exception): spec.validate_config(input_config) + + +@pytest.mark.parametrize( + "upstream_class_name, override_class_name", + [ + ("OauthConnectorInputSpecification", "OauthConnectorInputSpecification"), + ("OAuthConfigSpecification", "OAuthConfigSpecification"), + ("AdvancedAuth", "AdvancedAuth"), + ("ConnectorSpecification", "ConnectorSpecification"), + ], + ids=[ + "OauthConnectorInputSpecification", + "OAuthConfigSpecification", + "AdvancedAuth", + "ConnectorSpecification", + ], +) +def test_protocol_override_fields_in_sync( + upstream_class_name: str, override_class_name: str +) -> None: + """Ensure protocol override dataclasses stay compatible with their upstream counterparts. + + The airbyte_protocol.py file redeclares several dataclasses to add fields that the + upstream airbyte_protocol_dataclasses package does not yet include (e.g. scopes, + optional_scopes, scopes_join_strategy). If the upstream package adds new fields, this + test will fail to remind us to update the local overrides. + """ + import airbyte_protocol_dataclasses.models as upstream_models + + import airbyte_cdk.models.airbyte_protocol as override_models + + upstream_cls = getattr(upstream_models, upstream_class_name) + override_cls = getattr(override_models, override_class_name) + + upstream_fields = {f.name for f in dataclasses.fields(upstream_cls)} + override_fields = {f.name for f in dataclasses.fields(override_cls)} + + missing = upstream_fields - override_fields + assert not missing, ( + f"Upstream protocol added fields {missing} to {upstream_class_name} " + f"that are missing from the airbyte_protocol.py override. Update the override to match." + )