diff --git a/airbyte_cdk/models/airbyte_protocol.py b/airbyte_cdk/models/airbyte_protocol.py index 5c56244283..ec48e1ad53 100644 --- a/airbyte_cdk/models/airbyte_protocol.py +++ b/airbyte_cdk/models/airbyte_protocol.py @@ -75,11 +75,72 @@ class AirbyteStateMessage: destinationStats: Optional[AirbyteStateStats] = None # type: ignore [name-defined] +# The following dataclasses have been redeclared to include scopes, optional_scopes, +# and scopes_join_strategy fields that are used by declarative OAuth connectors. +# The protocol model (OauthConnectorInputSpecification) does not include these fields, +# so serpyco_rs silently drops them during deserialization. By overriding the model here +# and cascading through OAuthConfigSpecification → AdvancedAuth → ConnectorSpecification, +# the fields are preserved in the connector's spec output. +# This follows the same override pattern used above for AirbyteStateBlob. +@dataclass +class OauthConnectorInputSpecification: + consent_url: str + access_token_url: str + scope: Optional[str] = None + scopes: Optional[List[Dict[str, Any]]] = None + optional_scopes: Optional[List[Dict[str, Any]]] = None + # Stored as str (not ScopesJoinStrategy enum) because spec.py converts the enum + # to its .value before serialization. The protocol layer only sees plain strings. + scopes_join_strategy: Optional[str] = None + access_token_headers: Optional[Dict[str, Any]] = None + access_token_params: Optional[Dict[str, Any]] = None + extract_output: Optional[List[str]] = None + state: Optional[State] = None # type: ignore [name-defined] + client_id_key: Optional[str] = None + client_secret_key: Optional[str] = None + scope_key: Optional[str] = None + state_key: Optional[str] = None + auth_code_key: Optional[str] = None + redirect_uri_key: Optional[str] = None + token_expiry_key: Optional[str] = None + + +@dataclass +class OAuthConfigSpecification: + oauth_user_input_from_connector_config_specification: Optional[Dict[str, Any]] = None + oauth_connector_input_specification: Optional[OauthConnectorInputSpecification] = None + complete_oauth_output_specification: Optional[Dict[str, Any]] = None + complete_oauth_server_input_specification: Optional[Dict[str, Any]] = None + complete_oauth_server_output_specification: Optional[Dict[str, Any]] = None + + +@dataclass +class AdvancedAuth: + auth_flow_type: Optional[AuthFlowType] = None # type: ignore [name-defined] + predicate_key: Optional[List[str]] = None + predicate_value: Optional[str] = None + oauth_config_specification: Optional[OAuthConfigSpecification] = None + + +@dataclass +class ConnectorSpecification: + connectionSpecification: Dict[str, Any] + documentationUrl: Optional[str] = None + changelogUrl: Optional[str] = None + supportsIncremental: Optional[bool] = None + supportsNormalization: Optional[bool] = False + supportsDBT: Optional[bool] = False + supported_destination_sync_modes: Optional[List[DestinationSyncMode]] = None # type: ignore [name-defined] + authSpecification: Optional[AuthSpecification] = None # type: ignore [name-defined] + advanced_auth: Optional[AdvancedAuth] = None + protocol_version: Optional[str] = None + + @dataclass class AirbyteMessage: type: Type # type: ignore [name-defined] log: Optional[AirbyteLogMessage] = None # type: ignore [name-defined] - spec: Optional[ConnectorSpecification] = None # type: ignore [name-defined] + spec: Optional[ConnectorSpecification] = None connectionStatus: Optional[AirbyteConnectionStatus] = None # type: ignore [name-defined] catalog: Optional[AirbyteCatalog] = None # type: ignore [name-defined] record: Optional[AirbyteRecordMessage] = None # type: ignore [name-defined] diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 84aaa6c537..827a683f5a 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -3064,6 +3064,55 @@ definitions: The DeclarativeOAuth Specific string of the scopes needed to be grant for authenticated user. examples: - user:read user:read_orders workspaces:read + # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the + # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. + # The CDK schema defines the manifest contract; the platform reads these fields + # during the OAuth consent flow to build the authorization URL. + scopes: + title: Scopes + type: array + items: + type: object + required: + - scope + properties: + scope: + type: string + description: The OAuth scope string to request from the provider. + additionalProperties: true + description: |- + List of OAuth scope objects. When present, takes precedence over the `scope` string property. + The scope values are joined using the `scopes_join_strategy` (default: space) before being + sent to the OAuth provider. + examples: + - [{"scope": "user:read"}, {"scope": "user:write"}] + optional_scopes: + title: Optional Scopes + type: array + items: + type: object + required: + - scope + properties: + scope: + type: string + description: The OAuth scope string to request from the provider. + additionalProperties: true + description: |- + Optional OAuth scope objects that may or may not be granted. + examples: + - [{"scope": "admin:read"}] + scopes_join_strategy: + title: Scopes Join Strategy + type: string + enum: + - space + - comma + - plus + default: space + description: |- + The strategy used to join the `scopes` array into a single string for the OAuth request. + Defaults to `space` per RFC 6749. access_token_url: title: Access Token URL type: string diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 5d2f0521fa..a8db4d1221 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -20,6 +20,12 @@ class AuthFlowType(Enum): oauth1_0 = "oauth1.0" +class ScopesJoinStrategy(Enum): + space = "space" + comma = "comma" + plus = "plus" + + class BasicHttpAuthenticator(BaseModel): type: Literal["BasicHttpAuthenticator"] username: str = Field( @@ -827,6 +833,16 @@ class Config: max: int +class OAuthScope(BaseModel): + class Config: + extra = Extra.allow + + scope: str = Field( + ..., + description="The OAuth scope string to request from the provider.", + ) + + class OauthConnectorInputSpecification(BaseModel): class Config: extra = Extra.allow @@ -846,6 +862,27 @@ class Config: examples=["user:read user:read_orders workspaces:read"], title="Scopes", ) + # NOTE: scopes, optional_scopes, and scopes_join_strategy are processed by the + # platform OAuth handler (DeclarativeOAuthSpecHandler.kt), not by the CDK runtime. + # The CDK schema defines the manifest contract; the platform reads these fields + # during the OAuth consent flow to build the authorization URL. + scopes: Optional[List[OAuthScope]] = Field( + None, + description="List of OAuth scope objects. When present, takes precedence over the `scope` string property.\nThe scope values are joined using the `scopes_join_strategy` (default: space) before being\nsent to the OAuth provider.", + examples=[[{"scope": "user:read"}, {"scope": "user:write"}]], + title="Scopes", + ) + optional_scopes: Optional[List[OAuthScope]] = Field( + None, + description="Optional OAuth scope objects that may or may not be granted.", + examples=[[{"scope": "admin:read"}]], + title="Optional Scopes", + ) + scopes_join_strategy: Optional[ScopesJoinStrategy] = Field( + ScopesJoinStrategy.space, + description="The strategy used to join the `scopes` array into a single string for the OAuth request.\nDefaults to `space` per RFC 6749.", + title="Scopes Join Strategy", + ) access_token_url: str = Field( ..., description="The DeclarativeOAuth Specific URL templated string to obtain the `access_token`, `refresh_token` etc.\nThe placeholders are replaced during the processing to provide neccessary values.", diff --git a/airbyte_cdk/sources/declarative/spec/spec.py b/airbyte_cdk/sources/declarative/spec/spec.py index 20fb3c5c9d..9d328023e7 100644 --- a/airbyte_cdk/sources/declarative/spec/spec.py +++ b/airbyte_cdk/sources/declarative/spec/spec.py @@ -55,6 +55,16 @@ def generate_spec(self) -> ConnectorSpecification: obj["documentationUrl"] = self.documentation_url if self.advanced_auth: self.advanced_auth.auth_flow_type = self.advanced_auth.auth_flow_type.value # type: ignore # We know this is always assigned to an AuthFlow which has the auth_flow_type field + # Convert scopes_join_strategy enum to its string value (same pattern as auth_flow_type above) + oauth_spec = getattr(self.advanced_auth, "oauth_config_specification", None) + if oauth_spec: + oauth_input = getattr(oauth_spec, "oauth_connector_input_specification", None) + if ( + oauth_input + and hasattr(oauth_input, "scopes_join_strategy") + and oauth_input.scopes_join_strategy is not None + ): + oauth_input.scopes_join_strategy = oauth_input.scopes_join_strategy.value # type: ignore # Map CDK AuthFlow model to protocol AdvancedAuth model obj["advanced_auth"] = self.advanced_auth.dict() diff --git a/unit_tests/sources/declarative/spec/test_spec.py b/unit_tests/sources/declarative/spec/test_spec.py index 75287082c5..6516c146d1 100644 --- a/unit_tests/sources/declarative/spec/test_spec.py +++ b/unit_tests/sources/declarative/spec/test_spec.py @@ -2,6 +2,8 @@ # Copyright (c) 2025 Airbyte, Inc., all rights reserved. # +import dataclasses + import pytest from airbyte_cdk.models import ( @@ -131,6 +133,7 @@ oauth_connector_input_specification=model_declarative_oauth_connector_input_spec( consent_url="https://domain.host.com/endpoint/oauth?{client_id_key}={{client_id_key}}&{redirect_uri_key}={urlEncoder:{{redirect_uri_key}}}&{state_key}={{state_key}}", scope="reports:read campaigns:read", + scopes_join_strategy="space", # default from component schema, preserved through protocol override access_token_headers={"Content-Type": "application/json"}, access_token_params={"{auth_code_key}": "{{auth_code_key}}"}, access_token_url="https://domain.host.com/endpoint/v1/oauth2/access_token/", @@ -270,3 +273,45 @@ def test_given_invalid_config_value_when_validating_then_exception_is_raised() - with pytest.raises(Exception): spec.validate_config(input_config) + + +@pytest.mark.parametrize( + "upstream_class_name, override_class_name", + [ + ("OauthConnectorInputSpecification", "OauthConnectorInputSpecification"), + ("OAuthConfigSpecification", "OAuthConfigSpecification"), + ("AdvancedAuth", "AdvancedAuth"), + ("ConnectorSpecification", "ConnectorSpecification"), + ], + ids=[ + "OauthConnectorInputSpecification", + "OAuthConfigSpecification", + "AdvancedAuth", + "ConnectorSpecification", + ], +) +def test_protocol_override_fields_in_sync( + upstream_class_name: str, override_class_name: str +) -> None: + """Ensure protocol override dataclasses stay compatible with their upstream counterparts. + + The airbyte_protocol.py file redeclares several dataclasses to add fields that the + upstream airbyte_protocol_dataclasses package does not yet include (e.g. scopes, + optional_scopes, scopes_join_strategy). If the upstream package adds new fields, this + test will fail to remind us to update the local overrides. + """ + import airbyte_protocol_dataclasses.models as upstream_models + + import airbyte_cdk.models.airbyte_protocol as override_models + + upstream_cls = getattr(upstream_models, upstream_class_name) + override_cls = getattr(override_models, override_class_name) + + upstream_fields = {f.name for f in dataclasses.fields(upstream_cls)} + override_fields = {f.name for f in dataclasses.fields(override_cls)} + + missing = upstream_fields - override_fields + assert not missing, ( + f"Upstream protocol added fields {missing} to {upstream_class_name} " + f"that are missing from the airbyte_protocol.py override. Update the override to match." + )