diff --git a/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte_cdk/sources/utils/schema_helpers.py index ab7fbbf459..1bbcee626d 100644 --- a/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte_cdk/sources/utils/schema_helpers.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, MutableMapping, Tuple, cast import jsonref -from jsonschema import validate +from jsonschema import Draft7Validator from jsonschema.exceptions import ValidationError from pydantic.v1 import BaseModel, Field from referencing import Registry, Resource @@ -185,23 +185,47 @@ def _resolve_schema_references(self, raw_schema: dict[str, Any]) -> dict[str, An raise ValueError(f"Expected resolved to be a dict. Got {resolved}") +def _strip_nulls_from_optional_fields( + config: Mapping[str, Any], spec_schema: Mapping[str, Any] +) -> dict[str, Any]: + """Remove keys with None values that are not required by the schema. + + The platform may store optional config fields as explicit JSON null rather + than omitting the key. JSON Schema treats *absent* keys as valid for + non-required properties but rejects null unless the type includes "null". + Stripping these avoids spurious validation failures. + """ + required_fields = set(spec_schema.get("required", [])) + return {k: v for k, v in config.items() if v is not None or k in required_fields} + + +def _format_validation_error(error: ValidationError) -> str: + """Format a single validation error with its field path.""" + field_path = ".".join(str(p) for p in error.absolute_path) + if field_path: + return f"'{field_path}' - {error.message}" + return str(error.message) + + def check_config_against_spec_or_exit( config: Mapping[str, Any], spec: ConnectorSpecification ) -> None: - """ - Check config object against spec. In case of spec is invalid, throws - an exception with validation error description. + """Check config object against spec and raise on validation failure. - :param config - config loaded from file specified over command line - :param spec - spec object generated by connector + Strips null-valued optional fields before validation so that connectors + are not broken by platform-level null injection for unset fields. + Reports all validation errors, not just the first one. """ spec_schema = spec.connectionSpecification - try: - validate(instance=config, schema=spec_schema) - except ValidationError as validation_error: + sanitized_config = _strip_nulls_from_optional_fields(config, spec_schema) + validator = Draft7Validator(spec_schema) + errors = list(validator.iter_errors(sanitized_config)) + if errors: + error_details = [_format_validation_error(e) for e in errors] + message = "Config validation error: " + "; ".join(error_details) raise AirbyteTracedException( - message="Config validation error: " + validation_error.message, - internal_message=validation_error.message, + message=message, + internal_message="; ".join(e.message for e in errors), failure_type=FailureType.config_error, ) from None # required to prevent logging config secrets from the ValidationError's stacktrace diff --git a/unit_tests/sources/utils/test_schema_helpers.py b/unit_tests/sources/utils/test_schema_helpers.py index 3ccb13a909..ef0495809f 100644 --- a/unit_tests/sources/utils/test_schema_helpers.py +++ b/unit_tests/sources/utils/test_schema_helpers.py @@ -197,6 +197,111 @@ def test_shared_schemas_resolves_nested(): assert jsonref.JsonRef.replace_refs(actual_schema) +@pytest.mark.parametrize( + "config,spec_schema", + [ + pytest.param( + {"api_token": "valid", "optional_field": None}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "optional_field": {"type": "string"}, + }, + }, + id="null_optional_field_is_stripped", + ), + pytest.param( + {"api_token": "valid"}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "optional_field": {"type": "string"}, + }, + }, + id="absent_optional_field_passes", + ), + pytest.param( + {"api_token": "valid", "start_date": None, "end_date": None, "app_id": None}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "start_date": {"type": "string"}, + "end_date": {"type": "string"}, + "app_id": {"type": "string"}, + }, + }, + id="multiple_null_optional_fields_stripped", + ), + ], +) +def test_check_config_strips_null_optional_fields(config, spec_schema): + spec = ConnectorSpecificationSerializer.load({"connectionSpecification": spec_schema}) + check_config_against_spec_or_exit(config, spec) + + +@pytest.mark.parametrize( + "config,spec_schema,expected_substrings", + [ + pytest.param( + {"api_token": None, "optional_field": "present"}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "optional_field": {"type": "string"}, + }, + }, + ["api_token"], + id="null_required_field_reports_field_name", + ), + pytest.param( + {"credentials": {"token": 123}}, + { + "type": "object", + "required": ["credentials"], + "properties": { + "credentials": { + "type": "object", + "required": ["token"], + "properties": {"token": {"type": "string"}}, + } + }, + }, + ["credentials.token"], + id="nested_error_includes_full_path", + ), + pytest.param( + {"api_token": None, "region": None}, + { + "type": "object", + "required": ["api_token", "region"], + "properties": { + "api_token": {"type": "string"}, + "region": {"type": "string"}, + }, + }, + ["api_token", "region"], + id="multiple_errors_all_reported", + ), + ], +) +def test_check_config_error_message_includes_field_path(config, spec_schema, expected_substrings): + spec = ConnectorSpecificationSerializer.load({"connectionSpecification": spec_schema}) + with pytest_raises(AirbyteTracedException) as exc_info: + check_config_against_spec_or_exit(config, spec) + assert exc_info.value.failure_type == FailureType.config_error + assert "Config validation error:" in exc_info.value.message + for expected in expected_substrings: + assert expected in exc_info.value.message + + @pytest.mark.parametrize( "limit, record_count, expected", [