From 1cb2611aaa20cc9a7fceb08c747fd86cf2852a79 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:51:51 +0000 Subject: [PATCH 1/4] fix: strip null optional config fields before validation and include field path in error messages Config validation now strips null-valued non-required fields before running jsonschema validation. This prevents spurious 'None is not of type string' errors when the platform passes null for unset optional fields. Additionally, validation error messages now include the JSON path to the failing field, making config errors actionable for users. Co-Authored-By: gl_anatolii.yatsuk --- airbyte_cdk/sources/utils/schema_helpers.py | 30 ++++++++--- .../sources/utils/test_schema_helpers.py | 51 +++++++++++++++++++ 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte_cdk/sources/utils/schema_helpers.py index ab7fbbf459..4088ee2daa 100644 --- a/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte_cdk/sources/utils/schema_helpers.py @@ -185,22 +185,38 @@ def _resolve_schema_references(self, raw_schema: dict[str, Any]) -> dict[str, An raise ValueError(f"Expected resolved to be a dict. Got {resolved}") +def _strip_nulls_from_optional_fields(config: Mapping[str, Any], spec_schema: Mapping[str, Any]) -> dict[str, Any]: + """Remove keys with None values that are not required by the schema. + + The platform may store optional config fields as explicit JSON null rather + than omitting the key. JSON Schema treats *absent* keys as valid for + non-required properties but rejects null unless the type includes "null". + Stripping these avoids spurious validation failures. + """ + required_fields = set(spec_schema.get("required", [])) + return {k: v for k, v in config.items() if v is not None or k in required_fields} + + def check_config_against_spec_or_exit( config: Mapping[str, Any], spec: ConnectorSpecification ) -> None: - """ - Check config object against spec. In case of spec is invalid, throws - an exception with validation error description. + """Check config object against spec and raise on validation failure. - :param config - config loaded from file specified over command line - :param spec - spec object generated by connector + Strips null-valued optional fields before validation so that connectors + are not broken by platform-level null injection for unset fields. """ spec_schema = spec.connectionSpecification + sanitized_config = _strip_nulls_from_optional_fields(config, spec_schema) try: - validate(instance=config, schema=spec_schema) + validate(instance=sanitized_config, schema=spec_schema) except ValidationError as validation_error: + field_path = ".".join(str(p) for p in validation_error.absolute_path) + if field_path: + message = f"Config validation error: '{field_path}' - {validation_error.message}" + else: + message = f"Config validation error: {validation_error.message}" raise AirbyteTracedException( - message="Config validation error: " + validation_error.message, + message=message, internal_message=validation_error.message, failure_type=FailureType.config_error, ) from None # required to prevent logging config secrets from the ValidationError's stacktrace diff --git a/unit_tests/sources/utils/test_schema_helpers.py b/unit_tests/sources/utils/test_schema_helpers.py index 3ccb13a909..b00f8b8ef6 100644 --- a/unit_tests/sources/utils/test_schema_helpers.py +++ b/unit_tests/sources/utils/test_schema_helpers.py @@ -197,6 +197,57 @@ def test_shared_schemas_resolves_nested(): assert jsonref.JsonRef.replace_refs(actual_schema) +@pytest.mark.parametrize( + "config,spec_schema", + [ + pytest.param( + {"api_token": "valid", "optional_field": None}, + {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "optional_field": {"type": "string"}}}, + id="null_optional_field_is_stripped", + ), + pytest.param( + {"api_token": "valid"}, + {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "optional_field": {"type": "string"}}}, + id="absent_optional_field_passes", + ), + pytest.param( + {"api_token": "valid", "start_date": None, "end_date": None, "app_id": None}, + {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "start_date": {"type": "string"}, "end_date": {"type": "string"}, "app_id": {"type": "string"}}}, + id="multiple_null_optional_fields_stripped", + ), + ], +) +def test_check_config_strips_null_optional_fields(config, spec_schema): + spec = ConnectorSpecificationSerializer.load({"connectionSpecification": spec_schema}) + check_config_against_spec_or_exit(config, spec) + + +@pytest.mark.parametrize( + "config,spec_schema,expected_substring", + [ + pytest.param( + {"api_token": None, "optional_field": "present"}, + {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "optional_field": {"type": "string"}}}, + "api_token", + id="null_required_field_reports_field_name", + ), + pytest.param( + {"credentials": {"token": 123}}, + {"type": "object", "required": ["credentials"], "properties": {"credentials": {"type": "object", "required": ["token"], "properties": {"token": {"type": "string"}}}}}, + "credentials.token", + id="nested_error_includes_full_path", + ), + ], +) +def test_check_config_error_message_includes_field_path(config, spec_schema, expected_substring): + spec = ConnectorSpecificationSerializer.load({"connectionSpecification": spec_schema}) + with pytest_raises(AirbyteTracedException) as exc_info: + check_config_against_spec_or_exit(config, spec) + assert exc_info.value.failure_type == FailureType.config_error + assert expected_substring in exc_info.value.message + assert "Config validation error:" in exc_info.value.message + + @pytest.mark.parametrize( "limit, record_count, expected", [ From 2d951b501ce8f3303a51bb828b2abc9067957893 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:54:18 +0000 Subject: [PATCH 2/4] style: apply ruff formatting Co-Authored-By: gl_anatolii.yatsuk --- airbyte_cdk/sources/utils/schema_helpers.py | 4 +- .../sources/utils/test_schema_helpers.py | 50 +++++++++++++++++-- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte_cdk/sources/utils/schema_helpers.py index 4088ee2daa..17974b7df1 100644 --- a/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte_cdk/sources/utils/schema_helpers.py @@ -185,7 +185,9 @@ def _resolve_schema_references(self, raw_schema: dict[str, Any]) -> dict[str, An raise ValueError(f"Expected resolved to be a dict. Got {resolved}") -def _strip_nulls_from_optional_fields(config: Mapping[str, Any], spec_schema: Mapping[str, Any]) -> dict[str, Any]: +def _strip_nulls_from_optional_fields( + config: Mapping[str, Any], spec_schema: Mapping[str, Any] +) -> dict[str, Any]: """Remove keys with None values that are not required by the schema. The platform may store optional config fields as explicit JSON null rather diff --git a/unit_tests/sources/utils/test_schema_helpers.py b/unit_tests/sources/utils/test_schema_helpers.py index b00f8b8ef6..f7653c88ac 100644 --- a/unit_tests/sources/utils/test_schema_helpers.py +++ b/unit_tests/sources/utils/test_schema_helpers.py @@ -202,17 +202,40 @@ def test_shared_schemas_resolves_nested(): [ pytest.param( {"api_token": "valid", "optional_field": None}, - {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "optional_field": {"type": "string"}}}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "optional_field": {"type": "string"}, + }, + }, id="null_optional_field_is_stripped", ), pytest.param( {"api_token": "valid"}, - {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "optional_field": {"type": "string"}}}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "optional_field": {"type": "string"}, + }, + }, id="absent_optional_field_passes", ), pytest.param( {"api_token": "valid", "start_date": None, "end_date": None, "app_id": None}, - {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "start_date": {"type": "string"}, "end_date": {"type": "string"}, "app_id": {"type": "string"}}}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "start_date": {"type": "string"}, + "end_date": {"type": "string"}, + "app_id": {"type": "string"}, + }, + }, id="multiple_null_optional_fields_stripped", ), ], @@ -227,13 +250,30 @@ def test_check_config_strips_null_optional_fields(config, spec_schema): [ pytest.param( {"api_token": None, "optional_field": "present"}, - {"type": "object", "required": ["api_token"], "properties": {"api_token": {"type": "string"}, "optional_field": {"type": "string"}}}, + { + "type": "object", + "required": ["api_token"], + "properties": { + "api_token": {"type": "string"}, + "optional_field": {"type": "string"}, + }, + }, "api_token", id="null_required_field_reports_field_name", ), pytest.param( {"credentials": {"token": 123}}, - {"type": "object", "required": ["credentials"], "properties": {"credentials": {"type": "object", "required": ["token"], "properties": {"token": {"type": "string"}}}}}, + { + "type": "object", + "required": ["credentials"], + "properties": { + "credentials": { + "type": "object", + "required": ["token"], + "properties": {"token": {"type": "string"}}, + } + }, + }, "credentials.token", id="nested_error_includes_full_path", ), From 7e24f38b391289fc780037757f36428883c759c2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:56:09 +0000 Subject: [PATCH 3/4] fix: report all validation errors, not just the first one Uses Draft7Validator.iter_errors() to collect all config validation failures and reports them in a single error message separated by semicolons. Co-Authored-By: gl_anatolii.yatsuk --- airbyte_cdk/sources/utils/schema_helpers.py | 26 ++++++++++++------- .../sources/utils/test_schema_helpers.py | 24 +++++++++++++---- 2 files changed, 35 insertions(+), 15 deletions(-) diff --git a/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte_cdk/sources/utils/schema_helpers.py index 17974b7df1..18c590539b 100644 --- a/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte_cdk/sources/utils/schema_helpers.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, MutableMapping, Tuple, cast import jsonref -from jsonschema import validate +from jsonschema import Draft7Validator from jsonschema.exceptions import ValidationError from pydantic.v1 import BaseModel, Field from referencing import Registry, Resource @@ -199,6 +199,14 @@ def _strip_nulls_from_optional_fields( return {k: v for k, v in config.items() if v is not None or k in required_fields} +def _format_validation_error(error: ValidationError) -> str: + """Format a single validation error with its field path.""" + field_path = ".".join(str(p) for p in error.absolute_path) + if field_path: + return f"'{field_path}' - {error.message}" + return error.message + + def check_config_against_spec_or_exit( config: Mapping[str, Any], spec: ConnectorSpecification ) -> None: @@ -206,20 +214,18 @@ def check_config_against_spec_or_exit( Strips null-valued optional fields before validation so that connectors are not broken by platform-level null injection for unset fields. + Reports all validation errors, not just the first one. """ spec_schema = spec.connectionSpecification sanitized_config = _strip_nulls_from_optional_fields(config, spec_schema) - try: - validate(instance=sanitized_config, schema=spec_schema) - except ValidationError as validation_error: - field_path = ".".join(str(p) for p in validation_error.absolute_path) - if field_path: - message = f"Config validation error: '{field_path}' - {validation_error.message}" - else: - message = f"Config validation error: {validation_error.message}" + validator = Draft7Validator(spec_schema) + errors = list(validator.iter_errors(sanitized_config)) + if errors: + error_details = [_format_validation_error(e) for e in errors] + message = "Config validation error: " + "; ".join(error_details) raise AirbyteTracedException( message=message, - internal_message=validation_error.message, + internal_message="; ".join(e.message for e in errors), failure_type=FailureType.config_error, ) from None # required to prevent logging config secrets from the ValidationError's stacktrace diff --git a/unit_tests/sources/utils/test_schema_helpers.py b/unit_tests/sources/utils/test_schema_helpers.py index f7653c88ac..ef0495809f 100644 --- a/unit_tests/sources/utils/test_schema_helpers.py +++ b/unit_tests/sources/utils/test_schema_helpers.py @@ -246,7 +246,7 @@ def test_check_config_strips_null_optional_fields(config, spec_schema): @pytest.mark.parametrize( - "config,spec_schema,expected_substring", + "config,spec_schema,expected_substrings", [ pytest.param( {"api_token": None, "optional_field": "present"}, @@ -258,7 +258,7 @@ def test_check_config_strips_null_optional_fields(config, spec_schema): "optional_field": {"type": "string"}, }, }, - "api_token", + ["api_token"], id="null_required_field_reports_field_name", ), pytest.param( @@ -274,18 +274,32 @@ def test_check_config_strips_null_optional_fields(config, spec_schema): } }, }, - "credentials.token", + ["credentials.token"], id="nested_error_includes_full_path", ), + pytest.param( + {"api_token": None, "region": None}, + { + "type": "object", + "required": ["api_token", "region"], + "properties": { + "api_token": {"type": "string"}, + "region": {"type": "string"}, + }, + }, + ["api_token", "region"], + id="multiple_errors_all_reported", + ), ], ) -def test_check_config_error_message_includes_field_path(config, spec_schema, expected_substring): +def test_check_config_error_message_includes_field_path(config, spec_schema, expected_substrings): spec = ConnectorSpecificationSerializer.load({"connectionSpecification": spec_schema}) with pytest_raises(AirbyteTracedException) as exc_info: check_config_against_spec_or_exit(config, spec) assert exc_info.value.failure_type == FailureType.config_error - assert expected_substring in exc_info.value.message assert "Config validation error:" in exc_info.value.message + for expected in expected_substrings: + assert expected in exc_info.value.message @pytest.mark.parametrize( From d0d91d37f01bdae1ecd7367d7c376fe506bf3ca5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 9 Jun 2026 09:59:21 +0000 Subject: [PATCH 4/4] fix: resolve mypy no-any-return for ValidationError.message Co-Authored-By: gl_anatolii.yatsuk --- airbyte_cdk/sources/utils/schema_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/utils/schema_helpers.py b/airbyte_cdk/sources/utils/schema_helpers.py index 18c590539b..1bbcee626d 100644 --- a/airbyte_cdk/sources/utils/schema_helpers.py +++ b/airbyte_cdk/sources/utils/schema_helpers.py @@ -204,7 +204,7 @@ def _format_validation_error(error: ValidationError) -> str: field_path = ".".join(str(p) for p in error.absolute_path) if field_path: return f"'{field_path}' - {error.message}" - return error.message + return str(error.message) def check_config_against_spec_or_exit(