fix: strip null optional config fields before validation and include field path in error messages

airbyte_cdk/sources/utils/schema_helpers.py

@@ -11,7 +11,7 @@
 from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Mapping, MutableMapping, Tuple, cast
 
 import jsonref
-from jsonschema import validate
+from jsonschema import Draft7Validator
 from jsonschema.exceptions import ValidationError
 from pydantic.v1 import BaseModel, Field
 from referencing import Registry, Resource
@@ -185,23 +185,47 @@ def _resolve_schema_references(self, raw_schema: dict[str, Any]) -> dict[str, An
             raise ValueError(f"Expected resolved to be a dict. Got {resolved}")
 
 
+def _strip_nulls_from_optional_fields(
+    config: Mapping[str, Any], spec_schema: Mapping[str, Any]
+) -> dict[str, Any]:
+    """Remove keys with None values that are not required by the schema.
+
+    The platform may store optional config fields as explicit JSON null rather
+    than omitting the key. JSON Schema treats *absent* keys as valid for
+    non-required properties but rejects null unless the type includes "null".
+    Stripping these avoids spurious validation failures.
+    """
+    required_fields = set(spec_schema.get("required", []))
+    return {k: v for k, v in config.items() if v is not None or k in required_fields}
+
+
+def _format_validation_error(error: ValidationError) -> str:
+    """Format a single validation error with its field path."""
+    field_path = ".".join(str(p) for p in error.absolute_path)
+    if field_path:
+        return f"'{field_path}' - {error.message}"
+    return str(error.message)
+
+
 def check_config_against_spec_or_exit(
     config: Mapping[str, Any], spec: ConnectorSpecification
 ) -> None:
-    """
-    Check config object against spec. In case of spec is invalid, throws
-    an exception with validation error description.
+    """Check config object against spec and raise on validation failure.
 
-    :param config - config loaded from file specified over command line
-    :param spec - spec object generated by connector
+    Strips null-valued optional fields before validation so that connectors
+    are not broken by platform-level null injection for unset fields.
+    Reports all validation errors, not just the first one.
     """
     spec_schema = spec.connectionSpecification
-    try:
-        validate(instance=config, schema=spec_schema)
-    except ValidationError as validation_error:
+    sanitized_config = _strip_nulls_from_optional_fields(config, spec_schema)
+    validator = Draft7Validator(spec_schema)
+    errors = list(validator.iter_errors(sanitized_config))
+    if errors:
+        error_details = [_format_validation_error(e) for e in errors]
+        message = "Config validation error: " + "; ".join(error_details)
         raise AirbyteTracedException(
-            message="Config validation error: " + validation_error.message,
-            internal_message=validation_error.message,
+            message=message,
+            internal_message="; ".join(e.message for e in errors),
             failure_type=FailureType.config_error,
         ) from None  # required to prevent logging config secrets from the ValidationError's stacktrace
 

unit_tests/sources/utils/test_schema_helpers.py

@@ -197,6 +197,111 @@ def test_shared_schemas_resolves_nested():
         assert jsonref.JsonRef.replace_refs(actual_schema)
 
 
+@pytest.mark.parametrize(
+    "config,spec_schema",
+    [
+        pytest.param(
+            {"api_token": "valid", "optional_field": None},
+            {
+                "type": "object",
+                "required": ["api_token"],
+                "properties": {
+                    "api_token": {"type": "string"},
+                    "optional_field": {"type": "string"},
+                },
+            },
+            id="null_optional_field_is_stripped",
+        ),
+        pytest.param(
+            {"api_token": "valid"},
+            {
+                "type": "object",
+                "required": ["api_token"],
+                "properties": {
+                    "api_token": {"type": "string"},
+                    "optional_field": {"type": "string"},
+                },
+            },
+            id="absent_optional_field_passes",
+        ),
+        pytest.param(
+            {"api_token": "valid", "start_date": None, "end_date": None, "app_id": None},
+            {
+                "type": "object",
+                "required": ["api_token"],
+                "properties": {
+                    "api_token": {"type": "string"},
+                    "start_date": {"type": "string"},
+                    "end_date": {"type": "string"},
+                    "app_id": {"type": "string"},
+                },
+            },
+            id="multiple_null_optional_fields_stripped",
+        ),
+    ],
+)
+def test_check_config_strips_null_optional_fields(config, spec_schema):
+    spec = ConnectorSpecificationSerializer.load({"connectionSpecification": spec_schema})
+    check_config_against_spec_or_exit(config, spec)
+
+
+@pytest.mark.parametrize(
+    "config,spec_schema,expected_substrings",
+    [
+        pytest.param(
+            {"api_token": None, "optional_field": "present"},
+            {
+                "type": "object",
+                "required": ["api_token"],
+                "properties": {
+                    "api_token": {"type": "string"},
+                    "optional_field": {"type": "string"},
+                },
+            },
+            ["api_token"],
+            id="null_required_field_reports_field_name",
+        ),
+        pytest.param(
+            {"credentials": {"token": 123}},
+            {
+                "type": "object",
+                "required": ["credentials"],
+                "properties": {
+                    "credentials": {
+                        "type": "object",
+                        "required": ["token"],
+                        "properties": {"token": {"type": "string"}},
+                    }
+                },
+            },
+            ["credentials.token"],
+            id="nested_error_includes_full_path",
+        ),
+        pytest.param(
+            {"api_token": None, "region": None},
+            {
+                "type": "object",
+                "required": ["api_token", "region"],
+                "properties": {
+                    "api_token": {"type": "string"},
+                    "region": {"type": "string"},
+                },
+            },
+            ["api_token", "region"],
+            id="multiple_errors_all_reported",
+        ),
+    ],
+)
+def test_check_config_error_message_includes_field_path(config, spec_schema, expected_substrings):
+    spec = ConnectorSpecificationSerializer.load({"connectionSpecification": spec_schema})
+    with pytest_raises(AirbyteTracedException) as exc_info:
+        check_config_against_spec_or_exit(config, spec)
+    assert exc_info.value.failure_type == FailureType.config_error
+    assert "Config validation error:" in exc_info.value.message
+    for expected in expected_substrings:
+        assert expected in exc_info.value.message
+
+
 @pytest.mark.parametrize(
     "limit, record_count, expected",
     [