diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 251bce895d..6df783d1db 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2855,6 +2855,9 @@ definitions: - config - response - headers + - request_url + examples: + - "Skipping {{ request_url }} (HTTP 404)." error_message_contains: title: Error Message Substring description: Match the response if its error message contains the substring. @@ -2878,8 +2881,10 @@ definitions: interpolation_context: - response - headers + - request_url examples: - "{{ 'Too much requests' in response }}" + - "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}" - "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}" $parameters: type: object @@ -5752,6 +5757,11 @@ interpolation: x-ratelimit-limit: "600" x-ratelimit-remaining: "598" x-ratelimit-reset: "39" + - title: request_url + description: The URL of the response (the final URL after any redirects), with its query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so do not surface it for APIs that put a credential in the path. Available in HttpResponseFilter's error_message and predicate. + type: string + examples: + - "https://api.github.com/repos/airbytehq/airbyte/stargazers" - title: last_record description: Last record extracted from the response received from the API. type: object diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 2c3d864cd3..0317d52d65 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -653,6 +653,7 @@ class HttpResponseFilter(BaseModel): error_message: Optional[str] = Field( None, description="Error Message to display if the response matches the filter.", + examples=["Skipping {{ request_url }} (HTTP 404)."], title="Error Message", ) error_message_contains: Optional[str] = Field( @@ -673,6 +674,7 @@ class HttpResponseFilter(BaseModel): description="Match the response if the predicate evaluates to true.", examples=[ "{{ 'Too much requests' in response }}", + "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], title="Predicate", diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py index 866d6b7d25..dc4039c8e9 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py @@ -4,6 +4,7 @@ from dataclasses import InitVar, dataclass from typing import Any, Mapping, Optional, Set, Union +from urllib.parse import urlsplit, urlunsplit import requests @@ -36,6 +37,11 @@ class HttpResponseFilter: error_message_contains (str): error substring of matching requests predicate (str): predicate to apply to determine if a request is matching error_message (Union[InterpolatedString, str): error message to display if the response matches the filter + + `error_message` and `predicate` can interpolate `response` (the parsed JSON body), `headers` + and `request_url`, the URL the response came from (the final URL after any redirects) with its + query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so a + credential an API puts in the path is not removed. `error_message` can also interpolate `config`. """ config: Config @@ -138,6 +144,17 @@ def _matches_filter( return self.action # type: ignore # action is always cast to a ResponseAction not a str return None + @staticmethod + def _request_url(response: requests.Response) -> str: + # The query string is dropped because some APIs authenticate with a query parameter, and + # an error message can end up in a user-facing failure. + url = response.url + if not isinstance(url, str) or not url: + return "" + parts = urlsplit(url) + host = parts.netloc.rsplit("@", 1)[-1] + return urlunsplit((parts.scheme, host, parts.path, "", "")) + @staticmethod def _safe_response_json(response: requests.Response) -> dict[str, Any]: try: @@ -152,7 +169,10 @@ def _create_error_message(self, response: requests.Response) -> Optional[str]: :return: The evaluated error message string to be emitted """ return self.error_message.eval( # type: ignore[no-any-return, union-attr] - self.config, response=self._safe_response_json(response), headers=response.headers + self.config, + response=self._safe_response_json(response), + headers=response.headers, + request_url=self._request_url(response), ) def _response_matches_predicate(self, response: requests.Response) -> bool: @@ -163,6 +183,7 @@ def _response_matches_predicate(self, response: requests.Response) -> bool: None, # type: ignore[arg-type] response=self._safe_response_json(response), headers=response.headers, + request_url=self._request_url(response), ) ) if self.predicate diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py index 87e522d4ab..53c98fd596 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py @@ -237,3 +237,104 @@ def test_matches( assert actual_response_status.error_message == expected_error_resolution.error_message else: assert actual_response_status is None + + +@pytest.mark.parametrize( + "url, expected_request_url", + [ + pytest.param( + "https://api.github.com/repos/airbytehq/airbyte/stargazers?per_page=100&page=2", + "https://api.github.com/repos/airbytehq/airbyte/stargazers", + id="query_string_dropped", + ), + pytest.param( + "https://user:secret@api.example.com/v1/items?api_key=secret", + "https://api.example.com/v1/items", + id="credentials_dropped", + ), + pytest.param( + "https://user:pw@api.example.com:8443/v1/items?k=v#frag", + "https://api.example.com:8443/v1/items", + id="port_kept_fragment_dropped", + ), + pytest.param( + "https://[::1]:8443/v1/items?k=v", + "https://[::1]:8443/v1/items", + id="ipv6_host_and_port_kept", + ), + ], +) +def test_error_message_interpolates_request_url(requests_mock, url, expected_request_url): + requests_mock.get(url, status_code=404, json={"message": "Not Found"}) + response_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + http_codes={404}, + error_message="Skipping {{ request_url }}.", + ) + + resolution = response_filter.matches(requests.get(url)) + + assert resolution.response_action == ResponseAction.IGNORE + assert resolution.error_message == f"Skipping {expected_request_url}." + + +def test_predicate_interpolates_request_url(requests_mock): + requests_mock.get( + "https://api.github.com/repos/airbytehq/airbyte/stargazers", + status_code=404, + json={"message": "Not Found"}, + ) + requests_mock.get( + "https://api.github.com/repos/airbytehq/airbyte/events", + status_code=404, + json={"message": "Not Found"}, + ) + requests_mock.get( + "https://api.github.com/repos/airbytehq/airbyte-cdk/stargazers", + status_code=200, + json=[{"login": "octocat"}], + ) + response_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + predicate="{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}", + ) + + stargazers = response_filter.matches( + requests.get("https://api.github.com/repos/airbytehq/airbyte/stargazers") + ) + events = response_filter.matches( + requests.get("https://api.github.com/repos/airbytehq/airbyte/events") + ) + # The filter also runs on successful pages, so the predicate must not match on the URL alone. + successful_page = response_filter.matches( + requests.get("https://api.github.com/repos/airbytehq/airbyte-cdk/stargazers") + ) + + assert stargazers.response_action == ResponseAction.IGNORE + assert events is None + assert successful_page is None + + +def test_request_url_is_empty_when_response_has_no_url(): + response = requests.Response() + response.status_code = 404 + message_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + http_codes={404}, + error_message="Skipping [{{ request_url }}].", + ) + predicate_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + predicate="{{ request_url.endswith('/stargazers') }}", + ) + + assert message_filter.matches(response).error_message == "Skipping []." + assert predicate_filter.matches(response) is None