From 6fd2072c6006ac268c120f1c33f58427eb84a03c Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 14 Oct 2026 15:59:36 +0300 Subject: [PATCH 1/4] feat(declarative): expose `request_url` to HttpResponseFilter error_message and predicate A filter's error_message and predicate could only interpolate the config, the parsed response body and the response headers. When the body does not identify the resource (a bare 404, for example), the message cannot say which partition was skipped, and setting error_message on an IGNORE filter also replaces the CDK's own log line that carried the URL. request_url is the URL of the response with the query string and any userinfo removed, since some APIs authenticate through a query parameter and an error message can reach a user-facing failure. Co-Authored-By: Claude Opus 5.5 --- .../declarative_component_schema.yaml | 10 ++++ .../models/declarative_component_schema.py | 2 + .../error_handlers/http_response_filter.py | 22 +++++++- .../test_http_response_filter.py | 52 +++++++++++++++++++ 4 files changed, 85 insertions(+), 1 deletion(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 251bce895d..08572ccc0d 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2855,6 +2855,9 @@ definitions: - config - response - headers + - request_url + examples: + - "Skipping {{ request_url }} (HTTP 404)." error_message_contains: title: Error Message Substring description: Match the response if its error message contains the substring. @@ -2878,8 +2881,10 @@ definitions: interpolation_context: - response - headers + - request_url examples: - "{{ 'Too much requests' in response }}" + - "{{ request_url.endswith('/stargazers') }}" - "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}" $parameters: type: object @@ -5752,6 +5757,11 @@ interpolation: x-ratelimit-limit: "600" x-ratelimit-remaining: "598" x-ratelimit-reset: "39" + - title: request_url + description: The URL of the request that produced the response, without its query string or any credentials in it. Available in HttpResponseFilter's error_message and predicate. + type: string + examples: + - "https://api.github.com/repos/airbytehq/airbyte/stargazers" - title: last_record description: Last record extracted from the response received from the API. type: object diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 2c3d864cd3..afc1ad541c 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -653,6 +653,7 @@ class HttpResponseFilter(BaseModel): error_message: Optional[str] = Field( None, description="Error Message to display if the response matches the filter.", + examples=["Skipping {{ request_url }} (HTTP 404)."], title="Error Message", ) error_message_contains: Optional[str] = Field( @@ -674,6 +675,7 @@ class HttpResponseFilter(BaseModel): examples=[ "{{ 'Too much requests' in response }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", + "{{ request_url.endswith('/stargazers') }}", ], title="Predicate", ) diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py index 866d6b7d25..f42f9e6df5 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py @@ -4,6 +4,7 @@ from dataclasses import InitVar, dataclass from typing import Any, Mapping, Optional, Set, Union +from urllib.parse import urlsplit, urlunsplit import requests @@ -36,6 +37,10 @@ class HttpResponseFilter: error_message_contains (str): error substring of matching requests predicate (str): predicate to apply to determine if a request is matching error_message (Union[InterpolatedString, str): error message to display if the response matches the filter + + `error_message` and `predicate` can interpolate `response` (the parsed JSON body), `headers` + and `request_url`, the URL the response came from without its query string or credentials. + `error_message` can also interpolate `config`. """ config: Config @@ -138,6 +143,17 @@ def _matches_filter( return self.action # type: ignore # action is always cast to a ResponseAction not a str return None + @staticmethod + def _request_url(response: requests.Response) -> str: + # The query string is dropped because some APIs authenticate with a query parameter, and + # an error message can end up in a user-facing failure. + url = response.url + if not isinstance(url, str) or not url: + return "" + parts = urlsplit(url) + host = parts.netloc.rsplit("@", 1)[-1] + return urlunsplit((parts.scheme, host, parts.path, "", "")) + @staticmethod def _safe_response_json(response: requests.Response) -> dict[str, Any]: try: @@ -152,7 +168,10 @@ def _create_error_message(self, response: requests.Response) -> Optional[str]: :return: The evaluated error message string to be emitted """ return self.error_message.eval( # type: ignore[no-any-return, union-attr] - self.config, response=self._safe_response_json(response), headers=response.headers + self.config, + response=self._safe_response_json(response), + headers=response.headers, + request_url=self._request_url(response), ) def _response_matches_predicate(self, response: requests.Response) -> bool: @@ -163,6 +182,7 @@ def _response_matches_predicate(self, response: requests.Response) -> bool: None, # type: ignore[arg-type] response=self._safe_response_json(response), headers=response.headers, + request_url=self._request_url(response), ) ) if self.predicate diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py index 87e522d4ab..ac6f1bf096 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py @@ -237,3 +237,55 @@ def test_matches( assert actual_response_status.error_message == expected_error_resolution.error_message else: assert actual_response_status is None + + +@pytest.mark.parametrize( + "url, expected_request_url", + [ + pytest.param( + "https://api.github.com/repos/airbytehq/airbyte/stargazers?per_page=100&page=2", + "https://api.github.com/repos/airbytehq/airbyte/stargazers", + id="query_string_dropped", + ), + pytest.param( + "https://user:secret@api.example.com/v1/items?api_key=secret", + "https://api.example.com/v1/items", + id="credentials_dropped", + ), + ], +) +def test_error_message_interpolates_request_url(requests_mock, url, expected_request_url): + requests_mock.get(url, status_code=404, json={"message": "Not Found"}) + response_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + http_codes={404}, + error_message="Skipping {{ request_url }}.", + ) + + resolution = response_filter.matches(requests.get(url)) + + assert resolution.response_action == ResponseAction.IGNORE + assert resolution.error_message == f"Skipping {expected_request_url}." + + +def test_predicate_interpolates_request_url(requests_mock): + requests_mock.get("https://api.github.com/repos/airbytehq/airbyte/stargazers", status_code=404) + requests_mock.get("https://api.github.com/repos/airbytehq/airbyte/events", status_code=404) + response_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + predicate="{{ request_url.endswith('/stargazers') }}", + ) + + stargazers = response_filter.matches( + requests.get("https://api.github.com/repos/airbytehq/airbyte/stargazers") + ) + events = response_filter.matches( + requests.get("https://api.github.com/repos/airbytehq/airbyte/events") + ) + + assert stargazers.response_action == ResponseAction.IGNORE + assert events is None From e2f111c89eb1ee380d9bdf4e24869ac06d2a4967 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 14 Oct 2026 16:31:47 +0300 Subject: [PATCH 2/4] fix(declarative): describe exactly what request_url strips and pin its edge cases - Say that the query string, fragment and userinfo are removed and the path is kept, in the schema and the HttpResponseFilter docstring. - Match the YAML order of the predicate examples in the generated model. - Test a port, an IPv6 host and a response without a URL. Co-Authored-By: Claude Opus 5.5 --- .../declarative_component_schema.yaml | 2 +- .../models/declarative_component_schema.py | 2 +- .../error_handlers/http_response_filter.py | 5 +-- .../test_http_response_filter.py | 31 +++++++++++++++++++ 4 files changed, 36 insertions(+), 4 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 08572ccc0d..18bb107e3e 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -5758,7 +5758,7 @@ interpolation: x-ratelimit-remaining: "598" x-ratelimit-reset: "39" - title: request_url - description: The URL of the request that produced the response, without its query string or any credentials in it. Available in HttpResponseFilter's error_message and predicate. + description: The URL of the request that produced the response, with its query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so do not surface it for APIs that put a credential in the path. Available in HttpResponseFilter's error_message and predicate. type: string examples: - "https://api.github.com/repos/airbytehq/airbyte/stargazers" diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index afc1ad541c..0aa3a3d617 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -674,8 +674,8 @@ class HttpResponseFilter(BaseModel): description="Match the response if the predicate evaluates to true.", examples=[ "{{ 'Too much requests' in response }}", - "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", "{{ request_url.endswith('/stargazers') }}", + "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], title="Predicate", ) diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py index f42f9e6df5..1cfe2947e4 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py @@ -39,8 +39,9 @@ class HttpResponseFilter: error_message (Union[InterpolatedString, str): error message to display if the response matches the filter `error_message` and `predicate` can interpolate `response` (the parsed JSON body), `headers` - and `request_url`, the URL the response came from without its query string or credentials. - `error_message` can also interpolate `config`. + and `request_url`, the URL the response came from with its query string, fragment and userinfo + (user:password@) removed. The path is kept as-is, so a credential an API puts in the path is not + removed. `error_message` can also interpolate `config`. """ config: Config diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py index ac6f1bf096..0d084c541d 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py @@ -252,6 +252,16 @@ def test_matches( "https://api.example.com/v1/items", id="credentials_dropped", ), + pytest.param( + "https://user:pw@api.example.com:8443/v1/items?k=v#frag", + "https://api.example.com:8443/v1/items", + id="port_kept_fragment_dropped", + ), + pytest.param( + "https://[::1]:8443/v1/items?k=v", + "https://[::1]:8443/v1/items", + id="ipv6_host_and_port_kept", + ), ], ) def test_error_message_interpolates_request_url(requests_mock, url, expected_request_url): @@ -289,3 +299,24 @@ def test_predicate_interpolates_request_url(requests_mock): assert stargazers.response_action == ResponseAction.IGNORE assert events is None + + +def test_request_url_is_empty_when_response_has_no_url(): + response = requests.Response() + response.status_code = 404 + message_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + http_codes={404}, + error_message="Skipping [{{ request_url }}].", + ) + predicate_filter = HttpResponseFilter( + action=ResponseAction.IGNORE, + config={}, + parameters={}, + predicate="{{ request_url.endswith('/stargazers') }}", + ) + + assert message_filter.matches(response).error_message == "Skipping []." + assert predicate_filter.matches(response) is None From 551a94530f5da540bcf81707b1013d23467491a6 Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 14 Oct 2026 16:35:24 +0300 Subject: [PATCH 3/4] docs(declarative): scope the request_url predicate example to error responses HttpResponseFilter runs on every response, including successful pages, so a predicate that checks only the URL matches every 200 of that stream. Pair the example with an error-body condition and assert in the test that a successful page does not match. Co-Authored-By: Claude Opus 5.5 --- .../declarative_component_schema.yaml | 2 +- .../models/declarative_component_schema.py | 2 +- .../test_http_response_filter.py | 24 ++++++++++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 18bb107e3e..0fe6af880c 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2884,7 +2884,7 @@ definitions: - request_url examples: - "{{ 'Too much requests' in response }}" - - "{{ request_url.endswith('/stargazers') }}" + - "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}" - "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}" $parameters: type: object diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 0aa3a3d617..0317d52d65 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -674,7 +674,7 @@ class HttpResponseFilter(BaseModel): description="Match the response if the predicate evaluates to true.", examples=[ "{{ 'Too much requests' in response }}", - "{{ request_url.endswith('/stargazers') }}", + "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}", "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}", ], title="Predicate", diff --git a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py index 0d084c541d..53c98fd596 100644 --- a/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py +++ b/unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py @@ -281,13 +281,26 @@ def test_error_message_interpolates_request_url(requests_mock, url, expected_req def test_predicate_interpolates_request_url(requests_mock): - requests_mock.get("https://api.github.com/repos/airbytehq/airbyte/stargazers", status_code=404) - requests_mock.get("https://api.github.com/repos/airbytehq/airbyte/events", status_code=404) + requests_mock.get( + "https://api.github.com/repos/airbytehq/airbyte/stargazers", + status_code=404, + json={"message": "Not Found"}, + ) + requests_mock.get( + "https://api.github.com/repos/airbytehq/airbyte/events", + status_code=404, + json={"message": "Not Found"}, + ) + requests_mock.get( + "https://api.github.com/repos/airbytehq/airbyte-cdk/stargazers", + status_code=200, + json=[{"login": "octocat"}], + ) response_filter = HttpResponseFilter( action=ResponseAction.IGNORE, config={}, parameters={}, - predicate="{{ request_url.endswith('/stargazers') }}", + predicate="{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}", ) stargazers = response_filter.matches( @@ -296,9 +309,14 @@ def test_predicate_interpolates_request_url(requests_mock): events = response_filter.matches( requests.get("https://api.github.com/repos/airbytehq/airbyte/events") ) + # The filter also runs on successful pages, so the predicate must not match on the URL alone. + successful_page = response_filter.matches( + requests.get("https://api.github.com/repos/airbytehq/airbyte-cdk/stargazers") + ) assert stargazers.response_action == ResponseAction.IGNORE assert events is None + assert successful_page is None def test_request_url_is_empty_when_response_has_no_url(): From acb3e61d505c847cc313081975d8d295c5cb67ef Mon Sep 17 00:00:00 2001 From: Anatolii Yatsuk Date: Wed, 14 Oct 2026 16:38:48 +0300 Subject: [PATCH 4/4] docs(declarative): note that request_url is the final URL after redirects requests follows redirects, so `request_url` reflects the last hop rather than the URL the requester built (for example, GitHub redirects renamed repositories to /repositories//...). Co-Authored-By: Claude Opus 5.5 --- .../sources/declarative/declarative_component_schema.yaml | 2 +- .../requesters/error_handlers/http_response_filter.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 0fe6af880c..6df783d1db 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -5758,7 +5758,7 @@ interpolation: x-ratelimit-remaining: "598" x-ratelimit-reset: "39" - title: request_url - description: The URL of the request that produced the response, with its query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so do not surface it for APIs that put a credential in the path. Available in HttpResponseFilter's error_message and predicate. + description: The URL of the response (the final URL after any redirects), with its query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so do not surface it for APIs that put a credential in the path. Available in HttpResponseFilter's error_message and predicate. type: string examples: - "https://api.github.com/repos/airbytehq/airbyte/stargazers" diff --git a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py index 1cfe2947e4..dc4039c8e9 100644 --- a/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py +++ b/airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py @@ -39,9 +39,9 @@ class HttpResponseFilter: error_message (Union[InterpolatedString, str): error message to display if the response matches the filter `error_message` and `predicate` can interpolate `response` (the parsed JSON body), `headers` - and `request_url`, the URL the response came from with its query string, fragment and userinfo - (user:password@) removed. The path is kept as-is, so a credential an API puts in the path is not - removed. `error_message` can also interpolate `config`. + and `request_url`, the URL the response came from (the final URL after any redirects) with its + query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so a + credential an API puts in the path is not removed. `error_message` can also interpolate `config`. """ config: Config