feat(declarative): expose request_url to HttpResponseFilter error_message and predicate

airbyte_cdk/sources/declarative/declarative_component_schema.yaml

@@ -2855,6 +2855,9 @@ definitions:
           - config
           - response
           - headers
+          - request_url
+        examples:
+          - "Skipping {{ request_url }} (HTTP 404)."
       error_message_contains:
         title: Error Message Substring
         description: Match the response if its error message contains the substring.
@@ -2878,8 +2881,10 @@ definitions:
         interpolation_context:
           - response
           - headers
+          - request_url
         examples:
           - "{{ 'Too much requests' in response }}"
+          - "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}"
           - "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}"
       $parameters:
         type: object
@@ -5752,6 +5757,11 @@ interpolation:
           x-ratelimit-limit: "600"
           x-ratelimit-remaining: "598"
           x-ratelimit-reset: "39"
+    - title: request_url
+      description: The URL of the response (the final URL after any redirects), with its query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so do not surface it for APIs that put a credential in the path. Available in HttpResponseFilter's error_message and predicate.
+      type: string
+      examples:
+        - "https://api.github.com/repos/airbytehq/airbyte/stargazers"
     - title: last_record
       description: Last record extracted from the response received from the API.
       type: object

airbyte_cdk/sources/declarative/models/declarative_component_schema.py

@@ -653,6 +653,7 @@ class HttpResponseFilter(BaseModel):
     error_message: Optional[str] = Field(
         None,
         description="Error Message to display if the response matches the filter.",
+        examples=["Skipping {{ request_url }} (HTTP 404)."],
         title="Error Message",
     )
     error_message_contains: Optional[str] = Field(
@@ -673,6 +674,7 @@ class HttpResponseFilter(BaseModel):
         description="Match the response if the predicate evaluates to true.",
         examples=[
             "{{ 'Too much requests' in response }}",
+            "{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}",
             "{{ 'error_code' in response and response['error_code'] == 'ComplexityException' }}",
         ],
         title="Predicate",

airbyte_cdk/sources/declarative/requesters/error_handlers/http_response_filter.py

@@ -4,6 +4,7 @@
 
 from dataclasses import InitVar, dataclass
 from typing import Any, Mapping, Optional, Set, Union
+from urllib.parse import urlsplit, urlunsplit
 
 import requests
 
@@ -36,6 +37,11 @@ class HttpResponseFilter:
         error_message_contains (str): error substring of matching requests
         predicate (str): predicate to apply to determine if a request is matching
         error_message (Union[InterpolatedString, str): error message to display if the response matches the filter
+
+    `error_message` and `predicate` can interpolate `response` (the parsed JSON body), `headers`
+    and `request_url`, the URL the response came from (the final URL after any redirects) with its
+    query string, fragment and userinfo (user:password@) removed. The path is kept as-is, so a
+    credential an API puts in the path is not removed. `error_message` can also interpolate `config`.
     """
 
     config: Config
@@ -138,6 +144,17 @@ def _matches_filter(
             return self.action  # type: ignore # action is always cast to a ResponseAction not a str
         return None
 
+    @staticmethod
+    def _request_url(response: requests.Response) -> str:
+        # The query string is dropped because some APIs authenticate with a query parameter, and
+        # an error message can end up in a user-facing failure.
+        url = response.url
+        if not isinstance(url, str) or not url:
+            return ""
+        parts = urlsplit(url)
+        host = parts.netloc.rsplit("@", 1)[-1]
+        return urlunsplit((parts.scheme, host, parts.path, "", ""))
+
     @staticmethod
     def _safe_response_json(response: requests.Response) -> dict[str, Any]:
         try:
@@ -152,7 +169,10 @@ def _create_error_message(self, response: requests.Response) -> Optional[str]:
         :return: The evaluated error message string to be emitted
         """
         return self.error_message.eval(  # type: ignore[no-any-return, union-attr]
-            self.config, response=self._safe_response_json(response), headers=response.headers
+            self.config,
+            response=self._safe_response_json(response),
+            headers=response.headers,
+            request_url=self._request_url(response),
         )
 
     def _response_matches_predicate(self, response: requests.Response) -> bool:
@@ -163,6 +183,7 @@ def _response_matches_predicate(self, response: requests.Response) -> bool:
                     None,  # type: ignore[arg-type]
                     response=self._safe_response_json(response),
                     headers=response.headers,
+                    request_url=self._request_url(response),
                 )
             )
             if self.predicate

unit_tests/sources/declarative/requesters/error_handlers/test_http_response_filter.py

@@ -237,3 +237,104 @@ def test_matches(
         assert actual_response_status.error_message == expected_error_resolution.error_message
     else:
         assert actual_response_status is None
+
+
+@pytest.mark.parametrize(
+    "url, expected_request_url",
+    [
+        pytest.param(
+            "https://api.github.com/repos/airbytehq/airbyte/stargazers?per_page=100&page=2",
+            "https://api.github.com/repos/airbytehq/airbyte/stargazers",
+            id="query_string_dropped",
+        ),
+        pytest.param(
+            "https://user:secret@api.example.com/v1/items?api_key=secret",
+            "https://api.example.com/v1/items",
+            id="credentials_dropped",
+        ),
+        pytest.param(
+            "https://user:pw@api.example.com:8443/v1/items?k=v#frag",
+            "https://api.example.com:8443/v1/items",
+            id="port_kept_fragment_dropped",
+        ),
+        pytest.param(
+            "https://[::1]:8443/v1/items?k=v",
+            "https://[::1]:8443/v1/items",
+            id="ipv6_host_and_port_kept",
+        ),
+    ],
+)
+def test_error_message_interpolates_request_url(requests_mock, url, expected_request_url):
+    requests_mock.get(url, status_code=404, json={"message": "Not Found"})
+    response_filter = HttpResponseFilter(
+        action=ResponseAction.IGNORE,
+        config={},
+        parameters={},
+        http_codes={404},
+        error_message="Skipping {{ request_url }}.",
+    )
+
+    resolution = response_filter.matches(requests.get(url))
+
+    assert resolution.response_action == ResponseAction.IGNORE
+    assert resolution.error_message == f"Skipping {expected_request_url}."
+
+
+def test_predicate_interpolates_request_url(requests_mock):
+    requests_mock.get(
+        "https://api.github.com/repos/airbytehq/airbyte/stargazers",
+        status_code=404,
+        json={"message": "Not Found"},
+    )
+    requests_mock.get(
+        "https://api.github.com/repos/airbytehq/airbyte/events",
+        status_code=404,
+        json={"message": "Not Found"},
+    )
+    requests_mock.get(
+        "https://api.github.com/repos/airbytehq/airbyte-cdk/stargazers",
+        status_code=200,
+        json=[{"login": "octocat"}],
+    )
+    response_filter = HttpResponseFilter(
+        action=ResponseAction.IGNORE,
+        config={},
+        parameters={},
+        predicate="{{ response.get('message') == 'Not Found' and request_url.endswith('/stargazers') }}",
+    )
+
+    stargazers = response_filter.matches(
+        requests.get("https://api.github.com/repos/airbytehq/airbyte/stargazers")
+    )
+    events = response_filter.matches(
+        requests.get("https://api.github.com/repos/airbytehq/airbyte/events")
+    )
+    # The filter also runs on successful pages, so the predicate must not match on the URL alone.
+    successful_page = response_filter.matches(
+        requests.get("https://api.github.com/repos/airbytehq/airbyte-cdk/stargazers")
+    )
+
+    assert stargazers.response_action == ResponseAction.IGNORE
+    assert events is None
+    assert successful_page is None
+
+
+def test_request_url_is_empty_when_response_has_no_url():
+    response = requests.Response()
+    response.status_code = 404
+    message_filter = HttpResponseFilter(
+        action=ResponseAction.IGNORE,
+        config={},
+        parameters={},
+        http_codes={404},
+        error_message="Skipping [{{ request_url }}].",
+    )
+    predicate_filter = HttpResponseFilter(
+        action=ResponseAction.IGNORE,
+        config={},
+        parameters={},
+        predicate="{{ request_url.endswith('/stargazers') }}",
+    )
+
+    assert message_filter.matches(response).error_message == "Skipping []."
+    assert predicate_filter.matches(response) is None