diff --git a/airbyte_cdk/sources/streams/http/http_client.py b/airbyte_cdk/sources/streams/http/http_client.py index 7f22e0deed..9eae354943 100644 --- a/airbyte_cdk/sources/streams/http/http_client.py +++ b/airbyte_cdk/sources/streams/http/http_client.py @@ -560,6 +560,15 @@ def send_request( data=data, ) + env_settings = self._session.merge_environment_settings( + url=request.url, + proxies=request_kwargs.get("proxies"), + stream=request_kwargs.get("stream"), + verify=request_kwargs.get("verify"), + cert=request_kwargs.get("cert"), + ) + request_kwargs = {**request_kwargs, **env_settings} + response: requests.Response = self._send_with_retry( request=request, request_kwargs=request_kwargs, diff --git a/unit_tests/sources/streams/http/test_http.py b/unit_tests/sources/streams/http/test_http.py index d2bc4071e6..7512c3722e 100644 --- a/unit_tests/sources/streams/http/test_http.py +++ b/unit_tests/sources/streams/http/test_http.py @@ -92,7 +92,10 @@ def test_requests_native_token_authenticator(): def test_request_kwargs_used(mocker, requests_mock): stream = StubBasicReadHttpStream() - request_kwargs = {"cert": None, "proxies": "google.com"} + request_kwargs = { + "cert": None, + "proxies": {"http": "http://example.com", "https": "http://example.com"}, + } mocker.patch.object(stream, "request_kwargs", return_value=request_kwargs) send_mock = mocker.patch.object( stream._http_client._session, "send", wraps=stream._http_client._session.send @@ -101,8 +104,16 @@ def test_request_kwargs_used(mocker, requests_mock): list(stream.read_records(sync_mode=SyncMode.full_refresh)) - stream._http_client._session.send.assert_any_call(ANY, **request_kwargs) assert send_mock.call_count == 1 + call_args = send_mock.call_args_list[0] + call_kwargs = call_args.kwargs + + assert call_kwargs.get("cert") is None + + proxies = call_kwargs.get("proxies") + assert proxies is not None + assert proxies["http"] == "http://example.com" + assert proxies["https"] == "http://example.com" def test_stub_basic_read_http_stream_read_records(mocker): diff --git a/unit_tests/sources/streams/http/test_http_client.py b/unit_tests/sources/streams/http/test_http_client.py index d82548f058..d628a83ce6 100644 --- a/unit_tests/sources/streams/http/test_http_client.py +++ b/unit_tests/sources/streams/http/test_http_client.py @@ -1,6 +1,7 @@ # Copyright (c) 2024 Airbyte, Inc., all rights reserved. import logging +import os from datetime import timedelta from unittest.mock import MagicMock, patch @@ -746,6 +747,25 @@ def test_given_different_headers_then_response_is_not_cached(requests_mock): assert second_response.json()["test"] == "second response" +@patch.dict("os.environ", {"REQUESTS_CA_BUNDLE": "/path/to/ca-bundle.crt"}) +def test_send_request_respects_environment_variables(): + """Test that send_request respects REQUESTS_CA_BUNDLE environment variable.""" + http_client = HttpClient( + name="test", + logger=MagicMock(), + ) + + with patch.object(http_client, "_send_with_retry") as mock_send_with_retry: + http_client.send_request( + http_method="GET", url="https://api.example.com", request_kwargs={"timeout": 10} + ) + + passed_kwargs = mock_send_with_retry.call_args[1]["request_kwargs"] + + assert "verify" in passed_kwargs + assert passed_kwargs["verify"] == "/path/to/ca-bundle.crt" + + @pytest.mark.usefixtures("mock_sleep") @pytest.mark.parametrize( "response_code, expected_failure_type, error_message, exception_class",