Address feedback (#1)

Author: vdusek

docs/02_concepts/code/05_retries_async.py

@@ -9,6 +9,6 @@ async def main() -> None:
     apify_client = ApifyClientAsync(
         token=TOKEN,
         max_retries=8,
-        min_delay_between_retries=timedelta(milliseconds=500),  # 0.5s
-        timeout=timedelta(seconds=360),  # 6 mins
+        min_delay_between_retries=timedelta(milliseconds=500),
+        timeout=timedelta(seconds=360),
     )

docs/02_concepts/code/05_retries_sync.py

@@ -9,6 +9,6 @@ async def main() -> None:
     apify_client = ApifyClient(
         token=TOKEN,
         max_retries=8,
-        min_delay_between_retries=timedelta(milliseconds=500),  # 0.5s
-        timeout=timedelta(seconds=360),  # 6 mins
+        min_delay_between_retries=timedelta(milliseconds=500),
+        timeout=timedelta(seconds=360),
     )

scripts/check_async_docstrings.py

@@ -16,10 +16,6 @@
 
 # Go through every Python file in that directory
 for client_source_path in clients_path.glob('**/*.py'):
-    # Skip _http_clients package - sync and async are in separate files there
-    if '_http_clients' in str(client_source_path):
-        continue
-
     with open(client_source_path, encoding='utf-8') as source_file:
         # Read the source file and parse the code using Red Baron
         red = RedBaron(source_code=source_file.read())
@@ -31,8 +27,6 @@
 
         # Find the corresponding sync classes (same name, but without -Async)
         sync_class = red.find('ClassNode', name=async_class.name.replace('ClientAsync', 'Client'))
-        if not sync_class:
-            continue
 
         # Go through all methods in the async class
         for async_method in async_class.find_all('DefNode'):

scripts/fix_async_docstrings.py

@@ -11,23 +11,15 @@
 
 # Go through every Python file in that directory
 for client_source_path in clients_path.glob('**/*.py'):
-    # Skip _http_clients package - sync and async are in separate files there
-    if '_http_clients' in str(client_source_path):
-        continue
-
     with open(client_source_path, 'r+', encoding='utf-8') as source_file:
         # Read the source file and parse the code using Red Baron
         red = RedBaron(source_code=source_file.read())
 
         # Find all classes which end with "ClientAsync" (there should be at most 1 per file)
         async_class = red.find('ClassNode', name=re.compile('.*ClientAsync$'))
-        if not async_class:
-            continue
 
         # Find the corresponding sync classes (same name, but without -Async)
         sync_class = red.find('ClassNode', name=async_class.name.replace('ClientAsync', 'Client'))
-        if not sync_class:
-            continue
 
         # Go through all methods in the async class
         for async_method in async_class.find_all('DefNode'):

scripts/utils.py

@@ -49,6 +49,10 @@ def sync_to_async_docstring(docstring: str) -> str:
         (r'Client', r'ClientAsync'),
         (r'\bsynchronously\b', r'asynchronously'),
         (r'\bSynchronously\b', r'Asynchronously'),
+        (r'\bsynchronous\b', r'asynchronous'),
+        (r'\bSynchronous\b', r'Asynchronous'),
+        (r'Retry a function', r'Retry an async function'),
+        (r'Function to retry', r'Async function to retry'),
     ]
     res = docstring
     for pattern, replacement in substitutions:

src/apify_client/_apify_client.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+import warnings
 from functools import cached_property
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ClassVar
 
 from apify_client._client_registry import ClientRegistry, ClientRegistryAsync
 from apify_client._consts import (
@@ -11,7 +12,7 @@
     DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
     DEFAULT_TIMEOUT,
 )
-from apify_client._http_clients import AsyncHttpClient, SyncHttpClient
+from apify_client._http_clients import HttpClient, HttpClientAsync
 from apify_client._resource_clients import (
     ActorClient,
     ActorClientAsync,
@@ -81,6 +82,8 @@
 class ApifyClient:
     """The Apify API client."""
 
+    _OVERRIDABLE_DEFAULT_HEADERS: ClassVar[set[str]] = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'}
+
     def __init__(
         self,
         token: str | None = None,
@@ -90,6 +93,7 @@ def __init__(
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         timeout: timedelta = DEFAULT_TIMEOUT,
+        headers: dict[str, str] | None = None,
     ) -> None:
         """Initialize a new instance.
 
@@ -103,11 +107,15 @@ def __init__(
             min_delay_between_retries: How long will the client wait between retrying requests
                 (increases exponentially from this value).
             timeout: The socket timeout of the HTTP requests sent to the Apify API.
+            headers: Additional HTTP headers to include in all API requests.
         """
         # We need to do this because of mocking in tests and default mutable arguments.
         api_url = DEFAULT_API_URL if api_url is None else api_url
         api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
 
+        if headers:
+            self._check_custom_headers(headers)
+
         self._token = token
         """Apify API token for authentication."""
 
@@ -120,12 +128,13 @@ def __init__(
         self._statistics = ClientStatistics()
         """Collector for client request statistics."""
 
-        self._http_client = SyncHttpClient(
+        self._http_client = HttpClient(
             token=token,
             timeout=timeout or DEFAULT_TIMEOUT,
             max_retries=max_retries or DEFAULT_MAX_RETRIES,
             min_delay_between_retries=min_delay_between_retries or DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
             statistics=self._statistics,
+            headers=headers,
         )
         """HTTP client used to communicate with the Apify API."""
 
@@ -172,6 +181,18 @@ def _base_kwargs(self) -> dict:
             'client_registry': self._client_registry,
         }
 
+    def _check_custom_headers(self, headers: dict[str, str]) -> None:
+        """Warn if custom headers override important default headers."""
+        overwrite_headers = [key for key in headers if key.title() in self._OVERRIDABLE_DEFAULT_HEADERS]
+        if overwrite_headers:
+            warnings.warn(
+                f'{", ".join(overwrite_headers)} headers of {self.__class__.__name__} was overridden with an '
+                'explicit value. A wrong header value can lead to API errors, it is recommended to use the default '
+                f'value for following headers: {", ".join(self._OVERRIDABLE_DEFAULT_HEADERS)}.',
+                category=UserWarning,
+                stacklevel=3,
+            )
+
     @property
     def token(self) -> str | None:
         """The Apify API token used by the client."""
@@ -322,6 +343,8 @@ def store(self) -> StoreCollectionClient:
 class ApifyClientAsync:
     """The asynchronous version of the Apify API client."""
 
+    _OVERRIDABLE_DEFAULT_HEADERS: ClassVar[set[str]] = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'}
+
     def __init__(
         self,
         token: str | None = None,
@@ -331,6 +354,7 @@ def __init__(
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         timeout: timedelta = DEFAULT_TIMEOUT,
+        headers: dict[str, str] | None = None,
     ) -> None:
         """Initialize a new instance.
 
@@ -344,11 +368,15 @@ def __init__(
             min_delay_between_retries: How long will the client wait between retrying requests
                 (increases exponentially from this value).
             timeout: The socket timeout of the HTTP requests sent to the Apify API.
+            headers: Additional HTTP headers to include in all API requests.
         """
         # We need to do this because of mocking in tests and default mutable arguments.
         api_url = DEFAULT_API_URL if api_url is None else api_url
         api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
 
+        if headers:
+            self._check_custom_headers(headers)
+
         self._token = token
         """Apify API token for authentication."""
 
@@ -361,12 +389,13 @@ def __init__(
         self._statistics = ClientStatistics()
         """Collector for client request statistics."""
 
-        self._http_client = AsyncHttpClient(
+        self._http_client = HttpClientAsync(
             token=token,
             timeout=timeout or DEFAULT_TIMEOUT,
             max_retries=max_retries or DEFAULT_MAX_RETRIES,
             min_delay_between_retries=min_delay_between_retries or DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
             statistics=self._statistics,
+            headers=headers,
         )
         """HTTP client used to communicate with the Apify API."""
 
@@ -413,6 +442,18 @@ def _base_kwargs(self) -> dict:
             'client_registry': self._client_registry,
         }
 
+    def _check_custom_headers(self, headers: dict[str, str]) -> None:
+        """Warn if custom headers override important default headers."""
+        overwrite_headers = [key for key in headers if key.title() in self._OVERRIDABLE_DEFAULT_HEADERS]
+        if overwrite_headers:
+            warnings.warn(
+                f'{", ".join(overwrite_headers)} headers of {self.__class__.__name__} was overridden with an '
+                'explicit value. A wrong header value can lead to API errors, it is recommended to use the default '
+                f'value for following headers: {", ".join(self._OVERRIDABLE_DEFAULT_HEADERS)}.',
+                category=UserWarning,
+                stacklevel=3,
+            )
+
     @property
     def token(self) -> str | None:
         """The Apify API token used by the client."""

src/apify_client/_http_clients/__init__.py

@@ -1,5 +1,4 @@
-from ._async import AsyncHttpClient
-from ._sync import SyncHttpClient
+from ._http_client import HttpClient, HttpClientAsync
 
 __all__ = [
     'HttpClient',

src/apify_client/_http_clients/_base.py

@@ -34,6 +34,7 @@ def __init__(
         max_retries: int = DEFAULT_MAX_RETRIES,
         min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
         statistics: ClientStatistics | None = None,
+        headers: dict[str, str] | None = None,
     ) -> None:
         """Initialize the base HTTP client.
 
@@ -43,30 +44,31 @@ def __init__(
             max_retries: Maximum number of retries for failed requests.
             min_delay_between_retries: Minimum delay between retries.
             statistics: Statistics tracker for API calls. Created automatically if not provided.
+            headers: Additional HTTP headers to include in all requests.
         """
         self._timeout = timeout
         self._max_retries = max_retries
         self._min_delay_between_retries = min_delay_between_retries
         self._statistics = statistics or ClientStatistics()
 
         # Build headers for subclasses to use when creating their impit clients.
-        headers: dict[str, str] = {'Accept': 'application/json, */*'}
+        default_headers: dict[str, str] = {'Accept': 'application/json, */*'}
 
         workflow_key = os.getenv('APIFY_WORKFLOW_KEY')
         if workflow_key is not None:
-            headers['X-Apify-Workflow-Key'] = workflow_key
+            default_headers['X-Apify-Workflow-Key'] = workflow_key
 
         is_at_home = 'APIFY_IS_AT_HOME' in os.environ
         python_version = '.'.join([str(x) for x in sys.version_info[:3]])
         client_version = metadata.version('apify-client')
 
         user_agent = f'ApifyClient/{client_version} ({sys.platform}; Python/{python_version}); isAtHome/{is_at_home}'
-        headers['User-Agent'] = user_agent
+        default_headers['User-Agent'] = user_agent
 
         if token is not None:
-            headers['Authorization'] = f'Bearer {token}'
+            default_headers['Authorization'] = f'Bearer {token}'
 
-        self._headers = headers
+        self._headers = {**default_headers, **(headers or {})}
 
     @staticmethod
     def _parse_params(params: dict[str, Any] | None) -> dict[str, Any] | None:
@@ -109,9 +111,9 @@ def _prepare_request_call(
         self,
         headers: dict[str, str] | None = None,
         params: dict[str, Any] | None = None,
-        data: Any = None,
+        data: str | bytes | bytearray | None = None,
         json: JsonSerializable | None = None,
-    ) -> tuple[dict[str, str], dict[str, Any] | None, Any]:
+    ) -> tuple[dict[str, str], dict[str, Any] | None, bytes | None]:
         """Prepare headers, params, and body for an HTTP request. Serializes JSON and applies gzip compression."""
         if json is not None and data is not None:
             raise ValueError('Cannot pass both "json" and "data" parameters at the same time!')