Merge branch 'migration' of https://github.com/geetu040/openml-python into runs-migration-stacked

Author: Omswastik-11

openml/_api/clients/http.py

@@ -116,15 +116,13 @@ def __init__(  # noqa: PLR0913
         server: str,
         base_url: str,
         api_key: str,
-        timeout_seconds: int,
         retries: int,
         retry_policy: RetryPolicy,
         cache: HTTPCache | None = None,
     ) -> None:
         self.server = server
         self.base_url = base_url
         self.api_key = api_key
-        self.timeout_seconds = timeout_seconds
         self.retries = retries
         self.retry_policy = retry_policy
         self.cache = cache
@@ -238,6 +236,8 @@ def _validate_response(
             raise OpenMLServerError(f"URI too long! ({url})")
 
         retry_raise_e: Exception | None = None
+        code: int | None = None
+        message: str = ""
 
         try:
             code, message = self._parse_exception_response(response)
@@ -276,26 +276,25 @@ def _validate_response(
 
     def _request(  # noqa: PLR0913
         self,
+        session: requests.Session,
         method: str,
         url: str,
         params: Mapping[str, Any],
         data: Mapping[str, Any],
         headers: Mapping[str, str],
-        timeout: float | int,
         files: Mapping[str, Any] | None,
         **request_kwargs: Any,
     ) -> tuple[Response | None, Exception | None]:
         retry_raise_e: Exception | None = None
         response: Response | None = None
 
         try:
-            response = requests.request(
+            response = session.request(
                 method=method,
                 url=url,
                 params=params,
                 data=data,
                 headers=headers,
-                timeout=timeout,
                 files=files,
                 **request_kwargs,
             )
@@ -343,7 +342,6 @@ def request(
         headers = request_kwargs.pop("headers", {}).copy()
         headers.update(self.headers)
 
-        timeout = request_kwargs.pop("timeout", self.timeout_seconds)
         files = request_kwargs.pop("files", None)
 
         if use_cache and not reset_cache and self.cache is not None:
@@ -355,14 +353,15 @@ def request(
             except Exception:
                 raise  # propagate unexpected cache errors
 
+        session = requests.Session()
         for retry_counter in range(1, retries + 1):
             response, retry_raise_e = self._request(
+                session=session,
                 method=method,
                 url=url,
                 params=params,
                 data=data,
                 headers=headers,
-                timeout=timeout,
                 files=files,
                 **request_kwargs,
             )
@@ -377,6 +376,8 @@ def request(
             delay = self.retry_func(retry_counter)
             time.sleep(delay)
 
+        session.close()
+
         assert response is not None
 
         if use_cache and self.cache is not None:

openml/_api/setup/builder.py

@@ -39,7 +39,6 @@ def build(cls, config: Config) -> APIBackendBuilder:
             server=primary_api_config.server,
             base_url=primary_api_config.base_url,
             api_key=primary_api_config.api_key,
-            timeout_seconds=config.connection.timeout_seconds,
             retries=config.connection.retries,
             retry_policy=config.connection.retry_policy,
             cache=http_cache,
@@ -57,7 +56,6 @@ def build(cls, config: Config) -> APIBackendBuilder:
             server=fallback_api_config.server,
             base_url=fallback_api_config.base_url,
             api_key=fallback_api_config.api_key,
-            timeout_seconds=config.connection.timeout_seconds,
             retries=config.connection.retries,
             retry_policy=config.connection.retry_policy,
             cache=http_cache,

openml/_api/setup/config.py

@@ -19,7 +19,6 @@ class APIConfig:
 class ConnectionConfig:
     retries: int
     retry_policy: RetryPolicy
-    timeout_seconds: int
 
 
 @dataclass
@@ -52,7 +51,6 @@ class Config:
         default_factory=lambda: ConnectionConfig(
             retries=5,
             retry_policy=RetryPolicy.HUMAN,
-            timeout_seconds=10,
         )
     )
 

openml/testing.py

@@ -11,13 +11,12 @@
 import unittest
 from pathlib import Path
 from typing import ClassVar
-from urllib.parse import urljoin
 
 import requests
 
 import openml
 from openml._api import HTTPCache, HTTPClient, MinIOClient
-from openml.enums import RetryPolicy
+from openml.enums import APIVersion, RetryPolicy
 from openml.exceptions import OpenMLServerException
 from openml.tasks import TaskType
 
@@ -282,96 +281,42 @@ def _check_fold_timing_evaluations(  # noqa: PLR0913
                         assert evaluation <= max_val
 
 
-class TestAPIBase(unittest.TestCase):
-    server: str
-    base_url: str
-    api_key: str
-    timeout_seconds: int
-    retries: int
-    retry_policy: RetryPolicy
-    dir: str
-    ttl: int
+class TestAPIBase(TestBase):
     cache: HTTPCache
-    http_client: HTTPClient
-
-    def setUp(self) -> None:
-        self.server = "https://test.openml.org/"
-        self.base_url = "api/v1/xml"
-        self.api_key = "normaluser"
-        self.timeout_seconds = 10
-        self.retries = 3
-        self.retry_policy = RetryPolicy.HUMAN
-        self.dir = "test_cache"
-        self.ttl = 60 * 60 * 24 * 7
-
-        self.cache = self._get_http_cache(
-            path=Path(self.dir),
-            ttl=self.ttl,
-        )
-        self.http_client = self._get_http_client(
-            server=self.server,
-            base_url=self.base_url,
-            api_key=self.api_key,
-            timeout_seconds=self.timeout_seconds,
-            retries=self.retries,
-            retry_policy=self.retry_policy,
-            cache=self.cache,
-        )
-        self.minio_client = self._get_minio_client(path=Path(self.dir))
+    http_clients: dict[APIVersion, HTTPClient]
+    minio_client: MinIOClient
 
-        if self.cache.path.exists():
-            shutil.rmtree(self.cache.path)
+    def setUp(self, n_levels: int = 1, tmpdir_suffix: str = "") -> None:
+        super().setUp(n_levels=n_levels, tmpdir_suffix=tmpdir_suffix)
 
-    def tearDown(self) -> None:
-        if self.cache.path.exists():
-            shutil.rmtree(self.cache.path)
+        retries = self.connection_n_retries
+        retry_policy = RetryPolicy.HUMAN if self.retry_policy == "human" else RetryPolicy.ROBOT
+        ttl = openml._backend.get_config_value("cache.ttl")
+        cache_dir = self.static_cache_dir
 
-    def _get_http_cache(
-        self,
-        path: Path,
-        ttl: int,
-    ) -> HTTPCache:
-        return HTTPCache(
-            path=path,
+        self.cache = HTTPCache(
+            path=cache_dir,
             ttl=ttl,
         )
-
-    def _get_http_client(  # noqa: PLR0913
-        self,
-        server: str,
-        base_url: str,
-        api_key: str,
-        timeout_seconds: int,
-        retries: int,
-        retry_policy: RetryPolicy,
-        cache: HTTPCache | None = None,
-    ) -> HTTPClient:
-        return HTTPClient(
-            server=server,
-            base_url=base_url,
-            api_key=api_key,
-            timeout_seconds=timeout_seconds,
-            retries=retries,
-            retry_policy=retry_policy,
-            cache=cache,
-        )
-
-    def _get_minio_client(
-        self,
-        path: Path | None = None,
-    ) -> MinIOClient:
-        return MinIOClient(path=path)
-
-    def _get_url(
-        self,
-        server: str | None = None,
-        base_url: str | None = None,
-        path: str | None = None,
-    ) -> str:
-        server = server if server else self.server
-        base_url = base_url if base_url else self.base_url
-        path = path if path else ""
-        return urljoin(self.server, urljoin(self.base_url, path))
+        self.http_clients = {
+            APIVersion.V1: HTTPClient(
+                server="https://test.openml.org/",
+                base_url="api/v1/xml/",
+                api_key="normaluser",
+                retries=retries,
+                retry_policy=retry_policy,
+                cache=self.cache,
+            ),
+            APIVersion.V2: HTTPClient(
+                server="http://localhost:8002/",
+                base_url="",
+                api_key="",
+                retries=retries,
+                retry_policy=retry_policy,
+                cache=self.cache,
+            ),
+        }
+        self.minio_client = MinIOClient(path=cache_dir)
 
 
 def check_task_existence(

tests/test_api/test_http.py

@@ -4,11 +4,25 @@
 import pytest
 from openml.testing import TestAPIBase
 import os
+from urllib.parse import urljoin
+from openml.enums import APIVersion
+from openml._api import HTTPClient
 
 
 class TestHTTPClient(TestAPIBase):
+    http_client: HTTPClient
+
+    def setUp(self):
+        super().setUp()
+        self.http_client = self.http_clients[APIVersion.V1]
+
+    def _prepare_url(self, path: str | None = None) -> str:
+        server = self.http_client.server
+        base_url = self.http_client.base_url
+        return urljoin(server, urljoin(base_url, path))
+
     def test_cache(self):
-        url = self._get_url(path="task/31")
+        url = self._prepare_url(path="task/31")
         params = {"param1": "value1", "param2": "value2"}
 
         key = self.cache.get_key(url, params)
@@ -18,6 +32,7 @@ def test_cache(self):
             "test",
             "api",
             "v1",
+            "xml",
             "task",
             "31",
             "param1=value1&param2=value2",
@@ -68,7 +83,7 @@ def test_get_with_cache_creates_cache(self):
 
         # verify cache directory structure exists
         cache_key = self.cache.get_key(
-            self._get_url(path="task/1"),
+            self._prepare_url(path="task/1"),
             {},
         )
         cache_path = self.cache._key_to_path(cache_key)
@@ -94,7 +109,7 @@ def test_get_cache_expires(self):
         self.cache.ttl = 1
         path = "task/1"
 
-        url = self._get_url(path=path)
+        url = self._prepare_url(path=path)
         key = self.cache.get_key(url, {})
         cache_path = self.cache._key_to_path(key) / "meta.json"
 
@@ -115,7 +130,7 @@ def test_get_cache_expires(self):
     def test_get_reset_cache(self):
         path = "task/1"
 
-        url = self._get_url(path=path)
+        url = self._prepare_url(path=path)
         key = self.cache.get_key(url, {})
         cache_path = self.cache._key_to_path(key) / "meta.json"