Merge branch 'pr-1577' into migration (merge conflicts)

Author: geetu040

.github/workflows/test.yml

@@ -34,15 +34,27 @@ jobs:
         sklearn-only: ["true"]
 
         exclude:
-          # incompatible version combinations
+          # (python, sklearn) combinations for which there is no PyPI release
+          # scikit-learn 1.3
           - python-version: "3.13"
             scikit-learn: "1.3.*"
-          - python-version: "3.13"
-            scikit-learn: "1.4.*"
           - python-version: "3.14"
             scikit-learn: "1.3.*"
+          # scikit-learn 1.4
+          - python-version: "3.13"
+            scikit-learn: "1.4.*"
           - python-version: "3.14"
             scikit-learn: "1.4.*"
+          # scikit-learn 1.5
+          - python-version: "3.14"
+            scikit-learn: "1.5.*"
+          # scikit-learn 1.6
+          - python-version: "3.14"
+            scikit-learn: "1.6.*"
+          # scikit-learn 1.7 is installed with pandas 3
+          - python-version: "3.10"
+            scikit-learn: "1.7.*"
+
 
         include:
           # Full test run on ubuntu, 3.14
@@ -64,14 +76,6 @@ jobs:
             sklearn-only: "false"
             code-cov: true
 
-          # Pandas 2 run
-          - os: ubuntu-latest
-            python-version: "3.12"
-            scikit-learn: "1.5.*"
-            sklearn-only: "false"
-            pandas-version: "2.*"
-            code-cov: false
-
     steps:
     - uses: actions/checkout@v6
       with:
@@ -82,15 +86,21 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install test dependencies, scikit-learn, and optional pandas
+    - name: Install test dependencies, scikit-learn, and pandas
       shell: bash
       run: |
         python -m pip install --upgrade pip
         pip install -e .[test] scikit-learn==${{ matrix.scikit-learn }}
-        
-        if [ "${{ matrix.pandas-version }}" != "" ]; then
-          echo "Installing specific pandas version: ${{ matrix.pandas-version }}"
-          pip install "pandas==${{ matrix.pandas-version }}"
+
+        # scikit-learn 1.7+ requires pandas 3.x, earlier versions use pandas 2.x
+        version="${{ matrix.scikit-learn }}"
+        major=$(echo "$version" | cut -d. -f1)
+        minor=$(echo "$version" | cut -d. -f2)
+
+        if [[ "$major" -gt 1 ]] || { [[ "$major" -eq 1 ]] && [[ "$minor" -ge 7 ]]; }; then
+          pip install "pandas==3.*"
+        else
+          pip install "pandas==2.*"
         fi
 
     - name: Store repository status
@@ -101,22 +111,46 @@ jobs:
         echo "BEFORE=$git_status" >> $GITHUB_ENV
         echo "Repository status before tests: $git_status"
 
+    - name: Clone Services
+      if: matrix.os == 'ubuntu-latest'
+      id: clone-services
+      run: |
+        git clone --depth 1 https://github.com/openml/services.git
+
+    - name: Start Docker Services
+      id: start-services
+      if: matrix.os == 'ubuntu-latest'
+      working-directory: ./services
+      run: |
+        chmod -R a+rw ./data
+        chmod -R a+rw ./logs
+        docker compose --profile rest-api --profile minio --profile evaluation-engine up -d
+
+        echo "Waiting for PHP API to boot..."
+        timeout 60s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-php-rest-api)" == "healthy" ]; do sleep 5; done'
+
+        echo "Final Verification: Gateway Connectivity..."
+        curl -sSfL http://localhost:8000/api/v1/xml/data/1 | head -n 15
+
+        docker container ls
+
     - name: Show installed dependencies
       run: python -m pip list
 
     - name: Run tests on Ubuntu Test
       if: matrix.os == 'ubuntu-latest'
       env:
         OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
+        OPENML_USE_LOCAL_SERVICES: "true"
       run: |
         if [ "${{ matrix.code-cov }}" = "true" ]; then
           codecov="--cov=openml --long --cov-report=xml"
         fi
 
         if [ "${{ matrix.sklearn-only }}" = "true" ]; then
-          marks="sklearn and not production_server and not test_server"
+          marks="sklearn and not production_server"
         else
-          marks="not production_server and not test_server"
+          marks="not production_server"
         fi
 
         pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
@@ -125,15 +159,16 @@ jobs:
       if: matrix.os == 'ubuntu-latest'
       env:
         OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
+        OPENML_USE_LOCAL_SERVICES: "true"
       run: |
         if [ "${{ matrix.code-cov }}" = "true" ]; then
           codecov="--cov=openml --long --cov-report=xml"
         fi
 
         if [ "${{ matrix.sklearn-only }}" = "true" ]; then
-          marks="sklearn and production_server and not test_server"
+          marks="sklearn and production_server"
         else
-          marks="production_server and not test_server"
+          marks="production_server"
         fi
 
         pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
@@ -145,6 +180,25 @@ jobs:
       run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
         pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server"
 
+    - name: Upload coverage
+      if: matrix.code-cov && always()
+      uses: codecov/codecov-action@v4
+      with:
+        files: coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: true
+        verbose: true
+
+    - name: Dump server logs
+      if: always() && steps.start-services.outcome == 'success'
+      run: |
+        docker logs openml-php-rest-api -t
+
+    - name: Cleanup Docker setup
+      if: always() && steps.clone-services.outcome == 'success'
+      run: |
+        sudo rm -rf services
+
     - name: Check for files left behind by test
       if: matrix.os != 'windows-latest' && always()
       run: |
@@ -157,15 +211,6 @@ jobs:
             exit 1
         fi
 
-    - name: Upload coverage
-      if: matrix.code-cov && always()
-      uses: codecov/codecov-action@v4
-      with:
-        files: coverage.xml
-        token: ${{ secrets.CODECOV_TOKEN }}
-        fail_ci_if_error: true
-        verbose: true
-
   dummy_windows_py_sk024:
     name: (windows-latest, Py, sk0.24.*, sk-only:false)
     runs-on: ubuntu-latest

openml/base.py

@@ -8,7 +8,6 @@
 
 import xmltodict
 
-import openml
 import openml._api_calls
 
 from .utils import _get_rest_api_type_alias, _tag_openml_base

openml/evaluations/evaluation.py

@@ -3,7 +3,6 @@
 
 from dataclasses import asdict, dataclass
 
-import openml
 import openml.datasets
 import openml.flows
 import openml.runs

openml/setups/setup.py

@@ -4,7 +4,6 @@
 from dataclasses import asdict, dataclass
 from typing import Any
 
-import openml
 import openml.flows
 
 

openml/study/functions.py

@@ -8,7 +8,6 @@
 import pandas as pd
 import xmltodict
 
-import openml
 import openml._api_calls
 import openml.utils
 from openml.study.study import OpenMLBenchmarkSuite, OpenMLStudy

openml/tasks/task.py

@@ -1,15 +1,16 @@
 # License: BSD 3-Clause
 from __future__ import annotations
 
+import logging
 import warnings
 from abc import ABC
 from collections.abc import Sequence
 from enum import Enum
-from pathlib import Path
 from typing import TYPE_CHECKING, Any, ClassVar
 from typing_extensions import TypedDict
 
-import openml
+import arff
+
 import openml._api_calls
 from openml import datasets
 from openml.base import OpenMLBase
@@ -22,6 +23,9 @@
     import pandas as pd
 
 
+logger = logging.getLogger(__name__)
+
+
 # TODO(eddiebergman): Should use `auto()` but might be too late if these numbers are used
 # and stored on server.
 class TaskType(Enum):
@@ -178,18 +182,6 @@ def get_train_test_split_indices(
 
         return self.split.get(repeat=repeat, fold=fold, sample=sample)
 
-    def _download_split(self, cache_file: Path) -> None:
-        # TODO(eddiebergman): Not sure about this try to read and error approach
-        try:
-            with cache_file.open(encoding="utf8"):
-                pass
-        except OSError:
-            split_url = self.estimation_procedure["data_splits_url"]
-            openml._api_calls._download_text_file(
-                source=str(split_url),
-                output_path=str(cache_file),
-            )
-
     def download_split(self) -> OpenMLSplit:
         """Download the OpenML split for a given task."""
         # TODO(eddiebergman): Can this every be `None`?
@@ -199,9 +191,23 @@ def download_split(self) -> OpenMLSplit:
 
         try:
             split = OpenMLSplit._from_arff_file(cached_split_file)
-        except OSError:
+            logger.debug("Loaded file from cache: %s", str(cached_split_file))
+        except (OSError, arff.BadDataFormat):
+            logger.info("Failed to load file from cache: %s", str(cached_split_file))
+            if cached_split_file.exists():
+                logger.debug("Cleaning up old file")
+                cached_split_file.unlink()
             # Next, download and cache the associated split file
-            self._download_split(cached_split_file)
+            split_url = self.estimation_procedure["data_splits_url"]
+            openml._api_calls._download_text_file(
+                source=str(split_url),
+                output_path=str(cached_split_file),
+            )
+            if cached_split_file.exists():
+                logger.info("New file created of size %d", cached_split_file.stat().st_size)
+            else:
+                logger.info("Failed to create new file")
+
             split = OpenMLSplit._from_arff_file(cached_split_file)
 
         return split

pyproject.toml

@@ -126,6 +126,7 @@ version = {attr = "openml.__version__.__version__"}
 
 # https://docs.pytest.org/en/7.2.x/reference/reference.html#ini-options-ref
 [tool.pytest.ini_options]
+log_level="DEBUG"
 testpaths = ["tests"]
 minversion = "7.0"
 xfail_strict = true

tests/conftest.py

@@ -289,22 +289,35 @@ def as_robot() -> Iterator[None]:
 @pytest.fixture(autouse=True)
 def with_server(request):
     openml.config.set_api_version(APIVersion.V1)
+
     if "production_server" in request.keywords:
+        # use-production-server (remote)
         openml.config.set_servers("production")
-        yield
-        return
-    openml.config.set_servers("test")
+    elif os.getenv("OPENML_USE_LOCAL_SERVICES") == "true":
+        # use-test-server (local)
+        openml.config.set_servers("local")
+    else:
+        # use-test-server (remote)
+        openml.config.set_servers("test")
+
     yield
 
 
 @pytest.fixture(autouse=True)
 def with_test_cache(test_files_directory, request):
+    # Skip this fixture for TestBase subclasses - they manage their own cache directory
+    # in setUp()/tearDown(). Having both mechanisms fight over the global config
+    # causes race conditions.
+    if request.instance is not None and isinstance(request.instance, TestBase):
+        yield
+        return
+
     if not test_files_directory.exists():
         raise ValueError(
             f"Cannot find test cache dir, expected it to be {test_files_directory!s}!",
         )
     _root_cache_directory = openml.config._root_cache_directory
-    tmp_cache = test_files_directory / request.node.name
+    tmp_cache = test_files_directory / request.node.nodeid.replace("/", ".").replace("::", ".")
     openml.config.set_root_cache_directory(tmp_cache)
     yield
     openml.config.set_root_cache_directory(_root_cache_directory)

tests/test_datasets/test_dataset_functions.py

@@ -2005,4 +2005,4 @@ def test_get_dataset_parquet(requests_mock, test_files_directory, test_server_v1
     assert dataset._parquet_url is not None
     assert dataset.parquet_file is not None
     assert os.path.isfile(dataset.parquet_file)
-    assert dataset.data_file is None  # is alias for arff path
+    assert dataset.data_file is None  # is alias for arff path

tests/test_flows/test_flow.py

@@ -5,6 +5,7 @@
 import copy
 import hashlib
 import re
+import os
 import time
 from packaging.version import Version
 from unittest import mock
@@ -33,7 +34,6 @@
 from openml.testing import SimpleImputer, TestBase
 
 
-
 class TestFlow(TestBase):
     _multiprocess_can_split_ = True