Update configurability and docker documentation (#245)

Updates the configuration:
- configuration directory or files can now be set through environment
variables
 - the docker image will look at `/config` directory by default
- there is now docker image documentation, and it is pushed to Docker
Hub on release
 
I'm aware the introduced logging situation is not ideal, but that's
deferred to a follow up PR.

Author: PGijsbers

.github/workflows/docker-publish.yml

@@ -48,3 +48,12 @@ jobs:
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+      - name: Docker Hub Description
+        uses: peter-evans/dockerhub-description@v5
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+          repository: openml/rest-api
+          short-description: "Experimental OpenML REST API"
+          readme-filepath: docker/python/README.md

docker-compose.yaml

@@ -77,6 +77,9 @@ services:
     ports:
       - "8001:8000"
     volumes:
-      - .:/python-api
+      - .:/app
+      - ./src/config.toml:/config/config.toml
+    environment:
+      OPENML_REST_API_CONFIG_DIRECTORY: "/config"
     depends_on:
       - database

docker/python/Dockerfile

@@ -5,16 +5,21 @@ RUN apt-get update  \
 
 ENV VIRTUAL_ENV=/opt/venv
 RUN python -m venv $VIRTUAL_ENV
+# Prepending the path here means that `python` invocations below use the virtual environment
 ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 
 RUN python -m pip install uv
 
-WORKDIR /python-api
+WORKDIR /app
 COPY pyproject.toml .
 
 RUN uv pip install -e ".[dev]"
-COPY . /python-api
+COPY . /app
 
+RUN mkdir /config
+COPY ./src/config.toml /config/config.toml
+ENV OPENML_REST_API_CONFIG_FILE=/config/config.toml
+COPY ./docker/python/README.md /README.md
 
 EXPOSE 8000
 ENTRYPOINT ["python", "src/main.py"]

docker/python/README.md

@@ -0,0 +1,32 @@
+# OpenML REST API
+This is the work-in-progress Python-based REST API implementation for OpenML.
+This should not be considered production ready.
+New releases may contain breaking changes without deprecation warnings.
+
+## Usage
+The image is not intended to be used as a standalone image.
+Please reference either the docker compose file at [server-api](https://www.github.com/openml/server-api) for development purposes, or [services](https://www.github.com/openml/services) for deployment purposes.
+
+## Configuration
+Configuration is currently loaded from both a TOML file and a .env file.
+Environment variables are used for configurations that either shouldn't be shared (secrets), or that inform how to load the configuration.
+The TOML configuration file is used for all other settings.
+By default both of these files are loaded from the `/config` directory.
+
+### Configuration File
+A default configuration is available for reference at `/config/config.toml`, and can be used as reference.
+
+### Environment Variables
+Environment variables are used for configurations that either shouldn't be shared (secrets), or that inform how to load the configuration:
+
+ - `OPENML_REST_API_CONFIG_DIRECTORY`: points to the directory that contains configuration files (`config.toml`, `.env`) (default: `/config`)
+ - `OPENML_REST_API_CONFIG_FILE`: points to the file that contains the TOML configuration (default: not set). If set, takes precedence over `OPENML_REST_API_CONFIG_DIRECTORY`.
+ - `OPENML_REST_API_DOTENV_FILE`: points to the dot file that contains the environment variable settings (default: not set). If set, takes precedence over `OPENML_REST_API_CONFIG_DIRECTORY`.
+ - `OPENML_DATABASES_OPENML_USERNAME`: username for connecting to the `openml` database (default: `root`)
+ - `OPENML_DATABASES_OPENML_PASSWORD`: password for connecting to the `openml` database (default: `ok`)
+ - `OPENML_DATABASES_EXPDB_USERNAME`: username for connecting to the `openml_expdb` database (default: `root`)
+ - `OPENML_DATABASES_EXPDB_PASSWORD`: password for connecting to the `openml_expdb` database (default: `ok`)
+
+
+## Repository
+The code and dockerfile for this image are maintained [on GitHub](https://www.github.com/openml/server-api).

src/config.py

@@ -1,14 +1,37 @@
 import functools
+import logging
 import os
 import tomllib
 import typing
 from pathlib import Path
 
 from dotenv import load_dotenv
 
+logger = logging.getLogger(__name__)
+
 TomlTable = dict[str, typing.Any]
 
-CONFIG_PATH = Path(__file__).parent / "config.toml"
+CONFIG_DIRECTORY_ENV = "OPENML_REST_API_CONFIG_DIRECTORY"
+CONFIG_FILE_ENV = "OPENML_REST_API_CONFIG_FILE"
+DOTENV_FILE_ENV = "OPENML_REST_API_DOTENV_FILE"
+
+OPENML_DB_USERNAME_ENV = "OPENML_DATABASES_OPENML_USERNAME"
+OPENML_DB_PASSWORD_ENV = "OPENML_DATABASES_OPENML_PASSWORD"  # noqa: S105  # not a password
+EXPDB_DB_USERNAME_ENV = "OPENML_DATABASES_EXPDB_USERNAME"
+EXPDB_DB_PASSWORD_ENV = "OPENML_DATABASES_EXPDB_PASSWORD"  # noqa: S105  # not a password
+
+_config_directory = Path(os.getenv(CONFIG_DIRECTORY_ENV, Path(__file__).parent))
+_config_directory = _config_directory.expanduser().absolute()
+_config_file = Path(os.getenv(CONFIG_FILE_ENV, _config_directory / "config.toml"))
+_config_file = _config_file.expanduser().absolute()
+_dotenv_file = Path(os.getenv(DOTENV_FILE_ENV, _config_directory / ".env"))
+_dotenv_file = _dotenv_file.expanduser().absolute()
+
+logger.info("Configuration directory is '%s'", _config_directory)
+logger.info("Loading configuration file from '%s'", _config_file)
+logger.info("Loading environment variables from '%s'", _dotenv_file)
+
+load_dotenv(dotenv_path=_dotenv_file)
 
 
 def _apply_defaults_to_siblings(configuration: TomlTable) -> TomlTable:
@@ -25,35 +48,34 @@ def _load_configuration(file: Path) -> TomlTable:
     return tomllib.loads(file.read_text())
 
 
-def load_routing_configuration(file: Path = CONFIG_PATH) -> TomlTable:
+def load_routing_configuration(file: Path = _config_file) -> TomlTable:
     return typing.cast("TomlTable", _load_configuration(file)["routing"])
 
 
 @functools.cache
-def load_database_configuration(file: Path = CONFIG_PATH) -> TomlTable:
+def load_database_configuration(file: Path = _config_file) -> TomlTable:
     configuration = _load_configuration(file)
     database_configuration = _apply_defaults_to_siblings(
         configuration["databases"],
     )
-    load_dotenv()
     database_configuration["openml"]["username"] = os.environ.get(
-        "OPENML_DATABASES_OPENML_USERNAME",
+        OPENML_DB_USERNAME_ENV,
         "root",
     )
     database_configuration["openml"]["password"] = os.environ.get(
-        "OPENML_DATABASES_OPENML_PASSWORD",
+        OPENML_DB_PASSWORD_ENV,
         "ok",
     )
     database_configuration["expdb"]["username"] = os.environ.get(
-        "OPENML_DATABASES_EXPDB_USERNAME",
+        EXPDB_DB_USERNAME_ENV,
         "root",
     )
     database_configuration["expdb"]["password"] = os.environ.get(
-        "OPENML_DATABASES_EXPDB_PASSWORD",
+        EXPDB_DB_PASSWORD_ENV,
         "ok",
     )
     return database_configuration
 
 
-def load_configuration(file: Path = Path(__file__).parent / "config.toml") -> TomlTable:
+def load_configuration(file: Path = _config_file) -> TomlTable:
     return tomllib.loads(file.read_text())

src/main.py

@@ -1,4 +1,5 @@
 import argparse
+import logging
 
 import uvicorn
 from fastapi import FastAPI
@@ -58,6 +59,7 @@ def create_api() -> FastAPI:
 
 
 def main() -> None:
+    logging.basicConfig(level=logging.INFO)
     args = _parse_args()
     uvicorn.run(
         app="main:create_api",