diff --git a/.github/workflows/test-sdk-python.yml b/.github/workflows/test-sdk-python.yml index d672b216..1c3746e7 100644 --- a/.github/workflows/test-sdk-python.yml +++ b/.github/workflows/test-sdk-python.yml @@ -53,6 +53,32 @@ jobs: - name: make verify run: make verify + without-vendor-sdks: + name: Without vendor SDKs + runs-on: ubuntu-latest + defaults: + run: + working-directory: sdks/python + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.13" + + # 1.x still lists the vendor SDKs as dependencies, so they are removed again here; + # from 2.0 the plain install brings none and the uninstall step can go. + - name: Install the package without vendor SDKs + run: | + python -m pip install --upgrade pip + pip install . pytest pytest-asyncio + pip uninstall -y openai anthropic google-genai + + - name: Every evaluator without them + run: python -m pytest tests/unit/test_without_vendor_sdks.py -p no:cacheprovider + coverage: name: Coverage runs-on: ubuntu-latest diff --git a/README.md b/README.md index e1c4d842..61518ec3 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Check out the 2. Import an evaluator, pass your API key, and call `evaluate`
3. See the [TypeScript](./sdks/typescript/) and [Python](./sdks/python/) SDK READMEs for details | | [Python notebook](https://docs.learningcommons.org/evaluators/getting-started/quickstart#python-notebooks) | For quick prototyping | 1. Start Jupyter Lab: `jupyter lab` (opens at `http://localhost:8888`)
2. Browse into `evals/` and open the evaluator you want to try
3. Copy your text into the last code cell and run it

You can also use an IDE with Python and Jupyter support, such as VS Code with Microsoft's Python and Jupyter extensions. | | Local | To evaluate inputs locally on your machine | Clone this repository and follow the directions for your platform:
- [Setup on Mac/Linux](./docs/setup-mac-linux.md)
- [Setup on Windows](./docs/setup-windows.md) | diff --git a/demos/python/requirements.txt b/demos/python/requirements.txt index ab961e29..a005034d 100644 --- a/demos/python/requirements.txt +++ b/demos/python/requirements.txt @@ -3,4 +3,4 @@ uvicorn jinja2 python-multipart python-dotenv -learning-commons-evaluators>=1.0.0 +learning-commons-evaluators[all]>=1.0.0 diff --git a/sdks/python/README.md b/sdks/python/README.md index 559c0a60..b213685a 100644 --- a/sdks/python/README.md +++ b/sdks/python/README.md @@ -31,17 +31,29 @@ Python SDK for [Learning Commons evaluators](https://docs.learningcommons.org/ev ## Dependencies - Python 3.11+ -- The provider clients ship with the package (`openai`, `anthropic`, `google-genai`), along with `pydantic`, `httpx`, `pillow`, and `textstat`. +- `pydantic`, `httpx`, `pillow`, and `textstat`, plus the provider clients (`openai`, `anthropic`, `google-genai`), each also available as an extra (see [Installation](#installation)). - Credentials are constructor arguments. The SDK reads no provider keys from the environment, so a shell `GOOGLE_API_KEY` leaves `google_api_key` unset. A missing key raises `ConfigurationError` at construction, before any I/O. ## Installation -Install the SDK: +Install the SDK with the extra for each provider your evaluators call: ```bash -pip install learning-commons-evaluators +pip install "learning-commons-evaluators[google]" # one provider +pip install "learning-commons-evaluators[openai,google]" # several +pip install "learning-commons-evaluators[all]" # every provider ``` +| Extra | Installs | Evaluators that need it by default | +|---|---|---| +| `openai` | `openai` | Revision Accuracy, Revision Actionability, Revision Manageability, Sentence Structure, Strength Acknowledgment, Student Response Specificity, Tone Appropriateness, Vocabulary Complexity, Withholding Answers | +| `google` | `google-genai` | Background Knowledge Demands, Grade Level Appropriateness, Graphics Accuracy, Graphics Complexity, Meaning Directness, Organizational Structure, Purpose Clarity, Reference Knowledge Demands, Vocabulary Complexity | +| `anthropic` | `anthropic` | Critical Thinking, Math Standards Alignment | + +A `model_override` needs the extra for its provider instead. An evaluator whose provider's SDK is not installed raises `ConfigurationError` at construction, naming the extra to add. + +Version 1.x still installs all three provider clients without an extra, so these lines change nothing today. Version 2.0 will install them only through their extras, and the plain `pip install learning-commons-evaluators` will be for [bringing your own provider](#bring-your-own-provider), which needs none. + Pass the key each evaluator's default provider requires ([which keys](https://docs.learningcommons.org/evaluators/getting-started/quickstart#required-api-keys)): ```python @@ -301,10 +313,11 @@ class MyProvider: evaluator = ToneAppropriatenessEvaluator(llm_provider=MyProvider()) ``` -- Every LLM step of every evaluator, multi-step ones included, runs on it. No vendor API key is required; one passed anyway is unused. Non-LLM credentials still are: Math Standards Alignment needs `learning_commons_api_key` for the Knowledge Graph. +- Every LLM step of every evaluator, multi-step ones included, runs on it. No vendor SDK or vendor API key is required; one passed anyway is unused. Non-LLM credentials still are: Math Standards Alignment needs `learning_commons_api_key` for the Knowledge Graph. - It cannot be combined with `model_override`, and an object missing any of the three members raises `ConfigurationError` at construction. - Evaluators that attach images (the graphics evaluators) also need `supports_attachments = True` on the provider and an `attachments` keyword on `generate_structured`. - Telemetry and `evaluation.metadata.model` report your `label`. A failed call raises the usual `DependencyError` with `dependency="custom"` and your label as `model`, whatever the label says. +- For failures to be classified and retried like the built-in ones, raise an exception that carries the HTTP status as `status_code` (or a `response` with `status_code` and `headers`), or raise one of the SDK's own errors, such as `RateLimitError`. Anything else becomes a non-retryable `LLMProviderError`. ## More resources diff --git a/sdks/python/docs/local-development.md b/sdks/python/docs/local-development.md index d33a3897..6f17ad5f 100644 --- a/sdks/python/docs/local-development.md +++ b/sdks/python/docs/local-development.md @@ -45,10 +45,10 @@ From `sdks/python/`: ## Using the SDK before publishing ```bash -pip install -e /path/to/evaluators/sdks/python +pip install -e "/path/to/evaluators/sdks/python[all]" ``` -Editable install: changes to SDK source apply without reinstalling. +Editable install: changes to SDK source apply without reinstalling. Name the extras your evaluators need in place of `all`, or none if you inject your own `llm_provider`. ## Regenerating after a change under `evals/` diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml index 82c7b57f..17cd756d 100644 --- a/sdks/python/pyproject.toml +++ b/sdks/python/pyproject.toml @@ -65,7 +65,18 @@ dependencies = [ ] [project.optional-dependencies] +# One extra per provider, named by its `Provider` value, so the error for a missing SDK +# names the extra mechanically. Each repeats its pin from `dependencies` above, where the +# SDKs still sit so that no 1.x install changes; 2.0 drops them from there (DSCR-2625), +# leaving the base install for callers who inject their own `llm_provider`. Bump a pin in +# both places until then. +openai = ["openai>=2.47.0,<4"] +anthropic = ["anthropic>=1.0.0,<2"] +google = ["google-genai>=1.39.0,<3"] +all = ["learning-commons-evaluators[openai,anthropic,google]"] dev = [ + # The unit tests build vendor exceptions, and mypy reads the vendor types. + "learning-commons-evaluators[all]", "pytest>=7.0.0", "pytest-asyncio>=0.21.0", "pytest-cov>=5.0.0", diff --git a/sdks/python/src/learning_commons_evaluators/evaluators/base.py b/sdks/python/src/learning_commons_evaluators/evaluators/base.py index 80f6500e..13b13c4b 100644 --- a/sdks/python/src/learning_commons_evaluators/evaluators/base.py +++ b/sdks/python/src/learning_commons_evaluators/evaluators/base.py @@ -22,6 +22,7 @@ create_provider, provider_context, ) +from learning_commons_evaluators.providers.factory import missing_sdk_error, sdk_installed from learning_commons_evaluators.schemas.evaluator import EvaluationResult from learning_commons_evaluators.schemas.metadata import EvaluatorMetadata from learning_commons_evaluators.telemetry import ( @@ -46,8 +47,9 @@ class BaseEvaluator(ABC): evaluator = PurposeClarityEvaluator(EvaluatorConfig(google_api_key="...")) :raises ConfigurationError: for a malformed ``model_override``, an ``llm_provider`` that - does not implement :class:`LLMProvider` or is combined with ``model_override``, or a - missing credential the evaluator's contract requires (§3.1), before any I/O. + does not implement :class:`LLMProvider` or is combined with ``model_override``, a + vendor SDK a built-in adapter needs that is not installed, or a missing credential + the evaluator's contract requires (§3.1), before any I/O. """ #: The registry contract the evaluator is built from; set by the concrete class. @@ -71,6 +73,7 @@ def __init__(self, config: EvaluatorConfig | None = None, /, **fields: Any) -> N self._validate_llm_provider(config) self._override_provider = self._validate_model_override(config.model_override) + self._validate_sdks() self._validate_credentials() telemetry = config.telemetry_options @@ -164,6 +167,18 @@ def providers_in_use(self) -> tuple[Provider, ...]: return (self._override_provider,) return self.metadata.default_providers + def _validate_sdks(self) -> None: + """Every vendor SDK this instance's built-in adapters import is installed. + + Checked at construction like a missing key, and for all providers at once, so an + install without the right extra fails before any I/O naming everything to add, + rather than with a bare ``ImportError`` from inside the first evaluation. An + injected ``llm_provider`` uses no built-in adapter, so it needs no vendor SDK. + """ + missing = [provider for provider in self.providers_in_use if not sdk_installed(provider)] + if missing: + raise missing_sdk_error(missing, self.metadata.name) + def _credentials_satisfied_by_injection(self) -> frozenset[str]: """Credentials an injected dependency already carries, so the config need not. diff --git a/sdks/python/src/learning_commons_evaluators/providers/factory.py b/sdks/python/src/learning_commons_evaluators/providers/factory.py index feef75a1..28bd9a55 100644 --- a/sdks/python/src/learning_commons_evaluators/providers/factory.py +++ b/sdks/python/src/learning_commons_evaluators/providers/factory.py @@ -2,9 +2,46 @@ from __future__ import annotations +import importlib.util +from collections.abc import Sequence + from learning_commons_evaluators.errors import ConfigurationError from learning_commons_evaluators.providers.base import LLMProvider, Provider, ProviderConfig +#: The module each built-in adapter imports, and the distribution that provides it. The +#: extra that installs it is named after the provider: ``pip install "...[google]"``. +_SDKS: dict[Provider, tuple[str, str]] = { + Provider.OPENAI: ("openai", "openai"), + Provider.ANTHROPIC: ("anthropic", "anthropic"), + Provider.GOOGLE: ("google.genai", "google-genai"), +} + + +def sdk_installed(provider: Provider) -> bool: + """Whether ``provider``'s vendor SDK is importable, found without importing it.""" + try: + return importlib.util.find_spec(_SDKS[provider][0]) is not None + except ModuleNotFoundError: + # ``google.genai`` is looked up through ``google``, which may be absent itself. + return False + except ValueError: + # A module placed in ``sys.modules`` without a spec, as a stub in tests may be: it + # is importable, which is what the check is for. + return True + + +def missing_sdk_error(providers: Sequence[Provider], needed_by: str) -> ConfigurationError: + """The error for SDKs that are not installed, naming the extra that installs them.""" + missing = list(dict.fromkeys(providers)) + packages = " and ".join(_SDKS[provider][1] for provider in missing) + extras = ",".join(provider.value for provider in missing) + plural = len(missing) > 1 + return ConfigurationError( + f"{needed_by} needs the {packages} package{'s' if plural else ''}, which " + f"{'are' if plural else 'is'} not installed. Install {'them' if plural else 'it'} " + f'with: pip install "learning-commons-evaluators[{extras}]"' + ) + def create_provider(config: ProviderConfig) -> LLMProvider: """The built-in adapter for ``config.type``. @@ -12,8 +49,11 @@ def create_provider(config: ProviderConfig) -> LLMProvider: Adapters import their vendor SDK on construction, so a program that only ever uses one provider never loads the other two. - :raises ConfigurationError: for an unknown provider, an empty model, or a missing key. + :raises ConfigurationError: for an unknown provider, an empty model, a missing key, or + a vendor SDK that is not installed. """ + if isinstance(config.type, Provider) and not sdk_installed(config.type): + raise missing_sdk_error([config.type], f"The {config.type.value} provider") if config.type is Provider.OPENAI: from learning_commons_evaluators.providers.openai_sdk import OpenAIProvider @@ -33,4 +73,4 @@ def create_provider(config: ProviderConfig) -> LLMProvider: ) -__all__ = ["create_provider"] +__all__ = ["create_provider", "missing_sdk_error", "sdk_installed"] diff --git a/sdks/python/tests/unit/evaluators/test_config.py b/sdks/python/tests/unit/evaluators/test_config.py index 5733b14c..894941b4 100644 --- a/sdks/python/tests/unit/evaluators/test_config.py +++ b/sdks/python/tests/unit/evaluators/test_config.py @@ -15,9 +15,11 @@ PurposeClarityEvaluator, TelemetryOptions, ToneAppropriatenessEvaluator, + VocabularyComplexityEvaluator, ) from learning_commons_evaluators.evaluators.base import BaseEvaluator -from tests.unit.conftest import ProviderFactory +from learning_commons_evaluators.providers import ProviderConfig +from tests.unit.conftest import FakeProvider, ProviderFactory, sample_for from tests.unit.evaluators.test_single_step import contract as demo_contract from tests.unit.evaluators.test_single_step import define as define_demo @@ -201,3 +203,57 @@ def test_no_warning_without_an_override( with caplog.at_level(logging.WARNING, logger="learning_commons_evaluators"): PurposeClarityEvaluator(google_api_key="k") assert not [r for r in caplog.records if r.levelno == logging.WARNING] + + +class TestVendorSDKs: + """A built-in adapter's vendor SDK is checked at construction, like its key.""" + + @pytest.fixture + def installed(self, monkeypatch: pytest.MonkeyPatch) -> set[Provider]: + """The providers whose SDK counts as installed; none until a test adds some.""" + present: set[Provider] = set() + monkeypatch.setattr( + "learning_commons_evaluators.evaluators.base.sdk_installed", present.__contains__ + ) + return present + + def test_a_missing_sdk_names_the_evaluator_and_the_extra( + self, providers: ProviderFactory, installed: set[Provider] + ) -> None: + with pytest.raises(ConfigurationError) as exc_info: + PurposeClarityEvaluator(google_api_key="k") + assert str(exc_info.value) == ( + "Purpose Clarity Evaluator needs the google-genai package, which is not installed. " + 'Install it with: pip install "learning-commons-evaluators[google]"' + ) + + def test_every_missing_sdk_is_named_at_once( + self, providers: ProviderFactory, installed: set[Provider] + ) -> None: + with pytest.raises(ConfigurationError, match=r"\[openai,google\]"): + VocabularyComplexityEvaluator(openai_api_key="o", google_api_key="g") + installed.add(Provider.OPENAI) + with pytest.raises(ConfigurationError, match=r"needs the google-genai package.*\[google\]"): + VocabularyComplexityEvaluator(openai_api_key="o", google_api_key="g") + + def test_it_is_checked_before_the_keys( + self, providers: ProviderFactory, installed: set[Provider] + ) -> None: + # Installing comes before configuring, so the first error is the one to fix first. + with pytest.raises(ConfigurationError, match="google-genai"): + PurposeClarityEvaluator() + + def test_an_override_needs_its_own_providers_sdk_instead( + self, providers: ProviderFactory, installed: set[Provider] + ) -> None: + override = ModelOverride(provider=Provider.ANTHROPIC, model="claude-haiku-4-5-20251001") + with pytest.raises(ConfigurationError, match=r"\[anthropic\]"): + PurposeClarityEvaluator(anthropic_api_key="a", model_override=override) + installed.add(Provider.ANTHROPIC) + PurposeClarityEvaluator(anthropic_api_key="a", model_override=override) + + def test_an_injected_provider_needs_none( + self, providers: ProviderFactory, installed: set[Provider] + ) -> None: + injected = FakeProvider(ProviderConfig(type=Provider.OPENAI, model="m"), sample_for) + VocabularyComplexityEvaluator(llm_provider=injected) diff --git a/sdks/python/tests/unit/providers/test_factory.py b/sdks/python/tests/unit/providers/test_factory.py index b219f807..443b4e36 100644 --- a/sdks/python/tests/unit/providers/test_factory.py +++ b/sdks/python/tests/unit/providers/test_factory.py @@ -2,11 +2,17 @@ from __future__ import annotations +import importlib.util +import sys +import types +from typing import Any + import pytest from learning_commons_evaluators.errors import ConfigurationError from learning_commons_evaluators.providers import Provider, ProviderConfig, create_provider from learning_commons_evaluators.providers.anthropic_sdk import AnthropicProvider +from learning_commons_evaluators.providers.factory import missing_sdk_error, sdk_installed from learning_commons_evaluators.providers.google_genai import GoogleProvider from learning_commons_evaluators.providers.openai_sdk import OpenAIProvider @@ -70,3 +76,65 @@ def test_max_retries_reaches_the_vendor_client() -> None: ) assert anthropic_adapter._new_client is not None assert anthropic_adapter._new_client().max_retries == 0 + + +class TestVendorSDKs: + @pytest.fixture + def uninstalled(self, monkeypatch: pytest.MonkeyPatch) -> set[str]: + """Module names to treat as not installed; ``google.genai`` is found via ``google``.""" + gone: set[str] = set() + find_spec = importlib.util.find_spec + + def fake(name: str, package: str | None = None) -> Any: + if name in gone: + if name == "google.genai": + raise ModuleNotFoundError("No module named 'google'", name="google") + return None + return find_spec(name, package) + + monkeypatch.setattr(importlib.util, "find_spec", fake) + return gone + + @pytest.mark.parametrize("provider", list(Provider)) + def test_each_installed_sdk_is_found(self, provider: Provider) -> None: + assert sdk_installed(provider) + + @pytest.mark.parametrize( + ("provider", "module"), + [ + (Provider.OPENAI, "openai"), + (Provider.ANTHROPIC, "anthropic"), + (Provider.GOOGLE, "google.genai"), + ], + ) + def test_a_missing_one_is_not( + self, uninstalled: set[str], provider: Provider, module: str + ) -> None: + uninstalled.add(module) + assert not sdk_installed(provider) + assert all(sdk_installed(other) for other in Provider if other is not provider) + + def test_building_an_adapter_without_it_names_the_extra(self, uninstalled: set[str]) -> None: + uninstalled.add("google.genai") + with pytest.raises(ConfigurationError) as exc_info: + create_provider(ProviderConfig(type=Provider.GOOGLE, model="m", api_key="k")) + assert str(exc_info.value) == ( + "The google provider needs the google-genai package, which is not installed. " + 'Install it with: pip install "learning-commons-evaluators[google]"' + ) + + def test_the_error_names_every_missing_sdk_once(self) -> None: + error = missing_sdk_error( + [Provider.OPENAI, Provider.GOOGLE, Provider.OPENAI], "Vocabulary Complexity Evaluator" + ) + assert str(error) == ( + "Vocabulary Complexity Evaluator needs the openai and google-genai packages, which " + 'are not installed. Install them with: pip install "learning-commons-evaluators[openai,google]"' + ) + + def test_a_module_stubbed_into_sys_modules_counts_as_installed( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + # ``find_spec`` raises for a module that has no spec; it is importable all the same. + monkeypatch.setitem(sys.modules, "anthropic", types.ModuleType("anthropic")) + assert sdk_installed(Provider.ANTHROPIC) diff --git a/sdks/python/tests/unit/test_without_vendor_sdks.py b/sdks/python/tests/unit/test_without_vendor_sdks.py new file mode 100644 index 00000000..505fdd23 --- /dev/null +++ b/sdks/python/tests/unit/test_without_vendor_sdks.py @@ -0,0 +1,41 @@ +"""The package without its vendor SDKs, as an install with no extras will have it. + +The check runs in a child process that hides the SDKs before anything imports them, so +it holds whether or not they are installed here; CI also runs it where they are not. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + +#: Makes every vendor module unimportable, as if it were not installed, then runs the check. +SCRIPT = """ +import sys + +from tests.unit.without_vendor_sdks import is_vendor + + +class NotInstalled: + def find_spec(self, name, path=None, target=None): + if is_vendor(name): + raise ModuleNotFoundError(f"No module named {name!r}", name=name) + return None + + +sys.meta_path.insert(0, NotInstalled()) + +from tests.unit.without_vendor_sdks import main + +main() +""" + + +def test_every_evaluator_runs_on_an_injected_provider_and_refuses_without_one() -> None: + result = subprocess.run( + [sys.executable, "-c", SCRIPT], cwd=ROOT, capture_output=True, text=True, check=False + ) + assert result.returncode == 0, result.stderr diff --git a/sdks/python/tests/unit/without_vendor_sdks.py b/sdks/python/tests/unit/without_vendor_sdks.py new file mode 100644 index 00000000..a8725a1e --- /dev/null +++ b/sdks/python/tests/unit/without_vendor_sdks.py @@ -0,0 +1,61 @@ +"""Every evaluator, in a process where no vendor SDK can be imported. + +Run as its own process by ``test_without_vendor_sdks.py``, after hiding the SDKs, so it +holds for an install that has none: each evaluator refuses to build its own adapters, and +runs on an injected ``llm_provider`` without importing a vendor SDK on the way. +""" + +from __future__ import annotations + +import asyncio +import sys +from typing import Any + +from learning_commons_evaluators import ConfigurationError +from learning_commons_evaluators.evaluators.academic_standards_alignment.mathematics.math_standards_alignment import ( + MathStandardsAlignmentEvaluator, +) +from learning_commons_evaluators.evaluators.base import BaseEvaluator +from learning_commons_evaluators.evaluators.registry import EVALUATORS +from learning_commons_evaluators.providers import Provider, ProviderConfig +from tests.unit.conftest import FakeKnowledgeGraph, FakeProvider, sample_for +from tests.unit.telemetry.test_emission import evaluate_with, fixture_input + +#: The modules the built-in adapters import. +VENDOR_MODULES = ("openai", "anthropic", "google.genai") + + +def is_vendor(name: str) -> bool: + return any(name == module or name.startswith(f"{module}.") for module in VENDOR_MODULES) + + +def construct(evaluator: type[BaseEvaluator], **overrides: Any) -> BaseEvaluator: + keys = {f"{p.value}_api_key": "test-key" for p in evaluator.metadata.default_providers} + if evaluator is MathStandardsAlignmentEvaluator: + overrides.setdefault("knowledge_graph", FakeKnowledgeGraph()) + else: + keys.update({key: "test-key" for key in evaluator.metadata.required_credentials}) + return evaluator(**keys, telemetry=False, **overrides) + + +def main() -> None: + for evaluator in EVALUATORS: + try: + construct(evaluator) + except ConfigurationError as error: + assert "not installed" in str(error), (evaluator.metadata.id, error) + else: + raise AssertionError(f"{evaluator.metadata.id} built without its vendor SDK") + + injected = FakeProvider(ProviderConfig(type=Provider.OPENAI, model="m"), sample_for) + asyncio.run( + evaluate_with(construct(evaluator, llm_provider=injected), fixture_input(evaluator)) + ) + assert injected.calls, evaluator.metadata.id + + imported = sorted(name for name in sys.modules if is_vendor(name)) + assert not imported, f"vendor SDKs imported: {imported}" + + +if __name__ == "__main__": + main()