diff --git a/pyproject.toml b/pyproject.toml index 79121c63..0d53a6b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,6 @@ dependencies = [ # used in custom_types.py on py3.10; was missing before (latent # ImportError on 3.10), surfaced by ty/deptry during the migration 'backports.strenum; python_version < "3.11"', - "dask", "tqdm", "pybars3-wheel", # the osw CLI (src/osw/cli); a base dependency, not an extra, so `pip @@ -71,6 +70,10 @@ Download = "https://pypi.org/project/osw/#files" [project.optional-dependencies] wikitext = ["mwparserfromhell"] +# Only osw.utils.util.async_parallelize needs it, and nothing in the package calls +# that. Keeping it in the base dependencies made `import osw` impossible anywhere +# dask is unavailable - notably Pyodide, which has no dask build. +parallel = ["dask"] DB = [ "psycopg2", "sqlalchemy", @@ -106,7 +109,7 @@ workflow = [ "anyio>=4.9,<4.14", ] tutorial = ["osw[dataimport]"] -all = ["osw[dataimport,DB,UI,S3,wikitext,mcp]"] +all = ["osw[dataimport,DB,UI,S3,wikitext,mcp,parallel]"] [project.scripts] # command-line access to a live OSL instance, built from the same diff --git a/src/osw/utils/util.py b/src/osw/utils/util.py index 28830537..9672d111 100644 --- a/src/osw/utils/util.py +++ b/src/osw/utils/util.py @@ -12,8 +12,6 @@ from pathlib import Path from typing import IO, Callable, Dict, Iterable, List, Optional, Union -import dask -from dask.diagnostics import ProgressBar from tqdm.asyncio import tqdm _logger = logging.getLogger(__name__) @@ -585,7 +583,21 @@ def file_in_paths( def async_parallelize(func: Callable, iterable: Iterable, **kwargs): - """Work in progress""" + """Work in progress. Requires the optional ``parallel`` extra. + + dask is imported here rather than at module scope because this is the only + thing in osw that uses it. At module scope it made ``import osw`` fail + outright wherever dask is unavailable - util.py is imported unconditionally + by core, wtsite and wiki_tools - which rules out Pyodide, where there is no + dask build at all. + """ + # ty cannot resolve these: dask is the `parallel` extra and is deliberately + # absent from the dev environment, which is the condition this import exists to + # tolerate. Adding the extra to the dev group would make it resolve - that is what + # pyproject does for mcp - but it would install dask everywhere to type check the + # one place that is allowed to run without it. + import dask # ty: ignore[unresolved-import] + from dask.diagnostics import ProgressBar # ty: ignore[unresolved-import] async def inner(func_, iterable_, **kwargs_): async with AsyncMessageBuffer() as msg_buf: @@ -646,6 +658,11 @@ def wrapped_function(*args, **kwargs): from datetime import datetime from time import sleep + # Only the demo helpers below use dask; see async_parallelize for why it is + # not imported at module scope. Needs the optional `parallel` extra. + import dask # ty: ignore[unresolved-import] + from dask.diagnostics import ProgressBar # ty: ignore[unresolved-import] + print("Redirecting print messages to a buffer.") message_buffer = MessageBuffer() io_buffer = StringIO() diff --git a/uv.lock b/uv.lock index 88afba5a..628a3bbe 100644 --- a/uv.lock +++ b/uv.lock @@ -2072,7 +2072,6 @@ dependencies = [ { name = "backports-strenum", marker = "python_full_version < '3.11'" }, { name = "black" }, { name = "click" }, - { name = "dask" }, { name = "datamodel-code-generator" }, { name = "httpx" }, { name = "isort" }, @@ -2102,6 +2101,7 @@ dependencies = [ [package.optional-dependencies] all = [ { name = "boto3" }, + { name = "dask" }, { name = "deepl" }, { name = "geopy" }, { name = "mcp" }, @@ -2124,6 +2124,9 @@ mcp = [ { name = "mcp" }, { name = "mwparserfromhell" }, ] +parallel = [ + { name = "dask" }, +] s3 = [ { name = "boto3" }, ] @@ -2175,7 +2178,7 @@ requires-dist = [ { name = "black" }, { name = "boto3", marker = "extra == 's3'" }, { name = "click" }, - { name = "dask" }, + { name = "dask", marker = "extra == 'parallel'" }, { name = "datamodel-code-generator", specifier = "==0.51.0" }, { name = "deepl", marker = "extra == 'dataimport'" }, { name = "geopy", marker = "extra == 'dataimport'" }, @@ -2193,7 +2196,7 @@ requires-dist = [ { name = "opensemantic-base", specifier = ">=0.42.7" }, { name = "opensemantic-core", specifier = ">=0.57.4" }, { name = "osw", extras = ["dataimport"], marker = "extra == 'tutorial'" }, - { name = "osw", extras = ["dataimport", "db", "ui", "s3", "wikitext", "mcp"], marker = "extra == 'all'" }, + { name = "osw", extras = ["dataimport", "db", "ui", "s3", "wikitext", "mcp", "parallel"], marker = "extra == 'all'" }, { name = "osw", extras = ["wikitext"], marker = "extra == 'mcp'" }, { name = "prefect", marker = "extra == 'workflow'", specifier = ">=2.20.25,<3.0" }, { name = "psycopg2", marker = "extra == 'db'" }, @@ -2211,7 +2214,7 @@ requires-dist = [ { name = "typer" }, { name = "typing-extensions" }, ] -provides-extras = ["wikitext", "db", "s3", "dataimport", "ui", "mcp", "workflow", "tutorial", "all"] +provides-extras = ["wikitext", "parallel", "db", "s3", "dataimport", "ui", "mcp", "workflow", "tutorial", "all"] [package.metadata.requires-dev] dev = [