From 1f1232a2c2a87b5cf22dc996b6eaa3311b732e51 Mon Sep 17 00:00:00 2001 From: SimonTaurus Date: Sun, 1 Nov 2026 11:24:27 +0100 Subject: [PATCH 1/4] feat: make dask optional so osw imports without it - move dask into a new `parallel` extra and import it lazily. util.py is imported unconditionally by core, wtsite and wiki_tools, so a missing dask broke `import osw` outright - which ruled out Pyodide, where there is no dask build. - only async_parallelize (WIP, no callers in the package) and the __main__ demo helpers use it, so no public behaviour changes. - requires-python stays <3.14 on purpose: pydantic reports "Core Pydantic V1 functionality isn't compatible with Python 3.14 or greater", and osw, oold and opensemantic all build on pydantic.v1. Verified - 3.14 raises ConfigError on the first opensemantic model, 3.13 is fine. - verified: `import osw` with dask absent on 3.12 and 3.13, and in a Pyodide 3.13 kernel; suite unchanged vs origin/main (704 passed, same pre-existing 6 failures / 47 click-version errors before and after). --- pyproject.toml | 7 +++++-- src/osw/utils/util.py | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 79121c63..0d53a6b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,6 @@ dependencies = [ # used in custom_types.py on py3.10; was missing before (latent # ImportError on 3.10), surfaced by ty/deptry during the migration 'backports.strenum; python_version < "3.11"', - "dask", "tqdm", "pybars3-wheel", # the osw CLI (src/osw/cli); a base dependency, not an extra, so `pip @@ -71,6 +70,10 @@ Download = "https://pypi.org/project/osw/#files" [project.optional-dependencies] wikitext = ["mwparserfromhell"] +# Only osw.utils.util.async_parallelize needs it, and nothing in the package calls +# that. Keeping it in the base dependencies made `import osw` impossible anywhere +# dask is unavailable - notably Pyodide, which has no dask build. +parallel = ["dask"] DB = [ "psycopg2", "sqlalchemy", @@ -106,7 +109,7 @@ workflow = [ "anyio>=4.9,<4.14", ] tutorial = ["osw[dataimport]"] -all = ["osw[dataimport,DB,UI,S3,wikitext,mcp]"] +all = ["osw[dataimport,DB,UI,S3,wikitext,mcp,parallel]"] [project.scripts] # command-line access to a live OSL instance, built from the same diff --git a/src/osw/utils/util.py b/src/osw/utils/util.py index 28830537..4507388b 100644 --- a/src/osw/utils/util.py +++ b/src/osw/utils/util.py @@ -12,8 +12,6 @@ from pathlib import Path from typing import IO, Callable, Dict, Iterable, List, Optional, Union -import dask -from dask.diagnostics import ProgressBar from tqdm.asyncio import tqdm _logger = logging.getLogger(__name__) @@ -585,7 +583,16 @@ def file_in_paths( def async_parallelize(func: Callable, iterable: Iterable, **kwargs): - """Work in progress""" + """Work in progress. Requires the optional ``parallel`` extra. + + dask is imported here rather than at module scope because this is the only + thing in osw that uses it. At module scope it made ``import osw`` fail + outright wherever dask is unavailable - util.py is imported unconditionally + by core, wtsite and wiki_tools - which rules out Pyodide, where there is no + dask build at all. + """ + import dask + from dask.diagnostics import ProgressBar async def inner(func_, iterable_, **kwargs_): async with AsyncMessageBuffer() as msg_buf: @@ -643,6 +650,11 @@ def wrapped_function(*args, **kwargs): # testing redirect if __name__ == "__main__": + # Only the demo helpers below use dask; see async_parallelize for why it is + # not imported at module scope. Needs the optional `parallel` extra. + import dask + from dask.diagnostics import ProgressBar + from datetime import datetime from time import sleep From cd8b8047cd4e7fa3073ab2860ac2a702142943cc Mon Sep 17 00:00:00 2001 From: SimonTaurus Date: Thu, 5 Nov 2026 04:28:43 +0100 Subject: [PATCH 2/4] style: order the __main__ demo imports as ruff wants Stdlib before third party inside the block; the comment stays with the dask imports it explains. Caught by the quality job, not locally. --- src/osw/utils/util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/osw/utils/util.py b/src/osw/utils/util.py index 4507388b..4e44a2b5 100644 --- a/src/osw/utils/util.py +++ b/src/osw/utils/util.py @@ -650,14 +650,14 @@ def wrapped_function(*args, **kwargs): # testing redirect if __name__ == "__main__": + from datetime import datetime + from time import sleep + # Only the demo helpers below use dask; see async_parallelize for why it is # not imported at module scope. Needs the optional `parallel` extra. import dask from dask.diagnostics import ProgressBar - from datetime import datetime - from time import sleep - print("Redirecting print messages to a buffer.") message_buffer = MessageBuffer() io_buffer = StringIO() From 3fe98e484d575a10031090a3206119806643c82e Mon Sep 17 00:00:00 2001 From: SimonTaurus Date: Thu, 5 Nov 2026 04:36:09 +0100 Subject: [PATCH 3/4] chore: relock after moving dask to the parallel extra make check runs `uv lock --locked`, which fails on any pyproject change that the lockfile has not seen. Diff kept to the dask move: re-locking also pruned some zensical wheel urls, and --locked passes with those left alone, so they stay. --- uv.lock | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/uv.lock b/uv.lock index 88afba5a..628a3bbe 100644 --- a/uv.lock +++ b/uv.lock @@ -2072,7 +2072,6 @@ dependencies = [ { name = "backports-strenum", marker = "python_full_version < '3.11'" }, { name = "black" }, { name = "click" }, - { name = "dask" }, { name = "datamodel-code-generator" }, { name = "httpx" }, { name = "isort" }, @@ -2102,6 +2101,7 @@ dependencies = [ [package.optional-dependencies] all = [ { name = "boto3" }, + { name = "dask" }, { name = "deepl" }, { name = "geopy" }, { name = "mcp" }, @@ -2124,6 +2124,9 @@ mcp = [ { name = "mcp" }, { name = "mwparserfromhell" }, ] +parallel = [ + { name = "dask" }, +] s3 = [ { name = "boto3" }, ] @@ -2175,7 +2178,7 @@ requires-dist = [ { name = "black" }, { name = "boto3", marker = "extra == 's3'" }, { name = "click" }, - { name = "dask" }, + { name = "dask", marker = "extra == 'parallel'" }, { name = "datamodel-code-generator", specifier = "==0.51.0" }, { name = "deepl", marker = "extra == 'dataimport'" }, { name = "geopy", marker = "extra == 'dataimport'" }, @@ -2193,7 +2196,7 @@ requires-dist = [ { name = "opensemantic-base", specifier = ">=0.42.7" }, { name = "opensemantic-core", specifier = ">=0.57.4" }, { name = "osw", extras = ["dataimport"], marker = "extra == 'tutorial'" }, - { name = "osw", extras = ["dataimport", "db", "ui", "s3", "wikitext", "mcp"], marker = "extra == 'all'" }, + { name = "osw", extras = ["dataimport", "db", "ui", "s3", "wikitext", "mcp", "parallel"], marker = "extra == 'all'" }, { name = "osw", extras = ["wikitext"], marker = "extra == 'mcp'" }, { name = "prefect", marker = "extra == 'workflow'", specifier = ">=2.20.25,<3.0" }, { name = "psycopg2", marker = "extra == 'db'" }, @@ -2211,7 +2214,7 @@ requires-dist = [ { name = "typer" }, { name = "typing-extensions" }, ] -provides-extras = ["wikitext", "db", "s3", "dataimport", "ui", "mcp", "workflow", "tutorial", "all"] +provides-extras = ["wikitext", "parallel", "db", "s3", "dataimport", "ui", "mcp", "workflow", "tutorial", "all"] [package.metadata.requires-dev] dev = [ From fa103f12f8f89410ffe212f45f0eaa208efe6f6b Mon Sep 17 00:00:00 2001 From: SimonTaurus Date: Thu, 5 Nov 2026 04:40:48 +0100 Subject: [PATCH 4/4] fix: tell ty the dask imports are allowed to be unresolvable dask is the parallel extra, so uv sync --frozen does not install it and ty cannot resolve it. Suppressed at all four sites rather than added to the dev group: that is what pyproject does for mcp, but it would install dask into every job to type check the one function allowed to run without it. --- src/osw/utils/util.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/osw/utils/util.py b/src/osw/utils/util.py index 4e44a2b5..9672d111 100644 --- a/src/osw/utils/util.py +++ b/src/osw/utils/util.py @@ -591,8 +591,13 @@ def async_parallelize(func: Callable, iterable: Iterable, **kwargs): by core, wtsite and wiki_tools - which rules out Pyodide, where there is no dask build at all. """ - import dask - from dask.diagnostics import ProgressBar + # ty cannot resolve these: dask is the `parallel` extra and is deliberately + # absent from the dev environment, which is the condition this import exists to + # tolerate. Adding the extra to the dev group would make it resolve - that is what + # pyproject does for mcp - but it would install dask everywhere to type check the + # one place that is allowed to run without it. + import dask # ty: ignore[unresolved-import] + from dask.diagnostics import ProgressBar # ty: ignore[unresolved-import] async def inner(func_, iterable_, **kwargs_): async with AsyncMessageBuffer() as msg_buf: @@ -655,8 +660,8 @@ def wrapped_function(*args, **kwargs): # Only the demo helpers below use dask; see async_parallelize for why it is # not imported at module scope. Needs the optional `parallel` extra. - import dask - from dask.diagnostics import ProgressBar + import dask # ty: ignore[unresolved-import] + from dask.diagnostics import ProgressBar # ty: ignore[unresolved-import] print("Redirecting print messages to a buffer.") message_buffer = MessageBuffer()