Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,6 @@ dependencies = [
# used in custom_types.py on py3.10; was missing before (latent
# ImportError on 3.10), surfaced by ty/deptry during the migration
'backports.strenum; python_version < "3.11"',
"dask",
"tqdm",
"pybars3-wheel",
# the osw CLI (src/osw/cli); a base dependency, not an extra, so `pip
Expand All @@ -71,6 +70,10 @@ Download = "https://pypi.org/project/osw/#files"

[project.optional-dependencies]
wikitext = ["mwparserfromhell"]
# Only osw.utils.util.async_parallelize needs it, and nothing in the package calls
# that. Keeping it in the base dependencies made `import osw` impossible anywhere
# dask is unavailable - notably Pyodide, which has no dask build.
parallel = ["dask"]
DB = [
"psycopg2",
"sqlalchemy",
Expand Down Expand Up @@ -106,7 +109,7 @@ workflow = [
"anyio>=4.9,<4.14",
]
tutorial = ["osw[dataimport]"]
all = ["osw[dataimport,DB,UI,S3,wikitext,mcp]"]
all = ["osw[dataimport,DB,UI,S3,wikitext,mcp,parallel]"]

[project.scripts]
# command-line access to a live OSL instance, built from the same
Expand Down
23 changes: 20 additions & 3 deletions src/osw/utils/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,6 @@
from pathlib import Path
from typing import IO, Callable, Dict, Iterable, List, Optional, Union

import dask
from dask.diagnostics import ProgressBar
from tqdm.asyncio import tqdm

_logger = logging.getLogger(__name__)
Expand Down Expand Up @@ -585,7 +583,21 @@ def file_in_paths(


def async_parallelize(func: Callable, iterable: Iterable, **kwargs):
"""Work in progress"""
"""Work in progress. Requires the optional ``parallel`` extra.

dask is imported here rather than at module scope because this is the only
thing in osw that uses it. At module scope it made ``import osw`` fail
outright wherever dask is unavailable - util.py is imported unconditionally
by core, wtsite and wiki_tools - which rules out Pyodide, where there is no
dask build at all.
"""
# ty cannot resolve these: dask is the `parallel` extra and is deliberately
# absent from the dev environment, which is the condition this import exists to
# tolerate. Adding the extra to the dev group would make it resolve - that is what
# pyproject does for mcp - but it would install dask everywhere to type check the
# one place that is allowed to run without it.
import dask # ty: ignore[unresolved-import]
from dask.diagnostics import ProgressBar # ty: ignore[unresolved-import]

async def inner(func_, iterable_, **kwargs_):
async with AsyncMessageBuffer() as msg_buf:
Expand Down Expand Up @@ -646,6 +658,11 @@ def wrapped_function(*args, **kwargs):
from datetime import datetime
from time import sleep

# Only the demo helpers below use dask; see async_parallelize for why it is
# not imported at module scope. Needs the optional `parallel` extra.
import dask # ty: ignore[unresolved-import]
from dask.diagnostics import ProgressBar # ty: ignore[unresolved-import]

print("Redirecting print messages to a buffer.")
message_buffer = MessageBuffer()
io_buffer = StringIO()
Expand Down
11 changes: 7 additions & 4 deletions uv.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading