diff --git a/py/numcodecs-native-materialize/.gitignore b/py/numcodecs-native-materialize/.gitignore new file mode 100644 index 00000000..41c7d029 --- /dev/null +++ b/py/numcodecs-native-materialize/.gitignore @@ -0,0 +1,10 @@ +/.venv +/.mypy_cache +/.ruff_cache +/uv.lock + +/build +/site + +**/__pycache__ +**/*.egg-info diff --git a/py/numcodecs-native-materialize/.python-version b/py/numcodecs-native-materialize/.python-version new file mode 100644 index 00000000..c8cfe395 --- /dev/null +++ b/py/numcodecs-native-materialize/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/py/numcodecs-native-materialize/LICENSE b/py/numcodecs-native-materialize/LICENSE new file mode 120000 index 00000000..30cff740 --- /dev/null +++ b/py/numcodecs-native-materialize/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/py/numcodecs-native-materialize/README.md b/py/numcodecs-native-materialize/README.md new file mode 100644 index 00000000..6ff5d084 --- /dev/null +++ b/py/numcodecs-native-materialize/README.md @@ -0,0 +1,18 @@ +[![CI Status]][workflow] + +[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain + +# numcodecs-native-materialize + +Utility package to materialize instances of the `numcodecs-native-template` package. + +## License + +Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/). + +## Funding + +The `numcodecs-native-materialize` package has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe. + +Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054. diff --git a/py/numcodecs-native-materialize/pyproject.toml b/py/numcodecs-native-materialize/pyproject.toml new file mode 100644 index 00000000..1ba369c0 --- /dev/null +++ b/py/numcodecs-native-materialize/pyproject.toml @@ -0,0 +1,25 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "numcodecs_native_materialize" +version = "0.1.0" +description = "utility to materialize instances of the `numcodecs-native-template` package" + +authors = [{ name = "Juniper Tyree", email = "juniper.tyree@helsinki.fi" }] +maintainers = [{ name = "Juniper Tyree", email = "juniper.tyree@helsinki.fi" }] + +readme = "README.md" +license = { file = "LICENSE" } + +requires-python = ">=3.10" + +dependencies = [ + "toml~=0.10.0", +] + +[dependency-groups] +dev = [ + "ruff~=0.9.0", +] diff --git a/py/numcodecs-native-materialize/src/numcodecs_native_materialize/__main__.py b/py/numcodecs-native-materialize/src/numcodecs_native_materialize/__main__.py new file mode 100644 index 00000000..cc73e730 --- /dev/null +++ b/py/numcodecs-native-materialize/src/numcodecs_native_materialize/__main__.py @@ -0,0 +1,119 @@ +import argparse +import os +import re +import shlex +import shutil +import subprocess +from pathlib import Path + +import toml + +template_pattern = re.compile(r"(%[^%]+%)") + +repo_path = Path(__file__).parent.parent.parent.parent.parent +template_path = repo_path / "py" / "numcodecs-native-template" +dist_path = repo_path / "py" / "numcodecs-native-materialize" / "dist" + +parser = argparse.ArgumentParser(prog="numcodecs_native_materialize") +parser.add_argument( + "codec", + type=re.compile, + help="regex pattern for the codecs to materialize", +) +args = parser.parse_args() + +for c in (repo_path / "codecs").iterdir(): + codec = c.name + + if not args.codec or re.match(args.codec, codec) is None: + continue + + codec_crate_path = repo_path / "codecs" / codec + staging_path = ( + repo_path + / "py" + / "numcodecs-native-materialize" + / "staging" + / f"numcodecs-native-{codec}" + ) + + templates = { + "package-suffix": codec, + "package_suffix": codec.replace("-", "_"), + "crate-suffix": codec, + "crate-version": toml.load(codec_crate_path / "Cargo.toml")["package"][ + "version" + ], + "codec-id": f"{codec}.rs", + "codec-path": "".join(c.title() for c in codec.split("-")) + "Codec", + "CodecName": "".join(c.title() for c in codec.split("-")), + } + + if codec == "pco": + templates["codec-path"] = "Pcodec" + templates["CodecName"] = "Pco" + + for p in template_path.glob("**/*"): + if not p.is_file(): + continue + + np = str(staging_path / p.relative_to(template_path)) + for t, r in templates.items(): + np = np.replace(f"%{t}%", r) + np = Path(np) + + for m in template_pattern.finditer(str(np)): + raise Exception(f"unknown template {m.group(0)}") + + with p.open() as f: + c = f.read() + + for t, r in templates.items(): + c = c.replace(f"%{t}%", r) + + for line in c.splitlines(): + for m in template_pattern.finditer(line): + raise Exception(f"unknown template {m.group(0)}") + + np.parent.mkdir(parents=True, exist_ok=True) + + with np.open("w") as f: + f.write(c) + + shutil.rmtree(staging_path / ".venv", ignore_errors=True) + subprocess.run( + shlex.split("uv venv"), + check=True, + cwd=staging_path, + ) + subprocess.run( + shlex.split("uv pip install ."), + check=True, + cwd=staging_path, + env={**os.environ, "VIRTUAL_ENV": str(staging_path / ".venv")}, + ) + subprocess.run( + shlex.split( + f"uv run --no-sync python3 {Path(__file__).parent / 'stub.py'} " + + f"{'numcodecs_native_' + templates['package_suffix']} py" + ), + check=True, + cwd=staging_path, + env={**os.environ, "VIRTUAL_ENV": str(staging_path / ".venv")}, + ) + subprocess.run( + shlex.split( + f'uv run --no-sync python3 -c "from {"numcodecs_native_" + templates["package_suffix"]} ' + + f"import {templates['CodecName']} as Codec; " + + f'assert Codec.codec_id == {templates["codec-id"]!r}, Codec.codec_id"' + ), + check=True, + cwd=staging_path, + env={**os.environ, "VIRTUAL_ENV": str(staging_path / ".venv")}, + ) + shutil.rmtree(staging_path / ".venv", ignore_errors=True) + + subprocess.run( + shlex.split(f"uv build --directory {staging_path} --out-dir {dist_path}"), + check=True, + ) diff --git a/py/numcodecs-native-materialize/src/numcodecs_native_materialize/stub.py b/py/numcodecs-native-materialize/src/numcodecs_native_materialize/stub.py new file mode 100644 index 00000000..2d744a73 --- /dev/null +++ b/py/numcodecs-native-materialize/src/numcodecs_native_materialize/stub.py @@ -0,0 +1,327 @@ +# From: https://github.com/huggingface/candle/blob/main/candle-pyo3/stub.py +# See: https://raw.githubusercontent.com/huggingface/tokenizers/main/bindings/python/stub.py # noqa: E501 + +import argparse +import importlib +import inspect +import re +import sys +from pathlib import Path +from typing import ( + Any, + Optional, +) + +INDENT = " " * 4 + +LINK_USE_PATTERN = re.compile(r"\[([^]]+)\]") +LINK_DEF_PATTERN = re.compile(r"^\[([^]]+)\]: (.+)$") +INLINE_MATH_PATTERN = re.compile(r"(:?`\$)|(:?\$`)") + + +def do_indent(text: str, indent: str) -> str: + return text.replace("\n", f"\n{indent}") + + +def extract_docstring( + obj: Any, + cls: Optional[type] = None, +) -> Optional[str]: + docstring = getattr(obj, "__doc__", None) + + if docstring is not None or cls is None: + return docstring + + for m in cls.mro(): + entry = getattr(m, obj.__name__, None) + + if entry is None: + continue + + docstring = getattr(entry, "__doc__", None) + + if docstring is not None: + break + + return docstring + + +def process_docstring( + docstring: str, + module: str, +) -> str: + doc_lines = docstring.splitlines() + + link_defs = dict() + i = 0 + while i < len(doc_lines): + match = LINK_DEF_PATTERN.fullmatch(doc_lines[i]) + if match is not None and not match.group(1).startswith("^"): + link_defs[match.group(1)] = match.group(2) + doc_lines.pop(i) + else: + i += 1 + + math_block = False + + for i, line in enumerate(doc_lines): + if line.strip() == "```math" and not math_block: + line = line.replace("```math", "$$") + math_block = True + + if line.strip() == "```" and math_block: + line = line.replace("```", "$$") + math_block = False + + if not math_block: + line = INLINE_MATH_PATTERN.sub("$", line) + + matches = list(LINK_USE_PATTERN.finditer(line)) + for match in matches[::-1]: + next = None if match.end() >= len(line) else line[match.end()] + if next == "[" or next == "(": + continue + source = match.group(1) + if source.startswith("^"): + continue + target = link_defs.get(source) + if target is not None: + line = ( + line[: match.start()] + + f"[{source}]({target})" + + line[match.end() :] # noqa: E203 + ) + elif not source.startswith("Accessed: ") and source.startswith("`"): + source = source.replace("::", ".") + target = source.strip("`") + line = ( + line[: match.start()] + + f"[{source}][{module}.{target}]" + + line[match.end() :] # noqa: E203 + ) + + doc_lines[i] = line + + while len(doc_lines) > 0 and len(doc_lines[-1].strip()) == 0: + doc_lines.pop() + + return "\n".join(doc_lines) + + +def extract_and_process_docstring( + obj: Any, + cls: Optional[type] = None, + module: Optional[str] = None, +) -> Optional[str]: + docstring = extract_docstring(obj, cls=cls) + + if docstring is None: + return None + + if module is None: + module = obj.__module__ + + docstring = process_docstring(docstring, module) + + if len(docstring.strip()) == 0: + return None + + return docstring + + +def stub_function( + func: Any, + indent: str, + module: str, + name: str, + cls: Optional[type] = None, +) -> str: + text_signature = getattr(func, "__text_signature__", None) + + if text_signature is None: + text_signature = str(inspect.signature(func)) + else: + text_signature = ( + text_signature.replace("$self", "self") + .replace("$cls", "cls") + .lstrip() + .rstrip() + ) + + docstring = extract_and_process_docstring(func, cls=cls, module=module) + + string = f"{indent}def {name}{text_signature}:" + + if cls is not None: + if isinstance(inspect.getattr_static(cls, name, None), staticmethod): + string = f"{indent}@staticmethod\n{string}" + elif ( + isinstance(inspect.getattr_static(cls, name, None), classmethod) + or getattr(func, "__self__", None) is cls + ): + string = f"{indent}@classmethod\n{string}" + + if docstring is not None: + string += "\n" + string += f'{indent + INDENT}r"""\n' + string += f"{indent + INDENT}{do_indent(docstring, indent + INDENT)}\n" + string += f'{indent + INDENT}"""\n' + string += f"{indent + INDENT}...\n\n" + else: + string += " ...\n\n" + + return string + + +def stub_module( + obj: Any, + indent: str, +) -> str: + string = "" + + imports = set() + + for name, member in inspect.getmembers(obj): + if inspect.ismodule(member): + continue + if name.startswith("_"): + continue + if getattr(obj, "__all__", None) is None or name in obj.__all__: + string += stub_member( + member, indent, module=obj.__name__, imports=imports, name=name + ) + + if len(imports) > 0: + string = "\n".join(f"import {i}" for i in sorted(imports)) + "\n\n" + string + + if getattr(obj, "__all__", None) is not None: + string = f"__all__ = {sorted(set(obj.__all__))!r}\n\n" + string + + return string + + +def stub_class( + obj: type, + indent: str, + imports: set, +) -> str: + bases = [base for base in obj.__bases__ if base is not object] + + if len(bases) > 0: + inherit = [] + for base in bases: + if base.__module__ not in sys.modules: + continue + if getattr(sys.modules[base.__module__], base.__name__, None) is None: + continue + imports.add(base.__module__) + inherit.append(f"{base.__module__}.{base.__name__}") + inherit = f"({', '.join(inherit)})" + else: + inherit = "" + + string = f"class {obj.__name__}{inherit}:" + indent += INDENT + + body = "" + + docstring = extract_and_process_docstring(obj, cls=obj) + if docstring is not None: + body += f'{indent}r"""\n{indent}{do_indent(docstring, indent)}\n{indent}"""\n\n' + + # __init__ signature + if obj.__text_signature__: + text_signature = ( + obj.__text_signature__.replace("$self", "self") + .replace("$cls", "cls") + .lstrip() + .rstrip() + ) + body += f"{indent}def __init__{text_signature}: ...\n\n" + elif getattr(obj, "__init__", None) is not None: + body += stub_member( + obj.__init__, + indent=indent, + module=obj.__module__, + imports=imports, + name="__init__", + cls=obj, + ) + + for name, member in inspect.getmembers(obj): + if not name.startswith("_"): + body += stub_member( + member, + indent=indent, + module=obj.__module__, + imports=imports, + name=name, + cls=obj, + ) + + if len(body) == 0: + string += " ...\n\n" + else: + string += f"\n{body}\n\n" + + return string + + +def stub_member( + obj: Any, + indent: str, + module: str, + imports: set, + name: str, + cls: Optional[type] = None, +) -> str: + if inspect.ismodule(obj): + return stub_module(obj, indent) + + if inspect.isclass(obj): + return stub_class(obj, indent, imports) + + if inspect.isroutine(obj): + return stub_function(obj, indent, module, name, cls=cls) + + return f"{indent}{name} = {obj!r}\n\n" + + +def write_module_stubs(module: Any, directory: Path) -> None: + pyi_content = ( + stub_module( + module, + indent="", + ).strip() + + "\n" + ) + + directory.mkdir(parents=True, exist_ok=True) + + filename = directory / "__init__.pyi" + + with open(filename, "w") as f: + f.write(pyi_content) + + for name, member in inspect.getmembers(module): + if not inspect.ismodule(member): + continue + + if member.__name__ != f"{module.__name__}.{name}": + continue + + if name.startswith("_"): + continue + + write_module_stubs(member, directory / name) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("module") + parser.add_argument("output") + args = parser.parse_args() + + module = importlib.import_module(args.module) + + write_module_stubs(module, Path(args.output).joinpath(*args.module.split("."))) diff --git a/py/numcodecs-native-template/.gitignore b/py/numcodecs-native-template/.gitignore new file mode 100644 index 00000000..41c7d029 --- /dev/null +++ b/py/numcodecs-native-template/.gitignore @@ -0,0 +1,10 @@ +/.venv +/.mypy_cache +/.ruff_cache +/uv.lock + +/build +/site + +**/__pycache__ +**/*.egg-info diff --git a/py/numcodecs-native-template/.python-version b/py/numcodecs-native-template/.python-version new file mode 100644 index 00000000..c8cfe395 --- /dev/null +++ b/py/numcodecs-native-template/.python-version @@ -0,0 +1 @@ +3.10 diff --git a/py/numcodecs-native-template/Cargo.toml b/py/numcodecs-native-template/Cargo.toml new file mode 100644 index 00000000..7f05d4e6 --- /dev/null +++ b/py/numcodecs-native-template/Cargo.toml @@ -0,0 +1,20 @@ +[workspace] + +[package] +name = "numcodecs-native-%crate-suffix%" +version = "%crate-version%" +edition = "2024" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[lib] +name = "numcodecs_wasm" +crate-type = ["cdylib"] + +[dependencies] +numcodecs = { version = "0.3.2", default-features = false } +numcodecs-my-codec = { package = "numcodecs-%crate-suffix%", version = "%crate-version%", default-features = false } +numcodecs-python = { version = "0.7.2", default-features = false } +pyo3 = { version = "0.26", default-features = false, features = ["macros", "abi3-py310"] } +pyo3-error = { version = "0.6", default-features = false } +pyo3-log = { version = "0.13.0", default-features = false } diff --git a/py/numcodecs-native-template/LICENSE b/py/numcodecs-native-template/LICENSE new file mode 120000 index 00000000..30cff740 --- /dev/null +++ b/py/numcodecs-native-template/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/py/numcodecs-native-template/README.md b/py/numcodecs-native-template/README.md new file mode 100644 index 00000000..6baf1687 --- /dev/null +++ b/py/numcodecs-native-template/README.md @@ -0,0 +1,27 @@ +[![CI Status]][workflow] [![PyPi Release]][pypi] [![Read the Docs]][rtdocs] + +[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain + +[PyPi Release]: https://img.shields.io/pypi/v/numcodecs-native-%package-suffix%.svg +[pypi]: https://pypi.python.org/pypi/numcodecs-native-%package-suffix% + +[Read the Docs]: https://img.shields.io/readthedocs/numcodecs-wasm?label=readthedocs +[rtdocs]: https://numcodecs-wasm.readthedocs.io/en/stable/api/numcodecs_wasm_%package_suffix%/ + +# numcodecs-native-%package-suffix% + +[`numcodecs`] compression for the [`numcodecs-%crate-suffix%`] codec compiled to native. + +[`numcodecs`]: https://numcodecs.readthedocs.io/en/stable/ +[`numcodecs-%crate-suffix%`]: https://docs.rs/numcodecs-%crate-suffix%/ + +## License + +Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/). + +## Funding + +The `numcodecs-native-%package-suffix%` package has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe. + +Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054. diff --git a/py/numcodecs-native-template/py/numcodecs_native_%package_suffix%/__init__.py b/py/numcodecs-native-template/py/numcodecs_native_%package_suffix%/__init__.py new file mode 100644 index 00000000..9742cf44 --- /dev/null +++ b/py/numcodecs-native-template/py/numcodecs_native_%package_suffix%/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["%CodecName%"] + +from ._native import %CodecName% diff --git a/py/numcodecs-native-template/py/numcodecs_native_%package_suffix%/py.typed b/py/numcodecs-native-template/py/numcodecs_native_%package_suffix%/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/py/numcodecs-native-template/pyproject.toml b/py/numcodecs-native-template/pyproject.toml new file mode 100644 index 00000000..e48d1a0d --- /dev/null +++ b/py/numcodecs-native-template/pyproject.toml @@ -0,0 +1,34 @@ +[build-system] +requires = ["maturin>=1.8,<2.0"] +build-backend = "maturin" + +[project] +name = "numcodecs_native_%package_suffix%" +version = "%crate-version%" +description = "numcodecs-%crate-suffix% compression codec compiled to native" + +authors = [{ name = "Juniper Tyree", email = "juniper.tyree@helsinki.fi" }] +maintainers = [{ name = "Juniper Tyree", email = "juniper.tyree@helsinki.fi" }] + +readme = "README.md" +license = { file = "LICENSE" } + +requires-python = ">=3.10" + +dependencies = [ + "numcodecs>=0.13.0,<0.17", +] + +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + +[project.entry-points."numcodecs.codecs"] +"%codec-id%" = "numcodecs_native_%package_suffix%:%CodecName%" + +[tool.maturin] +python-source = "py" +module-name = "numcodecs_native_%package_suffix%._native" +features = ["pyo3/extension-module"] diff --git a/py/numcodecs-native-template/src/lib.rs b/py/numcodecs-native-template/src/lib.rs new file mode 100644 index 00000000..bb90659d --- /dev/null +++ b/py/numcodecs-native-template/src/lib.rs @@ -0,0 +1,18 @@ +use numcodecs::StaticCodecType; +use pyo3::prelude::*; +use numcodecs_my_codec::%codec-path% as MyCodec; + +#[pymodule] +#[pyo3(name = "_native")] +fn native<'py>(py: Python<'py>, module: &Bound<'py, PyModule>) -> Result<(), PyErr> { + let logger = pyo3_log::Logger::new(py, pyo3_log::Caching::Nothing)?; + logger + .install() + .map_err(|err| pyo3_error::PyErrChain::new(py, err))?; + + numcodecs_python::export_codec_class( + py, StaticCodecType::::of(), module.as_borrowed() + )?; + + Ok(()) +}