Skip to content

Commit 06c1541

Browse files
Merge branch 'main' into matmul-optim
2 parents 4ca52a2 + ade5b7b commit 06c1541

64 files changed

Lines changed: 17795 additions & 359 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.pre-commit-config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ repos:
1616
- id: trailing-whitespace
1717

1818
- repo: https://github.com/astral-sh/ruff-pre-commit
19-
rev: v0.15.4
19+
rev: v0.15.9
2020
hooks:
2121
- id: ruff-check
2222
args: ["--fix", "--show-fixes"]

AGENTS.md

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
# Repository Guidelines
2+
3+
## Project Structure & Module Organization
4+
The Python package lives in `src/blosc2/`, including the C/Cython extension sources
5+
(`blosc2_ext.*`) and core modules such as `core.py`, `ndarray.py`, and `schunk.py`.
6+
Tests are under `tests/`, with additional doctests enabled for select modules per
7+
`pytest.ini`. Documentation sources are in `doc/` and build output lands in `html/`.
8+
Examples are in `examples/`, and performance/benchmark scripts live in `bench/`.
9+
10+
## Build, Test, and Development Commands
11+
- `pip install .` builds the bundled C-Blosc2 and installs the package.
12+
- `pip install -e .` installs in editable mode for local development.
13+
- `CMAKE_PREFIX_PATH=/usr/local USE_SYSTEM_BLOSC2=1 pip install -e .` builds
14+
against a separately installed C-Blosc2.
15+
- `pytest` runs the default test suite (excludes `heavy` and `network` markers).
16+
- `pytest -m "heavy"` runs long-running tests.
17+
- `pytest -m "network"` runs tests requiring network access.
18+
- `cd doc && rm -rf ../html _build && python -m sphinx . ../html` builds docs.
19+
20+
## Coding Style & Naming Conventions
21+
Use Ruff for formatting and linting (line length 109). Enable pre-commit hooks:
22+
`python -m pip install pre-commit` then `pre-commit install`. Follow Python
23+
conventions: 4-space indentation, `snake_case` for functions/variables, and
24+
`PascalCase` for classes. Pytest discovery expects `tests/test_*.py` and
25+
`test_*` functions. Do not use leading underscores in module-level helper
26+
function names when those helpers are imported from other modules; reserve
27+
leading underscores for file-local implementation details. Avoid leading
28+
underscores in core module filenames under `src/blosc2/`; prefer non-underscored
29+
module names unless there is a strong reason to keep a module private.
30+
31+
For documentation and tutorial query examples, prefer the shortest idiom that
32+
matches the intended result type. Use `expr[:]` or `arr[mask][:]` when showing
33+
values, use `expr.compute()` when materializing an `NDArray`, and use
34+
`expr.compute(_use_index=False)` when demonstrating scan-vs-index behavior.
35+
Avoid `expr.compute()[:]` unless a NumPy array is specifically required.
36+
37+
## Testing Guidelines
38+
Pytest is required; warnings are treated as errors. The default configuration
39+
adds `--doctest-modules`, so keep doctest examples in `blosc2/core.py`,
40+
`blosc2/ndarray.py`, and `blosc2/schunk.py` accurate. Use markers `heavy` and
41+
`network` for slow or network-dependent tests.
42+
43+
## Commit & Pull Request Guidelines
44+
Recent commit messages are short, imperative sentences (e.g., “Add …”, “Fix …”)
45+
without ticket prefixes. For pull requests: branch from `main`, add tests for
46+
behavior changes, update docs for API changes, ensure the test suite passes,
47+
and avoid introducing new compiler warnings. Link issues when applicable and
48+
include clear reproduction steps for bug fixes.

CMakeLists.txt

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -41,16 +41,25 @@ add_custom_command(
4141
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/blosc2_ext.pyx"
4242
VERBATIM)
4343

44+
add_custom_command(
45+
OUTPUT indexing_ext.c
46+
COMMAND Python::Interpreter -m cython
47+
"${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/indexing_ext.pyx" --output-file indexing_ext.c
48+
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2/indexing_ext.pyx"
49+
VERBATIM)
50+
4451
# ...and add it to the target
4552
Python_add_library(blosc2_ext MODULE blosc2_ext.c WITH_SOABI)
4653
target_sources(blosc2_ext PRIVATE src/blosc2/matmul_kernels.c)
4754
target_include_directories(blosc2_ext PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/blosc2)
4855
if(UNIX)
4956
target_link_libraries(blosc2_ext PRIVATE ${CMAKE_DL_LIBS})
5057
endif()
58+
Python_add_library(indexing_ext MODULE indexing_ext.c WITH_SOABI)
5159

5260
# We need to link against NumPy
5361
target_link_libraries(blosc2_ext PRIVATE Python::NumPy)
62+
target_link_libraries(indexing_ext PRIVATE Python::NumPy)
5463

5564
# Fetch and build miniexpr library
5665
include(FetchContent)
@@ -68,7 +77,7 @@ endif()
6877

6978
FetchContent_Declare(miniexpr
7079
GIT_REPOSITORY https://github.com/Blosc/miniexpr.git
71-
GIT_TAG 37bf6982bf9619036b47f095b7005bc3c87a7447
80+
GIT_TAG f2faef741c4c507bf6a03167c72ce7f92c6f0ae8
7281
# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../miniexpr
7382
)
7483
FetchContent_MakeAvailable(miniexpr)
@@ -80,6 +89,7 @@ if(APPLE)
8089
endif()
8190

8291
target_compile_features(blosc2_ext PRIVATE c_std_11)
92+
target_compile_features(indexing_ext PRIVATE c_std_11)
8393
if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang")
8494
execute_process(
8595
COMMAND "${CMAKE_C_COMPILER}" -print-resource-dir
@@ -127,7 +137,7 @@ else()
127137
include(FetchContent)
128138
FetchContent_Declare(blosc2
129139
GIT_REPOSITORY https://github.com/Blosc/c-blosc2
130-
GIT_TAG 9200990b189c8357e5517860cfa9ef09cb117eae
140+
GIT_TAG 0568990388e6201240b170947d4c2199572f795d
131141
# SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../c-blosc2
132142
)
133143
FetchContent_MakeAvailable(blosc2)
@@ -156,7 +166,7 @@ endif()
156166

157167
# Python extension -> site-packages/blosc2
158168
install(
159-
TARGETS blosc2_ext
169+
TARGETS blosc2_ext indexing_ext
160170
LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/blosc2
161171
)
162172

bench/batch_store.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
NBATCHES = 10_000
2020
OBJECTS_PER_BATCH = 100
2121
TOTAL_OBJECTS = NBATCHES * OBJECTS_PER_BATCH
22-
BLOCKSIZE_MAX = 32
22+
ITEMS_PER_BLOCK = 32
2323
N_RANDOM_READS = 1_000
2424

2525

@@ -64,7 +64,7 @@ def build_store(
6464
storage = blosc2.Storage(mode="w")
6565
store = blosc2.BatchStore(
6666
storage=storage,
67-
max_blocksize=BLOCKSIZE_MAX,
67+
items_per_block=ITEMS_PER_BLOCK,
6868
serializer=serializer,
6969
cparams={
7070
"codec": codec,
@@ -84,7 +84,7 @@ def build_store(
8484
"use_dict": use_dict and codec in (blosc2.Codec.ZSTD, blosc2.Codec.LZ4, blosc2.Codec.LZ4HC),
8585
}
8686
with blosc2.BatchStore(
87-
storage=storage, max_blocksize=BLOCKSIZE_MAX, serializer=serializer, cparams=cparams
87+
storage=storage, items_per_block=ITEMS_PER_BLOCK, serializer=serializer, cparams=cparams
8888
) as store:
8989
for batch_index in range(NBATCHES):
9090
store.append(make_batch(batch_index))
@@ -132,7 +132,7 @@ def main() -> None:
132132
assert store is not None
133133
read_store = store
134134
else:
135-
read_store = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, max_blocksize=BLOCKSIZE_MAX)
135+
read_store = blosc2.BatchStore(urlpath=URLPATH, mode="r", contiguous=True, items_per_block=ITEMS_PER_BLOCK)
136136
samples, timings_ns = measure_random_reads(read_store)
137137
t0 = time.perf_counter()
138138
checksum = 0
@@ -147,7 +147,7 @@ def main() -> None:
147147
print(f" build time: {build_time_s:.3f} s")
148148
print(f" batches: {len(read_store)}")
149149
print(f" items: {TOTAL_OBJECTS}")
150-
print(f" max_blocksize: {read_store.max_blocksize}")
150+
print(f" items_per_block: {read_store.items_per_block}")
151151
print()
152152
print(read_store.info)
153153
print(f"Random scalar reads: {N_RANDOM_READS}")

0 commit comments

Comments
 (0)