Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@ sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev

in ubuntu

`boutdata` also uses [`h5py`](https://www.h5py.org) to read NetCDF-4 files,
because it opens files much faster than `netCDF4`. If `h5py` is not
installed, `netCDF4` is used. To always use `netCDF4` for reading, set the
environment variable `BOUTDATA_NETCDF_READER=netcdf4`.

# Install

`pip install boutdata`
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ dependencies = [
"natsort>=8.1.0",
"scipy >= 1.12.0",
"netCDF4",
"h5py",
]
dynamic = ["version"]

Expand Down
23 changes: 18 additions & 5 deletions src/boutdata/collect.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,17 @@ def collect(
do not have to re-open the dump files to read another variable
(default: None)

Notes
-----
NetCDF-4 dump files are read using h5py if it is installed, because it opens
files much faster than netCDF4. Set the environment variable
``BOUTDATA_NETCDF_READER=netcdf4`` to always use netCDF4 (see
:py:mod:`boututils.datafile`).

When reading many variables, it is still faster to pass a `datafile_cache`, or
to use :py:obj:`~boutdata.data.BoutData`, so that the files are only opened
once.

Examples
--------

Expand Down Expand Up @@ -282,9 +293,8 @@ def getDataFile(i):
f.close()
return result

if datafile_cache is None:
# close the DataFile if we are not keeping it in a cache
f.close()
# Keep the 0'th file open: it is read again in the loop over processors below
f0 = f

# Create a list with size of each dimension
ddims = [grid_info["sizes"][d] for d in dimensions]
Expand All @@ -301,7 +311,7 @@ def getDataFile(i):
is_fieldperp = False

for i in range(grid_info["npes"]):
f = getDataFile(i)
f = f0 if i == 0 else getDataFile(i)
temp_yindex, temp_f_attributes = _collect_from_one_proc(
i,
f,
Expand Down Expand Up @@ -664,7 +674,10 @@ def _collect_from_one_proc(
# No data for FieldPerp on this processor
return None, None

result[global_slices] = datafile.read(varname, ranges=local_slices)
# Attributes are not needed here, so do not create a BoutArray
result[global_slices] = datafile.read(
varname, ranges=local_slices, asBoutArray=False
)

if is_fieldperp:
return temp_yindex, f_attributes
Expand Down
12 changes: 12 additions & 0 deletions src/boutdata/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
import pytest

import boututils.datafile


@pytest.fixture(params=["h5py", "netcdf4"])
def netcdf_reader(request, monkeypatch):
"""Run a test with each of the libraries that can be used to read NetCDF files"""
if request.param == "h5py":
pytest.importorskip("h5py")
monkeypatch.setattr(boututils.datafile, "netcdf_reader", request.param)
return request.param
28 changes: 28 additions & 0 deletions src/boutdata/tests/test_collect.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@

# Note - using tmp_path fixture requires pytest>=3.9.0

# Run every test with each library that can read NetCDF files
pytestmark = pytest.mark.usefixtures("netcdf_reader")


collect_kwargs_list = [
pytest.param({"xguards": True, "yguards": "include_upper"}, id="collect_guards"),
pytest.param({"xguards": False, "yguards": False}, id="collect_noguards"),
Expand Down Expand Up @@ -1279,3 +1283,27 @@ def test_disconnected_doublenull_with_compression(
collect_kwargs=collect_kwargs,
squash_kwargs=squash_kwargs,
)

@pytest.mark.parametrize("scenario", ["core_min", "single_null_full"])
def test_collect_opens_each_file_once(
self, tmp_path, scenario, request, monkeypatch
):
"""Check that collecting a field opens each dump file exactly once"""
data_path, _, _ = request.getfixturevalue(scenario)
symlink_dump_files(data_path, tmp_path)

with DataFile(tmp_path / "BOUT.dmp.0.nc") as f:
npes = int(f["NXPE"]) * int(f["NYPE"])

opened_files = []
original_init = DataFile.__init__

def counting_init(self, filename=None, *args, **kwargs):
opened_files.append(filename)
original_init(self, filename, *args, **kwargs)

monkeypatch.setattr(DataFile, "__init__", counting_init)

collect("field3d_t_1", path=tmp_path, info=False)

assert len(opened_files) == npes
Loading