From 30681d3f5c90ee200cd1c8bd591971906012ba28 Mon Sep 17 00:00:00 2001 From: toto Date: Tue, 3 Nov 2026 13:36:27 +0100 Subject: [PATCH 1/5] Reuse the 0'th dump file in collect instead of reopening it collect() opened file 0 to read the grid info, closed it, then opened it again in the loop over processors. Keep it open so that each dump file is opened exactly once per call, and add a test counting the opens. Co-Authored-By: Claude Opus 5.5 --- src/boutdata/collect.py | 7 +++---- src/boutdata/tests/test_collect.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/boutdata/collect.py b/src/boutdata/collect.py index 30d9950ea..9f8e5e27c 100644 --- a/src/boutdata/collect.py +++ b/src/boutdata/collect.py @@ -282,9 +282,8 @@ def getDataFile(i): f.close() return result - if datafile_cache is None: - # close the DataFile if we are not keeping it in a cache - f.close() + # Keep the 0'th file open: it is read again in the loop over processors below + f0 = f # Create a list with size of each dimension ddims = [grid_info["sizes"][d] for d in dimensions] @@ -301,7 +300,7 @@ def getDataFile(i): is_fieldperp = False for i in range(grid_info["npes"]): - f = getDataFile(i) + f = f0 if i == 0 else getDataFile(i) temp_yindex, temp_f_attributes = _collect_from_one_proc( i, f, diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py index 2862d4e15..9b73274a2 100644 --- a/src/boutdata/tests/test_collect.py +++ b/src/boutdata/tests/test_collect.py @@ -1279,3 +1279,27 @@ def test_disconnected_doublenull_with_compression( collect_kwargs=collect_kwargs, squash_kwargs=squash_kwargs, ) + + @pytest.mark.parametrize("scenario", ["core_min", "single_null_full"]) + def test_collect_opens_each_file_once( + self, tmp_path, scenario, request, monkeypatch + ): + """Check that collecting a field opens each dump file exactly once""" + data_path, _, _ = request.getfixturevalue(scenario) + symlink_dump_files(data_path, tmp_path) + + with DataFile(tmp_path / "BOUT.dmp.0.nc") as f: + npes = int(f["NXPE"]) * int(f["NYPE"]) + + opened_files = [] + original_init = DataFile.__init__ + + def counting_init(self, filename=None, *args, **kwargs): + opened_files.append(filename) + original_init(self, filename, *args, **kwargs) + + monkeypatch.setattr(DataFile, "__init__", counting_init) + + collect("field3d_t_1", path=tmp_path, info=False) + + assert len(opened_files) == npes From 6e894286162d4c9fc0e337802934c000182c87d3 Mon Sep 17 00:00:00 2001 From: toto Date: Tue, 3 Nov 2026 13:58:26 +0100 Subject: [PATCH 2/5] Read NetCDF-4 files with h5py when possible netCDF4 reads the metadata of every variable when it opens a file, which takes ~30 ms for a typical BOUT++ dump file. collect() opens every dump file for every variable, so opening files dominated its run time. Add DataFile_netCDF_h5py, a read-only DataFile backend that reads NetCDF-4 (HDF5) files with h5py. It gives the same results as DataFile_netCDF: variable order, dimensions, sizes, attributes, fill value masking. Variables it cannot read identically (string arrays, packed data, variables shorter than an unlimited dimension) are read with netCDF4. Metadata is cached per file, keyed on path, modification time, size and inode, so it is read again if the file changes. DataFile uses the new backend for read-only .nc/.ncdf HDF5 files when h5py is installed. The BOUTDATA_NETCDF_READER environment variable (or boututils.datafile.netcdf_reader) selects "auto", "netcdf4" or "h5py". test_collect.py and test_restart.py now run with both readers, and test_datafile.py checks that the readers give identical results. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + src/boutdata/tests/conftest.py | 12 + src/boutdata/tests/test_collect.py | 4 + src/boutdata/tests/test_datafile.py | 317 +++++++++++++++++++ src/boutdata/tests/test_restart.py | 4 + src/boututils/datafile.py | 470 +++++++++++++++++++++++++++- 6 files changed, 807 insertions(+), 1 deletion(-) create mode 100644 src/boutdata/tests/conftest.py create mode 100644 src/boutdata/tests/test_datafile.py diff --git a/pyproject.toml b/pyproject.toml index 57bfd5aa8..2b176dc1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "natsort>=8.1.0", "scipy >= 1.12.0", "netCDF4", + "h5py", ] dynamic = ["version"] diff --git a/src/boutdata/tests/conftest.py b/src/boutdata/tests/conftest.py new file mode 100644 index 000000000..15ebefbd5 --- /dev/null +++ b/src/boutdata/tests/conftest.py @@ -0,0 +1,12 @@ +import pytest + +import boututils.datafile + + +@pytest.fixture(params=["h5py", "netcdf4"]) +def netcdf_reader(request, monkeypatch): + """Run a test with each of the libraries that can be used to read NetCDF files""" + if request.param == "h5py": + pytest.importorskip("h5py") + monkeypatch.setattr(boututils.datafile, "netcdf_reader", request.param) + return request.param diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py index 9b73274a2..8cfce45ec 100644 --- a/src/boutdata/tests/test_collect.py +++ b/src/boutdata/tests/test_collect.py @@ -23,6 +23,10 @@ # Note - using tmp_path fixture requires pytest>=3.9.0 +# Run every test with each library that can read NetCDF files +pytestmark = pytest.mark.usefixtures("netcdf_reader") + + collect_kwargs_list = [ pytest.param({"xguards": True, "yguards": "include_upper"}, id="collect_guards"), pytest.param({"xguards": False, "yguards": False}, id="collect_noguards"), diff --git a/src/boutdata/tests/test_datafile.py b/src/boutdata/tests/test_datafile.py new file mode 100644 index 000000000..d4cfd1f21 --- /dev/null +++ b/src/boutdata/tests/test_datafile.py @@ -0,0 +1,317 @@ +import subprocess +import sys + +import numpy as np +import pytest +from netCDF4 import Dataset, default_fillvals + +import boututils.datafile +from boutdata.tests.make_test_data import ( + create_dump_file, + create_restart_file, + make_grid_info, +) +from boututils.datafile import DataFile, DataFile_netCDF + +h5py = pytest.importorskip("h5py") +from boututils.datafile import DataFile_netCDF_h5py + + +def create_unusual_file(path): + """Create a NetCDF-4 file with features not found in the BOUT++ test files""" + with Dataset(path, "w") as f: + f.createDimension("t", None) + f.createDimension("x", 4) + f.createDimension("y", 3) + f.createDimension("char5", 5) + f.setncattr("title", "unusual file") + f.setncattr("numbers", np.array([1.0, 2.0])) + f.setncattr("one_number", np.int32(3)) + f.setncattr_string("strings", ["a", "bc"]) + + # Time-dependent variable with fewer time points than another variable + v = f.createVariable("short_t", float, ("t", "x")) + v[:] = np.arange(8.0).reshape(2, 4) + v = f.createVariable("long_t", float, ("t", "y")) + v[:] = np.arange(9.0).reshape(3, 3) + v.setncattr("multiple", np.array([1, 2, 3], dtype=np.int32)) + v.setncattr("empty_string", "") + v.setncattr_string("vlen_string", "vlen") + + # Coordinate variable, which is its own dimension + v = f.createVariable("t", float, ("t",)) + v[:] = [0.0, 1.0, 2.0] + + # Variable with the name of a dimension, but different dimensions + v = f.createVariable("y", float, ("x",)) + v[:] = np.arange(4.0) + + # Fill values, with and without the _FillValue attribute + v = f.createVariable("with_fill", float, ("x", "y"), fill_value=-1.0) + v[:] = np.arange(12.0).reshape(4, 3) + v[1, 1] = -1.0 + v = f.createVariable("default_fill", float, ("x", "y")) + data = np.arange(12.0).reshape(4, 3) + data[2, 0] = default_fillvals["f8"] + v[:] = data + v = f.createVariable("with_nan_fill", float, ("x",), fill_value=np.nan) + v[:] = [0.0, np.nan, 2.0, 3.0] + v = f.createVariable("byte_var", np.int8, ("x",)) + v[:] = [1, 2, 3, 4] + + # Variables that netCDF4 converts when reading + v = f.createVariable("packed", np.int16, ("x",)) + v.setncattr("scale_factor", 0.5) + v.setncattr("add_offset", 1.0) + v[:] = [1.0, 2.0, 3.0, 4.0] + + # Strings + v = f.createVariable("string_scalar", str, ()) + v[0] = "hello" + v = f.createVariable("string_array", str, ("x",)) + v[:] = np.array(["a", "bb", "ccc", "dddd"], dtype=object) + v = f.createVariable("chars", "S1", ("t", "char5")) + v[:] = np.array([list("abcde"), list("fghij"), list("klmno")], dtype="S1") + + # Variables in a group are not listed by netCDF4 + g = f.createGroup("section") + v = g.createVariable("in_group", float, ("x",)) + v[:] = np.arange(4.0) + + v = f.createVariable("int_scalar", np.int32, ()) + v.assignValue(7) + + +@pytest.fixture(scope="module") +def test_files(tmp_path_factory): + """NetCDF-4 files to compare the two readers on""" + tmp_path = tmp_path_factory.mktemp("datafile") + grid_info = make_grid_info() + rng = np.random.default_rng(42) + create_dump_file( + i=0, + tmpdir=tmp_path, + rng=rng, + grid_info=grid_info, + boundaries=["xinner", "xouter", "ylower", "yupper"], + fieldperp_global_yind=3, + ) + create_restart_file( + i=0, tmpdir=tmp_path, rng=rng, grid_info=grid_info, fieldperp_global_yind=3 + ) + create_unusual_file(tmp_path / "unusual.nc") + return { + "dump": tmp_path / "BOUT.dmp.0.nc", + "restart": tmp_path / "BOUT.restart.0.nc", + "unusual": tmp_path / "unusual.nc", + } + + +def assert_same(expected, actual, what): + """Check that two results are identical, including their types""" + assert type(actual) is type(expected), what + if isinstance(expected, np.ndarray): + assert actual.dtype == expected.dtype, what + assert actual.shape == expected.shape, what + np.testing.assert_array_equal( + np.asarray(actual), np.asarray(expected), err_msg=what + ) + np.testing.assert_array_equal( + np.ma.getmaskarray(actual), np.ma.getmaskarray(expected), err_msg=what + ) + if hasattr(expected, "attributes"): + assert_same(expected.attributes, actual.attributes, f"{what}.attributes") + elif isinstance(expected, dict): + assert list(actual) == list(expected), what + for key in expected: + assert_same(expected[key], actual[key], f"{what}[{key}]") + elif isinstance(expected, list): + assert len(actual) == len(expected), what + for i, (e, a) in enumerate(zip(expected, actual)): + assert_same(e, a, f"{what}[{i}]") + elif isinstance(expected, float) and np.isnan(expected): + assert np.isnan(actual), what + else: + assert actual == expected, what + + +def call_or_error(method, *args, **kwargs): + """Result of a call, or the name of the exception it raised""" + try: + return method(*args, **kwargs) + except Exception as e: # noqa: BLE001 + return type(e).__name__ + + +@pytest.mark.parametrize("file", ["dump", "restart", "unusual"]) +def test_h5py_reader_matches_netcdf4(test_files, file): + path = test_files[file] + with DataFile_netCDF(path) as expected, DataFile_netCDF_h5py(path) as actual: + check_same_contents(expected, actual) + + +def check_same_contents(expected, actual): + assert_same(expected.list(), actual.list(), "list()") + assert_same( + expected.list_file_attributes(), + actual.list_file_attributes(), + "list_file_attributes()", + ) + for name in expected.list_file_attributes(): + assert_same( + expected.read_file_attribute(name), + actual.read_file_attribute(name), + f"read_file_attribute({name})", + ) + with pytest.raises(AttributeError): + actual.read_file_attribute("not_an_attribute") + + for name in expected.list(): + assert_same(expected.dimensions(name), actual.dimensions(name), f"{name} dims") + assert_same(expected.size(name), actual.size(name), f"{name} size") + assert_same(expected.ndims(name), actual.ndims(name), f"{name} ndims") + assert_same(expected.attributes(name), actual.attributes(name), f"{name} attrs") + for asBoutArray in [True, False]: + assert_same( + expected.read(name, asBoutArray=asBoutArray), + actual.read(name, asBoutArray=asBoutArray), + f"{name} read(asBoutArray={asBoutArray})", + ) + + ndims = expected.ndims(name) + if ndims == 0: + continue + ranges_list = [ + [slice(1, None, 2)] + [slice(0, 2)] * (ndims - 1), + [slice(0, 1)] + [None] * (ndims - 1), + [0, 1] * ndims, + [slice(None, None, -1)] * ndims, + ] + for ranges in ranges_list: + for asBoutArray in [True, False]: + assert_same( + call_or_error(expected.read, name, ranges, asBoutArray), + call_or_error(actual.read, name, ranges, asBoutArray), + f"{name} read({ranges}, asBoutArray={asBoutArray})", + ) + + # Variables that do not exist, or are only NetCDF dimensions + for name in ["not_a_variable", "x", "section", "section/in_group"]: + assert actual.read(name) is None, name + assert actual.size(name) == [], name + with pytest.raises(ValueError): + actual.dimensions(name) + + +def test_h5py_reader_case_insensitive(test_files): + path = test_files["dump"] + with DataFile_netCDF(path) as expected, DataFile_netCDF_h5py(path) as actual: + assert_same(expected.read("FIELD3D_T_1"), actual.read("FIELD3D_T_1"), "read") + assert_same( + call_or_error(expected.attributes, "FIELD3D_T_1"), + call_or_error(actual.attributes, "FIELD3D_T_1"), + "attributes", + ) + + +def test_h5py_reader_is_read_only(test_files): + with DataFile_netCDF_h5py(test_files["dump"]) as f: + with pytest.raises(Exception, match="File not writeable"): + f.write("new_variable", np.zeros(3)) + with pytest.raises(Exception, match="File not writeable"): + f.write_file_attribute("new_attribute", 1) + with pytest.raises(ValueError): + DataFile_netCDF_h5py(test_files["dump"], write=True) + + +def test_h5py_reader_attributes_are_copies(test_files): + """Changing returned attributes must not change the cached metadata""" + with DataFile_netCDF_h5py(test_files["dump"]) as f: + first = f.attributes("field3d_t_1") + first["new_attribute"] = 1 + with DataFile_netCDF_h5py(test_files["dump"]) as f: + second = f.attributes("field3d_t_1") + assert "new_attribute" not in second + + +def test_h5py_reader_sees_changed_file(tmp_path): + """Cached metadata must not be used after a file has been changed""" + path = tmp_path / "changing.nc" + with Dataset(path, "w") as f: + f.createDimension("x", 2) + f.createVariable("a", float, ("x",))[:] = [1.0, 2.0] + with DataFile_netCDF_h5py(path) as f: + assert f.list() == ["a"] + + with Dataset(path, "a") as f: + f.createVariable("b", float, ("x",))[:] = [3.0, 4.0] + with DataFile_netCDF_h5py(path) as f: + assert f.list() == ["a", "b"] + + +@pytest.fixture +def netcdf3_file(tmp_path): + path = tmp_path / "netcdf3.nc" + with Dataset(path, "w", format="NETCDF3_CLASSIC") as f: + f.createDimension("x", 2) + f.createVariable("a", float, ("x",))[:] = [1.0, 2.0] + return path + + +def test_dispatch_auto(test_files, netcdf3_file, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "auto") + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF_h5py + # NetCDF-3 files are not HDF5 files + with DataFile(netcdf3_file) as f: + assert type(f.impl) is DataFile_netCDF + # Writing needs netCDF4 + with DataFile(netcdf3_file, write=True) as f: + assert type(f.impl) is DataFile_netCDF + with DataFile(test_files["dump"].parent / "new.nc", create=True) as f: + assert type(f.impl) is DataFile_netCDF + + monkeypatch.setattr(boututils.datafile, "has_h5py", False) + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF + + +def test_dispatch_netcdf4(test_files, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "netcdf4") + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF + + +def test_dispatch_h5py(test_files, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "h5py") + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF_h5py + + monkeypatch.setattr(boututils.datafile, "has_h5py", False) + with pytest.raises(ImportError): + DataFile(test_files["dump"]) + + +def test_dispatch_invalid(test_files, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "not_a_library") + with pytest.raises(ValueError): + DataFile(test_files["dump"]) + + +@pytest.mark.parametrize( + ("reader", "expected"), + [("netcdf4", "DataFile_netCDF"), ("auto", "DataFile_netCDF_h5py")], +) +def test_dispatch_environment_variable(test_files, monkeypatch, reader, expected): + monkeypatch.setenv("BOUTDATA_NETCDF_READER", reader) + code = ( + "from boututils.datafile import DataFile; " + f"print(type(DataFile({str(test_files['dump'])!r}).impl).__name__)" + ) + result = subprocess.run( + [sys.executable, "-W", "ignore", "-c", code], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == expected diff --git a/src/boutdata/tests/test_restart.py b/src/boutdata/tests/test_restart.py index 8edf31b6a..290ca1d9e 100644 --- a/src/boutdata/tests/test_restart.py +++ b/src/boutdata/tests/test_restart.py @@ -2,6 +2,7 @@ import numpy as np import numpy.testing as npt +import pytest from boutdata import restart from boutdata.collect import collect @@ -12,6 +13,9 @@ make_grid_info, ) +# Run every test with each library that can read NetCDF files +pytestmark = pytest.mark.usefixtures("netcdf_reader") + def check_redistributed_data( expected, diff --git a/src/boututils/datafile.py b/src/boututils/datafile.py index a8eef2b99..0c7f5b641 100644 --- a/src/boututils/datafile.py +++ b/src/boututils/datafile.py @@ -8,6 +8,15 @@ - ``h5py`` (for HDF5 files) - ``netCDF4`` (preferred NetCDF library) +NetCDF-4 files are HDF5 files. When ``h5py`` is available, NetCDF-4 files +opened read-only are read with ``h5py``, which opens files much faster than +``netCDF4``. The reader can be chosen with the environment variable +``BOUTDATA_NETCDF_READER`` (or by setting ``boututils.datafile.netcdf_reader``): + +- ``"auto"`` (default): use ``h5py`` if it is installed, otherwise ``netCDF4`` +- ``"netcdf4"``: always use ``netCDF4`` +- ``"h5py"``: always use ``h5py`` for NetCDF-4 files opened read-only + NOTE ---- NetCDF and HDF5 include unlimited dimensions, but this library is just @@ -23,6 +32,7 @@ """ +import os import pathlib import numpy as np @@ -31,7 +41,7 @@ from boututils.boutwarnings import alwayswarn try: - from netCDF4 import Dataset + from netCDF4 import Dataset, default_fillvals has_netCDF = True except ImportError: @@ -46,6 +56,26 @@ except ImportError: has_h5py = False +# Which library to use to read NetCDF-4 files: "auto", "netcdf4" or "h5py" +netcdf_reader = os.environ.get("BOUTDATA_NETCDF_READER", "auto") + + +def _use_h5py_reader(filename, write, create): + """Should the NetCDF file `filename` be read with h5py?""" + if netcdf_reader not in ("auto", "netcdf4", "h5py"): + raise ValueError( + f"Invalid netcdf_reader '{netcdf_reader}'. Should be 'auto', 'netcdf4' " + "or 'h5py'" + ) + if write or create or netcdf_reader == "netcdf4": + return False + if netcdf_reader == "h5py" and not has_h5py: + raise ImportError("netcdf_reader is 'h5py' but h5py is not available") + if not has_h5py or filename.suffix not in (".nc", ".ncdf"): + return False + # NetCDF-3 files are not HDF5 files, so need to be read with netCDF4 + return h5py.is_hdf5(filename) + class DataFile: """File I/O class @@ -97,6 +127,8 @@ def __init__( self.impl = DataFile_HDF5( filename=filename, write=write, create=create, format=format ) + elif _use_h5py_reader(filename, write, create): + self.impl = DataFile_netCDF_h5py(filename=filename) else: self.impl = DataFile_netCDF( filename=filename, @@ -720,6 +752,442 @@ def attributes(self, varname): return attributes +class DataFile_netCDF_h5py(DataFile_netCDF): + """Read-only access to NetCDF-4 files using h5py + + NetCDF-4 files are HDF5 files, and h5py opens them much faster than netCDF4, + which reads the metadata of all variables when opening a file. Results are the + same as from DataFile_netCDF. Variables that h5py cannot read in the same way + as netCDF4 (e.g. strings arrays or packed data) are read using DataFile_netCDF. + """ + + # Attributes used internally by NetCDF-4 and HDF5 dimension scales, which netCDF4 + # does not show + _hidden_attributes = ( + "_Netcdf4Coordinates", + "_Netcdf4Dimid", + "_NCProperties", + "_IsNetcdf4", + "_SuperblockVersion", + "_nc3_strict", + "DIMENSION_LIST", + "REFERENCE_LIST", + "CLASS", + "NAME", + ) + + # Attributes that make netCDF4 modify the data it reads + _netcdf4_conversion_attributes = ( + "scale_factor", + "add_offset", + "missing_value", + "valid_min", + "valid_max", + "valid_range", + "_Unsigned", + "_Encoding", + ) + + _netcdf4_file = None + + # Metadata (list of variables, dimensions, attributes) of files that have been + # opened, so that it only needs to be read once, rather than every time a file is + # opened. Keys are (path, modification time, size, inode), so the metadata is read + # again if a file changes. + _metadata_cache = {} # noqa: RUF012 + _metadata_cache_max_files = 10000 + + def open(self, filename, write=False, create=False, format=None): + if write or create: + raise ValueError("DataFile_netCDF_h5py can only read files") + self.handle = h5py.File(filename, mode="r") + self.writeable = False + + stat = os.stat(filename) + cache_key = ( + os.path.abspath(filename), + stat.st_mtime_ns, + stat.st_size, + stat.st_ino, + ) + if len(self._metadata_cache) >= self._metadata_cache_max_files: + self._metadata_cache.clear() + self._metadata = self._metadata_cache.setdefault( + cache_key, + { + "contents": None, + "attributes": {}, + "needs_netcdf4": {}, + "dimension_lengths": {}, + }, + ) + + def close(self): + if self.handle is not None: + self.handle.close() + self.handle = None + if self._netcdf4_file is not None: + self._netcdf4_file.close() + self._netcdf4_file = None + + def __init__(self, filename=None, write=False, create=False, format=None): + if not has_h5py: + message = "DataFile: No supported HDF5 python-modules available" + raise ImportError(message) + self._filename = filename + self._kwargs = {} + if filename is not None: + self.open(filename, write=write, create=create, format=format) + self._attributes_cache = {} + + def _get_netcdf4_file(self): + """DataFile_netCDF for the same file, for variables h5py cannot read""" + if self._netcdf4_file is None: + self._netcdf4_file = DataFile_netCDF(self._filename) + return self._netcdf4_file + + def _read_name_attribute(self, name): + """Read the "NAME" attribute of dimension scale `name`, or None if not a scale + + Uses the low-level h5py API, because this is much faster than opening the + dataset. + """ + name = name.encode() + try: + if not h5py.h5a.exists(self.handle.id, b"NAME", obj_name=name): + return None + except (KeyError, RuntimeError): + # No object called `name` + return None + attribute = h5py.h5a.open(self.handle.id, b"NAME", obj_name=name) + value = np.empty((), dtype=attribute.dtype) + attribute.read(value) + return bytes(value) + + def _is_dimension_only(self, name): + """Is `name` a NetCDF dimension that is not also a variable?""" + scale_name = self._read_name_attribute(name) + return scale_name is not None and scale_name.startswith( + b"This is a netCDF dimension but not" + ) + + def _get_variable(self, name): + """Get the h5py.Dataset of a variable, or None if there is no such variable""" + if self.handle is None or "/" in name or name.startswith("_nc4_non_coord_"): + return None + if self._is_dimension_only(name): + # A variable with the same name as a dimension, but different + # dimensions, is renamed by NetCDF + name = "_nc4_non_coord_" + name + try: + var = self.handle[name] + except KeyError: + return None + if not isinstance(var, h5py.Dataset): + return None + return var + + def _needs_netcdf4(self, name, var): + """Does variable `var` have to be read with netCDF4 to get the same result?""" + needs_netcdf4 = self._metadata["needs_netcdf4"] + if name not in needs_netcdf4: + needs_netcdf4[name] = self._check_needs_netcdf4(var) + return needs_netcdf4[name] + + def _check_needs_netcdf4(self, var): + if var.dtype.kind not in "biuf": + # Only scalar strings are handled here + return not (var.shape == () and h5py.check_string_dtype(var.dtype)) + if ( + var.shape != () + and "DIMENSION_LIST" not in var.attrs + and (var.attrs.get("CLASS") != b"DIMENSION_SCALE" or len(var.shape) != 1) + ): + # Not a NetCDF-4 variable, and not a 1d coordinate variable + return True + if any(a in var.attrs for a in self._netcdf4_conversion_attributes): + return True + # netCDF4 pads variables that are shorter than an unlimited dimension with + # fill values + return any( + maxsize is None and size < length + for size, maxsize, length in zip( + var.shape, var.maxshape, self._variable_dimension_lengths(var) + ) + ) + + @staticmethod + def _convert_attribute(value): + """Convert an attribute read by h5py to the value netCDF4 would give""" + if isinstance(value, h5py.Empty): + return "" + if isinstance(value, bytes | np.bytes_): + return value.decode("utf-8", "replace").replace("\x00", "") + if isinstance(value, np.ndarray): + if value.dtype.kind in "OSU": + strings = [DataFile_netCDF_h5py._convert_attribute(v) for v in value] + return strings[0] if len(strings) == 1 else strings + if value.shape == (1,): + return value[0] + return value + + def _mask_fill_values(self, var, data): + """Mask fill values in the same way as netCDF4""" + fill_value = var.attrs.get("_FillValue") + if fill_value is None: + no_fill = ( + var.id.get_create_plist().get_fill_time() == h5py.h5d.FILL_TIME_NEVER + ) + if no_fill or var.dtype.str[1:] in ("i1", "u1"): + return np.ma.masked_array(data) + fill_value = default_fillvals[var.dtype.str[1:]] + if np.isnan(fill_value): + return np.ma.masked_array(data, mask=np.isnan(data)) + return np.ma.masked_array(data, mask=(data == fill_value)) + + def read(self, name, ranges=None, asBoutArray=True): + """Read a variable from the file.""" + if self.handle is None: + return None + + var = self._get_variable(name) + n = name + if var is None: + # Not found. Try to find using case-insensitive search + for n in self.list(): + if n.lower() == name.lower(): + print(f"WARNING: Reading '{n}' instead of '{name}'") + var = self._get_variable(n) + if var is None: + return None + + if self._needs_netcdf4(n, var): + return self._get_netcdf4_file().read(n, ranges, asBoutArray) + + if asBoutArray: + attributes = self.attributes(n) + + ndims = len(var.shape) + if ndims == 0: + if h5py.check_string_dtype(var.dtype): + data = var.asstr()[()] + else: + data = np.asarray(var[()]) + if not asBoutArray: + data = self._mask_fill_values(var, data) + if asBoutArray: + data = BoutArray(data, attributes=attributes) + return data + + if ranges: + if len(ranges) == 2 * ndims: + # Reform list of pairs of ints into slices + ranges = [slice(a, b) for a, b in zip(ranges[::2], ranges[1::2])] + elif len(ranges) != ndims: + raise ValueError( + "Incorrect number of elements in ranges argument " + f"(got {ranges}, expected {ndims} or {2 * ndims})" + ) + ranges = ranges[:ndims] + if any( + r is None or (isinstance(r, slice) and (r.step or 1) < 0) + for r in ranges + ): + # Indexing that h5py does not support in the same way as netCDF4 + return self._get_netcdf4_file().read(n, ranges, asBoutArray) + data = var[tuple(ranges)] + else: + data = var[...] + + if asBoutArray: + return BoutArray(data, attributes=attributes) + return self._mask_fill_values(var, data) + + def sync(self): + self.handle.flush() + + def _get_contents(self): + """Scan the file for its variables and dimensions + + Returns + ------- + variables : list of str + Names of the variables, in the same order as netCDF4 lists them + dimension_names : dict + Map from NetCDF dimension IDs to dimension names + """ + if self._metadata["contents"] is not None: + return self._metadata["contents"] + + datasets = [] + + def add_dataset(name, info): + # visit() is recursive, so skip objects inside groups + if info.type == h5py.h5o.TYPE_DATASET and b"/" not in name: + datasets.append(name.decode()) + + # Use the creation order of the variables, like netCDF4, if it was saved + try: + h5py.h5o.visit( + self.handle.id, add_dataset, info=True, idx_type=h5py.h5.INDEX_CRT_ORDER + ) + except (KeyError, RuntimeError, ValueError): + datasets = [] + h5py.h5o.visit( + self.handle.id, add_dataset, info=True, idx_type=h5py.h5.INDEX_NAME + ) + + variables = [] + dimension_names = {} + for name in datasets: + scale_name = self._read_name_attribute(name) + if scale_name is not None: + # A dimension scale, i.e. a NetCDF dimension + dimension_id = self.handle[name].attrs.get("_Netcdf4Dimid") + if dimension_id is not None: + dimension_names[int(dimension_id)] = name + if scale_name.startswith(b"This is a netCDF dimension but not"): + continue + variables.append(name.removeprefix("_nc4_non_coord_")) + + self._metadata["contents"] = variables, dimension_names + return self._metadata["contents"] + + def list(self): + if self.handle is None: + return [] + return list(self._get_contents()[0]) + + def dimensions(self, varname): + if self.handle is None: + return None + var = self._get_variable(varname) + if var is None: + raise ValueError("No such variable") + if self._needs_netcdf4(varname, var): + return self._get_netcdf4_file().dimensions(varname) + if var.shape == (): + return () + if "DIMENSION_LIST" not in var.attrs: + # 1d coordinate variable: the variable is its own dimension + return (varname,) + dimension_names = self._get_contents()[1] + dimension_ids = var.attrs.get("_Netcdf4Coordinates") + if dimension_ids is not None and all( + i in dimension_names for i in dimension_ids + ): + return tuple(dimension_names[i] for i in dimension_ids) + # Look up the names of the dimension scales. This is slow, because HDF5 has to + # search the file for the name of each referenced object + return tuple( + self.handle[refs[0]].name.removeprefix("/") + for refs in var.attrs["DIMENSION_LIST"] + ) + + def ndims(self, varname): + if self.handle is None: + raise ValueError("File not open") + return len(self.dimensions(varname)) + + def size(self, varname): + if self.handle is None: + return [] + var = self._get_variable(varname) + if var is None: + return [] + if self._needs_netcdf4(varname, var): + return self._get_netcdf4_file().size(varname) + # Use the size of the dimensions, like netCDF4 + return self._variable_dimension_lengths(var) + + def _variable_dimension_lengths(self, var): + """Lengths of the dimensions of `var`, as NetCDF defines them""" + if var.shape == (): + return [] + if "DIMENSION_LIST" not in var.attrs: + # 1d coordinate variable, which is its own dimension + return [self._dimension_length(var)] + return [ + self._dimension_length(self.handle[refs[0]]) + for refs in var.attrs["DIMENSION_LIST"] + ] + + def _dimension_length(self, dimension): + """Length of a dimension, given its dimension scale dataset""" + if dimension.maxshape[0] is not None: + return dimension.shape[0] + + # Unlimited dimension: NetCDF uses the longest variable that uses it + dimension_lengths = self._metadata["dimension_lengths"] + dimension_id = int(dimension.attrs.get("_Netcdf4Dimid", -1)) + if dimension_id in dimension_lengths: + return dimension_lengths[dimension_id] + length = dimension.shape[0] + if "REFERENCE_LIST" in dimension.attrs: + for ref, axis in dimension.attrs["REFERENCE_LIST"]: + length = max(length, self.handle[ref].shape[axis]) + if dimension_id >= 0: + dimension_lengths[dimension_id] = length + return length + + def write(self, name, data, info=False, *, dims=None): + raise Exception("File not writeable. Open with write=True keyword") + + def read_file_attribute(self, name): + if name in self._hidden_attributes or name not in self.handle.attrs: + raise AttributeError(f"DataFile (netCDF4) has no file attribute {name}") + return self._convert_attribute(self.handle.attrs[name]) + + def write_file_attribute(self, name, value): + raise Exception("File not writeable. Open with write=True keyword") + + def list_file_attributes(self): + return [a for a in self.handle.attrs if a not in self._hidden_attributes] + + def attributes(self, varname): + try: + return self._attributes_cache[varname] + except KeyError: + # Need to build the attributes dictionary for this variable + if self.handle is None: + return None + shared_attributes = self._metadata["attributes"] + if varname in shared_attributes: + # Copy, so that changing the result does not change the shared cache + attributes = dict(shared_attributes[varname]) + self._attributes_cache[varname] = attributes + return attributes + + var = self._get_variable(varname) + n = varname + if var is None: + # Not found. Try to find using case-insensitive search + for n in self.list(): + if n.lower() == varname.lower(): + print(f"WARNING: Reading '{n}' instead of '{varname}'") + var = self._get_variable(n) + if var is None: + return None + + if self._needs_netcdf4(n, var): + attributes = self._get_netcdf4_file().attributes(varname) + else: + attributes = { + attrname: self._convert_attribute(value) + for attrname, value in var.attrs.items() + if attrname not in self._hidden_attributes + } + if "bout_type" not in attributes: + attributes["bout_type"] = self._bout_type_from_dimensions(varname) + + # Save the attributes for this variable to the caches + self._attributes_cache[varname] = attributes + if n == varname: + shared_attributes[varname] = dict(attributes) + + return attributes + + class DataFile_HDF5(DataFile): handle = None From 29eb2b20d4857b8e25d118c92c1ab759a8fc8c13 Mon Sep 17 00:00:00 2001 From: toto Date: Tue, 3 Nov 2026 13:58:26 +0100 Subject: [PATCH 3/5] Do not create a BoutArray for each processor's part of a variable The attributes of the BoutArray returned by DataFile.read() were not used when collecting a variable from the files of each processor, so read the data with asBoutArray=False. Co-Authored-By: Claude Opus 5.5 --- src/boutdata/collect.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/boutdata/collect.py b/src/boutdata/collect.py index 9f8e5e27c..555a8414e 100644 --- a/src/boutdata/collect.py +++ b/src/boutdata/collect.py @@ -663,7 +663,10 @@ def _collect_from_one_proc( # No data for FieldPerp on this processor return None, None - result[global_slices] = datafile.read(varname, ranges=local_slices) + # Attributes are not needed here, so do not create a BoutArray + result[global_slices] = datafile.read( + varname, ranges=local_slices, asBoutArray=False + ) if is_fieldperp: return temp_yindex, f_attributes From bbbd116f74f9160420c4b33f0b5c174145b1aebd Mon Sep 17 00:00:00 2001 From: toto Date: Tue, 3 Nov 2026 13:58:39 +0100 Subject: [PATCH 4/5] Document reading NetCDF-4 files with h5py Co-Authored-By: Claude Opus 5.5 --- README.md | 5 +++++ src/boutdata/collect.py | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/README.md b/README.md index 04172ab00..b4ace2f34 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev in ubuntu +`boutdata` also uses [`h5py`](https://www.h5py.org) to read NetCDF-4 files, +because it opens files much faster than `netCDF4`. If `h5py` is not +installed, `netCDF4` is used. To always use `netCDF4` for reading, set the +environment variable `BOUTDATA_NETCDF_READER=netcdf4`. + # Install `pip install boutdata` diff --git a/src/boutdata/collect.py b/src/boutdata/collect.py index 555a8414e..e0c55219d 100644 --- a/src/boutdata/collect.py +++ b/src/boutdata/collect.py @@ -166,6 +166,17 @@ def collect( do not have to re-open the dump files to read another variable (default: None) + Notes + ----- + NetCDF-4 dump files are read using h5py if it is installed, because it opens + files much faster than netCDF4. Set the environment variable + ``BOUTDATA_NETCDF_READER=netcdf4`` to always use netCDF4 (see + :py:mod:`boututils.datafile`). + + When reading many variables, it is still faster to pass a `datafile_cache`, or + to use :py:obj:`~boutdata.data.BoutData`, so that the files are only opened + once. + Examples -------- From ab50230a42c7d2f3078eff122c1caae44bcba798 Mon Sep 17 00:00:00 2001 From: toto Date: Tue, 3 Nov 2026 14:23:11 +0100 Subject: [PATCH 5/5] Read bool and non-native byte order variables with netCDF4 Leave these to netCDF4 rather than relying on h5py giving identical results for them. Co-Authored-By: Claude Opus 5.5 --- src/boututils/datafile.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/boututils/datafile.py b/src/boututils/datafile.py index 0c7f5b641..03c60c285 100644 --- a/src/boututils/datafile.py +++ b/src/boututils/datafile.py @@ -895,9 +895,12 @@ def _needs_netcdf4(self, name, var): return needs_netcdf4[name] def _check_needs_netcdf4(self, var): - if var.dtype.kind not in "biuf": + if var.dtype.kind not in "iuf": # Only scalar strings are handled here return not (var.shape == () and h5py.check_string_dtype(var.dtype)) + if not var.dtype.isnative: + # Leave byte-swapping to netCDF4 + return True if ( var.shape != () and "DIMENSION_LIST" not in var.attrs