diff --git a/README.md b/README.md index 04172ab00..b4ace2f34 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,11 @@ sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev in ubuntu +`boutdata` also uses [`h5py`](https://www.h5py.org) to read NetCDF-4 files, +because it opens files much faster than `netCDF4`. If `h5py` is not +installed, `netCDF4` is used. To always use `netCDF4` for reading, set the +environment variable `BOUTDATA_NETCDF_READER=netcdf4`. + # Install `pip install boutdata` diff --git a/pyproject.toml b/pyproject.toml index 57bfd5aa8..2b176dc1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "natsort>=8.1.0", "scipy >= 1.12.0", "netCDF4", + "h5py", ] dynamic = ["version"] diff --git a/src/boutdata/collect.py b/src/boutdata/collect.py index 30d9950ea..e0c55219d 100644 --- a/src/boutdata/collect.py +++ b/src/boutdata/collect.py @@ -166,6 +166,17 @@ def collect( do not have to re-open the dump files to read another variable (default: None) + Notes + ----- + NetCDF-4 dump files are read using h5py if it is installed, because it opens + files much faster than netCDF4. Set the environment variable + ``BOUTDATA_NETCDF_READER=netcdf4`` to always use netCDF4 (see + :py:mod:`boututils.datafile`). + + When reading many variables, it is still faster to pass a `datafile_cache`, or + to use :py:obj:`~boutdata.data.BoutData`, so that the files are only opened + once. + Examples -------- @@ -282,9 +293,8 @@ def getDataFile(i): f.close() return result - if datafile_cache is None: - # close the DataFile if we are not keeping it in a cache - f.close() + # Keep the 0'th file open: it is read again in the loop over processors below + f0 = f # Create a list with size of each dimension ddims = [grid_info["sizes"][d] for d in dimensions] @@ -301,7 +311,7 @@ def getDataFile(i): is_fieldperp = False for i in range(grid_info["npes"]): - f = getDataFile(i) + f = f0 if i == 0 else getDataFile(i) temp_yindex, temp_f_attributes = _collect_from_one_proc( i, f, @@ -664,7 +674,10 @@ def _collect_from_one_proc( # No data for FieldPerp on this processor return None, None - result[global_slices] = datafile.read(varname, ranges=local_slices) + # Attributes are not needed here, so do not create a BoutArray + result[global_slices] = datafile.read( + varname, ranges=local_slices, asBoutArray=False + ) if is_fieldperp: return temp_yindex, f_attributes diff --git a/src/boutdata/tests/conftest.py b/src/boutdata/tests/conftest.py new file mode 100644 index 000000000..15ebefbd5 --- /dev/null +++ b/src/boutdata/tests/conftest.py @@ -0,0 +1,12 @@ +import pytest + +import boututils.datafile + + +@pytest.fixture(params=["h5py", "netcdf4"]) +def netcdf_reader(request, monkeypatch): + """Run a test with each of the libraries that can be used to read NetCDF files""" + if request.param == "h5py": + pytest.importorskip("h5py") + monkeypatch.setattr(boututils.datafile, "netcdf_reader", request.param) + return request.param diff --git a/src/boutdata/tests/test_collect.py b/src/boutdata/tests/test_collect.py index 2862d4e15..8cfce45ec 100644 --- a/src/boutdata/tests/test_collect.py +++ b/src/boutdata/tests/test_collect.py @@ -23,6 +23,10 @@ # Note - using tmp_path fixture requires pytest>=3.9.0 +# Run every test with each library that can read NetCDF files +pytestmark = pytest.mark.usefixtures("netcdf_reader") + + collect_kwargs_list = [ pytest.param({"xguards": True, "yguards": "include_upper"}, id="collect_guards"), pytest.param({"xguards": False, "yguards": False}, id="collect_noguards"), @@ -1279,3 +1283,27 @@ def test_disconnected_doublenull_with_compression( collect_kwargs=collect_kwargs, squash_kwargs=squash_kwargs, ) + + @pytest.mark.parametrize("scenario", ["core_min", "single_null_full"]) + def test_collect_opens_each_file_once( + self, tmp_path, scenario, request, monkeypatch + ): + """Check that collecting a field opens each dump file exactly once""" + data_path, _, _ = request.getfixturevalue(scenario) + symlink_dump_files(data_path, tmp_path) + + with DataFile(tmp_path / "BOUT.dmp.0.nc") as f: + npes = int(f["NXPE"]) * int(f["NYPE"]) + + opened_files = [] + original_init = DataFile.__init__ + + def counting_init(self, filename=None, *args, **kwargs): + opened_files.append(filename) + original_init(self, filename, *args, **kwargs) + + monkeypatch.setattr(DataFile, "__init__", counting_init) + + collect("field3d_t_1", path=tmp_path, info=False) + + assert len(opened_files) == npes diff --git a/src/boutdata/tests/test_datafile.py b/src/boutdata/tests/test_datafile.py new file mode 100644 index 000000000..d4cfd1f21 --- /dev/null +++ b/src/boutdata/tests/test_datafile.py @@ -0,0 +1,317 @@ +import subprocess +import sys + +import numpy as np +import pytest +from netCDF4 import Dataset, default_fillvals + +import boututils.datafile +from boutdata.tests.make_test_data import ( + create_dump_file, + create_restart_file, + make_grid_info, +) +from boututils.datafile import DataFile, DataFile_netCDF + +h5py = pytest.importorskip("h5py") +from boututils.datafile import DataFile_netCDF_h5py + + +def create_unusual_file(path): + """Create a NetCDF-4 file with features not found in the BOUT++ test files""" + with Dataset(path, "w") as f: + f.createDimension("t", None) + f.createDimension("x", 4) + f.createDimension("y", 3) + f.createDimension("char5", 5) + f.setncattr("title", "unusual file") + f.setncattr("numbers", np.array([1.0, 2.0])) + f.setncattr("one_number", np.int32(3)) + f.setncattr_string("strings", ["a", "bc"]) + + # Time-dependent variable with fewer time points than another variable + v = f.createVariable("short_t", float, ("t", "x")) + v[:] = np.arange(8.0).reshape(2, 4) + v = f.createVariable("long_t", float, ("t", "y")) + v[:] = np.arange(9.0).reshape(3, 3) + v.setncattr("multiple", np.array([1, 2, 3], dtype=np.int32)) + v.setncattr("empty_string", "") + v.setncattr_string("vlen_string", "vlen") + + # Coordinate variable, which is its own dimension + v = f.createVariable("t", float, ("t",)) + v[:] = [0.0, 1.0, 2.0] + + # Variable with the name of a dimension, but different dimensions + v = f.createVariable("y", float, ("x",)) + v[:] = np.arange(4.0) + + # Fill values, with and without the _FillValue attribute + v = f.createVariable("with_fill", float, ("x", "y"), fill_value=-1.0) + v[:] = np.arange(12.0).reshape(4, 3) + v[1, 1] = -1.0 + v = f.createVariable("default_fill", float, ("x", "y")) + data = np.arange(12.0).reshape(4, 3) + data[2, 0] = default_fillvals["f8"] + v[:] = data + v = f.createVariable("with_nan_fill", float, ("x",), fill_value=np.nan) + v[:] = [0.0, np.nan, 2.0, 3.0] + v = f.createVariable("byte_var", np.int8, ("x",)) + v[:] = [1, 2, 3, 4] + + # Variables that netCDF4 converts when reading + v = f.createVariable("packed", np.int16, ("x",)) + v.setncattr("scale_factor", 0.5) + v.setncattr("add_offset", 1.0) + v[:] = [1.0, 2.0, 3.0, 4.0] + + # Strings + v = f.createVariable("string_scalar", str, ()) + v[0] = "hello" + v = f.createVariable("string_array", str, ("x",)) + v[:] = np.array(["a", "bb", "ccc", "dddd"], dtype=object) + v = f.createVariable("chars", "S1", ("t", "char5")) + v[:] = np.array([list("abcde"), list("fghij"), list("klmno")], dtype="S1") + + # Variables in a group are not listed by netCDF4 + g = f.createGroup("section") + v = g.createVariable("in_group", float, ("x",)) + v[:] = np.arange(4.0) + + v = f.createVariable("int_scalar", np.int32, ()) + v.assignValue(7) + + +@pytest.fixture(scope="module") +def test_files(tmp_path_factory): + """NetCDF-4 files to compare the two readers on""" + tmp_path = tmp_path_factory.mktemp("datafile") + grid_info = make_grid_info() + rng = np.random.default_rng(42) + create_dump_file( + i=0, + tmpdir=tmp_path, + rng=rng, + grid_info=grid_info, + boundaries=["xinner", "xouter", "ylower", "yupper"], + fieldperp_global_yind=3, + ) + create_restart_file( + i=0, tmpdir=tmp_path, rng=rng, grid_info=grid_info, fieldperp_global_yind=3 + ) + create_unusual_file(tmp_path / "unusual.nc") + return { + "dump": tmp_path / "BOUT.dmp.0.nc", + "restart": tmp_path / "BOUT.restart.0.nc", + "unusual": tmp_path / "unusual.nc", + } + + +def assert_same(expected, actual, what): + """Check that two results are identical, including their types""" + assert type(actual) is type(expected), what + if isinstance(expected, np.ndarray): + assert actual.dtype == expected.dtype, what + assert actual.shape == expected.shape, what + np.testing.assert_array_equal( + np.asarray(actual), np.asarray(expected), err_msg=what + ) + np.testing.assert_array_equal( + np.ma.getmaskarray(actual), np.ma.getmaskarray(expected), err_msg=what + ) + if hasattr(expected, "attributes"): + assert_same(expected.attributes, actual.attributes, f"{what}.attributes") + elif isinstance(expected, dict): + assert list(actual) == list(expected), what + for key in expected: + assert_same(expected[key], actual[key], f"{what}[{key}]") + elif isinstance(expected, list): + assert len(actual) == len(expected), what + for i, (e, a) in enumerate(zip(expected, actual)): + assert_same(e, a, f"{what}[{i}]") + elif isinstance(expected, float) and np.isnan(expected): + assert np.isnan(actual), what + else: + assert actual == expected, what + + +def call_or_error(method, *args, **kwargs): + """Result of a call, or the name of the exception it raised""" + try: + return method(*args, **kwargs) + except Exception as e: # noqa: BLE001 + return type(e).__name__ + + +@pytest.mark.parametrize("file", ["dump", "restart", "unusual"]) +def test_h5py_reader_matches_netcdf4(test_files, file): + path = test_files[file] + with DataFile_netCDF(path) as expected, DataFile_netCDF_h5py(path) as actual: + check_same_contents(expected, actual) + + +def check_same_contents(expected, actual): + assert_same(expected.list(), actual.list(), "list()") + assert_same( + expected.list_file_attributes(), + actual.list_file_attributes(), + "list_file_attributes()", + ) + for name in expected.list_file_attributes(): + assert_same( + expected.read_file_attribute(name), + actual.read_file_attribute(name), + f"read_file_attribute({name})", + ) + with pytest.raises(AttributeError): + actual.read_file_attribute("not_an_attribute") + + for name in expected.list(): + assert_same(expected.dimensions(name), actual.dimensions(name), f"{name} dims") + assert_same(expected.size(name), actual.size(name), f"{name} size") + assert_same(expected.ndims(name), actual.ndims(name), f"{name} ndims") + assert_same(expected.attributes(name), actual.attributes(name), f"{name} attrs") + for asBoutArray in [True, False]: + assert_same( + expected.read(name, asBoutArray=asBoutArray), + actual.read(name, asBoutArray=asBoutArray), + f"{name} read(asBoutArray={asBoutArray})", + ) + + ndims = expected.ndims(name) + if ndims == 0: + continue + ranges_list = [ + [slice(1, None, 2)] + [slice(0, 2)] * (ndims - 1), + [slice(0, 1)] + [None] * (ndims - 1), + [0, 1] * ndims, + [slice(None, None, -1)] * ndims, + ] + for ranges in ranges_list: + for asBoutArray in [True, False]: + assert_same( + call_or_error(expected.read, name, ranges, asBoutArray), + call_or_error(actual.read, name, ranges, asBoutArray), + f"{name} read({ranges}, asBoutArray={asBoutArray})", + ) + + # Variables that do not exist, or are only NetCDF dimensions + for name in ["not_a_variable", "x", "section", "section/in_group"]: + assert actual.read(name) is None, name + assert actual.size(name) == [], name + with pytest.raises(ValueError): + actual.dimensions(name) + + +def test_h5py_reader_case_insensitive(test_files): + path = test_files["dump"] + with DataFile_netCDF(path) as expected, DataFile_netCDF_h5py(path) as actual: + assert_same(expected.read("FIELD3D_T_1"), actual.read("FIELD3D_T_1"), "read") + assert_same( + call_or_error(expected.attributes, "FIELD3D_T_1"), + call_or_error(actual.attributes, "FIELD3D_T_1"), + "attributes", + ) + + +def test_h5py_reader_is_read_only(test_files): + with DataFile_netCDF_h5py(test_files["dump"]) as f: + with pytest.raises(Exception, match="File not writeable"): + f.write("new_variable", np.zeros(3)) + with pytest.raises(Exception, match="File not writeable"): + f.write_file_attribute("new_attribute", 1) + with pytest.raises(ValueError): + DataFile_netCDF_h5py(test_files["dump"], write=True) + + +def test_h5py_reader_attributes_are_copies(test_files): + """Changing returned attributes must not change the cached metadata""" + with DataFile_netCDF_h5py(test_files["dump"]) as f: + first = f.attributes("field3d_t_1") + first["new_attribute"] = 1 + with DataFile_netCDF_h5py(test_files["dump"]) as f: + second = f.attributes("field3d_t_1") + assert "new_attribute" not in second + + +def test_h5py_reader_sees_changed_file(tmp_path): + """Cached metadata must not be used after a file has been changed""" + path = tmp_path / "changing.nc" + with Dataset(path, "w") as f: + f.createDimension("x", 2) + f.createVariable("a", float, ("x",))[:] = [1.0, 2.0] + with DataFile_netCDF_h5py(path) as f: + assert f.list() == ["a"] + + with Dataset(path, "a") as f: + f.createVariable("b", float, ("x",))[:] = [3.0, 4.0] + with DataFile_netCDF_h5py(path) as f: + assert f.list() == ["a", "b"] + + +@pytest.fixture +def netcdf3_file(tmp_path): + path = tmp_path / "netcdf3.nc" + with Dataset(path, "w", format="NETCDF3_CLASSIC") as f: + f.createDimension("x", 2) + f.createVariable("a", float, ("x",))[:] = [1.0, 2.0] + return path + + +def test_dispatch_auto(test_files, netcdf3_file, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "auto") + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF_h5py + # NetCDF-3 files are not HDF5 files + with DataFile(netcdf3_file) as f: + assert type(f.impl) is DataFile_netCDF + # Writing needs netCDF4 + with DataFile(netcdf3_file, write=True) as f: + assert type(f.impl) is DataFile_netCDF + with DataFile(test_files["dump"].parent / "new.nc", create=True) as f: + assert type(f.impl) is DataFile_netCDF + + monkeypatch.setattr(boututils.datafile, "has_h5py", False) + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF + + +def test_dispatch_netcdf4(test_files, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "netcdf4") + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF + + +def test_dispatch_h5py(test_files, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "h5py") + with DataFile(test_files["dump"]) as f: + assert type(f.impl) is DataFile_netCDF_h5py + + monkeypatch.setattr(boututils.datafile, "has_h5py", False) + with pytest.raises(ImportError): + DataFile(test_files["dump"]) + + +def test_dispatch_invalid(test_files, monkeypatch): + monkeypatch.setattr(boututils.datafile, "netcdf_reader", "not_a_library") + with pytest.raises(ValueError): + DataFile(test_files["dump"]) + + +@pytest.mark.parametrize( + ("reader", "expected"), + [("netcdf4", "DataFile_netCDF"), ("auto", "DataFile_netCDF_h5py")], +) +def test_dispatch_environment_variable(test_files, monkeypatch, reader, expected): + monkeypatch.setenv("BOUTDATA_NETCDF_READER", reader) + code = ( + "from boututils.datafile import DataFile; " + f"print(type(DataFile({str(test_files['dump'])!r}).impl).__name__)" + ) + result = subprocess.run( + [sys.executable, "-W", "ignore", "-c", code], + capture_output=True, + text=True, + check=True, + ) + assert result.stdout.strip() == expected diff --git a/src/boutdata/tests/test_restart.py b/src/boutdata/tests/test_restart.py index 8edf31b6a..290ca1d9e 100644 --- a/src/boutdata/tests/test_restart.py +++ b/src/boutdata/tests/test_restart.py @@ -2,6 +2,7 @@ import numpy as np import numpy.testing as npt +import pytest from boutdata import restart from boutdata.collect import collect @@ -12,6 +13,9 @@ make_grid_info, ) +# Run every test with each library that can read NetCDF files +pytestmark = pytest.mark.usefixtures("netcdf_reader") + def check_redistributed_data( expected, diff --git a/src/boututils/datafile.py b/src/boututils/datafile.py index a8eef2b99..03c60c285 100644 --- a/src/boututils/datafile.py +++ b/src/boututils/datafile.py @@ -8,6 +8,15 @@ - ``h5py`` (for HDF5 files) - ``netCDF4`` (preferred NetCDF library) +NetCDF-4 files are HDF5 files. When ``h5py`` is available, NetCDF-4 files +opened read-only are read with ``h5py``, which opens files much faster than +``netCDF4``. The reader can be chosen with the environment variable +``BOUTDATA_NETCDF_READER`` (or by setting ``boututils.datafile.netcdf_reader``): + +- ``"auto"`` (default): use ``h5py`` if it is installed, otherwise ``netCDF4`` +- ``"netcdf4"``: always use ``netCDF4`` +- ``"h5py"``: always use ``h5py`` for NetCDF-4 files opened read-only + NOTE ---- NetCDF and HDF5 include unlimited dimensions, but this library is just @@ -23,6 +32,7 @@ """ +import os import pathlib import numpy as np @@ -31,7 +41,7 @@ from boututils.boutwarnings import alwayswarn try: - from netCDF4 import Dataset + from netCDF4 import Dataset, default_fillvals has_netCDF = True except ImportError: @@ -46,6 +56,26 @@ except ImportError: has_h5py = False +# Which library to use to read NetCDF-4 files: "auto", "netcdf4" or "h5py" +netcdf_reader = os.environ.get("BOUTDATA_NETCDF_READER", "auto") + + +def _use_h5py_reader(filename, write, create): + """Should the NetCDF file `filename` be read with h5py?""" + if netcdf_reader not in ("auto", "netcdf4", "h5py"): + raise ValueError( + f"Invalid netcdf_reader '{netcdf_reader}'. Should be 'auto', 'netcdf4' " + "or 'h5py'" + ) + if write or create or netcdf_reader == "netcdf4": + return False + if netcdf_reader == "h5py" and not has_h5py: + raise ImportError("netcdf_reader is 'h5py' but h5py is not available") + if not has_h5py or filename.suffix not in (".nc", ".ncdf"): + return False + # NetCDF-3 files are not HDF5 files, so need to be read with netCDF4 + return h5py.is_hdf5(filename) + class DataFile: """File I/O class @@ -97,6 +127,8 @@ def __init__( self.impl = DataFile_HDF5( filename=filename, write=write, create=create, format=format ) + elif _use_h5py_reader(filename, write, create): + self.impl = DataFile_netCDF_h5py(filename=filename) else: self.impl = DataFile_netCDF( filename=filename, @@ -720,6 +752,445 @@ def attributes(self, varname): return attributes +class DataFile_netCDF_h5py(DataFile_netCDF): + """Read-only access to NetCDF-4 files using h5py + + NetCDF-4 files are HDF5 files, and h5py opens them much faster than netCDF4, + which reads the metadata of all variables when opening a file. Results are the + same as from DataFile_netCDF. Variables that h5py cannot read in the same way + as netCDF4 (e.g. strings arrays or packed data) are read using DataFile_netCDF. + """ + + # Attributes used internally by NetCDF-4 and HDF5 dimension scales, which netCDF4 + # does not show + _hidden_attributes = ( + "_Netcdf4Coordinates", + "_Netcdf4Dimid", + "_NCProperties", + "_IsNetcdf4", + "_SuperblockVersion", + "_nc3_strict", + "DIMENSION_LIST", + "REFERENCE_LIST", + "CLASS", + "NAME", + ) + + # Attributes that make netCDF4 modify the data it reads + _netcdf4_conversion_attributes = ( + "scale_factor", + "add_offset", + "missing_value", + "valid_min", + "valid_max", + "valid_range", + "_Unsigned", + "_Encoding", + ) + + _netcdf4_file = None + + # Metadata (list of variables, dimensions, attributes) of files that have been + # opened, so that it only needs to be read once, rather than every time a file is + # opened. Keys are (path, modification time, size, inode), so the metadata is read + # again if a file changes. + _metadata_cache = {} # noqa: RUF012 + _metadata_cache_max_files = 10000 + + def open(self, filename, write=False, create=False, format=None): + if write or create: + raise ValueError("DataFile_netCDF_h5py can only read files") + self.handle = h5py.File(filename, mode="r") + self.writeable = False + + stat = os.stat(filename) + cache_key = ( + os.path.abspath(filename), + stat.st_mtime_ns, + stat.st_size, + stat.st_ino, + ) + if len(self._metadata_cache) >= self._metadata_cache_max_files: + self._metadata_cache.clear() + self._metadata = self._metadata_cache.setdefault( + cache_key, + { + "contents": None, + "attributes": {}, + "needs_netcdf4": {}, + "dimension_lengths": {}, + }, + ) + + def close(self): + if self.handle is not None: + self.handle.close() + self.handle = None + if self._netcdf4_file is not None: + self._netcdf4_file.close() + self._netcdf4_file = None + + def __init__(self, filename=None, write=False, create=False, format=None): + if not has_h5py: + message = "DataFile: No supported HDF5 python-modules available" + raise ImportError(message) + self._filename = filename + self._kwargs = {} + if filename is not None: + self.open(filename, write=write, create=create, format=format) + self._attributes_cache = {} + + def _get_netcdf4_file(self): + """DataFile_netCDF for the same file, for variables h5py cannot read""" + if self._netcdf4_file is None: + self._netcdf4_file = DataFile_netCDF(self._filename) + return self._netcdf4_file + + def _read_name_attribute(self, name): + """Read the "NAME" attribute of dimension scale `name`, or None if not a scale + + Uses the low-level h5py API, because this is much faster than opening the + dataset. + """ + name = name.encode() + try: + if not h5py.h5a.exists(self.handle.id, b"NAME", obj_name=name): + return None + except (KeyError, RuntimeError): + # No object called `name` + return None + attribute = h5py.h5a.open(self.handle.id, b"NAME", obj_name=name) + value = np.empty((), dtype=attribute.dtype) + attribute.read(value) + return bytes(value) + + def _is_dimension_only(self, name): + """Is `name` a NetCDF dimension that is not also a variable?""" + scale_name = self._read_name_attribute(name) + return scale_name is not None and scale_name.startswith( + b"This is a netCDF dimension but not" + ) + + def _get_variable(self, name): + """Get the h5py.Dataset of a variable, or None if there is no such variable""" + if self.handle is None or "/" in name or name.startswith("_nc4_non_coord_"): + return None + if self._is_dimension_only(name): + # A variable with the same name as a dimension, but different + # dimensions, is renamed by NetCDF + name = "_nc4_non_coord_" + name + try: + var = self.handle[name] + except KeyError: + return None + if not isinstance(var, h5py.Dataset): + return None + return var + + def _needs_netcdf4(self, name, var): + """Does variable `var` have to be read with netCDF4 to get the same result?""" + needs_netcdf4 = self._metadata["needs_netcdf4"] + if name not in needs_netcdf4: + needs_netcdf4[name] = self._check_needs_netcdf4(var) + return needs_netcdf4[name] + + def _check_needs_netcdf4(self, var): + if var.dtype.kind not in "iuf": + # Only scalar strings are handled here + return not (var.shape == () and h5py.check_string_dtype(var.dtype)) + if not var.dtype.isnative: + # Leave byte-swapping to netCDF4 + return True + if ( + var.shape != () + and "DIMENSION_LIST" not in var.attrs + and (var.attrs.get("CLASS") != b"DIMENSION_SCALE" or len(var.shape) != 1) + ): + # Not a NetCDF-4 variable, and not a 1d coordinate variable + return True + if any(a in var.attrs for a in self._netcdf4_conversion_attributes): + return True + # netCDF4 pads variables that are shorter than an unlimited dimension with + # fill values + return any( + maxsize is None and size < length + for size, maxsize, length in zip( + var.shape, var.maxshape, self._variable_dimension_lengths(var) + ) + ) + + @staticmethod + def _convert_attribute(value): + """Convert an attribute read by h5py to the value netCDF4 would give""" + if isinstance(value, h5py.Empty): + return "" + if isinstance(value, bytes | np.bytes_): + return value.decode("utf-8", "replace").replace("\x00", "") + if isinstance(value, np.ndarray): + if value.dtype.kind in "OSU": + strings = [DataFile_netCDF_h5py._convert_attribute(v) for v in value] + return strings[0] if len(strings) == 1 else strings + if value.shape == (1,): + return value[0] + return value + + def _mask_fill_values(self, var, data): + """Mask fill values in the same way as netCDF4""" + fill_value = var.attrs.get("_FillValue") + if fill_value is None: + no_fill = ( + var.id.get_create_plist().get_fill_time() == h5py.h5d.FILL_TIME_NEVER + ) + if no_fill or var.dtype.str[1:] in ("i1", "u1"): + return np.ma.masked_array(data) + fill_value = default_fillvals[var.dtype.str[1:]] + if np.isnan(fill_value): + return np.ma.masked_array(data, mask=np.isnan(data)) + return np.ma.masked_array(data, mask=(data == fill_value)) + + def read(self, name, ranges=None, asBoutArray=True): + """Read a variable from the file.""" + if self.handle is None: + return None + + var = self._get_variable(name) + n = name + if var is None: + # Not found. Try to find using case-insensitive search + for n in self.list(): + if n.lower() == name.lower(): + print(f"WARNING: Reading '{n}' instead of '{name}'") + var = self._get_variable(n) + if var is None: + return None + + if self._needs_netcdf4(n, var): + return self._get_netcdf4_file().read(n, ranges, asBoutArray) + + if asBoutArray: + attributes = self.attributes(n) + + ndims = len(var.shape) + if ndims == 0: + if h5py.check_string_dtype(var.dtype): + data = var.asstr()[()] + else: + data = np.asarray(var[()]) + if not asBoutArray: + data = self._mask_fill_values(var, data) + if asBoutArray: + data = BoutArray(data, attributes=attributes) + return data + + if ranges: + if len(ranges) == 2 * ndims: + # Reform list of pairs of ints into slices + ranges = [slice(a, b) for a, b in zip(ranges[::2], ranges[1::2])] + elif len(ranges) != ndims: + raise ValueError( + "Incorrect number of elements in ranges argument " + f"(got {ranges}, expected {ndims} or {2 * ndims})" + ) + ranges = ranges[:ndims] + if any( + r is None or (isinstance(r, slice) and (r.step or 1) < 0) + for r in ranges + ): + # Indexing that h5py does not support in the same way as netCDF4 + return self._get_netcdf4_file().read(n, ranges, asBoutArray) + data = var[tuple(ranges)] + else: + data = var[...] + + if asBoutArray: + return BoutArray(data, attributes=attributes) + return self._mask_fill_values(var, data) + + def sync(self): + self.handle.flush() + + def _get_contents(self): + """Scan the file for its variables and dimensions + + Returns + ------- + variables : list of str + Names of the variables, in the same order as netCDF4 lists them + dimension_names : dict + Map from NetCDF dimension IDs to dimension names + """ + if self._metadata["contents"] is not None: + return self._metadata["contents"] + + datasets = [] + + def add_dataset(name, info): + # visit() is recursive, so skip objects inside groups + if info.type == h5py.h5o.TYPE_DATASET and b"/" not in name: + datasets.append(name.decode()) + + # Use the creation order of the variables, like netCDF4, if it was saved + try: + h5py.h5o.visit( + self.handle.id, add_dataset, info=True, idx_type=h5py.h5.INDEX_CRT_ORDER + ) + except (KeyError, RuntimeError, ValueError): + datasets = [] + h5py.h5o.visit( + self.handle.id, add_dataset, info=True, idx_type=h5py.h5.INDEX_NAME + ) + + variables = [] + dimension_names = {} + for name in datasets: + scale_name = self._read_name_attribute(name) + if scale_name is not None: + # A dimension scale, i.e. a NetCDF dimension + dimension_id = self.handle[name].attrs.get("_Netcdf4Dimid") + if dimension_id is not None: + dimension_names[int(dimension_id)] = name + if scale_name.startswith(b"This is a netCDF dimension but not"): + continue + variables.append(name.removeprefix("_nc4_non_coord_")) + + self._metadata["contents"] = variables, dimension_names + return self._metadata["contents"] + + def list(self): + if self.handle is None: + return [] + return list(self._get_contents()[0]) + + def dimensions(self, varname): + if self.handle is None: + return None + var = self._get_variable(varname) + if var is None: + raise ValueError("No such variable") + if self._needs_netcdf4(varname, var): + return self._get_netcdf4_file().dimensions(varname) + if var.shape == (): + return () + if "DIMENSION_LIST" not in var.attrs: + # 1d coordinate variable: the variable is its own dimension + return (varname,) + dimension_names = self._get_contents()[1] + dimension_ids = var.attrs.get("_Netcdf4Coordinates") + if dimension_ids is not None and all( + i in dimension_names for i in dimension_ids + ): + return tuple(dimension_names[i] for i in dimension_ids) + # Look up the names of the dimension scales. This is slow, because HDF5 has to + # search the file for the name of each referenced object + return tuple( + self.handle[refs[0]].name.removeprefix("/") + for refs in var.attrs["DIMENSION_LIST"] + ) + + def ndims(self, varname): + if self.handle is None: + raise ValueError("File not open") + return len(self.dimensions(varname)) + + def size(self, varname): + if self.handle is None: + return [] + var = self._get_variable(varname) + if var is None: + return [] + if self._needs_netcdf4(varname, var): + return self._get_netcdf4_file().size(varname) + # Use the size of the dimensions, like netCDF4 + return self._variable_dimension_lengths(var) + + def _variable_dimension_lengths(self, var): + """Lengths of the dimensions of `var`, as NetCDF defines them""" + if var.shape == (): + return [] + if "DIMENSION_LIST" not in var.attrs: + # 1d coordinate variable, which is its own dimension + return [self._dimension_length(var)] + return [ + self._dimension_length(self.handle[refs[0]]) + for refs in var.attrs["DIMENSION_LIST"] + ] + + def _dimension_length(self, dimension): + """Length of a dimension, given its dimension scale dataset""" + if dimension.maxshape[0] is not None: + return dimension.shape[0] + + # Unlimited dimension: NetCDF uses the longest variable that uses it + dimension_lengths = self._metadata["dimension_lengths"] + dimension_id = int(dimension.attrs.get("_Netcdf4Dimid", -1)) + if dimension_id in dimension_lengths: + return dimension_lengths[dimension_id] + length = dimension.shape[0] + if "REFERENCE_LIST" in dimension.attrs: + for ref, axis in dimension.attrs["REFERENCE_LIST"]: + length = max(length, self.handle[ref].shape[axis]) + if dimension_id >= 0: + dimension_lengths[dimension_id] = length + return length + + def write(self, name, data, info=False, *, dims=None): + raise Exception("File not writeable. Open with write=True keyword") + + def read_file_attribute(self, name): + if name in self._hidden_attributes or name not in self.handle.attrs: + raise AttributeError(f"DataFile (netCDF4) has no file attribute {name}") + return self._convert_attribute(self.handle.attrs[name]) + + def write_file_attribute(self, name, value): + raise Exception("File not writeable. Open with write=True keyword") + + def list_file_attributes(self): + return [a for a in self.handle.attrs if a not in self._hidden_attributes] + + def attributes(self, varname): + try: + return self._attributes_cache[varname] + except KeyError: + # Need to build the attributes dictionary for this variable + if self.handle is None: + return None + shared_attributes = self._metadata["attributes"] + if varname in shared_attributes: + # Copy, so that changing the result does not change the shared cache + attributes = dict(shared_attributes[varname]) + self._attributes_cache[varname] = attributes + return attributes + + var = self._get_variable(varname) + n = varname + if var is None: + # Not found. Try to find using case-insensitive search + for n in self.list(): + if n.lower() == varname.lower(): + print(f"WARNING: Reading '{n}' instead of '{varname}'") + var = self._get_variable(n) + if var is None: + return None + + if self._needs_netcdf4(n, var): + attributes = self._get_netcdf4_file().attributes(varname) + else: + attributes = { + attrname: self._convert_attribute(value) + for attrname, value in var.attrs.items() + if attrname not in self._hidden_attributes + } + if "bout_type" not in attributes: + attributes["bout_type"] = self._bout_type_from_dimensions(varname) + + # Save the attributes for this variable to the caches + self._attributes_cache[varname] = attributes + if n == varname: + shared_attributes[varname] = dict(attributes) + + return attributes + + class DataFile_HDF5(DataFile): handle = None