diff --git a/HISTORY.rst b/HISTORY.rst index 8e5a931..4c1c39f 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -2,6 +2,10 @@ History ======= +1.2.3 (2026-02-03) +------------------ +* Fix: Restrict netCDF4 version to not equal 1.7.4 due to a bug related to strings and string variables (#133) + 1.2.2 (2025-05-09) ------------------ * Docs: Improve layout of header (#125) diff --git a/pyproject.toml b/pyproject.toml index ede3f9f..97739dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sofar" -version = "1.2.2" +version = "1.2.3" description = "Maybe the most complete python package for SOFA files so far." readme = "README.md" license = {file = "LICENSE"} @@ -26,7 +26,7 @@ classifiers = [ "Programming Language :: Python :: 3.13", ] dependencies = [ - 'netCDF4', + 'netCDF4!=1.7.4', 'numpy>=1.14.0', 'beautifulsoup4', 'requests', @@ -126,7 +126,7 @@ convention = "numpy" [tool.bumpversion] -current_version = "1.2.2" +current_version = "1.2.3" parse = "(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" serialize = ["{major}.{minor}.{patch}"] search = "{current_version}" diff --git a/sofar/__init__.py b/sofar/__init__.py index f9ba34e..a83fe6c 100644 --- a/sofar/__init__.py +++ b/sofar/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __author__ = """The pyfar developers""" __email__ = 'info@pyfar.org' -__version__ = '1.2.2' +__version__ = '1.2.3' from .sofa import Sofa diff --git a/sofar/sofa.py b/sofar/sofa.py index 3701b4e..d629feb 100644 --- a/sofar/sofa.py +++ b/sofar/sofa.py @@ -76,8 +76,9 @@ class Sofa(): will be a scalar inside SOFA objects after reading from disk. - For more examples refer to the `Quick tour of SOFA and sofar` at - https://sofar.readthedocs.io/en/stable/ + For more examples refer to the + `sofar and SOFA `__ + notebook in the pyfar example gallery. """ # these have to be set here, because they are used in __setattr__ and diff --git a/sofar/sofastream.py b/sofar/sofastream.py index 72fdd45..ede6dc9 100644 --- a/sofar/sofastream.py +++ b/sofar/sofastream.py @@ -5,6 +5,25 @@ from netCDF4 import Dataset import numpy as np +import sofar as sf + +from .io import _format_value_from_netcdf + + +class _LazyArray(np.ndarray): + """Represent a NetCDF numeric variable without reading its data.""" + + def __new__(cls, shape, dtype): + data = np.array(0, dtype=dtype) + strides = (0, ) * len(shape) + return np.lib.stride_tricks.as_strided( + data, shape=shape, strides=strides).view(cls) + + def copy(self, order='C'): + """Return a view instead of materializing the full array.""" + _ = order + return self + class SofaStream(): """ @@ -12,7 +31,8 @@ class SofaStream(): file into memory. :class:`SofaStream` opens a SOFA-file and retrieves only the requested - data. + data. It can also verify the file against the SOFA convention using the + same checks as :py:func:`~sofar.Sofa.verify`. If you want to use all the data from a SOFA-file use :class:`Sofa` class and :func:`read_sofa` function instead. @@ -83,6 +103,134 @@ def __exit__(self, *args): """ self._file.close() + def verify(self, issue_handling="raise", mode="write"): + """ + Verify a SOFA file against the SOFA standard. + + See :py:func:`~sofar.Sofa.verify` for details. + + This method checks whether mandatory data are present, names, data + types, and dimensions follow the SOFA standard, and attribute values + are consistent with the SOFA standard. Missing mandatory data raise an + error when ``issue_handling='raise'``; otherwise, they are added with + their default value and a warning is given. + + Numeric variables are represented by lazy placeholders, so only their + type and dimensions are checked without loading their contents into + memory. String variables are loaded because their values and lengths + can be relevant for convention verification. + + A detailed set of validation rules can be found at + https://github.com/pyfar/sofar/tree/main/sofar/verification_rules + + Parameters + ---------- + issue_handling : str, optional + Defines how detected issues are handled. + + ``'raise'`` + Warnings and errors are raised if issues are detected. + ``'print'`` + Issues are printed without raising warnings and errors. + ``'return'`` + Issues are returned as a string but neither raised nor printed. + + The default is ``'raise'``. + mode : str, optional + The SOFA standard is more strict for writing data than for reading + data. + + ``'write'`` + All units (e.g., ``'meter'``) must be lower case. + ``'read'`` + Units can contain upper case letters (e.g., ``'Meter'``). + + The default is ``'write'``. + + Returns + ------- + issues : str, None + Detected issues as a string. None if no issues were detected. Note + that this is only returned if ``issue_handling='return'``. + """ + return self._verify_open_file(issue_handling, mode) + + def _verify_open_file(self, issue_handling, mode): + """Verify the currently opened NetCDF file.""" + return self._read_open_file_for_verification().verify( + issue_handling=issue_handling, mode=mode) + + def _read_open_file_for_verification(self): + """Read the currently opened file using lazy numeric variables.""" + # get SOFA object with default values and convention metadata + sofa = sf.Sofa( + self._file.SOFAConventions, + version=self._file.SOFAConventionsVersion, + verify=False) + sofa.protected = False + + # NetCDF attribute _Encoding is skipped when reading SOFA files + skip = ["_Encoding"] + all_attr = [] + + # load global attributes + for attr in self._file.ncattrs(): + value = getattr(self._file, attr) + key = f"GLOBAL_{attr}" + all_attr.append(key) + + if not hasattr(sofa, key): + sofa._add_custom_api_entry(key, value, None, None, "attribute") + sofa.protected = False + else: + setattr(sofa, key, value) + + # load variable metadata and string values + for var in self._file.variables.keys(): + netcdf_variable = self._file[var] + key = var.replace(".", "_") + all_attr.append(key) + + if netcdf_variable.datatype == "S1": + value = _format_value_from_netcdf(netcdf_variable[:], var) + dtype = "string" + else: + value = _LazyArray( + netcdf_variable.shape, netcdf_variable.dtype) + dtype = "double" + + if hasattr(sofa, key): + setattr(sofa, key, value) + else: + dimensions = "".join(list(netcdf_variable.dimensions)) + sofa._add_custom_api_entry( + key, value, None, dimensions, dtype) + sofa.protected = False + + # load variable attributes + for attr in [a for a in netcdf_variable.ncattrs() + if a not in skip]: + value = getattr(netcdf_variable, attr) + attr_key = key + "_" + attr + all_attr.append(attr_key) + + if not hasattr(sofa, attr_key): + sofa._add_custom_api_entry( + attr_key, value, None, None, "attribute") + sofa.protected = False + else: + setattr(sofa, attr_key, value) + + # remove default entries that were not contained in the file + attrs = [attr for attr in sofa.__dict__.keys() + if not attr.startswith("_")] + for attr in attrs: + if attr not in all_attr: + delattr(sofa, attr) + + sofa.protected = True + return sofa + def __getattr__(self, name): """ Executed when accessing data within a with statement diff --git a/sofar/utils.py b/sofar/utils.py index db70d72..136bcd2 100644 --- a/sofar/utils.py +++ b/sofar/utils.py @@ -262,7 +262,7 @@ def _atleast_nd(array, ndim): def _nd_newaxis(array, ndim): """Append dimensions to the end of an array until array.ndim == ndim.""" - array = np.array(array) + array = np.asarray(array) for _ in range(ndim - array.ndim): array = array[..., np.newaxis] diff --git a/tests/test_sofastream.py b/tests/test_sofastream.py index f7b7ac0..8faedcd 100644 --- a/tests/test_sofastream.py +++ b/tests/test_sofastream.py @@ -39,6 +39,45 @@ def test_sofastream_attribute_error(temp_sofa_file): _ = file.Wrong_Attribute +def test_sofastream_verify_does_not_call_read_sofa( + temp_sofa_file, monkeypatch): + """Verify SofaStream does not load the full SOFA file.""" + + def fail_read_sofa(*_args, **_kwargs): + raise AssertionError("SofaStream.verify must not call read_sofa") + + monkeypatch.setattr(sf, "read_sofa", fail_read_sofa) + + with SofaStream(temp_sofa_file) as file: + file.verify() + + +def test_sofastream_read_open_file_for_verification(temp_sofa_file): + """Verify the lazy SOFA object retains file metadata.""" + expected = sf.read_sofa(temp_sofa_file, verify=False) + + with SofaStream(temp_sofa_file) as file: + actual = file._read_open_file_for_verification() + + expected_keys = {key for key in expected.__dict__ + if not key.startswith("_")} + actual_keys = {key for key in actual.__dict__ + if not key.startswith("_")} + assert actual_keys == expected_keys + + for key in expected_keys: + expected_value = getattr(expected, key) + actual_value = getattr(actual, key) + if isinstance(actual_value, np.ndarray): + expected_array = np.atleast_1d(expected_value) + assert actual_value.shape == expected_array.shape + assert actual_value.dtype == expected_array.dtype + if np.issubdtype(expected_array.dtype, np.str_): + np.testing.assert_array_equal(actual_value, expected_array) + else: + assert actual_value == expected_value + + def test_sofastream_inspect(capfd, temp_sofa_file): tempdir = TemporaryDirectory()