Up tests

Author: Descanonge

docs/source/conf.py

@@ -17,7 +17,7 @@
 ## Project information
 
 project = "Neba"
-copyright = "2023, Clément Haëck"
+copyright = "2026, Clément Haëck"
 author = "Clément Haëck"
 
 version = neba.__version__

src/neba/data/source.py

@@ -83,7 +83,7 @@ def root_directory(self) -> Path:
 
         return rootdir
 
-    def get_source(self, relative: bool = False, _warn: bool = True) -> list[str]:  # type: ignore
+    def get_source(self, relative: bool = False, _warn: bool = True) -> list[str]:  # type: ignore[override]
         """Return list of filenames.
 
         Parameters

src/neba/data/xarray.py

@@ -5,7 +5,7 @@
 import logging
 import os
 from collections.abc import Callable, Hashable, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Literal, assert_never, cast, overload
+from typing import TYPE_CHECKING, Any, Literal, cast, overload
 
 import xarray as xr
 
@@ -162,7 +162,7 @@ def send_single_call(
             kwargs = self.to_zarr_kwargs | kwargs
             return ds.to_zarr(outfile, **kwargs)
 
-        assert_never(format)
+        raise ValueError(f"File format '{format}' not supported.")
 
     def add_metadata(
         self,
@@ -247,11 +247,8 @@ def send_calls_together(
             grouped_calls = calls[slc]
             delayed = [self.send_single_call(c, **kwargs) for c in grouped_calls]
 
-            # Compute them all at once
-            # This loop is super important, this create all the futures for computation
-            # and remove them as soon as they are completed (and the variable `future`
-            # goes out of scope). That way the data does not pile up, it is freed.
-            # We only care about the side effect of writing to disk, not the result data.
+            # Futures are deleted as soon as they go out of scope. They do not pile up
+            # but we still return only when all are completed.
             for future in distributed.as_completed(client.compute(delayed)):
                 log.debug("\t\tfuture completed: %s", future)
 
@@ -510,7 +507,7 @@ def split_by_time(
                     "Resampling frequency is equal to that of dataset. "
                     "Will not resample."
                 )
-                return [ds_unit for _, ds_unit in ds.groupby("time", squeeze=False)]
+                return [ds.isel(time=[i]) for i in range(ds.time.size)]
 
         resample = ds.resample(time=freq)
         return [ds_unit for _, ds_unit in resample]

tests/conftest.py

@@ -19,6 +19,9 @@
 settings.load_profile(os.getenv("HYPOTHESIS_PROFILE", "dev").lower())
 
 
+# register dask.distributed fixtures
+pytest_plugins = "distributed.utils_test"
+
 def pytest_configure(config: pytest.Config):
     config.addinivalue_line("markers", "todo")
 

tests/data/test_interface.py

@@ -240,8 +240,29 @@ def setup(self):
                 is_setup.add("B")
                 super().setup()
 
+            def _lines(self):
+                return "SourceB_repr"
+
+
         class MyDataInterface(DataInterface):
             Source = ModuleMix.create([SourceA, SourceB])
 
         MyDataInterface()
         assert is_setup == {"A", "B", "C"}
+
+    def test_repr(self):
+
+        class MyDataInterface(DataInterface):
+            class SourceA(SourceAbstract):
+                def _lines(self):
+                    return ["SourceA_repr"]
+            class SourceB(SourceAbstract):
+                def _lines(self):
+                    return ["SourceB_repr"]
+
+            Source = ModuleMix.create([SourceA, SourceB])
+
+        di = MyDataInterface()
+        assert repr(di.source) == "SourceA\n\tSourceA_repr\nSourceB\n\tSourceB_repr"
+
+    # more tests in test_saource.TestModuleMix

tests/data/test_params.py

@@ -300,31 +300,49 @@ class sub(Section):
             di.sub.b = 1
 
 
-def test_autocached():
-    class MyDataInterface(DataInterface):
-        Parameters = ParametersDict
+class TestCachedModule:
 
-        class Loader(LoaderAbstract, CachedModule):
-            @property
-            @autocached
-            def test_property(self):
-                return 0
+    def get_interface(self):
+        class MyDataInterface(DataInterface):
+            Parameters = ParametersDict
+
+            class Loader(LoaderAbstract, CachedModule):
+                @property
+                @autocached
+                def test_property(self):
+                    return 0
+
+                @autocached
+                def test_method(self):
+                    return 1
+
+        return MyDataInterface
+
+    def test_autocached(self):
+        di = self.get_interface()()
+        assert len(di.loader.cache) == 0
+
+        _ = di.loader.test_property
+        assert di.loader.cache == dict(test_property=0)
+
+        _ = di.loader.test_method()
+        assert di.loader.cache == dict(test_property=0, test_method=1)
 
-            @autocached
-            def test_method(self):
-                return 1
+        di.trigger_callbacks()
+        assert len(di.loader.cache) == 0
 
-    di = MyDataInterface()
-    assert len(di.loader.cache) == 0
+    def test_disable(self):
+        di_cls = self.get_interface()
+        di_cls.Loader._add_void_callback = False
+        di = di_cls()
 
-    _ = di.loader.test_property
-    assert di.loader.cache == dict(test_property=0)
+        assert len(di._reset_callbacks) == 0
 
-    _ = di.loader.test_method()
-    assert di.loader.cache == dict(test_property=0, test_method=1)
+        _ = di.loader.test_property
+        assert "test_property" in di.loader.cache
+        di.trigger_callbacks()
+        assert "test_property" in di.loader.cache
 
-    di.trigger_callbacks()
-    assert len(di.loader.cache) == 0
 
 
 class TestParamsExcursion:

tests/data/test_source.py

@@ -51,7 +51,23 @@ def test_intersection(self):
         mix = mix_cls()
         assert mix.get_source() == ["b", "c"]
 
-    def test_file_select(self):
+    def test_no_select(self):
+        SourceA = get_simple_source("SourceA", ["a", "b", "c"])
+        SourceB = get_simple_source("SourceB", ["b", "c", "d", "e"])
+        mix_cls = SourceIntersection.create([SourceA, SourceB])
+        mix = mix_cls()
+        with pytest.raises(ValueError):
+            mix.select()
+
+        def select(mod, *args, **kwargs):
+            return "SourceA"
+
+        mix.set_select(select)
+        assert isinstance(mix.select(), SourceA)
+
+    def test_select_unbound(self):
+        """Select function is defined outside of interface."""
+
         class SourceA(SimpleSource):
             source_loc = "a"
 
@@ -81,6 +97,42 @@ class DataInterfaceMix(DataInterface):
 
         assert di.source.apply_select("get_filename") == "file_a_0"
 
+    def get_interface(self):
+        class DataInterfaceMix(DataInterface):
+            Parameters = ParametersDict
+
+            class SourceA(SimpleSource):
+                source_loc = "a"
+
+                def get_filename(self, **fixes):
+                    param = fixes.get("param", self.parameters["param"])
+                    return f"file_a_{param}"
+
+            class SourceB(SimpleSource):
+                source_loc = "b"
+
+                def get_filename(self, **fixes):
+                    param = fixes.get("param", self.parameters["param"])
+                    return f"file_b_{param}"
+
+            @staticmethod
+            def select(module, **kwargs):
+                return kwargs.get("selected", module.parameters["selected"])
+
+            Source = SourceUnion.create([SourceA, SourceB], select_func=select)
+
+        return DataInterfaceMix
+
+    def test_select_bound(self):
+        """Select is defined as static method."""
+        di = self.get_interface()(param=0, selected="SourceA")
+        assert di.source.apply_select("get_filename") == "file_a_0"
+
+    def test_file_select(self):
+        di = self.get_interface()(param=0, selected="SourceA")
+
+        assert di.source.apply_select("get_filename") == "file_a_0"
+
         di.parameters["param"] = 1
         di.parameters["selected"] = "SourceB"
         assert di.source.apply_select("get_filename") == "file_b_1"
@@ -94,8 +146,49 @@ class DataInterfaceMix(DataInterface):
             == "file_a_2"
         )
 
-        # automatic dispatch
-        assert di.source.get_filename() == "file_b_1"
+    def test_apply(self):
+        di = self.get_interface()(param=0, selected="SourceA")
+        assert di.source.apply("get_filename", all=True, param=1) == [
+            "file_a_1",
+            "file_b_1",
+        ]
+
+        assert di.source.apply("get_filename", all=False, param=1) == "file_a_1"
+
+    def test_automatic_dispatch(self):
+        di = self.get_interface()(param=0, selected="SourceA")
+
+        assert di.source.get_filename() == "file_a_0"
+        di.parameters["selected"] = "SourceB"
+        assert di.source.get_filename() == "file_b_0"
+
+        # disabled
+        di.source._auto_dispatch_getattr = False
+        with pytest.raises(AttributeError):
+            di.source.get_filename()
+
+        # attribute does not exist in base classes
+        with pytest.raises(AttributeError):
+            di.source.unknown_attribute()
+
+        # exception in selection function: no infinie recursion
+        def select(mod, **kwargs):
+            raise ValueError
+
+        di.source.set_select(select)
+        with pytest.raises(ValueError):
+            di.source.select()
+        with pytest.raises(AttributeError):
+            di.source.get_filename()
+
+    def test_bad_select(self):
+        di = self.get_interface()
+
+        def select(mod, **kwargs):
+            return "NonExistentBaseModule"
+
+        with pytest.raises(AttributeError):
+            di.source.select()
 
 
 def setup_multiple_files(tmpdir, var: str = "A") -> list[str]:
@@ -129,6 +222,7 @@ class MyDataInterface(DataInterface):
             Parameters = ParametersDict
 
             class Source(GlobSource):
+                # here we test root_directory as a simple str
                 def get_root_directory(self):
                     return str(tmpdir)
 
@@ -140,6 +234,11 @@ def get_glob_pattern(self):
         di = MyDataInterface(var="A")
         assert di.get_source() == ref_filenames
 
+        # test relative
+        assert di.get_source(relative=True) == [
+            f.removeprefix(str(tmpdir) + "/") for f in ref_filenames
+        ]
+
         # check files cached
         assert di.source.cache["datafiles"] == ref_filenames
 
@@ -155,8 +254,9 @@ class MyDataInterface(DataInterface):
             Parameters = ParametersDict
 
             class Source(FileFinderSource):
+                # here we test root_directory as a list
                 def get_root_directory(self):
-                    return str(tmpdir)
+                    return [str(tmpdir), "subdir"]
 
                 def get_filename_pattern(self):
                     var = self.parameters["var"]
@@ -165,11 +265,16 @@ def get_filename_pattern(self):
         return MyDataInterface
 
     def test_get_source(self, tmpdir):
-        ref_filenames = setup_multiple_files(tmpdir, var="A")
+        ref_filenames = setup_multiple_files(tmpdir / "subdir", var="A")
 
         di = self.setup_interface(tmpdir)(var="A")
         assert di.get_source() == ref_filenames
 
+        # test relative
+        assert di.get_source(relative=True) == [
+            f.removeprefix(str(tmpdir / "subdir") + "/") for f in ref_filenames
+        ]
+
         # check files cached
         assert di.source.cache["datafiles"] == ref_filenames
 
@@ -179,7 +284,7 @@ def test_get_source(self, tmpdir):
         assert len(di.get_source()) == 0
 
     def test_fixes(self, tmpdir):
-        ref_filenames = setup_multiple_files(tmpdir, var="A")
+        ref_filenames = setup_multiple_files(tmpdir / "subdir", var="A")
 
         di = self.setup_interface(tmpdir)(var="A", Y="2010")