restrict npsl metadata discovery to http transfer

Author: kenjimarshall

.pre-commit-config.yaml

@@ -9,11 +9,14 @@ repos:
     rev: v0.19.0
     hooks:
       - id: yamlfmt
-  - repo: https://github.com/psf/black
-    rev: 25.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.15.14
     hooks:
-      - id: black
-        types_or: [python, jupyter]
+      # Run the linter.
+      - id: ruff-check
+      # Run the formatter.
+      - id: ruff-format
   - repo: https://github.com/kynan/nbstripout
     rev: 0.9.1
     hooks:

bil/api/__init__.py

@@ -4,10 +4,7 @@
 
 from __future__ import annotations
 
-from typing import Any
-from . import abstracts
-from . import formatS
-from . import formatNPSL
+from . import abstracts, formatNPSL, formatS
 from .utils import fetch
 
 IMPLEMENTATIONS: dict[str, type[formatS.Study] | type[formatNPSL.Study]] = {

bil/api/abstracts.py

@@ -7,14 +7,15 @@
 import functools
 from abc import ABC, abstractmethod
 from functools import cached_property
-from typing import Any, Callable, Iterable, TypeVar
 from pathlib import Path
+from typing import Any, Callable, Iterable, TypeVar
 
+import h5py
 import numpy as np
 import pandas as pd
-import h5py
 
 from .utils import fetch
+from .utils.subject import Subject
 
 SDR_VERSIONED_URL: str = (
     "https://stacks.stanford.edu/v{version}/file/zz618yg1930/version/{version}/data"
@@ -219,13 +220,19 @@ def around(self, time_ms: int, t_before: int, t_after: int) -> "Span":
         Returns:
             New aligned span.
         """
+        if t_before == 0 and t_after == 0:
+            raise ValueError("Must specify non-zero t_before or t_after!")
+        if t_before < 0:
+            raise ValueError("Cannot have negative t_before")
+        if t_after < 0:
+            raise ValueError("Cannot have negative t_after")
         start = time_ms - t_before
-        end = time_ms + t_after + 1
+        end = time_ms + t_after
 
         span = self
         if self.start + end > self.stop:
             span = span.extend(t_after=self.start + end - self.stop)
-            end = -1
+            end = -1  # go to end of span
         if start < 0:
             span = span.extend(t_before=abs(start))
             if end != -1:
@@ -280,9 +287,7 @@ def spanset_cls(self) -> type["SpanSet"]:
     def spanarray_cls(self) -> type["SpanSet"]:
         """SpanArray instantiation class."""
 
-    def __init__(
-        self, spans: list[Span], **kwargs: Any  # pylint: disable=unused-argument
-    ) -> None:
+    def __init__(self, spans: list[Span], **kwargs: Any) -> None:
         """Initialize SpanSet.
 
         Args:
@@ -412,7 +417,12 @@ def around(
                 target_t = list(time_ms)[index]
             else:
                 target_t = time_ms
-            span_arr.append(span.around(int(target_t), t_before, t_after))
+            try:
+                # Check to make sure we are number-like
+                target_t = int(str(target_t))
+            except Exception as e:
+                raise ValueError from e
+            span_arr.append(span.around(target_t, t_before, t_after))
         return self._make_spanarray(span_arr)
 
     def map(self, func: Callable, *args: Any, **kwargs: Any) -> list[Any]:
@@ -449,8 +459,7 @@ class StudyMixin(ABC):
         quiet: Verbosity flag.
     """
 
-    # For linting purposes!
-    # span_cls: type[Span]
+    head: str
 
     def __init__(
         self, study_id: str, download_dir: str, quiet: bool = False, **kwargs: Any
@@ -505,7 +514,7 @@ def __getitem__(self, key: Any) -> Any:
         """
         if self.df is None:
             return self.by_time(key)
-        return super().__getitem__(key)  # type: ignore[misc]
+        return super().__getitem__(key)  # ty: ignore[unresolved-attribute]
 
     @cached_property
     @abstractmethod
@@ -573,6 +582,7 @@ class HeadH5Study(StudyMixin):
     """Base class for studies with remote HDF5 head files."""
 
     data_paths: dict[str, list[str]] = {}
+    subject: Subject
 
     def __init__(
         self,
@@ -664,7 +674,7 @@ def _make_span(self, row: pd.Series) -> Span:
         """
         start = int(row["ms_start"])
         stop = int(row["ms_end"])
-        return self.span_cls(
+        return self.span_cls(  # ty: ignore[unresolved-attribute]
             start=start, stop=stop, study_id=self.study_id, study=self, metadata=row
         )
 
@@ -699,17 +709,21 @@ def __init__(
         """
         self.deposition_version = deposition_version
         super().__init__(
-            study_id=study_id,
-            download_dir=f"{download_dir}/{study_id}",
-            quiet=quiet,
-            spans=(),
+            study_id=study_id,  # ty: ignore[unknown-argument]
+            download_dir=f"{download_dir}/{study_id}",  # ty: ignore[unknown-argument]
+            quiet=quiet,  # ty: ignore[unknown-argument]
+            spans=(),  # ty: ignore[unknown-argument]
             **kwargs,
-        )  # type: ignore[call-arg]
+        )
 
     @cached_property
     def fetcher(self) -> fetch.FetcherHTTPS:
         """HTTPS fetch manager."""
-        return fetch.FetcherHTTPS(self.url, self.download_dir, quiet=self.quiet)
+        return fetch.FetcherHTTPS(
+            self.url,  # ty: ignore[unresolved-attribute]
+            self.download_dir,  # ty: ignore[unresolved-attribute]
+            quiet=self.quiet,  # ty: ignore[unresolved-attribute]
+        )
 
     @property
     def _default_url(self) -> str:
@@ -735,7 +749,7 @@ def __init__(self, spans: list[Span], *args: Any, **kwargs: Any) -> None:
             raise ValueError("SpanArray requires spans of the same length")
 
         # init
-        super().__init__(spans=spans_list, *args, **kwargs)  # type: ignore[call-arg]
+        super().__init__(spans=spans_list, *args, **kwargs)  # ty: ignore[unknown-argument]
 
     def _wrap(self, func: str, *args: Any, **kwargs: Any) -> np.ndarray:
         """Wrap results into numpy array.
@@ -748,7 +762,7 @@ def _wrap(self, func: str, *args: Any, **kwargs: Any) -> np.ndarray:
         Returns:
             Numpy array of results.
         """
-        return np.array(super()._wrap(func, *args, **kwargs))  # type: ignore[misc]
+        return np.array(super()._wrap(func, *args, **kwargs))  # ty: ignore[unresolved-attribute]
 
 
 class DataCatalog:

bil/api/formatNPSL.py

@@ -6,18 +6,19 @@
 """
 
 from __future__ import annotations
+
 from functools import cached_property
-from typing import Any, TYPE_CHECKING
-import numpy as np
+from typing import TYPE_CHECKING, Any
+
 import h5py
+import numpy as np
 import pandas as pd
 
 from . import abstracts
 from .utils import subject
-from .utils import b_helpers
 
 if TYPE_CHECKING:
-    from .abstracts import StudyMixin
+    from .abstracts import HeadH5Study
 
 SUBJECTS: dict[str, subject.Subject] = {
     "L": subject.Subject("L", [subject.UTAH], ["m1"]),
@@ -51,7 +52,7 @@ def raster(self, region: str | None = None) -> np.ndarray:
 class Span(DataCatalog, abstracts.Span):
     """Span implementation for format NPSL."""
 
-    study: StudyMixin
+    study: HeadH5Study
 
     def _get(self, h5_dataset: h5py.Dataset) -> np.ndarray:
         """Slice and return data from an HDF5 dataset.
@@ -85,9 +86,9 @@ def _get_regions(self, prefix: str, region: str | None = None) -> np.ndarray:
             else:
                 suffixes = [""]  # Default fallback
         elif region.lower() in ["m1", "pmd"]:
-            assert (
-                region in self.study.subject.regions
-            ), f"subject {self.study.subject.name} does not have region {region}!"
+            assert region in self.study.subject.regions, (
+                f"subject {self.study.subject.name} does not have region {region}!"
+            )
             # Find index of region to determine suffix
             idx = self.study.subject.regions.index(region.lower())
             suffixes = ["" if idx == 0 else str(idx + 1)]
@@ -218,15 +219,12 @@ def _init_meta(self) -> None:
         """Initialize metadata by parsing the HDF5 head file."""
         if self._df is not None:
             return
-        if self.fetcher.check_file_exists("df/trial.csv"):
-            path = self.fetcher.get_file("df/trial.csv")
-            dataframe = pd.read_csv(path)
-            dataframe = dataframe.sort_values(by="number").reset_index(drop=True)
-            self.df = dataframe
-            return
-        df = b_helpers.df_from_h5(self.head, self.study_id)
-        df = df.sort_values(by="number").reset_index(drop=True)
-        self.df = df
+
+        path = self.fetcher.get_file("trial.csv")
+        dataframe = pd.read_csv(path)
+        dataframe = dataframe.sort_values(by="number").reset_index(drop=True)
+        self.df = dataframe
+        return
 
 
 class Study(abstracts.PublicMixin, StudyBase, SpanSet):

bil/api/formatS.py

@@ -3,23 +3,24 @@
 """
 
 from __future__ import annotations
-from typing import Any, TYPE_CHECKING
+
 import sqlite3
 import warnings
 from functools import cached_property
 from pathlib import Path
+from typing import TYPE_CHECKING, Any
 
-import yaml
+import h5py
 import numpy as np
 import pandas as pd
-import h5py
+import yaml
 
 from . import abstracts
 from .abstracts import needs_data
 from .utils import subject
 
 if TYPE_CHECKING:
-    from .abstracts import StudyMixin
+    from .abstracts import HeadH5Study
 
 # Channel mapping for Utah arrays, accounting for spatial orientation
 U_M1_MAP = subject.UTAH_MAP.copy()[::-1, ::-1].T
@@ -246,7 +247,7 @@ class Span(DataCatalog, abstracts.Span):
     neural data and behavioral signals.
     """
 
-    study: StudyMixin
+    study: HeadH5Study
 
     def _stream(self, region: str | None = None) -> np.ndarray:
         """Identify relevant neural data streams.
@@ -746,10 +747,11 @@ def _init_meta(self) -> None:
         """Fetch and load trial metadata."""
         if self._df is not None:
             return
-        assert self.fetcher.check_file_exists(
-            "df/trial.df.xz"
-        ), f"trial information not available; does the dataset {self.study_id} exist?"
+        assert self.fetcher.check_file_exists("df/trial.df.xz"), (
+            f"trial information not available; does the dataset {self.study_id} exist?"
+        )
         path = self.fetcher.get_file("df/trial.df.xz")
         dataframe = pd.read_pickle(path)
+        dataframe = pd.DataFrame(dataframe)
         dataframe = dataframe.sort_values(by="number").reset_index(drop=True)
         self.df = dataframe