Merge branch 'master' into update-ci

zm711 · web-flow · commit 5c49aa9bc43c · 2025-06-06T11:25:20.000-04:00
diff --git a/.github/workflows/io-test.yml b/.github/workflows/io-test.yml
@@ -119,13 +119,6 @@ jobs:
       #  run: |
       #    pip install --no-dependencies -e .
 
-      - name: Install wine
-        run: |
-          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
-          sudo dpkg --add-architecture i386
-          sudo apt-get update -qq
-          sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
-
       - name: Pip list
         run: |
           pip list
diff --git a/.github/workflows/plexon2-testing.yml b/.github/workflows/plexon2-testing.yml
@@ -0,0 +1,109 @@
+name: NeoPlexon2Test
+
+on:
+  workflow_dispatch:
+    schedule:
+      - cron: "0 12 * * 0" # weekly at noon UTC on Sundays
+
+
+jobs:
+  build-and-test:
+    name: Test on (${{ inputs.os }}) (${{ matrix.python-version}}) (${{ matrix.numpy-version }})
+    runs-on: ${{ inputs.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ['3.9', '3.12']
+        numpy-version: ['1.26', '2.0']
+    defaults:
+      # by default run in bash mode (required for conda usage)
+      run:
+        shell: bash -l {0}
+    steps:
+
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Get current year-month
+        id: date
+        run: echo "date=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
+
+      - name: Get ephy_testing_data current head hash
+        # the key depend on the last commit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
+        id: ephy_testing_data
+        run: |
+          echo "dataset_hash=$(git ls-remote https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git HEAD | cut -f1)" >> $GITHUB_OUTPUT
+
+      - uses: actions/cache/restore@v4
+        # Loading cache of ephys_testing_dataset
+        id: cache-datasets
+        with:
+          path: ~/ephy_testing_data
+          key: ${{ runner.os }}-datasets-${{ steps.ephy_testing_data.outputs.dataset_hash }}
+          restore-keys: ${{ runner.os }}-datasets-
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: neo-test-env-${{ matrix.python-version }}
+          python-version: "${{ matrix.python-version }}"
+
+      - name: Install testing dependencies
+        # testing environment is only created from yml if no cache was found
+        # restore-key hits should result in `cache-hit` == 'false'
+        #if: steps.cache-conda-env.outputs.cache-hit != 'true'
+        run: |
+          conda install pip numpy=${{ matrix.numpy-version }} -c conda-forge
+        # this command is for updating cache. We are resting removal.
+        # conda env update --name neo-test-env-${{ matrix.python-version }} --file environment_testing.yml --prune
+
+      - name: Install git-annex
+        # this is the trick from the spikeinterface repo for getting git-annex to work with datalad
+        # see https://github.com/SpikeInterface/spikeinterface/pull/3877 for more info
+        shell: bash
+        run: |
+          pip install datalad-installer
+          datalad-installer --sudo ok git-annex --method datalad/packages
+          git config --global filter.annex.process "git-annex filter-process"  # recommended for efficiency
+
+      - name: Configure git
+        run: |
+          git config --global user.email "neo_ci@fake_mail.com"
+          git config --global user.name "neo CI"
+
+      - name: Python version
+        run: |
+          which python
+          python --version
+
+      - name: Install neo including dependencies
+        # installation with dependencies is only required if no cache was found
+        # restore-key hits should result in `cache-hit` == 'false'
+        # if: steps.cache-conda-env.outputs.cache-hit != 'true'
+        run: |
+          pip install --upgrade -e .
+          pip install .[test]
+
+
+      - name: Install wine
+        run: |
+          sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list
+          sudo dpkg --add-architecture i386
+          sudo apt-get update -qq
+          sudo apt-get install -yqq --allow-downgrades libc6:i386 libgcc-s1:i386 libstdc++6:i386 wine
+
+      - name: Pip list
+        run: |
+          pip list
+
+      - name: Conda list
+        run: |
+          conda list
+
+      - name: Test with pytest
+        env:
+          HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_local_plugin_path
+          PLEXON2_TEST: true
+        run: |
+          # only neo.rawio and neo.io
+          pytest --cov=neo neo/test/rawiotest
+          pytest --cov=neo neo/test/iotest
diff --git a/doc/source/authors.rst b/doc/source/authors.rst
@@ -71,7 +71,7 @@ and may not be the current affiliation of a contributor.
 * Daniel P. Crepeau [30]
 * Divyansh Gupta [31]
 * Nate Dolensek [3]
-* Philipp Hornauer [32]
+* Philipp Hornauer [32, 45]
 * Robert Wolff [42]
 * Jules Lebert [33]
 * Benjamin Heasly
@@ -140,6 +140,7 @@ and may not be the current affiliation of a contributor.
 42. Istituto Italiano di Tecnologia (IIT), Genoa, Italy
 43. University of Genoa, Italy
 44. AquiNeuro, SAS
+45. Maxwell Biosystems AG
 
 
 
diff --git a/neo/rawio/blackrockrawio.py b/neo/rawio/blackrockrawio.py
@@ -597,10 +597,17 @@ def _parse_header(self):
             for c in range(spike_channels.size):
                 st_ann = seg_ann["spikes"][c]
                 channel_id, unit_id = self.internal_unit_ids[c]
-                unit_tag = {0: "unclassified", 255: "noise"}.get(unit_id, str(unit_id))
                 st_ann["channel_id"] = channel_id
                 st_ann["unit_id"] = unit_id
-                st_ann["unit_tag"] = unit_tag
+                if unit_id == 0:
+                    st_ann["unit_classification"] = "unclassified"
+                elif 1 <= unit_id <= 16:
+                    st_ann["unit_classification"] = "sorted"
+                elif unit_id == 255:
+                    st_ann["unit_classification"] = "noise"
+                else:  # 17-254 are reserved
+                    st_ann["unit_classification"] = "reserved"
+                st_ann["unit_tag"] = st_ann["unit_classification"]
                 st_ann["description"] = f"SpikeTrain channel_id: {channel_id}, unit_id: {unit_id}"
                 st_ann["file_origin"] = self._filenames["nev"] + ".nev"
 
@@ -1258,9 +1265,7 @@ def __read_nev_data(self, nev_data_masks, nev_data_types):
         # based on blackrock's own code this is okay so applying an int to round down is necessary to obtain the
         # memory map of full packets and toss the partial packet.
         # See reference: https://github.com/BlackrockNeurotech/Python-Utilities/blob/fa75aa671680306788e10d3d8dd625f9da4ea4f6/brpylib/brpylib.py#L580-L587
-        n_packets = int(
-            (self.__get_file_size(filename) - header_size) / data_size
-        )
+        n_packets = int((self.__get_file_size(filename) - header_size) / data_size)
 
         raw_data = np.memmap(
             filename,
diff --git a/neo/rawio/maxwellrawio.py b/neo/rawio/maxwellrawio.py
@@ -242,18 +242,20 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, strea
             raise (e)
 
 
-_hdf_maxwell_error = """Maxwell file format is based on HDF5.
-The internal compression requires a custom plugin!!!
-This is a big pain for the end user.
-You, as a end user, should ask Maxwell company to change this.
-Please visit this page and install the missing decompression libraries:
-https://share.mxwbio.com/d/4742248b2e674a85be97/
-Then, link the decompression library by setting the `HDF5_PLUGIN_PATH` to your
-installation location, e.g. via
+_hdf_maxwell_error = """The MaxWell file compression requires a custom plugin.
+You can use the auto_install_maxwell_hdf5_compression_plugin() function or 
+(if it fails) install it manually:
+Download the missing decompression library:
+https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35
+Then, link the decompression library by setting the `HDF5_PLUGIN_PATH` to its location, 
+e.g. directly in Python via:
 os.environ['HDF5_PLUGIN_PATH'] = '/path/to/custom/hdf5/plugin/'
-
-Alternatively, you can use the auto_install_maxwell_hdf5_compression_plugin() below
-function that do it automagically.
+or in your shell via:
+export HDF5_PLUGIN_PATH=/path/to/custom/hdf5/plugin/
+You can also set the `HDF5_PLUGIN_PATH` environment variable in your shell
+configuration file (e.g. .bashrc, .bash_profile, .zshrc, etc.) to make it
+permanent. 
+See https://mxw.bio/MxW_Doc_Installing_Decompression_Library_to_load_MaxLab_Live_Recordings for more details.
 """
 
 
@@ -267,13 +269,17 @@ def auto_install_maxwell_hdf5_compression_plugin(hdf5_plugin_path=None, force_do
     hdf5_plugin_path.mkdir(exist_ok=True)
 
     if platform.system() == "Linux":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FLinux%2Flibcompression.so&dl=1"
+        remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FLinux%2Flibcompression.so&dl=1"
         local_lib = hdf5_plugin_path / "libcompression.so"
     elif platform.system() == "Darwin":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FMacOS%2Flibcompression.dylib&dl=1"
+        if platform.machine() == "arm64":
+            remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FMacOS%2FMac_arm64%2Flibcompression.dylib&dl=1"
+        else:
+            # Assuming x86_64 for MacOS
+            remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FMacOS%2FMac_x86_64%2Flibcompression.dylib&dl=1"
         local_lib = hdf5_plugin_path / "libcompression.dylib"
     elif platform.system() == "Windows":
-        remote_lib = "https://share.mxwbio.com/d/4742248b2e674a85be97/files/?p=%2FWindows%2Fcompression.dll&dl=1"
+        remote_lib = "https://share.mxwbio.com/d/7f2d1e98a1724a1b8b35/files/?p=%2FWindows%2Fcompression.dll&dl=1"
         local_lib = hdf5_plugin_path / "compression.dll"
 
     if not force_download and local_lib.is_file():
diff --git a/neo/rawio/spikeglxrawio.py b/neo/rawio/spikeglxrawio.py
@@ -53,6 +53,7 @@
 from pathlib import Path
 import os
 import re
+from warnings import warn
 
 import numpy as np
 
@@ -76,7 +77,9 @@ class SpikeGLXRawIO(BaseRawWithBufferApiIO):
     dirname: str, default: ''
         The spikeglx folder containing meta/bin files
     load_sync_channel: bool, default: False
-        The last channel (SY0) of each stream is a fake channel used for synchronisation
+        Can be used to load the synch stream as the last channel of the neural data.
+        This option is deprecated and will be removed in version 0.15.
+        From versions higher than 0.14.1 the sync channel is always loaded as a separate stream.
     load_channel_location: bool, default: False
         If True probeinterface is used to load the channel locations from the directory
 
@@ -109,6 +112,13 @@ def __init__(self, dirname="", load_sync_channel=False, load_channel_location=Fa
         BaseRawWithBufferApiIO.__init__(self)
         self.dirname = dirname
         self.load_sync_channel = load_sync_channel
+        if load_sync_channel:
+            warn(
+                "The load_sync_channel=True option is deprecated and will be removed in version 0.15 \n"
+                "The sync channel is now loaded as a separate stream by default and should be accessed as such. ",
+                DeprecationWarning,
+                stacklevel=2,
+            )
         self.load_channel_location = load_channel_location
 
     def _source_name(self):
@@ -152,6 +162,8 @@ def _parse_header(self):
         signal_buffers = []
         signal_streams = []
         signal_channels = []
+        sync_stream_id_to_buffer_id = {}
+
         for stream_name in stream_names:
             # take first segment
             info = self.signals_info_dict[0, stream_name]
@@ -168,6 +180,21 @@ def _parse_header(self):
             for local_chan in range(info["num_chan"]):
                 chan_name = info["channel_names"][local_chan]
                 chan_id = f"{stream_name}#{chan_name}"
+
+                # Sync channel
+                if (
+                    "nidq" not in stream_name
+                    and "SY0" in chan_name
+                    and not self.load_sync_channel
+                    and local_chan == info["num_chan"] - 1
+                ):
+                    # This is a sync channel and should be added as its own stream
+                    sync_stream_id = f"{stream_name}-SYNC"
+                    sync_stream_id_to_buffer_id[sync_stream_id] = buffer_id
+                    stream_id_for_chan = sync_stream_id
+                else:
+                    stream_id_for_chan = stream_id
+
                 signal_channels.append(
                     (
                         chan_name,
@@ -177,25 +204,33 @@ def _parse_header(self):
                         info["units"],
                         info["channel_gains"][local_chan],
                         info["channel_offsets"][local_chan],
-                        stream_id,
+                        stream_id_for_chan,
                         buffer_id,
                     )
                 )
 
-            # all channel by dafult unless load_sync_channel=False
+            # all channel by default unless load_sync_channel=False
             self._stream_buffer_slice[stream_id] = None
+
             # check sync channel validity
             if "nidq" not in stream_name:
                 if not self.load_sync_channel and info["has_sync_trace"]:
-                    # the last channel is remove from the stream but not from the buffer
-                    last_chan = signal_channels[-1]
-                    last_chan = last_chan[:-2] + ("", buffer_id)
-                    signal_channels = signal_channels[:-1] + [last_chan]
+                    # the last channel is removed from the stream but not from the buffer
                     self._stream_buffer_slice[stream_id] = slice(0, -1)
+
+                    # Add a buffer slice for the sync channel
+                    sync_stream_id = f"{stream_name}-SYNC"
+                    self._stream_buffer_slice[sync_stream_id] = slice(-1, None)
+
                 if self.load_sync_channel and not info["has_sync_trace"]:
                     raise ValueError("SYNC channel is not present in the recording. " "Set load_sync_channel to False")
 
         signal_buffers = np.array(signal_buffers, dtype=_signal_buffer_dtype)
+
+        # Add sync channels as their own streams
+        for sync_stream_id, buffer_id in sync_stream_id_to_buffer_id.items():
+            signal_streams.append((sync_stream_id, sync_stream_id, buffer_id))
+
         signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype)
         signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype)
 
@@ -237,6 +272,14 @@ def _parse_header(self):
                 t_start = frame_start / sampling_frequency
 
                 self._t_starts[stream_name][seg_index] = t_start
+
+                # This need special logic because sync not present in stream_names
+                if f"{stream_name}-SYNC" in signal_streams["name"]:
+                    sync_stream_name = f"{stream_name}-SYNC"
+                    if sync_stream_name not in self._t_starts:
+                        self._t_starts[sync_stream_name] = {}
+                    self._t_starts[sync_stream_name][seg_index] = t_start
+
                 t_stop = info["sample_length"] / info["sampling_rate"]
                 self._t_stops[seg_index] = max(self._t_stops[seg_index], t_stop)
 
@@ -266,6 +309,10 @@ def _parse_header(self):
                     # need probeinterface to be installed
                     import probeinterface
 
+                    # Skip for sync streams
+                    if "SYNC" in stream_name:
+                        continue
+
                     info = self.signals_info_dict[seg_index, stream_name]
                     if "imroTbl" in info["meta"] and info["stream_kind"] == "ap":
                         # only for ap channel
@@ -529,7 +576,7 @@ def extract_stream_info(meta_file, meta):
         if (
             "imDatPrb_type" not in meta
             or meta["imDatPrb_type"] == "0"
-            or meta["imDatPrb_type"] in ("1015", "1016", "1022", "1030", "1031", "1032", "1100", "1121", "1300")
+            or meta["imDatPrb_type"] in ("1015", "1016", "1022", "1030", "1031", "1032", "1100", "1121", "1123","1300")
         ):
             # This work with NP 1.0 case with different metadata versions
             # https://github.com/billkarsh/SpikeGLX/blob/15ec8898e17829f9f08c226bf04f46281f106e5f/Markdown/Metadata_30.md
diff --git a/neo/test/iotest/test_plexon2io.py b/neo/test/iotest/test_plexon2io.py
@@ -3,6 +3,7 @@
 """
 
 import unittest
+import os
 
 from neo.io import Plexon2IO
 from neo.test.iotest.common_io_test import BaseTestIO
@@ -17,8 +18,9 @@
 except (ImportError, TimeoutError):
     HAVE_PYPL2 = False
 
+TEST_PLEXON2 = bool(os.getenv("PLEXON2_TEST"))
 
-@unittest.skipUnless(HAVE_PYPL2, "requires pypl package and all its dependencies")
+@unittest.skipUnless(HAVE_PYPL2 and TEST_PLEXON2, "requires pypl package and all its dependencies")
 class TestPlexon2IO(BaseTestIO, unittest.TestCase):
     entities_to_download = TestPlexon2RawIO.entities_to_download
     entities_to_test = TestPlexon2RawIO.entities_to_test
diff --git a/neo/test/rawiotest/test_plexon2rawio.py b/neo/test/rawiotest/test_plexon2rawio.py
diff --git a/neo/test/rawiotest/test_spikeglxrawio.py b/neo/test/rawiotest/test_spikeglxrawio.py
