update job name, setup singularity, isolate gpu tests

install sing deps

Author: chyumin

.github/workflows/test_containers_gpu.yml

@@ -22,7 +22,7 @@ jobs:
             --cloud-region=us-east-2 \
             --cloud-type=g4dn.xlarge \
             --labels=cml-gpu
-  train-model:
+  test-gpu-sorters:
     needs: deploy-runner
     runs-on: [self-hosted, cml-gpu]
     timeout-minutes: 4320 # 72h
@@ -31,9 +31,14 @@ jobs:
       options: --gpus all
     steps:
       - uses: actions/checkout@v3
+
+      - uses: eWaterCycle/setup-singularity@v7
+        with:
+          singularity-version: 3.8.3
+
       - name: Run test singularity containers with GPU
         env:
           REPO_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
         run: |
           pip install -r requirements_test.txt
-          pytest -sv tests/test_singularity_containers.py::test_kilosort3
+          pytest -sv tests/test_singularity_containers.py::test_tridesclous

.github/workflows/test_containers_singularity.yml

@@ -20,6 +20,21 @@ jobs:
         with:
           python-version: 3.8
 
+
+      - name: Install singularity dependencies
+        run: |
+          sudo apt-get update && sudo apt-get install -y \
+          build-essential \
+          libssl-dev \
+          uuid-dev \
+          libgpgme11-dev \
+          squashfs-tools \
+          libseccomp-dev \
+          wget \
+          pkg-config \
+          git \
+          cryptsetup
+
       - uses: eWaterCycle/setup-singularity@v7
         with:
           singularity-version: 3.8.3

tests/test_containers_gpu.py

@@ -0,0 +1,57 @@
+import os
+import shutil
+
+import pytest
+
+import spikeinterface.extractors as se
+import spikeinterface.sorters as ss
+
+os.environ['SINGULARITY_DISABLE_CACHE'] = 'true'
+
+
+@pytest.fixture(autouse=True)
+def work_dir(request, tmp_path):
+    """
+    This fixture, along with "run_kwargs" creates one folder per
+    test function using built-in tmp_path pytest fixture
+
+    The tmp_path will be the working directory for the test function
+
+    At the end of the each test function, a clean up will be done
+    """
+    os.chdir(tmp_path)
+    yield
+    os.chdir(request.config.invocation_dir)
+    shutil.rmtree(str(tmp_path))
+
+
+@pytest.fixture
+def run_kwargs(work_dir):
+    test_recording, _ = se.toy_example(
+        duration=30,
+        seed=0,
+        num_channels=64,
+        num_segments=1
+    )
+    test_recording = test_recording.save(name='toy')
+    return dict(recording=test_recording, verbose=True, singularity_image=True)
+
+
+def test_kilosort2(run_kwargs):
+    sorting = ss.run_kilosort2(output_folder="kilosort2", **run_kwargs)
+    print(sorting)
+
+
+def test_kilosort2_5(run_kwargs):
+    sorting = ss.run_kilosort2_5(output_folder="kilosort2_5", **run_kwargs)
+    print(sorting)
+
+
+def test_kilosort3(run_kwargs):
+    sorting = ss.run_kilosort3(output_folder="kilosort3", **run_kwargs)
+    print(sorting)
+
+
+def test_pykilosort(run_kwargs):
+    sorting = ss.run_pykilosort(output_folder="pykilosort", **run_kwargs)
+    print(sorting)

tests/test_singularity_containers.py

@@ -75,27 +75,3 @@ def test_hdsort(run_kwargs):
 def test_kilosort1(run_kwargs):
     sorting = ss.run_kilosort(output_folder="kilosort", useGPU=False, **run_kwargs)
     print(sorting)
-
-
-@pytest.mark.skip(reason="GPU required")
-def test_kilosort2(run_kwargs):
-    sorting = ss.run_kilosort2(output_folder="kilosort2", **run_kwargs)
-    print(sorting)
-
-
-@pytest.mark.skip(reason="GPU required")
-def test_kilosort2_5(run_kwargs):
-    sorting = ss.run_kilosort2_5(output_folder="kilosort2_5", **run_kwargs)
-    print(sorting)
-
-
-@pytest.mark.skip(reason="GPU required")
-def test_kilosort3(run_kwargs):
-    sorting = ss.run_kilosort3(output_folder="kilosort3", **run_kwargs)
-    print(sorting)
-
-
-@pytest.mark.skip(reason="GPU required")
-def test_pykilosort(run_kwargs):
-    sorting = ss.run_pykilosort(output_folder="pykilosort", **run_kwargs)
-    print(sorting)