Merge pull request #1819 from dandi/bids-ignore

Upload .bidsignore files as part of BIDS datasets

Author: yarikoptic

dandi/consts.py

@@ -201,6 +201,8 @@ def urls(self) -> Iterator[str]:
 
 BIDS_DATASET_DESCRIPTION = "dataset_description.json"
 
+BIDS_IGNORE_FILE = ".bidsignore"
+
 # Fields which would be used to compose organized filenames
 # TODO: add full description into command --help etc
 # Order matters!

dandi/files/__init__.py

@@ -17,7 +17,11 @@
 from pathlib import Path
 
 from dandi import get_logger
-from dandi.consts import BIDS_DATASET_DESCRIPTION, dandiset_metadata_file
+from dandi.consts import (
+    BIDS_DATASET_DESCRIPTION,
+    BIDS_IGNORE_FILE,
+    dandiset_metadata_file,
+)
 from dandi.exceptions import UnknownAssetError
 
 from ._private import BIDSFileFactory, DandiFileFactory
@@ -110,7 +114,9 @@ def find_dandi_files(
     while path_queue:
         p, bidsdd = path_queue.popleft()
         if p.name.startswith("."):
-            continue
+            # Allow .bidsignore files within BIDS datasets to be uploaded
+            if not (p.name == BIDS_IGNORE_FILE and bidsdd is not None):
+                continue
         if p.is_dir():
             if p.is_symlink():
                 lgr.warning("%s: Ignoring unsupported symbolic link to directory", p)

dandi/tests/fixtures.py

@@ -687,6 +687,7 @@ def bids_dandiset(new_dandiset: SampleDandiset, bids_examples: Path) -> SampleDa
         ignore=shutil.ignore_patterns(dandiset_metadata_file),
     )
     (new_dandiset.dspath / "CHANGES").write_text("0.1.0 2014-11-03\n")
+    (new_dandiset.dspath / ".bidsignore").write_text("dandiset.yaml\n")
     return new_dandiset
 
 

dandi/tests/test_files.py

@@ -179,6 +179,7 @@ def test_find_dandi_files_with_bids(tmp_path: Path) -> None:
         dandiset_metadata_file,
         "foo.txt",
         "bar.nwb",
+        "bids1/.bidsignore",
         "bids1/dataset_description.json",
         "bids1/file.txt",
         "bids1/subdir/quux.nwb",
@@ -196,6 +197,12 @@ def test_find_dandi_files_with_bids(tmp_path: Path) -> None:
 
     assert files == [
         NWBAsset(filepath=tmp_path / "bar.nwb", path="bar.nwb", dandiset_path=tmp_path),
+        GenericBIDSAsset(
+            filepath=tmp_path / "bids1" / ".bidsignore",
+            path="bids1/.bidsignore",
+            dandiset_path=tmp_path,
+            bids_dataset_description_ref=ANY,  # type: ignore[arg-type]
+        ),
         BIDSDatasetDescriptionAsset(
             filepath=tmp_path / "bids1" / "dataset_description.json",
             path="bids1/dataset_description.json",
@@ -246,9 +253,15 @@ def test_find_dandi_files_with_bids(tmp_path: Path) -> None:
         ),
     ]
 
-    bidsdd = files[1]
+    bidsdd = files[2]
     assert isinstance(bidsdd, BIDSDatasetDescriptionAsset)
     assert sorted(bidsdd.dataset_files, key=attrgetter("filepath")) == [
+        GenericBIDSAsset(
+            filepath=tmp_path / "bids1" / ".bidsignore",
+            path="bids1/.bidsignore",
+            dandiset_path=tmp_path,
+            bids_dataset_description_ref=ANY,  # type: ignore[arg-type]
+        ),
         GenericBIDSAsset(
             filepath=tmp_path / "bids1" / "file.txt",
             path="bids1/file.txt",
@@ -271,7 +284,7 @@ def test_find_dandi_files_with_bids(tmp_path: Path) -> None:
     for asset in bidsdd.dataset_files:
         assert asset.bids_dataset_description is bidsdd
 
-    bidsdd = files[5]
+    bidsdd = files[6]
     assert isinstance(bidsdd, BIDSDatasetDescriptionAsset)
     assert sorted(bidsdd.dataset_files, key=attrgetter("filepath")) == [
         GenericBIDSAsset(

dandi/tests/test_upload.py

@@ -226,6 +226,8 @@ def test_upload_bids_validation_ignore(
     dandiset.get_asset_by_path("dataset_description.json")
     # actual data file?
     dandiset.get_asset_by_path("sub-Sub1/anat/sub-Sub1_T1w.nii.gz")
+    # .bidsignore file?
+    dandiset.get_asset_by_path(".bidsignore")
 
 
 def test_upload_bids_metadata(
@@ -276,6 +278,8 @@ def test_upload_bids(
     dandiset.get_asset_by_path("dataset_description.json")
     # actual data file?
     dandiset.get_asset_by_path("sub-Sub1/anat/sub-Sub1_T1w.nii.gz")
+    # .bidsignore file should be uploaded automatically
+    dandiset.get_asset_by_path(".bidsignore")
 
 
 def test_upload_bids_non_nwb_file(bids_dandiset: SampleDandiset) -> None: