Find NDIcalc document definitions in installed calculator packages

.github/workflows/ci.yml

@@ -333,11 +333,33 @@ jobs:
 
       - name: Import NDI and its sibling packages
         run: |
-          "$RUNNER_TEMP/venv/bin/python" -c "import ndi, did, ndr, vlt, vhlib, ndicompress, ndi_ontology"
+          "$RUNNER_TEMP/venv/bin/python" -c "import ndi, did, ndr, vlt, vhlib, ndicompress, ndi_ontology, ndicalc_vis_matlab"
 
       - name: Validate installation
         # Run from outside the checkout so `import ndi` resolves to the
         # installed wheel, not the source tree.
         working-directory: ${{ runner.temp }}
         run: |
           "$RUNNER_TEMP/venv/bin/python" -m ndi check
+
+      - name: Resolve an NDIcalc document definition
+        # NDIcalc-vis-matlab's definitions are not in this repo: they come
+        # from the ndicalc-vis-matlab package, found through its `ndi.calc`
+        # entry point (ndi_common_PathConstants.CalcDoc). Prove a plain
+        # install finds one, both directly and through DID's
+        # $NDICALCDOCUMENTPATH placeholder. Run from outside the checkout for
+        # the same reason as the step above.
+        working-directory: ${{ runner.temp }}
+        run: |
+          "$RUNNER_TEMP/venv/bin/python" - <<'PY'
+          from did.validate import resolve_definition_path
+          from ndi.common import ndi_common_PathConstants
+          from ndi.document import ndi_document
+
+          print("CalcDoc:", ndi_common_PathConstants.CalcDoc)
+          doc = ndi_document("vision/contrast_tuning")
+          assert "contrast_tuning" in doc.document_properties, doc.document_properties
+          path = resolve_definition_path("$NDICALCDOCUMENTPATH/vision/contrast_tuning.json")
+          assert path, "DID could not resolve $NDICALCDOCUMENTPATH/vision/contrast_tuning.json"
+          print("vision/contrast_tuning resolved from", path)
+          PY

.gitignore

@@ -53,17 +53,3 @@ pythonArtifacts.tar.gz
 # Generated tutorial HTML (keep locally for review, not in repo)
 tutorials/*.html
 *.sqlite
-
-# Document definitions copied in by ndi_install.py from NDIcalc-vis-matlab
-# (install_ndi_common_docs). These are that repo's files, regenerated by every
-# install and by CI, so tracking them here would vendor a second copy that
-# immediately starts drifting from the dependency that owns them. Scoped to the
-# specific subtrees the installer writes -- the rest of database_documents/ and
-# schema_documents/ is this repo's own and stays tracked. If NDIcalc-vis-matlab
-# ever ships another subtree, add it here too.
-src/ndi/ndi_common/database_documents/calc/
-src/ndi/ndi_common/database_documents/neuro/
-src/ndi/ndi_common/database_documents/vision/
-src/ndi/ndi_common/schema_documents/calc/
-src/ndi/ndi_common/schema_documents/neuro/
-src/ndi/ndi_common/schema_documents/vision/

ndi_install.py

@@ -41,14 +41,6 @@
         "python_path": ".",
         "description": "VH-Lab data utilities and file formats (not on PyPI)",
     },
-    {
-        "name": "NDIcalc-vis-matlab",
-        "repo": "https://github.com/VH-Lab/NDIcalc-vis-matlab.git",
-        "branch": "main",
-        "python_path": "",
-        "ndi_common": True,
-        "description": "NDI calculator and visualization document definitions",
-    },
     {
         "name": "Pyraview",
         "repo": "https://github.com/VH-Lab/Pyraview.git",
@@ -370,56 +362,6 @@ def write_pth_file(site_packages: Path, tools_dir: Path) -> Path | None:
         return None
 
 
-# ---------------------------------------------------------------------------
-# ndi_common document definitions from external dependencies
-# ---------------------------------------------------------------------------
-
-
-def install_ndi_common_docs(tools_dir: Path, ndi_root: Path) -> bool:
-    """Copy ndi_common/{database,schema}_documents from external deps.
-
-    Some dependencies (e.g. NDIcalc-vis-matlab) ship document type
-    definitions that NDI-python needs at runtime.  This copies their
-    ``ndi_common/database_documents`` and ``ndi_common/schema_documents``
-    trees into NDI-python's own ``ndi_common`` folder so they are
-    discoverable via ``ndi_common_PathConstants.DOCUMENT_PATH``.
-    """
-    import shutil
-
-    ndi_common = ndi_root / "src" / "ndi" / "ndi_common"
-    ok = True
-
-    for dep in DEPENDENCIES:
-        if not dep.get("ndi_common"):
-            continue
-        dep_dir = tools_dir / dep["name"]
-        dep_common = dep_dir / "ndi_common"
-        if not dep_common.is_dir():
-            warn(f"{dep['name']}: ndi_common folder not found at {dep_common}")
-            ok = False
-            continue
-
-        for sub in ("database_documents", "schema_documents"):
-            src = dep_common / sub
-            dst = ndi_common / sub
-            if not src.is_dir():
-                continue
-            count = 0
-            for src_file in src.rglob("*"):
-                if src_file.is_dir():
-                    continue
-                rel = src_file.relative_to(src)
-                dst_file = dst / rel
-                dst_file.parent.mkdir(parents=True, exist_ok=True)
-                shutil.copy2(src_file, dst_file)
-                count += 1
-            detail(f"Copied {count} {sub} files from {dep['name']}")
-
-        success(f"Installed document definitions from {dep['name']}")
-
-    return ok
-
-
 # ---------------------------------------------------------------------------
 # pip installation
 # ---------------------------------------------------------------------------
@@ -516,6 +458,7 @@ def validate() -> tuple[int, int]:
         ("DID (did.implementations.sqlitedb)", "did.implementations.sqlitedb"),
         ("DID (did.datastructures)", "did.datastructures"),
         ("vhlab-toolbox (vlt)", "vlt"),
+        ("ndicalc-vis-matlab (calculator documents)", "ndicalc_vis_matlab"),
         ("pyraview (signal pyramids)", "pyraview"),
         ("numpy", "numpy"),
         ("networkx", "networkx"),
@@ -706,9 +649,6 @@ def main() -> int:
         fail("Installation failed: see the pip errors above")
         return 1
 
-    # Copy document definitions from external dependencies
-    install_ndi_common_docs(tools_dir, ndi_root)
-
     # ── Step 5: Validate ───────────────────────────────────────────────
     if args.no_validate:
         print("\n[5/5] Validation skipped (--no-validate)")

pyproject.toml

@@ -48,6 +48,12 @@ dependencies = [
     # now lives; NDI-matlab pulls the MATLAB half in the same way, through its
     # requirements.txt.
     "ndi-ontology @ https://github.com/Waltham-Data-Science/ndi-ontology-python/archive/refs/heads/main.zip",
+    # NDIcalc-vis-matlab's calculator document definitions and schemas
+    # (ndi_common/{database,schema}_documents/{calc,neuro,vision}). NDI finds
+    # them through the package's `ndi.calc` entry point -- see
+    # ndi.fun.find_calc_directories and ndi_common_PathConstants.CalcDoc --
+    # the way NDI-matlab finds sibling NDIcalc*-matlab toolboxes.
+    "ndicalc-vis-matlab @ https://github.com/VH-Lab/NDIcalc-vis-matlab/archive/refs/heads/main.zip",
     "numpy>=1.20.0",
     "networkx>=2.6",
     "jsonschema>=4.0.0",

src/ndi/check.py

@@ -82,10 +82,14 @@ def check(name: str, ok: bool, detail: str = "") -> None:
 
         definitions = _DIDPathConstants.DEFINITIONS
         missing = [k for k in _DID_PLACEHOLDERS if k not in definitions]
+
+        def _all_dirs(value) -> bool:
+            # The calculator placeholders hold a list, as in MATLAB.
+            values = value if isinstance(value, (list, tuple)) else [value]
+            return all(Path(str(v)).is_dir() for v in values)
+
         unresolved = [
-            k
-            for k in _DID_PLACEHOLDERS
-            if k in definitions and not Path(str(definitions[k])).is_dir()
+            k for k in _DID_PLACEHOLDERS if k in definitions and not _all_dirs(definitions[k])
         ]
         if missing:
             check(
@@ -108,6 +112,36 @@ def check(name: str, ok: bool, detail: str = "") -> None:
     except Exception as e:
         check("DID path globals ($NDI...PATH)", False, str(e))
 
+    # Calculator document definitions. They live in separate packages
+    # (ndicalc-vis-matlab, a runtime dependency) found through the ndi.calc
+    # entry-point group, so a broken or missing one leaves every calculator
+    # document type unresolvable while NDI's own types still work.
+    try:
+        from ndi.common import ndi_common_PathConstants
+
+        calc_doc = ndi_common_PathConstants.CalcDoc
+        absent = [str(p) for p in calc_doc if not p.is_dir()]
+        if not calc_doc:
+            check(
+                "NDI calculator documents (ndi.calc)",
+                False,
+                "no calculator package found; install ndicalc-vis-matlab",
+            )
+        elif absent:
+            check(
+                "NDI calculator documents (ndi.calc)",
+                False,
+                f"no such directory: {', '.join(absent)}",
+            )
+        else:
+            check(
+                "NDI calculator documents (ndi.calc)",
+                True,
+                f"{len(calc_doc)} package(s)",
+            )
+    except Exception as e:
+        check("NDI calculator documents (ndi.calc)", False, str(e))
+
     # NDR and NDI-compress are real runtime dependencies that nothing else here
     # imports, so a missing or broken one used to go unreported until a read.
     ok, detail = _try_import("ndr")