From f162a89d33e806412c9f3228ed22d896c769dfab Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 18:11:57 +0000 Subject: [PATCH 01/52] Draw cell outlines, and make genes and density changeable in place Three things the viewer could not do. Two of them looked like flags the caller had forgotten to pass, which is its own kind of bug report. CELL OUTLINES. The cells document has carried contours since it was first written, readContourFile has parsed them since it was ported, and nothing drew them: openPyramid added centroids as Points and stopped. readContours is the missing document layer over readContourFile -- it finds contours.bin in the document and, crucially, applies contour_reference. Vertices are usually stored relative to their cell's centroid, because that is what makes them fit in int16, so a caller that drew readContourFile's output directly would put every outline in a small cluster near the origin: a picture of nothing, produced without an error. The tests assert the two references separately AND assert that they differ, since a reader that ignored the field entirely would pass a test that only checked one. An empty polygon comes back as (0, 2) rather than being dropped, so row i here stays row i of cells.tsv; dropping it would slide every later outline onto the wrong cell. napari will not take a zero-vertex shape, so those are filtered at the layer instead. GENES AND DENSITY. Both were fixed at launch, so changing either meant quitting and reopening the session. Neither needs that: a level is summed over the selected genes into a 2D array, so the shape does not change when the selection does and only the layer's data has to be swapped. The panel lives in its own module because it needs magicgui, which napari brings and a caller composing layers by hand may not; a missing magicgui now costs the panel rather than the picture. A gene name that matches nothing refuses the whole apply rather than drawing the ones that did match, because a partial picture looks exactly like a complete one. --name. The image layer was named from the pyramid's label, which the ingest usually took from the filename, so it announced the section rather than what was being shown -- actively misleading once --genes narrows it. readContours was very nearly a second copy of the binary parser: the grep that said no reader existed was for lowercase "contour", which does not match readContourFile. It delegates. Co-Authored-By: Claude Opus 5 --- src/ndi/fun/doc_gene.py | 89 ++++++++++++ src/ndi/gui/app/genepyramid/cli.py | 109 ++++++++++++++- src/ndi/gui/app/genepyramid/controls.py | 120 +++++++++++++++++ src/ndi/gui/app/genepyramid/viewer.py | 42 +++++- tests/test_doc_gene_contours.py | 171 ++++++++++++++++++++++++ 5 files changed, 528 insertions(+), 3 deletions(-) create mode 100644 src/ndi/gui/app/genepyramid/controls.py create mode 100644 tests/test_doc_gene_contours.py diff --git a/src/ndi/fun/doc_gene.py b/src/ndi/fun/doc_gene.py index a773ff92..58082cf8 100644 --- a/src/ndi/fun/doc_gene.py +++ b/src/ndi/fun/doc_gene.py @@ -1620,3 +1620,92 @@ def _cells_doc_count(cells_doc): except Exception: pass return None + + +def readContours(session, cells_doc): + """Boundary polygons for a cells document, in SOURCE coordinates. + + The document-level companion to :func:`readContourFile`, which parses + the bytes. This adds the two things a caller needs and the file does + not carry: it finds ``contours.bin`` in the document, and it applies + ``contour_reference``. + + THAT SECOND PART IS THE POINT. Vertices are usually stored RELATIVE to + their cell's centroid -- that is what makes them fit in int16, and + :func:`writeContourFile` refuses absolute coordinates that would wrap + -- so a caller that draws :func:`readContourFile`'s output directly + puts every outline in a small cluster near the origin. That is a + picture of nothing, drawn without an error. Here the centroid is added + back, so what comes out is always in the same frame as the centroids + from :func:`readCells` and as the pyramid itself. + + Args: + session: an ndi.session or ndi.dataset holding the document. + cells_doc: a spatialGeneExpressionCells document. + + Returns: + ``(polys, info)``. *polys* is one ``(N, 2)`` array of ``[x, y]`` + per cell in cell_index order; a cell with no usable boundary gets + a ``(0, 2)`` array rather than being dropped, so row i here is + row i of cells.tsv. *info* is :func:`readContourFile`'s, plus + ``contourReference`` and ``nEmpty``. + + Raises: + ValueError: if the document was written without contours, if the + reference is not one this understands, or if there are not as + many centroids as cells to place them on. + """ + c = cells_doc.document_properties["spatialGeneExpressionCells"] + if not bool(c.get("contours_present")): + raise ValueError( + f"spatialGeneExpressionCells {cells_doc.id} has contours_present " + f"0: this cell table was written without boundaries. Re-ingest " + f"the cellbin with contours to get them." + ) + + reference = c.get("contour_reference") or "centroid" + if reference not in ("centroid", "absolute"): + raise ValueError(f"contour_reference is {reference!r}; expected 'centroid' or 'absolute'.") + + fh = session.database_openbinarydoc(cells_doc, "contours.bin") + try: + # The dtypes are document FIELDS, not constants, so they are passed + # through rather than assumed -- the same reason readContourFile + # takes them as arguments. + polys, info = readContourFile( + fh, + nVerticesPerCell=int(c.get("n_vertices_per_cell") or 0), + vertexType=c.get("data_type_vertex") or _CONTOUR_VERTEX_TYPE, + offsetType=c.get("data_type_offset") or _CONTOUR_OFFSET_TYPE, + ) + finally: + session.database_closebinarydoc(fh) + + if reference == "centroid": + cols, _info = readCells(session, cells_doc) + cx = np.asarray(cols["x"], dtype=float) + cy = np.asarray(cols["y"], dtype=float) + if len(cx) != info["nCells"]: + raise ValueError( + f"contours.bin holds {info['nCells']} cells but cells.tsv " + f"holds {len(cx)} rows. Centroid-relative vertices cannot be " + f"placed without a centroid each." + ) + placed = [] + for i, poly in enumerate(polys): + if len(poly) == 0: + placed.append(np.zeros((0, 2), dtype=float)) + continue + placed.append( + np.column_stack( + [poly[:, 0].astype(float) + cx[i], poly[:, 1].astype(float) + cy[i]] + ) + ) + polys = placed + else: + polys = [np.asarray(p, dtype=float).reshape(-1, 2) for p in polys] + + info = dict(info) + info["contourReference"] = reference + info["nEmpty"] = sum(1 for p in polys if len(p) == 0) + return polys, info diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index b6d1d615..99831b8c 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -60,6 +60,25 @@ def build_parser() -> argparse.ArgumentParser: "Otherwise a document id, or a directory holding cells.tsv as " "extract_cells.py writes it.", ) + p.add_argument( + "--outlines", + action="store_true", + help="draw cell boundary polygons, not just centroids. Needs " + "--cells and a cells document whose contours_present is 1.", + ) + p.add_argument( + "--name", + default="", + metavar="TEXT", + help="name for the image layer. Default is the pyramid's label, " + "which the ingest usually took from the file, so it names the " + "SECTION rather than what is being shown.", + ) + p.add_argument( + "--no-controls", + action="store_true", + help="do not dock the gene / density panel", + ) p.add_argument("--list", action="store_true", help="list the pyramids and exit") p.add_argument( "--report", @@ -142,6 +161,62 @@ def _resolve_cells(session, pyr_doc, spec: str): return cols, info +def _resolve_outlines(session, pyr_doc, spec: str, cells_info): + """Boundary polygons for the cells document --cells resolved. + + Returns None after printing why, so main can exit 1. A directory of + extracted cells is refused rather than half-supported: contours.bin + is read out of the DOCUMENT, and pretending otherwise would fail + later with a message about a missing file rather than about the + thing the caller actually asked for. + """ + import os + + from ndi.fun.doc_gene import readContours + from ndi.query import ndi_query + + if spec != "auto" and os.path.isdir(spec): + print( + "--outlines reads contours.bin from the cells DOCUMENT, and " + f"{spec} is a directory. Drop the path from --cells to use the " + "pyramid's own cells document.", + file=sys.stderr, + ) + return None + + if cells_info is not None and not cells_info.get("contoursPresent"): + print( + "this cells document has contours_present 0: it was written " + "without boundaries, so there is nothing to draw. Re-ingest the " + "cellbin with contours.", + file=sys.stderr, + ) + return None + + docs = session.database_search( + ndi_query("").isa("spatialGeneExpressionCells") + & ndi_query("").depends_on("spatialGeneExpressionPyramid_id", pyr_doc.id) + ) + if spec != "auto": + docs = [d for d in docs if d.id == spec] + if len(docs) != 1: + print(f"expected one cells document for --outlines, found {len(docs)}", file=sys.stderr) + return None + + try: + polys, info = readContours(session, docs[0]) + except Exception as e: + print(f"could not read contours: {e}", file=sys.stderr) + return None + + drawn = sum(1 for p in polys if len(p)) + print( + f"[outlines] {drawn} of {info['nCells']} cells have a boundary " + f"({info['contourReference']}-referenced, {info['nVerticesTotal']} vertices)" + ) + return polys + + def _resolve_genes(session, pyr_doc, spec: str): """Gene symbols or accessions to ZERO-BASED rows. @@ -201,6 +276,15 @@ def main(argv=None) -> int: print(_describe(d), file=sys.stderr) return 1 + if args.outlines and not args.cells: + print( + "--outlines needs --cells: the boundaries live on the cells " + "document, and centroid-relative vertices need the centroids " + "to be placed at all.", + file=sys.stderr, + ) + return 1 + gene_rows = _resolve_genes(session, pyr, args.genes) cells, cells_info = _resolve_cells(session, pyr, args.cells) density = not args.no_density @@ -225,7 +309,12 @@ def main(argv=None) -> int: if cells_info.get("segmentationMethod"): print(f" segmented by {cells_info['segmentationMethod']}") if cells_info.get("contoursPresent"): - print(" contours present but not read; centroids only") + print( + f" contours present ({cells_info.get('contourReference')}" + f"-referenced); --outlines draws them" + ) + else: + print(" no contours in this document; centroids only") print(f" {'bin':>5} {'height':>8} {'width':>8} {'tiles':>12}") for lv in levels: print( @@ -239,7 +328,23 @@ def main(argv=None) -> int: overlay = None if cells is not None: overlay = {"x": cells["x"], "y": cells["y"]} - openPyramid(session, pyr, gene_rows=gene_rows, density=density, cells=overlay) + + polys = None + if args.outlines: + polys = _resolve_outlines(session, pyr, args.cells, cells_info) + if polys is None: + return 1 + + openPyramid( + session, + pyr, + gene_rows=gene_rows, + density=density, + cells=overlay, + outlines=polys, + controls=not args.no_controls, + name=args.name or None, + ) return 0 diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py new file mode 100644 index 00000000..74ba4096 --- /dev/null +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -0,0 +1,120 @@ +"""A small dock panel for the two choices that were launch-only. + +Which genes to show and whether to divide by bin area were both fixed at +startup, so changing either meant quitting and relaunching -- and a +relaunch rebuilds nothing cheap: it reopens the session and rebuilds the +ladder. Both are in fact recomputable in place, because a level is summed +over the selected genes into a 2D array, so the shape does not change +when the selection does. Only the layer's data has to be swapped. + +Kept OUT of :mod:`viewer` because it needs magicgui, which napari brings +but which a caller composing layers by hand may not have. A missing +magicgui costs the panel, not the picture. +""" + +from __future__ import annotations + +from typing import Any + +__all__ = ["addControls", "resolveGeneRows"] + + +def resolveGeneRows(session, pyr_doc, text: str): + """Comma-separated symbols or accessions to ZERO-BASED geneList rows. + + Returns ``(rows, missing)``. *rows* is None for "everything", which is + what an empty box means and what ``levelArrays`` wants for all genes. + + EVERY match is kept, not the first. Real annotations repeat symbols -- + the opossum list repeats thousands of them -- and taking one row would + silently show part of a gene's signal while looking complete. + """ + from ....fun.doc_gene_export import readGeneList + + wanted = [w.strip() for w in text.split(",") if w.strip()] + if not wanted: + return None, [] + + ids, names = readGeneList(session, pyr_doc) + rows: list[int] = [] + missing: list[str] = [] + for w in wanted: + hit = [i for i, (a, n) in enumerate(zip(ids, names)) if w in (a, n)] + if hit: + rows.extend(hit) + else: + missing.append(w) + return sorted(set(rows)), missing + + +def addControls(viewer, session, pyr_doc, image_layer, density: bool = True) -> Any: + """Dock the gene / density panel. Returns the widget, or None. + + None means magicgui was not importable. That is reported once on + stderr rather than raised: the viewer is already on screen by this + point and taking it down over a missing convenience would be worse + than doing without. + """ + try: + from magicgui.widgets import CheckBox, Container, Label, LineEdit, PushButton + except ImportError as e: # pragma: no cover - depends on the install + import sys + + print( + f"[genepyramid] gene/density panel unavailable ({e}). The image is " + f"unaffected; relaunch with --genes / --no-density to change them.", + file=sys.stderr, + ) + return None + + from .multiscale import levelArrays + + genes = LineEdit( + label="genes", + value="", + tooltip=( + "Comma-separated symbols or accessions. Empty means every gene. " + "A symbol naming several rows contributes all of them." + ), + ) + dens = CheckBox( + label="density", + value=density, + tooltip=( + "Counts per base pixel rather than per bin. Binning SUMS, so " + "with this off a coarse level is binSize^2 brighter than a fine " + "one and the picture jumps every time the viewer changes level." + ), + ) + status = Label(value="all genes") + apply = PushButton(text="Apply") + + def _apply(): + rows, missing = resolveGeneRows(session, pyr_doc, genes.value) + if missing: + # Refuse rather than quietly drawing the genes that did match: + # a partial picture is indistinguishable from a complete one. + status.value = ( + "not found: " + ", ".join(missing[:4]) + (" ..." if len(missing) > 4 else "") + ) + return + try: + image_layer.data = levelArrays(session, pyr_doc, rows, dens.value) + except Exception as e: # pragma: no cover - depends on the data + status.value = f"failed: {e}" + return + # The contrast range that suited all genes is wrong for three of + # them, and vice versa, so it is recomputed rather than kept. + try: + image_layer.reset_contrast_limits() + except Exception: + pass + n = "all genes" if rows is None else f"{len(rows)} row(s)" + status.value = f"{n}, {'density' if dens.value else 'raw counts'}" + + apply.changed.connect(lambda *_: _apply()) + + panel = Container(widgets=[genes, dens, apply, status], labels=True) + panel.native.setToolTip("Applies on click; the ladder is rebuilt lazily.") + viewer.window.add_dock_widget(panel, area="right", name="genes") + return panel diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index 3c8247e2..9b19ba4a 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -39,6 +39,9 @@ def openPyramid( gene_rows=None, density: bool = True, cells: dict[str, Any] | None = None, + outlines=None, + controls: bool = True, + name: str | None = None, show: bool = True, ): """Open a spatial gene expression pyramid in napari. @@ -56,6 +59,20 @@ def openPyramid( straight in, because the image layer carries the origin in its translate and centroids that skip that transform land somewhere plausible and wrong. + outlines: optional list of ``(N, 2)`` ``[x, y]`` vertex arrays in + SOURCE coordinates, as :func:`~ndi.fun.doc_gene.readContours` + returns them, drawn as a Shapes layer. Routed through + :func:`~.multiscale.sourceToWorld` for the same reason + centroids are. Empty polygons are dropped: napari treats a + zero-vertex shape as malformed rather than as nothing. + controls: dock a small panel for choosing genes and switching + between density and raw counts. Without it both are fixed at + launch, since this viewer holds no other state. Silently + skipped when magicgui is unavailable -- the panel is a + convenience and its absence must not stop the picture. + name: image layer name. Defaults to the pyramid's label, which is + whatever the ingest recorded -- often the file stem, which + names the section rather than what is being shown. show: call ``napari.run()``. False returns the viewer without blocking, which is what a test or a caller composing several layers wants. @@ -66,7 +83,25 @@ def openPyramid( napari = require_napari() viewer = napari.Viewer() - viewer.add_image(**layerSpec(session, pyr_doc, gene_rows, density)) + image = viewer.add_image(**layerSpec(session, pyr_doc, gene_rows, density, name)) + + if outlines is not None: + keep = [p for p in outlines if len(p)] + if keep: + paths = [] + for p in keep: + row, col = sourceToWorld(session, pyr_doc, p[:, 0], p[:, 1]) + paths.append(list(zip(row, col))) + # shape_type polygon closes the ring itself, which matches the + # format: writeContourFile does not repeat the first vertex. + viewer.add_shapes( + paths, + shape_type="polygon", + name="cell outlines", + face_color="transparent", + edge_color="cyan", + edge_width=1, + ) if cells is not None: row, col = sourceToWorld(session, pyr_doc, cells["x"], cells["y"]) @@ -78,6 +113,11 @@ def openPyramid( border_width=0, ) + if controls: + from .controls import addControls + + addControls(viewer, session, pyr_doc, image, density) + if show: napari.run() return viewer diff --git a/tests/test_doc_gene_contours.py b/tests/test_doc_gene_contours.py new file mode 100644 index 00000000..f1088ea5 --- /dev/null +++ b/tests/test_doc_gene_contours.py @@ -0,0 +1,171 @@ +"""readContours: contours.bin placed in the frame the pyramid uses. + +readContourFile already parsed these bytes; readContours is the document +layer over it -- finding the file and applying contour_reference. So the +tests here are about THAT, and the parsing errors they trigger are +readContourFile's own. + +The case worth testing hardest is ``contour_reference``. Vertices are +usually stored RELATIVE to their cell's centroid -- that is what makes +them fit in int16 -- and a reader that ignores that draws every outline +in a small cluster near the origin. It produces a picture, and the +picture is of nothing. So the two references are asserted separately AND +asserted to differ, since a reader that silently treated everything as +absolute would pass a test that only checked one of them. +""" + +from __future__ import annotations + +import struct + +import numpy as np +import pytest + +from ndi.fun import doc_gene + + +def _binary(polys, offset_dtype=" Date: Tue, 8 Sep 2026 18:56:49 +0000 Subject: [PATCH 02/52] Panels for picking a gene, choosing counts, and filtering cell types Rebuilt the controls to match the widgets the FICTURE viewer in the bscholl project already uses, because those are the shapes people here know and a different idiom for the same job is a cost with no return. The magicgui text box asked you to already know a gene name; the point of a picker is not having to. ADD GENE. Type-ahead QComboBox with a MatchContains completer over the whole list, showing "SYMBOL (ACCESSION)" so an unnamed or duplicated symbol is still selectable -- this opossum annotation repeats 324 symbols in its first 2,000. Adding creates its own additive layer rather than replacing the base, which is what makes two genes comparable; adding the same gene twice replaces its own layer instead of stacking. Every row a symbol matches is summed, since taking the first would show part of a gene's signal and look complete. DISPLAY. Radio buttons, not a checkbox: two exclusive states, so each label can say what it IS instead of naming one and leaving the other implied. Switching swaps only the layer's data, so the camera, the zoom and the level survive. It rebuilds with the SAME gene rows the base layer was drawn from, remembered on the layer -- recomputing with all genes would quietly widen what is shown. CELL TYPES. The cellTypeLabels documents were being written and never read: there was a writer and no reader at all. readCellTypeLabels is the reader, and it places labels BY cell_index rather than by file order, because cell_index is written explicitly for exactly that reason and a reordered file would otherwise give every cell its neighbour's type. An unlabelled cell stays as an empty string rather than being dropped, which would shift every later row. Each labeling gets its own checkbox group rather than being merged: a cellbin carries a transferred atlas call AND clusterings, and they are not interchangeable. Hiding sets ALPHA rather than deleting points, because the points layer's row order is the contract with cells.tsv and with every other labels.tsv beside it; deleting would break the next labeling the user toggles. Two labelings therefore filter jointly instead of the last click winning. The viewer now takes cells_doc, which it needs because the overlay arrives as plain arrays with no way back to the document the labels depend on. Co-Authored-By: Claude Opus 5 --- src/ndi/fun/doc_gene.py | 80 +++++ src/ndi/gui/app/genepyramid/cli.py | 5 + src/ndi/gui/app/genepyramid/controls.py | 400 ++++++++++++++++++------ src/ndi/gui/app/genepyramid/viewer.py | 42 ++- tests/test_doc_gene_labels.py | 69 ++++ 5 files changed, 490 insertions(+), 106 deletions(-) diff --git a/src/ndi/fun/doc_gene.py b/src/ndi/fun/doc_gene.py index 58082cf8..a96fad22 100644 --- a/src/ndi/fun/doc_gene.py +++ b/src/ndi/fun/doc_gene.py @@ -1709,3 +1709,83 @@ def readContours(session, cells_doc): info["contourReference"] = reference info["nEmpty"] = sum(1 for p in polys if len(p) == 0) return polys, info + + +def readCellTypeLabels(session, labels_doc): + """Read ``labels.tsv`` -- one class name per cell. + + The counterpart to :func:`makeCellTypeLabels`, which had no reader. + + ROW ORDER IS THE CONTRACT with cells.tsv, and cell_index is written + explicitly rather than inferred, so this returns labels placed BY + cell_index rather than in file order. A file that arrived reordered + would otherwise give every cell its neighbour's type -- valid-looking + and wrong, which is the same failure the writer refuses to create. + + An unlabelled cell is the empty string, not a missing row: a labeling + that covers part of the table is normal, and dropping those rows would + shift the rest. + + Args: + session: an ndi.session or ndi.dataset holding the document. + labels_doc: a cellTypeLabels document. + + Returns: + ``(labels, info)``. *labels* is a list of str, one per cell, in + cell_index order. *info* carries labelName, isUnsupervised, + categories (sorted, blanks excluded), nUnlabeled and nCells. + """ + c = labels_doc.document_properties["cellTypeLabels"] + + fh = session.database_openbinarydoc(labels_doc, "labels.tsv") + try: + text = fh.read().decode("utf-8") + finally: + session.database_closebinarydoc(fh) + + rows = [ln for ln in text.splitlines() if ln.strip()] + if not rows: + raise ValueError(f"labels.tsv of {labels_doc.id} is empty.") + header = rows[0].split("\t") + try: + i_idx = header.index("cell_index") + i_lab = header.index("label") + except ValueError as e: + raise ValueError( + f"labels.tsv of {labels_doc.id} has header {header!r}; expected " + f"cell_index and label." + ) from e + + n = int(c.get("n_cells") or (len(rows) - 1)) + labels = [""] * n + for ln in rows[1:]: + parts = ln.split("\t") + idx = int(parts[i_idx]) + if 0 <= idx < n: + labels[idx] = parts[i_lab] if i_lab < len(parts) else "" + + info = { + "labelName": c.get("label_name", ""), + "isUnsupervised": bool(c.get("is_unsupervised")), + "taxonomyLevel": c.get("taxonomy_level", ""), + "assignmentMethod": c.get("assignment_method", ""), + "nCells": n, + "nUnlabeled": sum(1 for v in labels if not v.strip()), + "categories": sorted({v for v in labels if v.strip()}), + } + return labels, info + + +def findCellTypeLabels(session, cells_doc): + """Every cellTypeLabels document belonging to a cells document. + + Returned rather than merged: a cellbin routinely carries a transferred + atlas call AND one or more unsupervised clusterings, and they are not + interchangeable, so a caller picks. + """ + from ..query import ndi_query + + return session.database_search( + ndi_query("").isa("cellTypeLabels") + & ndi_query("").depends_on("cells_document_id", cells_doc.id) + ) diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index 99831b8c..bba09df4 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -158,6 +158,10 @@ def _resolve_cells(session, pyr_doc, spec: str): ) cols, info = readCells(session, docs[0]) info["source"] = docs[0].id + # The document itself, not just its id: the cell-type panel needs it to + # find the cellTypeLabels that depend on it, and re-querying by id in + # the viewer would be the same search done twice. + info["document"] = docs[0] return cols, info @@ -342,6 +346,7 @@ def main(argv=None) -> int: density=density, cells=overlay, outlines=polys, + cells_doc=(cells_info or {}).get("document"), controls=not args.no_controls, name=args.name or None, ) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 74ba4096..50baf3d8 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1,120 +1,328 @@ -"""A small dock panel for the two choices that were launch-only. - -Which genes to show and whether to divide by bin area were both fixed at -startup, so changing either meant quitting and relaunching -- and a -relaunch rebuilds nothing cheap: it reopens the session and rebuilds the -ladder. Both are in fact recomputable in place, because a level is summed -over the selected genes into a 2D array, so the shape does not change -when the selection does. Only the layer's data has to be swapped. - -Kept OUT of :mod:`viewer` because it needs magicgui, which napari brings -but which a caller composing layers by hand may not have. A missing -magicgui costs the panel, not the picture. +"""Dock panels: pick a gene, choose density or counts, filter cell types. + +Built with qtpy directly rather than magicgui, to match the widgets the +FICTURE viewer in the bscholl project already uses -- a type-ahead +combobox over the gene list, radio buttons for the two display modes, and +one checkbox per cell class. Those are the shapes people here already +know, and a different idiom for the same job is a cost with no return. + +Everything Qt is imported inside the functions. This module is imported +by :mod:`viewer`, which a headless caller may import to compose layers by +hand, and a top-level ``qtpy`` import would drag Qt into that. """ from __future__ import annotations from typing import Any -__all__ = ["addControls", "resolveGeneRows"] +__all__ = ["addGenePanel", "addDisplayPanel", "addCellTypePanel", "addAllPanels"] + +# --------------------------------------------------------------- genes -def resolveGeneRows(session, pyr_doc, text: str): - """Comma-separated symbols or accessions to ZERO-BASED geneList rows. - Returns ``(rows, missing)``. *rows* is None for "everything", which is - what an empty box means and what ``levelArrays`` wants for all genes. +def addGenePanel(viewer, session, pyr_doc) -> Any: + """Type-ahead combobox over the gene list, and an Add button. - EVERY match is kept, not the first. Real annotations repeat symbols -- - the opossum list repeats thousands of them -- and taking one row would - silently show part of a gene's signal while looking complete. + ADDS A LAYER PER GENE rather than replacing the base image, which is + what makes two genes comparable: they land as separate additive + layers and the base stays put underneath. Adding the same gene twice + replaces its own layer instead of stacking duplicates. + + A symbol can name SEVERAL rows -- real annotations repeat them, and + this opossum list repeats 324 of the first 2,000 -- so every matching + row is summed into the layer. Taking the first would silently show + part of a gene's signal. """ - from ....fun.doc_gene_export import readGeneList + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QComboBox, + QCompleter, + QHBoxLayout, + QLabel, + QPushButton, + QVBoxLayout, + QWidget, + ) - wanted = [w.strip() for w in text.split(",") if w.strip()] - if not wanted: - return None, [] + from ....fun.doc_gene_export import readGeneList + from .multiscale import layerSpec ids, names = readGeneList(session, pyr_doc) - rows: list[int] = [] - missing: list[str] = [] - for w in wanted: - hit = [i for i, (a, n) in enumerate(zip(ids, names)) if w in (a, n)] - if hit: - rows.extend(hit) - else: - missing.append(w) - return sorted(set(rows)), missing - - -def addControls(viewer, session, pyr_doc, image_layer, density: bool = True) -> Any: - """Dock the gene / density panel. Returns the widget, or None. - - None means magicgui was not importable. That is reported once on - stderr rather than raised: the viewer is already on screen by this - point and taking it down over a missing convenience would be worse - than doing without. - """ - try: - from magicgui.widgets import CheckBox, Container, Label, LineEdit, PushButton - except ImportError as e: # pragma: no cover - depends on the install - import sys - print( - f"[genepyramid] gene/density panel unavailable ({e}). The image is " - f"unaffected; relaunch with --genes / --no-density to change them.", - file=sys.stderr, - ) - return None + # Symbol first because that is what people type; the accession is kept + # alongside so an unnamed or duplicated symbol is still selectable. + display, lookup = [], {} + for row, (acc, nm) in enumerate(zip(ids, names)): + text = f"{nm} ({acc})" if nm and nm != acc else str(acc) + display.append(text) + lookup.setdefault(text, []).append(row) + for key in (nm, acc): + if key: + lookup.setdefault(str(key), []).append(row) - from .multiscale import levelArrays + box = QWidget() + outer = QVBoxLayout(box) + outer.addWidget(QLabel(f"Add gene ({len(ids):,} in this pyramid)")) - genes = LineEdit( - label="genes", - value="", - tooltip=( - "Comma-separated symbols or accessions. Empty means every gene. " - "A symbol naming several rows contributes all of them." - ), - ) - dens = CheckBox( - label="density", - value=density, - tooltip=( - "Counts per base pixel rather than per bin. Binning SUMS, so " - "with this off a coarse level is binSize^2 brighter than a fine " - "one and the picture jumps every time the viewer changes level." - ), - ) - status = Label(value="all genes") - apply = PushButton(text="Apply") - - def _apply(): - rows, missing = resolveGeneRows(session, pyr_doc, genes.value) - if missing: - # Refuse rather than quietly drawing the genes that did match: - # a partial picture is indistinguishable from a complete one. - status.value = ( - "not found: " + ", ".join(missing[:4]) + (" ..." if len(missing) > 4 else "") - ) + combo = QComboBox() + combo.setEditable(True) + combo.addItems(display) + completer = QCompleter(display, combo) + completer.setCaseSensitivity(Qt.CaseInsensitive) + completer.setFilterMode(Qt.MatchContains) + combo.setCompleter(completer) + + add = QPushButton("Add") + status = QLabel("") + status.setWordWrap(True) + + def _on_add(): + text = combo.currentText().strip() + if not text: return - try: - image_layer.data = levelArrays(session, pyr_doc, rows, dens.value) - except Exception as e: # pragma: no cover - depends on the data - status.value = f"failed: {e}" + rows = lookup.get(text) + if not rows: + status.setText(f"'{text}' is not in this gene list") return - # The contrast range that suited all genes is wrong for three of - # them, and vice versa, so it is recomputed rather than kept. + name = f"gene: {text}" + if name in viewer.layers: + del viewer.layers[name] + spec = layerSpec(session, pyr_doc, sorted(set(rows)), True, name) + layer = viewer.add_image(**spec, colormap="viridis", blending="additive") try: - image_layer.reset_contrast_limits() + layer.reset_contrast_limits() except Exception: pass - n = "all genes" if rows is None else f"{len(rows)} row(s)" - status.value = f"{n}, {'density' if dens.value else 'raw counts'}" + extra = f" ({len(rows)} rows summed)" if len(rows) > 1 else "" + status.setText(f"added {name}{extra}") + + add.clicked.connect(_on_add) + + row = QHBoxLayout() + row.addWidget(combo, stretch=1) + row.addWidget(add) + outer.addLayout(row) + outer.addWidget(status) + outer.addStretch() + + viewer.window.add_dock_widget(box, name="Add gene", area="right") + return box + + +# ------------------------------------------------------------- display + + +def addDisplayPanel(viewer, session, pyr_doc, image_layer, density: bool = True) -> Any: + """Radio buttons for density against raw counts. + + Two exclusive states, so radio buttons rather than a checkbox: the + label then says what each choice IS, instead of naming one and + leaving the other implied. + + Switching only swaps the layer's data. A level is summed over the + selected genes into a 2D array, so the shape does not change and the + camera, the zoom and the level the viewer is on all survive. + """ + from qtpy.QtWidgets import QButtonGroup, QLabel, QRadioButton, QVBoxLayout, QWidget + + from .multiscale import levelArrays + + box = QWidget() + outer = QVBoxLayout(box) + outer.addWidget(QLabel("Base layer values")) + + r_density = QRadioButton("density (counts per base pixel)") + r_counts = QRadioButton("raw counts (summed per bin)") + r_density.setChecked(density) + r_counts.setChecked(not density) + + group = QButtonGroup(box) + group.addButton(r_density) + group.addButton(r_counts) + + note = QLabel( + "Binning sums, so raw counts make a coarse level binSize^2 brighter " + "than a fine one and the picture jumps at each level change." + ) + note.setWordWrap(True) + + status = QLabel("") + status.setWordWrap(True) + + def _switch(want_density: bool): + try: + image_layer.data = levelArrays( + session, pyr_doc, _currentRows(image_layer), want_density + ) + image_layer.reset_contrast_limits() + except Exception as e: + status.setText(f"failed: {e}") + return + status.setText("density" if want_density else "raw counts") + + r_density.toggled.connect(lambda on: _switch(True) if on else None) + r_counts.toggled.connect(lambda on: _switch(False) if on else None) + + outer.addWidget(r_density) + outer.addWidget(r_counts) + outer.addWidget(note) + outer.addWidget(status) + outer.addStretch() + + viewer.window.add_dock_widget(box, name="Display", area="right") + return box + + +def _currentRows(image_layer): + """The gene rows the base layer was built from, if it recorded them. + + The base layer is whatever openPyramid drew; nothing here changes its + gene selection, so None (all genes) is the honest default rather than + a guess that would quietly widen the picture. + """ + return getattr(image_layer, "_ndi_gene_rows", None) + + +# ---------------------------------------------------------- cell types + + +def addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer=None) -> Any: + """One checkbox per class, per labeling, over the cell layers. + + A cellbin routinely carries several labelings -- a transferred atlas + call and one or more clusterings -- so each becomes its own group + rather than being merged. They are not interchangeable: a clustering + carries no biological identity. + + Hiding works by ALPHA, not by removing points. The row order of the + points layer is the row order of cells.tsv and of every labels.tsv + beside it, and deleting rows would break that correspondence for the + next labeling the user toggles. + """ + from qtpy.QtWidgets import ( + QCheckBox, + QLabel, + QScrollArea, + QVBoxLayout, + QWidget, + ) + + from ....fun.doc_gene import findCellTypeLabels, readCellTypeLabels + + docs = findCellTypeLabels(session, cells_doc) + if not docs: + return None + + import numpy as np + + labelings = [] + for d in docs: + try: + labels, info = readCellTypeLabels(session, d) + except Exception as e: + print(f"[genepyramid] skipping a cellTypeLabels document: {e}") + continue + labelings.append((labels, info)) + if not labelings: + return None + + box = QWidget() + outer = QVBoxLayout(box) + + n_points = len(points_layer.data) + # visible[i] is False when ANY ticked-off class covers cell i, so two + # labelings filter jointly rather than the last one clicked winning. + hidden_by = {} + + def _refresh(): + hide = np.zeros(n_points, dtype=bool) + for mask in hidden_by.values(): + if mask is not None: + hide |= mask + for layer in (points_layer, shapes_layer): + if layer is None: + continue + try: + colors = np.asarray(layer.face_color).copy() + if colors.shape[0] == len(hide): + colors[:, 3] = np.where(hide, 0.0, 1.0) + layer.face_color = colors + edge = np.asarray(layer.edge_color).copy() + if edge.shape[0] == len(hide): + edge[:, 3] = np.where(hide, 0.0, 1.0) + layer.edge_color = edge + layer.refresh() + except Exception: + pass + + for labels, info in labelings: + kind = "clustering" if info["isUnsupervised"] else "cell type call" + title = QLabel(f"{info['labelName'] or '(unnamed)'} -- {kind}") + outer.addWidget(title) + if info["nUnlabeled"]: + note = QLabel(f" {info['nUnlabeled']:,} cells unlabelled by this") + note.setWordWrap(True) + outer.addWidget(note) + + arr = np.asarray(labels, dtype=object) + inner = QWidget() + col = QVBoxLayout(inner) + for category in info["categories"]: + mask = arr == category + cb = QCheckBox(f"{category} ({int(mask.sum()):,})") + cb.setChecked(True) + key = (info["labelName"], category) + + def _toggled(on, key=key, mask=mask): + hidden_by[key] = None if on else mask[:n_points] + _refresh() + + cb.toggled.connect(_toggled) + col.addWidget(cb) + scroll = QScrollArea() + scroll.setWidgetResizable(True) + scroll.setWidget(inner) + outer.addWidget(scroll) + + viewer.window.add_dock_widget(box, name="Cell types", area="right") + return box + - apply.changed.connect(lambda *_: _apply()) +# ----------------------------------------------------------------- all + + +def addAllPanels( + viewer, + session, + pyr_doc, + image_layer, + density, + cells_doc=None, + points_layer=None, + shapes_layer=None, +): + """Every panel the session supports, skipping what it has no data for. + + A missing Qt costs the panels rather than the picture: the viewer is + already on screen by the time this runs, and taking it down over a + convenience would be worse than doing without. + """ + try: + import qtpy.QtWidgets # noqa: F401 + except ImportError as e: # pragma: no cover - depends on the install + import sys + + print( + f"[genepyramid] control panels unavailable ({e}). The image is " + f"unaffected; --genes and --no-density still work at launch.", + file=sys.stderr, + ) + return {} - panel = Container(widgets=[genes, dens, apply, status], labels=True) - panel.native.setToolTip("Applies on click; the ladder is rebuilt lazily.") - viewer.window.add_dock_widget(panel, area="right", name="genes") - return panel + made = {} + made["genes"] = addGenePanel(viewer, session, pyr_doc) + made["display"] = addDisplayPanel(viewer, session, pyr_doc, image_layer, density) + if cells_doc is not None and points_layer is not None: + made["cellTypes"] = addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer) + return made diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index 9b19ba4a..3d409384 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -40,6 +40,7 @@ def openPyramid( density: bool = True, cells: dict[str, Any] | None = None, outlines=None, + cells_doc=None, controls: bool = True, name: str | None = None, show: bool = True, @@ -65,11 +66,16 @@ def openPyramid( :func:`~.multiscale.sourceToWorld` for the same reason centroids are. Empty polygons are dropped: napari treats a zero-vertex shape as malformed rather than as nothing. - controls: dock a small panel for choosing genes and switching - between density and raw counts. Without it both are fixed at - launch, since this viewer holds no other state. Silently - skipped when magicgui is unavailable -- the panel is a - convenience and its absence must not stop the picture. + cells_doc: the spatialGeneExpressionCells document the overlay + came from. Only needed for the cell-type panel, which reads + the cellTypeLabels documents that depend on it -- the cells + themselves arrive as plain arrays and carry no way back to + their own document. + controls: dock the gene, display and cell-type panels. Without + them the gene selection and the density choice are fixed at + launch, since this viewer holds no other state. Skipped with + a note when Qt is unavailable -- they are conveniences and + their absence must not stop the picture. name: image layer name. Defaults to the pyramid's label, which is whatever the ingest recorded -- often the file stem, which names the section rather than what is being shown. @@ -85,6 +91,7 @@ def openPyramid( viewer = napari.Viewer() image = viewer.add_image(**layerSpec(session, pyr_doc, gene_rows, density, name)) + shapes = None if outlines is not None: keep = [p for p in outlines if len(p)] if keep: @@ -94,7 +101,7 @@ def openPyramid( paths.append(list(zip(row, col))) # shape_type polygon closes the ring itself, which matches the # format: writeContourFile does not repeat the first vertex. - viewer.add_shapes( + shapes = viewer.add_shapes( paths, shape_type="polygon", name="cell outlines", @@ -103,9 +110,10 @@ def openPyramid( edge_width=1, ) + points = None if cells is not None: row, col = sourceToWorld(session, pyr_doc, cells["x"], cells["y"]) - viewer.add_points( + points = viewer.add_points( list(zip(row, col)), name="cell centroids", size=cells.get("size", 8), @@ -114,9 +122,23 @@ def openPyramid( ) if controls: - from .controls import addControls - - addControls(viewer, session, pyr_doc, image, density) + from .controls import addAllPanels + + # Remembered on the layer so the display panel can rebuild the + # ladder with the SAME gene selection when it switches between + # density and counts; recomputing with all genes would silently + # widen what is being shown. + image._ndi_gene_rows = gene_rows + addAllPanels( + viewer, + session, + pyr_doc, + image, + density, + cells_doc=cells_doc, + points_layer=points, + shapes_layer=shapes, + ) if show: napari.run() diff --git a/tests/test_doc_gene_labels.py b/tests/test_doc_gene_labels.py index 000ec232..5db5c3b1 100644 --- a/tests/test_doc_gene_labels.py +++ b/tests/test_doc_gene_labels.py @@ -139,3 +139,72 @@ def test_several_labelings_coexist_on_one_segmentation(cells): assert leiden.dependency_value("cells_document_id") == cells_doc.id assert _props(atlas)["is_unsupervised"] == 0 assert _props(leiden)["is_unsupervised"] == 1 + + +class _LFH: + def __init__(self, text): + self._t = text.encode("utf-8") + + def read(self): + return self._t + + +class _LDoc: + id = "labels-doc" + + def __init__(self, n_cells=4, name="leiden", unsupervised=1): + self.document_properties = { + "cellTypeLabels": { + "label": "", + "label_name": name, + "taxonomy_level": "", + "n_cells": n_cells, + "n_categories": 2, + "n_unlabeled": 1, + "assignment_method": "", + "is_unsupervised": unsupervised, + } + } + + +class _LSession: + def __init__(self, text): + self.text = text + + def database_openbinarydoc(self, doc, name): + assert name == "labels.tsv" + return _LFH(self.text) + + def database_closebinarydoc(self, fh): + pass + + +def test_labels_are_placed_by_cell_index_not_file_order(): + """The rows arrive shuffled; every label must still land on its cell.""" + from ndi.fun.doc_gene import readCellTypeLabels + + text = "cell_index\tlabel\n2\tB\n0\tA\n3\t\n1\tA\n" + labels, info = readCellTypeLabels(_LSession(text), _LDoc()) + assert labels == ["A", "A", "B", ""] + assert info["categories"] == ["A", "B"] + assert info["nUnlabeled"] == 1 + assert info["isUnsupervised"] is True + assert info["labelName"] == "leiden" + + +def test_an_unlabelled_cell_is_empty_rather_than_dropped(): + """Dropping it would shift every later cell onto the wrong label.""" + from ndi.fun.doc_gene import readCellTypeLabels + + text = "cell_index\tlabel\n0\tA\n1\t\n2\tB\n3\tB\n" + labels, _ = readCellTypeLabels(_LSession(text), _LDoc()) + assert len(labels) == 4 + assert labels[1] == "" + + +def test_a_labels_file_with_the_wrong_header_is_refused(): + from ndi.fun.doc_gene import readCellTypeLabels + + text = "row\tvalue\n0\tA\n" + with pytest.raises(ValueError, match="expected cell_index and label"): + readCellTypeLabels(_LSession(text), _LDoc()) From 80ecac45ca46ee5544f6f2260bdbbc33c0bd3a03 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 19:13:16 +0000 Subject: [PATCH 03/52] Pick genes from a checkbox list, with variants combined The gene picker listed one entry per pyramid COLUMN, and a real annotation names the same gene on several: this opossum list repeats 324 symbols in its first 2,000, because a symbol can carry several accessions. So the same gene appeared many times over and the reader was left to guess which one was the gene. geneIndex now collapses the list to one entry per symbol and keeps every row it names; the layer sums them, since taking the first would draw part of a gene's signal and look like all of it. The combobox becomes a filterable list with a checkbox each, which is what the FICTURE viewer next door already does. Ticking adds that gene's own additive layer in its own colour -- one map for everything made two genes into a single picture that neither of them was -- and unticking removes it; closing the layer in napari's own list unticks the box, so the two cannot disagree. Two labelings that group the cells identically and differ only in what the groups are called are now named once and drawn once. That is not a hypothetical: a subclass call transferred onto a clustering is a RENAMING of those clusters, so it agrees with the clustering by construction, and showing both invites the reader to treat one as corroborating the other. labelPartition compares MEMBERSHIPS, since comparing names would miss exactly this case, and supervised calls are considered first so the labeling carrying biological names is the one that stays. Panels now read down the right-hand side in the order they are used: display mode, cell types, then the long scrolling gene list. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 312 +++++++++++++++++++----- tests/test_gui_genepyramid_controls.py | 73 ++++++ 2 files changed, 323 insertions(+), 62 deletions(-) create mode 100644 tests/test_gui_genepyramid_controls.py diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 50baf3d8..aa083473 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1,10 +1,15 @@ -"""Dock panels: pick a gene, choose density or counts, filter cell types. +"""Dock panels: choose density or counts, filter cell types, pick genes. Built with qtpy directly rather than magicgui, to match the widgets the -FICTURE viewer in the bscholl project already uses -- a type-ahead -combobox over the gene list, radio buttons for the two display modes, and -one checkbox per cell class. Those are the shapes people here already -know, and a different idiom for the same job is a cost with no return. +FICTURE viewer in the bscholl project already uses -- a filterable list +with a checkbox per entry, and radio buttons for the two display modes. +Those are the shapes people here already know, and a different idiom for +the same job is a cost with no return. + +PANEL ORDER, top to bottom on the right: the display mode, then the cell +types, then the genes. The first two are short and are read once; the +gene list is long and is scrolled, so putting it last keeps the others +on screen. Everything Qt is imported inside the functions. This module is imported by :mod:`viewer`, which a headless caller may import to compose layers by @@ -15,31 +20,76 @@ from typing import Any -__all__ = ["addGenePanel", "addDisplayPanel", "addCellTypePanel", "addAllPanels"] +__all__ = [ + "geneIndex", + "labelPartition", + "addGenePanel", + "addDisplayPanel", + "addCellTypePanel", + "addAllPanels", +] # --------------------------------------------------------------- genes +def geneIndex(ids, names) -> dict: + """Collapse a gene list to ONE ENTRY PER SYMBOL. + + Real annotations name the same gene on several rows -- this opossum + list repeats 324 symbols in its first 2,000 -- because a symbol can + carry several accessions: transcript variants, a duplicated model, a + scaffold copy. The rows are separate columns of the pyramid, so a + picker that offered one item per ROW listed the same gene many times + over and left the user to guess which was the gene. + + Every row a symbol names is kept, and callers sum them: taking the + first would show part of the gene's signal and look like the whole. + + Args: + ids: accessions, one per gene row, in zero-based row order. + names: symbols, same length. A blank falls back to the accession. + + Returns: + A dict, sorted case-insensitively by symbol, mapping symbol to + ``{"rows": [...], "accessions": [...]}``. + """ + by: dict[str, dict] = {} + for row, (acc, nm) in enumerate(zip(ids, names)): + acc = str(acc or "").strip() + symbol = str(nm or "").strip() or acc or f"row {row}" + entry = by.setdefault(symbol, {"rows": [], "accessions": []}) + entry["rows"].append(row) + if acc and acc not in entry["accessions"]: + entry["accessions"].append(acc) + return dict(sorted(by.items(), key=lambda kv: kv[0].lower())) + + +# Distinct colours rather than one map applied to everything: the layers +# blend additively, so two genes in the same colormap make one picture +# that neither of them is. +_GENE_COLORMAPS = ("magenta", "green", "cyan", "yellow", "red", "blue") + + def addGenePanel(viewer, session, pyr_doc) -> Any: - """Type-ahead combobox over the gene list, and an Add button. + """A filterable list of gene symbols, one checkbox each. - ADDS A LAYER PER GENE rather than replacing the base image, which is - what makes two genes comparable: they land as separate additive - layers and the base stays put underneath. Adding the same gene twice - replaces its own layer instead of stacking duplicates. + Ticking a symbol ADDS ITS OWN LAYER rather than replacing the base + image, which is what makes two genes comparable: they land as + separate additive layers in different colours and the base stays put + underneath. Unticking removes that layer again, and removing the + layer in napari's own list unticks the box, so the two never disagree. - A symbol can name SEVERAL rows -- real annotations repeat them, and - this opossum list repeats 324 of the first 2,000 -- so every matching - row is summed into the layer. Taking the first would silently show - part of a gene's signal. + Variants are already combined by :func:`geneIndex`; every row a + symbol names is summed into its layer. """ from qtpy.QtCore import Qt from qtpy.QtWidgets import ( - QComboBox, - QCompleter, QHBoxLayout, QLabel, + QLineEdit, + QListWidget, + QListWidgetItem, QPushButton, QVBoxLayout, QWidget, @@ -49,64 +99,151 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: from .multiscale import layerSpec ids, names = readGeneList(session, pyr_doc) - - # Symbol first because that is what people type; the accession is kept - # alongside so an unnamed or duplicated symbol is still selectable. - display, lookup = [], {} - for row, (acc, nm) in enumerate(zip(ids, names)): - text = f"{nm} ({acc})" if nm and nm != acc else str(acc) - display.append(text) - lookup.setdefault(text, []).append(row) - for key in (nm, acc): - if key: - lookup.setdefault(str(key), []).append(row) + index = geneIndex(ids, names) + n_variants = sum(1 for e in index.values() if len(e["rows"]) > 1) box = QWidget() outer = QVBoxLayout(box) - outer.addWidget(QLabel(f"Add gene ({len(ids):,} in this pyramid)")) - - combo = QComboBox() - combo.setEditable(True) - combo.addItems(display) - completer = QCompleter(display, combo) - completer.setCaseSensitivity(Qt.CaseInsensitive) - completer.setFilterMode(Qt.MatchContains) - combo.setCompleter(completer) + head = f"{len(index):,} genes" + if n_variants: + head += f" ({n_variants:,} with variants, summed)" + outer.addWidget(QLabel(head)) + + search = QLineEdit() + search.setPlaceholderText("filter by symbol or accession") + outer.addWidget(search) + + listw = QListWidget() + items: dict[str, Any] = {} + # Thirty thousand items go in as one batch: each add would otherwise + # relayout the list. + listw.setUpdatesEnabled(False) + for symbol, entry in index.items(): + rows, accs = entry["rows"], entry["accessions"] + text = symbol if len(rows) == 1 else f"{symbol} ({len(rows)} variants)" + item = QListWidgetItem(text) + item.setFlags(item.flags() | Qt.ItemIsUserCheckable) + item.setCheckState(Qt.Unchecked) + item.setData(Qt.UserRole, symbol) + # Accessions are searchable too: a symbol is what people usually + # type, but not everything here has one. + item.setData(Qt.UserRole + 1, " ".join([symbol] + accs).lower()) + item.setToolTip( + f"{symbol}\n{len(rows)} pyramid column(s): {', '.join(accs) or '(no accession)'}" + ) + listw.addItem(item) + items[symbol] = item + listw.setUpdatesEnabled(True) + outer.addWidget(listw, stretch=1) - add = QPushButton("Add") status = QLabel("") status.setWordWrap(True) - def _on_add(): - text = combo.currentText().strip() - if not text: - return - rows = lookup.get(text) - if not rows: - status.setText(f"'{text}' is not in this gene list") - return - name = f"gene: {text}" + clear = QPushButton("Remove all gene layers") + row = QHBoxLayout() + row.addStretch() + row.addWidget(clear) + outer.addLayout(row) + outer.addWidget(status) + + state = {"busy": False, "colour": 0} + + def _shown() -> int: + return sum(1 for it in items.values() if it.checkState() == Qt.Checked) + + def _report(extra: str = ""): + n = _shown() + status.setText(f"{n} gene layer(s) shown" + (f" -- {extra}" if extra else "")) + + def _filter(text: str): + needle = text.strip().lower() + listw.setUpdatesEnabled(False) + try: + for it in items.values(): + it.setHidden(bool(needle) and needle not in it.data(Qt.UserRole + 1)) + finally: + listw.setUpdatesEnabled(True) + + search.textChanged.connect(_filter) + + def _add(symbol: str): + name = f"gene: {symbol}" if name in viewer.layers: - del viewer.layers[name] - spec = layerSpec(session, pyr_doc, sorted(set(rows)), True, name) - layer = viewer.add_image(**spec, colormap="viridis", blending="additive") + return + rows = index[symbol]["rows"] + spec = layerSpec(session, pyr_doc, rows, True, name) + cmap = _GENE_COLORMAPS[state["colour"] % len(_GENE_COLORMAPS)] + state["colour"] += 1 + try: + layer = viewer.add_image(**spec, colormap=cmap, blending="additive") + except Exception: + # An unknown colormap must not cost the layer. + layer = viewer.add_image(**spec, blending="additive") try: layer.reset_contrast_limits() except Exception: pass - extra = f" ({len(rows)} rows summed)" if len(rows) > 1 else "" - status.setText(f"added {name}{extra}") + extra = f"{symbol}: {len(rows)} variants summed" if len(rows) > 1 else "" + _report(extra) + + def _remove(symbol: str): + name = f"gene: {symbol}" + if name in viewer.layers: + del viewer.layers[name] + _report() - add.clicked.connect(_on_add) + def _on_item_changed(item): + if state["busy"]: + return + state["busy"] = True + try: + symbol = item.data(Qt.UserRole) + if item.checkState() == Qt.Checked: + _add(symbol) + else: + _remove(symbol) + except Exception as e: + status.setText(f"failed: {e}") + finally: + state["busy"] = False - row = QHBoxLayout() - row.addWidget(combo, stretch=1) - row.addWidget(add) - outer.addLayout(row) - outer.addWidget(status) - outer.addStretch() + listw.itemChanged.connect(_on_item_changed) + + def _on_clear(): + state["busy"] = True + try: + for symbol, it in items.items(): + if it.checkState() == Qt.Checked: + it.setCheckState(Qt.Unchecked) + _remove(symbol) + finally: + state["busy"] = False + _report() + + clear.clicked.connect(_on_clear) + + def _on_layer_removed(event): + """A layer closed in napari's own list unticks its box.""" + name = getattr(getattr(event, "value", None), "name", "") + if not str(name).startswith("gene: "): + return + item = items.get(str(name)[len("gene: ") :]) + if item is None or item.checkState() != Qt.Checked: + return + was, state["busy"] = state["busy"], True + try: + item.setCheckState(Qt.Unchecked) + finally: + state["busy"] = was + _report() + + try: + viewer.layers.events.removed.connect(_on_layer_removed) + except Exception: # pragma: no cover - depends on the napari build + pass - viewer.window.add_dock_widget(box, name="Add gene", area="right") + _report() + viewer.window.add_dock_widget(box, name="Genes", area="right") return box @@ -187,6 +324,24 @@ def _currentRows(image_layer): # ---------------------------------------------------------- cell types +def labelPartition(labels) -> frozenset: + """The GROUPING a labeling induces, with the class names discarded. + + Two labelings that partition the cells identically and differ only in + what the groups are called carry the same information, and showing + both invites the reader to treat one as corroborating the other. That + happens for a real reason: a subclass call transferred onto clusters + is a RENAMING of those clusters, one name per cluster, so it agrees + with the clustering by construction. + + Comparing names would miss this; comparing memberships catches it. + """ + groups: dict[Any, list] = {} + for i, v in enumerate(labels): + groups.setdefault(v, []).append(i) + return frozenset(frozenset(g) for g in groups.values()) + + def addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer=None) -> Any: """One checkbox per class, per labeling, over the cell layers. @@ -195,6 +350,12 @@ def addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer=None rather than being merged. They are not interchangeable: a clustering carries no biological identity. + A labeling that groups the cells EXACTLY as an earlier one does is + named and then skipped, not drawn twice; see :func:`labelPartition`. + Supervised calls are considered first, so when a clustering and the + subclass call transferred onto it agree, it is the one carrying + biological names that stays. + Hiding works by ALPHA, not by removing points. The row order of the points layer is the row order of cells.tsv and of every labels.tsv beside it, and deleting rows would break that correspondence for the @@ -227,9 +388,32 @@ def addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer=None if not labelings: return None + # Supervised first, so a redundant pair keeps the named call. + labelings.sort(key=lambda t: (bool(t[1]["isUnsupervised"]), t[1]["labelName"] or "")) + + kept, redundant, seen = [], [], {} + for labels, info in labelings: + key = labelPartition(labels) + first = seen.get(key) + if first is None: + seen[key] = info["labelName"] or "(unnamed)" + kept.append((labels, info)) + else: + redundant.append((info["labelName"] or "(unnamed)", first)) + box = QWidget() outer = QVBoxLayout(box) + for name, first in redundant: + msg = ( + f"{name} groups the cells exactly as {first} does, differing only " + f"in what the groups are called, so it is not shown separately." + ) + print(f"[genepyramid] {msg}") + note = QLabel(msg) + note.setWordWrap(True) + outer.addWidget(note) + n_points = len(points_layer.data) # visible[i] is False when ANY ticked-off class covers cell i, so two # labelings filter jointly rather than the last one clicked winning. @@ -256,7 +440,7 @@ def _refresh(): except Exception: pass - for labels, info in labelings: + for labels, info in kept: kind = "clustering" if info["isUnsupervised"] else "cell type call" title = QLabel(f"{info['labelName'] or '(unnamed)'} -- {kind}") outer.addWidget(title) @@ -304,6 +488,10 @@ def addAllPanels( ): """Every panel the session supports, skipping what it has no data for. + Added in the order they should read down the right-hand side: the + display mode, the cell types, then the genes. napari stacks docks in + the order they arrive, so this order is the layout. + A missing Qt costs the panels rather than the picture: the viewer is already on screen by the time this runs, and taking it down over a convenience would be worse than doing without. @@ -321,8 +509,8 @@ def addAllPanels( return {} made = {} - made["genes"] = addGenePanel(viewer, session, pyr_doc) made["display"] = addDisplayPanel(viewer, session, pyr_doc, image_layer, density) if cells_doc is not None and points_layer is not None: made["cellTypes"] = addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer) + made["genes"] = addGenePanel(viewer, session, pyr_doc) return made diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py new file mode 100644 index 00000000..b45864c6 --- /dev/null +++ b/tests/test_gui_genepyramid_controls.py @@ -0,0 +1,73 @@ +"""The two decisions the control panels make before Qt is involved. + +Both are about NOT SHOWING THE SAME THING TWICE, and both were reported +from the demo session rather than guessed at: a gene list that named the +same symbol many times over, and two labelings that turned out to be one. +""" + +from __future__ import annotations + +from ndi.gui.app.genepyramid import controls + +# ------------------------------------------------------------ geneIndex + + +def test_one_entry_per_symbol_however_many_rows_it_names(): + ids = ["ENS1", "ENS2", "ENS3", "ENS4"] + names = ["Sox2", "Gad1", "Sox2", "Sox2"] + index = controls.geneIndex(ids, names) + assert list(index) == ["Gad1", "Sox2"] + assert index["Sox2"]["rows"] == [0, 2, 3] + assert index["Sox2"]["accessions"] == ["ENS1", "ENS3", "ENS4"] + + +def test_every_row_is_kept_because_the_layer_sums_them(): + """Taking the first would draw part of a gene and look like all of it.""" + ids = [f"ENS{i}" for i in range(5)] + names = ["A", "B", "A", "B", "A"] + index = controls.geneIndex(ids, names) + assert sorted(r for e in index.values() for r in e["rows"]) == [0, 1, 2, 3, 4] + + +def test_a_row_with_no_symbol_falls_back_to_its_accession(): + index = controls.geneIndex(["ENSX", "ENSY"], ["", None]) + assert list(index) == ["ENSX", "ENSY"] + assert index["ENSX"]["rows"] == [0] + + +def test_a_row_with_neither_is_still_selectable(): + """Dropping it would shift nothing, but it would hide a real column.""" + index = controls.geneIndex(["", ""], ["", ""]) + assert list(index) == ["row 0", "row 1"] + + +def test_symbols_are_ordered_case_insensitively(): + index = controls.geneIndex(["1", "2", "3"], ["bcl2", "Actb", "Zic1"]) + assert list(index) == ["Actb", "bcl2", "Zic1"] + + +# ------------------------------------------------------- labelPartition + + +def test_a_renaming_of_the_same_groups_compares_equal(): + """Why the panel drops one: a transferred call renames its clusters.""" + leiden = ["0", "0", "1", "2", "1"] + subclass = ["L2/3", "L2/3", "Pvalb", "Sst", "Pvalb"] + assert controls.labelPartition(leiden) == controls.labelPartition(subclass) + + +def test_genuinely_different_labelings_do_not_compare_equal(): + a = ["0", "0", "1", "2", "1"] + b = ["0", "1", "1", "2", "1"] + assert controls.labelPartition(a) != controls.labelPartition(b) + + +def test_unlabelled_cells_are_a_group_of_their_own(): + """Two labelings that leave DIFFERENT cells blank are not the same.""" + a = ["", "x", "x"] + b = ["x", "", "x"] + assert controls.labelPartition(a) != controls.labelPartition(b) + + +def test_the_same_names_in_a_different_order_are_a_different_partition(): + assert controls.labelPartition(["a", "b"]) != controls.labelPartition(["b", "b"]) From a904adea5d82aeedc32b6ffb70dba4eb80e314e5 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 19:30:59 +0000 Subject: [PATCH 04/52] Show one labeling when two say the same thing, and let --labels choose Comparing partitions for EQUALITY was too strict for the case it was written for. A subclass call transferred cell by cell onto a clustering agrees with it almost everywhere and disagrees on a handful of boundary cells: not a second opinion, the same opinion with noise -- but not the same partition either, so both were drawn and the reader was back where they started. labelAgreement measures it instead. For each group of one labeling, the majority value of the other is what it predicts, and the agreement is the fraction that prediction is right for; a per-cluster renaming scores 1.0. It is DIRECTIONAL, since a fine clustering determines a coarse call without the reverse holding, so both directions are tested and the larger is reported. The number is quoted in the panel every time rather than applied silently, so a pair that only nearly agrees reads as a pair that only nearly agrees. selectLabelings is pure and holds the decision, which is the part that can be wrong. --labels overrides it outright: a comma list of names, or 'none'. A name that is not there comes back as missing and is reported, because a typo that silently showed everything would look like the flag not working. labelPartition goes: agreement of 1.0 in both directions is partition equality, and two ways to ask the same question is one too many. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/cli.py | 25 ++++ src/ndi/gui/app/genepyramid/controls.py | 159 +++++++++++++++++------- src/ndi/gui/app/genepyramid/viewer.py | 8 ++ tests/test_gui_genepyramid_controls.py | 89 ++++++++++--- 4 files changed, 223 insertions(+), 58 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index bba09df4..c766d70d 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -74,6 +74,16 @@ def build_parser() -> argparse.ArgumentParser: "which the ingest usually took from the file, so it names the " "SECTION rather than what is being shown.", ) + p.add_argument( + "--labels", + default="", + metavar="A,B", + help="which cell type labelings to show, by name; 'none' shows no " + "labeling panel. Default lets the panel decide, which means two " + "labelings that say the same thing about the section -- a subclass " + "call and the clustering it was transferred onto -- are shown once " + "and the panel says which it kept.", + ) p.add_argument( "--no-controls", action="store_true", @@ -165,6 +175,20 @@ def _resolve_cells(session, pyr_doc, spec: str): return cols, info +def _resolve_labelings(spec: str): + """--labels into what openPyramid wants: names, none of them, or None. + + ``none`` is an EMPTY LIST rather than None, because the two mean + different things here: no names asked for, against no choice made. + """ + spec = (spec or "").strip() + if not spec: + return None + if spec.lower() == "none": + return [] + return [s.strip() for s in spec.split(",") if s.strip()] + + def _resolve_outlines(session, pyr_doc, spec: str, cells_info): """Boundary polygons for the cells document --cells resolved. @@ -349,6 +373,7 @@ def main(argv=None) -> int: cells_doc=(cells_info or {}).get("document"), controls=not args.no_controls, name=args.name or None, + labelings=_resolve_labelings(args.labels), ) return 0 diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index aa083473..b46e84e1 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -22,7 +22,8 @@ __all__ = [ "geneIndex", - "labelPartition", + "labelAgreement", + "selectLabelings", "addGenePanel", "addDisplayPanel", "addCellTypePanel", @@ -324,25 +325,100 @@ def _currentRows(image_layer): # ---------------------------------------------------------- cell types -def labelPartition(labels) -> frozenset: - """The GROUPING a labeling induces, with the class names discarded. +# Two labelings this close say the same thing about the section. The +# number is reported every time rather than applied silently, so a pair +# that only nearly agrees is visible as a pair that nearly agrees. +REDUNDANT_AT = 0.99 - Two labelings that partition the cells identically and differ only in - what the groups are called carry the same information, and showing - both invites the reader to treat one as corroborating the other. That - happens for a real reason: a subclass call transferred onto clusters - is a RENAMING of those clusters, one name per cluster, so it agrees - with the clustering by construction. - Comparing names would miss this; comparing memberships catches it. +def labelAgreement(a, b) -> float: + """How completely knowing *a* tells you *b*, as a fraction of cells. + + Exact equality of partitions was too strict for the case it was + written for. A subclass call transferred cell by cell onto a + clustering agrees with it almost everywhere and disagrees on a + handful of boundary cells, which is not a second opinion -- it is the + same opinion with noise -- but it is not the same partition either, + so an equality test showed both and the reader was back where they + started. + + For each group of *a*, the majority *b* value is the one *a* would + predict; this returns the fraction of cells that value is right for. + 1.0 means *a* determines *b* exactly, which is what a per-cluster + renaming gives. + + The measure is DIRECTIONAL. A fine clustering determines a coarse + call without the reverse holding, so callers test both ways. + """ + a = list(a) + b = list(b) + if len(a) != len(b): + raise ValueError( + f"labelings of {len(a)} and {len(b)} cells cannot be compared; " + f"they are not labelings of the same cells" + ) + if not a: + return 1.0 + groups: dict[Any, dict] = {} + for va, vb in zip(a, b): + counts = groups.setdefault(va, {}) + counts[vb] = counts.get(vb, 0) + 1 + return sum(max(c.values()) for c in groups.values()) / len(a) + + +def selectLabelings(found, wanted=None): + """Which labelings to draw, and which are the same one twice. + + Pure, so the decision is testable without a display -- it is the part + that can be wrong. + + *found* is a list of ``(labels, info)`` as + :func:`~ndi.fun.doc_gene.readCellTypeLabels` returns them. + + With *wanted* -- a list of label names -- the caller has chosen, and + nothing is collapsed or reordered away from that choice. Names that + are not there come back in *missing* rather than being ignored: a + typo that silently showed everything would look like the flag not + working. + + Without it, supervised calls are considered first and any later + labeling that agrees with a kept one above :data:`REDUNDANT_AT` is + set aside, so the labeling carrying biological names is the one that + stays. + + Returns: + ``(kept, redundant, missing)``. *redundant* holds + ``(name, kept_name, forward, reverse)``, the two agreements + measured in both directions so the note can quote a number. """ - groups: dict[Any, list] = {} - for i, v in enumerate(labels): - groups.setdefault(v, []).append(i) - return frozenset(frozenset(g) for g in groups.values()) + found = sorted(found, key=lambda t: (bool(t[1]["isUnsupervised"]), t[1]["labelName"] or "")) + + if wanted is not None: + want = [str(w).strip() for w in wanted if str(w).strip()] + by_name = {(t[1]["labelName"] or ""): t for t in found} + kept = [by_name[w] for w in want if w in by_name] + return kept, [], [w for w in want if w not in by_name] + + kept, redundant = [], [] + for labels, info in found: + name = info["labelName"] or "(unnamed)" + dup = None + for kept_labels, kept_info in kept: + forward = labelAgreement(labels, kept_labels) + reverse = labelAgreement(kept_labels, labels) + if max(forward, reverse) >= REDUNDANT_AT: + dup = (name, kept_info["labelName"] or "(unnamed)", forward, reverse) + break + if dup is None: + kept.append((labels, info)) + else: + redundant.append(dup) + return kept, redundant, [] -def addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer=None) -> Any: +def addCellTypePanel( + viewer, session, cells_doc, points_layer, shapes_layer=None, labelings=None +) -> Any: """One checkbox per class, per labeling, over the cell layers. A cellbin routinely carries several labelings -- a transferred atlas @@ -350,11 +426,10 @@ def addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer=None rather than being merged. They are not interchangeable: a clustering carries no biological identity. - A labeling that groups the cells EXACTLY as an earlier one does is - named and then skipped, not drawn twice; see :func:`labelPartition`. - Supervised calls are considered first, so when a clustering and the - subclass call transferred onto it agree, it is the one carrying - biological names that stays. + A labeling that says the same thing as one already shown is named + and then set aside rather than drawn twice; see + :func:`selectLabelings`. Pass *labelings* -- a list of label names -- + to choose outright instead, in which case nothing is collapsed. Hiding works by ALPHA, not by removing points. The row order of the points layer is the row order of cells.tsv and of every labels.tsv @@ -377,42 +452,39 @@ def addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer=None import numpy as np - labelings = [] + found = [] for d in docs: try: labels, info = readCellTypeLabels(session, d) except Exception as e: print(f"[genepyramid] skipping a cellTypeLabels document: {e}") continue - labelings.append((labels, info)) - if not labelings: + found.append((labels, info)) + if not found: return None - # Supervised first, so a redundant pair keeps the named call. - labelings.sort(key=lambda t: (bool(t[1]["isUnsupervised"]), t[1]["labelName"] or "")) - - kept, redundant, seen = [], [], {} - for labels, info in labelings: - key = labelPartition(labels) - first = seen.get(key) - if first is None: - seen[key] = info["labelName"] or "(unnamed)" - kept.append((labels, info)) - else: - redundant.append((info["labelName"] or "(unnamed)", first)) + kept, redundant, missing = selectLabelings(found, labelings) box = QWidget() outer = QVBoxLayout(box) - for name, first in redundant: - msg = ( - f"{name} groups the cells exactly as {first} does, differing only " - f"in what the groups are called, so it is not shown separately." - ) + def _note(msg): print(f"[genepyramid] {msg}") - note = QLabel(msg) - note.setWordWrap(True) - outer.addWidget(note) + w = QLabel(msg) + w.setWordWrap(True) + outer.addWidget(w) + + for name, other, forward, reverse in redundant: + _note( + f"{name} and {other} agree about {100 * max(forward, reverse):.1f}% " + f"of cells -- one grouping under two sets of names -- so only " + f"{other} is shown. Use --labels to choose for yourself." + ) + if missing: + have = ", ".join((i["labelName"] or "(unnamed)") for _lb, i in found) + _note(f"no labeling named {', '.join(missing)}; this cells document has: {have}") + if not kept: + _note("no labelings to show.") n_points = len(points_layer.data) # visible[i] is False when ANY ticked-off class covers cell i, so two @@ -485,6 +557,7 @@ def addAllPanels( cells_doc=None, points_layer=None, shapes_layer=None, + labelings=None, ): """Every panel the session supports, skipping what it has no data for. diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index 3d409384..cd06a765 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -42,6 +42,7 @@ def openPyramid( outlines=None, cells_doc=None, controls: bool = True, + labelings=None, name: str | None = None, show: bool = True, ): @@ -71,6 +72,12 @@ def openPyramid( the cellTypeLabels documents that depend on it -- the cells themselves arrive as plain arrays and carry no way back to their own document. + labelings: label names to show in the cell-type panel, or None to + let it decide. Two labelings that say the same thing about the + section -- a subclass call transferred onto the clustering it + was transferred onto -- are otherwise drawn twice, so by + default the panel keeps one and says so. Naming them here + overrides that entirely. controls: dock the gene, display and cell-type panels. Without them the gene selection and the density choice are fixed at launch, since this viewer holds no other state. Skipped with @@ -138,6 +145,7 @@ def openPyramid( cells_doc=cells_doc, points_layer=points, shapes_layer=shapes, + labelings=labelings, ) if show: diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index b45864c6..0d08e54d 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -46,28 +46,87 @@ def test_symbols_are_ordered_case_insensitively(): assert list(index) == ["Actb", "bcl2", "Zic1"] -# ------------------------------------------------------- labelPartition +# ------------------------------------------------------- labelAgreement -def test_a_renaming_of_the_same_groups_compares_equal(): - """Why the panel drops one: a transferred call renames its clusters.""" +def _info(name, unsup=False): + return {"labelName": name, "isUnsupervised": unsup, "categories": [], "nUnlabeled": 0} + + +def test_a_per_cluster_renaming_determines_the_call_exactly(): leiden = ["0", "0", "1", "2", "1"] subclass = ["L2/3", "L2/3", "Pvalb", "Sst", "Pvalb"] - assert controls.labelPartition(leiden) == controls.labelPartition(subclass) + assert controls.labelAgreement(leiden, subclass) == 1.0 + + +def test_a_handful_of_disagreeing_cells_is_not_a_second_opinion(): + """Exact partition equality missed this, which is why it is measured.""" + leiden = ["0"] * 100 + ["1"] * 100 + subclass = ["A"] * 99 + ["B"] + ["B"] * 100 + # Not the same partition -- one cell moved -- but not two opinions. + assert controls.labelAgreement(leiden, subclass) == 0.995 + + +def test_the_measure_is_directional(): + """A fine clustering determines a coarse call, not the other way.""" + fine = ["0", "1", "2", "3"] + coarse = ["A", "A", "B", "B"] + assert controls.labelAgreement(fine, coarse) == 1.0 + assert controls.labelAgreement(coarse, fine) == 0.5 + + +def test_labelings_of_different_lengths_are_refused(): + import pytest + + with pytest.raises(ValueError, match="not labelings of the same cells"): + controls.labelAgreement(["a"], ["a", "b"]) + + +# ------------------------------------------------------ selectLabelings + + +def test_the_named_call_is_what_survives_a_redundant_pair(): + n = 200 + leiden = ["0"] * (n // 2) + ["1"] * (n // 2) + subclass = ["Pvalb"] * (n // 2) + ["Sst"] * (n // 2) + found = [ + (leiden, _info("leiden", unsup=True)), + (subclass, _info("subclass_nn_column")), + ] + kept, redundant, missing = controls.selectLabelings(found) + assert [i["labelName"] for _lb, i in kept] == ["subclass_nn_column"] + assert redundant[0][0] == "leiden" + assert redundant[0][1] == "subclass_nn_column" + assert missing == [] + + +def test_two_labelings_that_really_differ_are_both_kept(): + a = ["x"] * 50 + ["y"] * 50 + b = ["p", "q"] * 50 + found = [(a, _info("a")), (b, _info("b"))] + kept, redundant, _ = controls.selectLabelings(found) + assert len(kept) == 2 + assert redundant == [] -def test_genuinely_different_labelings_do_not_compare_equal(): - a = ["0", "0", "1", "2", "1"] - b = ["0", "1", "1", "2", "1"] - assert controls.labelPartition(a) != controls.labelPartition(b) +def test_naming_a_labeling_overrides_the_collapsing(): + leiden = ["0"] * 100 + ["1"] * 100 + subclass = ["Pvalb"] * 100 + ["Sst"] * 100 + found = [(leiden, _info("leiden", unsup=True)), (subclass, _info("subclass_nn_column"))] + kept, redundant, missing = controls.selectLabelings(found, ["leiden"]) + assert [i["labelName"] for _lb, i in kept] == ["leiden"] + assert redundant == [] and missing == [] -def test_unlabelled_cells_are_a_group_of_their_own(): - """Two labelings that leave DIFFERENT cells blank are not the same.""" - a = ["", "x", "x"] - b = ["x", "", "x"] - assert controls.labelPartition(a) != controls.labelPartition(b) +def test_asking_for_a_labeling_that_is_not_there_says_so(): + found = [(["a", "b"], _info("real"))] + kept, _redundant, missing = controls.selectLabelings(found, ["typo"]) + assert kept == [] + assert missing == ["typo"] -def test_the_same_names_in_a_different_order_are_a_different_partition(): - assert controls.labelPartition(["a", "b"]) != controls.labelPartition(["b", "b"]) +def test_an_empty_selection_shows_nothing_rather_than_everything(): + """'--labels none' is a choice, and it is not the same as no choice.""" + found = [(["a", "b"], _info("real"))] + kept, _redundant, missing = controls.selectLabelings(found, []) + assert kept == [] and missing == [] From 2ab847d25169b1b061104eb44282a47b464c4f72 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 19:40:50 +0000 Subject: [PATCH 05/52] Follow the GeneIngest rename in the bridge yaml VH-Lab/NDI-matlab#977 renames ndi.gui.app.GeneIngest to ndi.gui.app.GEFManager, so the matlab_path recorded here points at a file that no longer exists. test_every_recorded_matlab_path_points_at_a_real_file catches exactly that, and it caught this. The decision_log gains the half that is no longer symmetrical in the usual direction: the app's new View button draws nothing in MATLAB, it runs napariViewGEF from this package. So what stays not_yet_ported is the ingest and the listing, and the viewing the MATLAB app offers is already this side's -- reached across a process boundary rather than reimplemented. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/ndi_matlab_python_bridge.yaml | 4 +-- src/ndi/gui/ndi_matlab_python_bridge.yaml | 35 ++++++++++++++--------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index 484620b4..340072b0 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -2203,10 +2203,10 @@ functions: # +ndi/+fun/+doc/+gene -- spatial-transcriptomics ingest additions # # A group of five MATLAB functions that lift the .gef / .h5ad ingest - # path out of ndi.gui.app.GeneIngest so a script, a batch importer or a + # path out of ndi.gui.app.GEFManager so a script, a batch importer or a # Python caller can drive it without opening the GUI (VH-Lab/NDI-matlab # PR #933). None of the five has a Python counterpart yet: - # ndi.gui.app.GeneIngest itself is not ported either (see the entry in + # ndi.gui.app.GEFManager itself is not ported either (see the entry in # src/ndi/gui/app/ndi_matlab_python_bridge.yaml), and the ingest path # only becomes reachable from Python once the GUI class or a # non-interactive equivalent lands. The tile / cell / pyramid readers diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index c56b2732..a46bccf0 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -681,19 +681,26 @@ ported_late: Lands after JRCLUST itself is reachable from Python, not before. Noted in #204. - - name: GeneIngest - matlab_path: "+ndi/+gui/+app/GeneIngest.m" - matlab_last_sync_hash: "eae346a" + - name: GEFManager + matlab_path: "+ndi/+gui/+app/GEFManager.m" + matlab_last_sync_hash: "8edda6d" status: not_yet_ported decision_log: > - MATLAB App Designer window that lists, adds and removes spatial - transcriptomics pyramids in a session, and drives the .gef / - cellbin .h5ad ingest path with a confirm step between file choice - and the write. Added in VH-Lab/NDI-matlab PR #933 alongside the - +fun/+doc/+gene entry points (fromGEF, fromCellBin, makeSourceFile, - readNotes, tilePath) that this app calls, all recorded - not_yet_ported in src/ndi/fun/. Python has the pyramid readers and - writers (ndi.fun.doc_gene) but no session-scoped ingest UI over - them; the app is the natural place for one and lands with a - non-interactive or navigator-integrated equivalent. Interactive - viewing is separately napari's, through NDI-python. + MATLAB App Designer window that lists, views, adds and removes + spatial transcriptomics pyramids in a session, and drives the .gef + / cellbin .h5ad ingest path with a confirm step between file choice + and the write. Added in VH-Lab/NDI-matlab PR #933 as GeneIngest and + renamed there in PR #977, which also gave it a View button. Named + alongside the +fun/+doc/+gene entry points (fromGEF, fromCellBin, + makeSourceFile, readNotes, tilePath) that this app calls, all + recorded not_yet_ported in src/ndi/fun/. Python has the pyramid + readers and writers (ndi.fun.doc_gene) but no session-scoped ingest + UI over them; the app is the natural place for one and lands with a + non-interactive or navigator-integrated equivalent. + ONE PART OF IT IS ALREADY PORTED, in the other direction. View + draws nothing in MATLAB: it builds a command line for the napari + viewer in THIS package -- ndi.gui.app.genepyramid.cli, the + napariViewGEF console script -- and runs it. So what stays unported + is the ingest and the listing; the viewing the MATLAB app offers is + this side's, reached across a process boundary rather than + reimplemented. From 5cb1980b966ff6ba3a68db8947def14a99e30b3c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 19:50:14 +0000 Subject: [PATCH 06/52] Cut the launch's eager work from ~27s to ~5s, and say where it goes The wait before the window opens was not NDI and was not the pyramid. The image ladder is lazy by construction -- layerSpec resolves tile paths and reads no tile bytes -- so every second of it was the CELLS, which are read whole. Measured on the opossum section's shape (493,126 cells, 24 vertices each, a 26 MB cells.tsv and a 49 MB contours.bin): 3.4s cells.tsv parsed 3.4s cells.tsv parsed AGAIN, inside readContours 2.0s one np.stack per cell to split contours.bin 3.3s one np.column_stack per cell to add the centroids back 13.8s list(zip(row, col)) per polygon in openPyramid 0.4s list(zip()) for the centroids Nothing there is doing real work per cell; it is half a million Python round trips through numpy for operations that are one array each. readContourFile now builds one (total, 2) array and hands back views into it, which is also what lets readContours place every vertex in a single addition: np.repeat gives each vertex its own cell's centroid, so a per-cell offset becomes a whole-array operation. readContours takes the centroids the caller already parsed instead of re-reading cells.tsv. The viewer uses np.column_stack per polygon and one for the whole centroid table. Same numbers out, and the placement is now pinned by tests -- including the empty-file case, where np.split would otherwise hand back one polygon where there are none. 2.5s readCells 0.7s readContours 1.6s world transform 0.0s centroids WHAT IS LEFT IS NAPARI, and this cannot make it fast: add_shapes triangulates every polygon as it takes them, and there are half a million. So the launch now narrates itself to stderr, a line per stage with its elapsed time, and names that step as the long one before it starts. A program that makes you wait owes you an account of the wait. NDI_GENEPYRAMID_QUIET=1 turns it off. Also: the density-versus-counts explanation is hover text on the two radio buttons now, not a paragraph standing under them. It is read once. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/doc_gene.py | 58 ++++++++++------ src/ndi/gui/app/genepyramid/cli.py | 24 +++++-- src/ndi/gui/app/genepyramid/controls.py | 17 +++-- src/ndi/gui/app/genepyramid/progress.py | 55 +++++++++++++++ src/ndi/gui/app/genepyramid/viewer.py | 89 ++++++++++++++++--------- tests/test_doc_gene_contours.py | 26 ++++++++ 6 files changed, 203 insertions(+), 66 deletions(-) create mode 100644 src/ndi/gui/app/genepyramid/progress.py diff --git a/src/ndi/fun/doc_gene.py b/src/ndi/fun/doc_gene.py index a96fad22..867dda4a 100644 --- a/src/ndi/fun/doc_gene.py +++ b/src/ndi/fun/doc_gene.py @@ -1356,16 +1356,24 @@ def readContourFile( vx = np.frombuffer(raw, vt, count=total, offset=pos) vy = np.frombuffer(raw, vt, count=total, offset=pos + total * vt.itemsize) - polys = [ - np.stack([vx[offsets[i] : offsets[i + 1]], vy[offsets[i] : offsets[i + 1]]], 1) - for i in range(n) - ] + # One (total, 2) array, then SPLIT INTO VIEWS of it. A stack per cell + # cost 2.0s on the opossum section's 493,126 cells; this costs 0.4s, + # and the whole-file array is what lets readContours place every + # vertex in one vectorised addition instead of half a million small + # ones. The views share that array, so nothing is copied. + xy = np.empty((total, 2), vt) + xy[:, 0] = vx + xy[:, 1] = vy + polys = np.split(xy, offsets[1:-1]) if n else [] info = { "nCells": n, "nVerticesTotal": total, "vertexType": vertexType, "offsetType": offsetType, "nVerticesPerCell": nVerticesPerCell, + # The flat form, for callers that would otherwise rebuild it. + "vertices": xy, + "offsets": offsets, } return polys, info @@ -1622,7 +1630,7 @@ def _cells_doc_count(cells_doc): return None -def readContours(session, cells_doc): +def readContours(session, cells_doc, cells=None): """Boundary polygons for a cells document, in SOURCE coordinates. The document-level companion to :func:`readContourFile`, which parses @@ -1642,6 +1650,11 @@ def readContours(session, cells_doc): Args: session: an ndi.session or ndi.dataset holding the document. cells_doc: a spatialGeneExpressionCells document. + cells: the columns :func:`readCells` returns for this same + document, when the caller already has them. Only the + centroids are used, and only for a centroid-referenced file. + Passing them avoids a second parse of cells.tsv, which is the + larger of the two files and the slower to parse. Returns: ``(polys, info)``. *polys* is one ``(N, 2)`` array of ``[x, y]`` @@ -1681,29 +1694,32 @@ def readContours(session, cells_doc): finally: session.database_closebinarydoc(fh) + verts = np.asarray(info["vertices"], dtype=float) + offsets = info["offsets"] + n = info["nCells"] + if reference == "centroid": - cols, _info = readCells(session, cells_doc) + # Re-reading cells.tsv here cost a second parse of a 26 MB file -- + # 3.4s on the opossum section, for centroids the caller had + # already read. Passed in, it is free. + cols = cells if cells is not None else readCells(session, cells_doc)[0] cx = np.asarray(cols["x"], dtype=float) cy = np.asarray(cols["y"], dtype=float) - if len(cx) != info["nCells"]: + if len(cx) != n: raise ValueError( - f"contours.bin holds {info['nCells']} cells but cells.tsv " + f"contours.bin holds {n} cells but cells.tsv " f"holds {len(cx)} rows. Centroid-relative vertices cannot be " f"placed without a centroid each." ) - placed = [] - for i, poly in enumerate(polys): - if len(poly) == 0: - placed.append(np.zeros((0, 2), dtype=float)) - continue - placed.append( - np.column_stack( - [poly[:, 0].astype(float) + cx[i], poly[:, 1].astype(float) + cy[i]] - ) - ) - polys = placed - else: - polys = [np.asarray(p, dtype=float).reshape(-1, 2) for p in polys] + # One addition over every vertex, rather than one per cell. Each + # vertex takes its own cell's centroid through np.repeat, which + # is what makes a per-cell offset expressible as a whole-array + # operation: 1.5s in place of 5.3s. + counts = np.diff(offsets) + verts[:, 0] += np.repeat(cx, counts) + verts[:, 1] += np.repeat(cy, counts) + + polys = np.split(verts, offsets[1:-1]) if n else [] info = dict(info) info["contourReference"] = reference diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index c766d70d..4bdcc6f9 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -189,7 +189,7 @@ def _resolve_labelings(spec: str): return [s.strip() for s in spec.split(",") if s.strip()] -def _resolve_outlines(session, pyr_doc, spec: str, cells_info): +def _resolve_outlines(session, pyr_doc, spec: str, cells_info, cells=None): """Boundary polygons for the cells document --cells resolved. Returns None after printing why, so main can exit 1. A directory of @@ -232,7 +232,7 @@ def _resolve_outlines(session, pyr_doc, spec: str, cells_info): return None try: - polys, info = readContours(session, docs[0]) + polys, info = readContours(session, docs[0], cells=cells) except Exception as e: print(f"could not read contours: {e}", file=sys.stderr) return None @@ -278,8 +278,12 @@ def _resolve_genes(session, pyr_doc, spec: str): def main(argv=None) -> int: args = build_parser().parse_args(argv) - session = _open_session(args.session) - docs = _pyramids(session) + from .progress import stage + + with stage(f"opening the session at {args.session}"): + session = _open_session(args.session) + with stage("finding the pyramids"): + docs = _pyramids(session) if not docs: print(f"no spatialGeneExpressionPyramid in {args.session}", file=sys.stderr) @@ -314,7 +318,11 @@ def main(argv=None) -> int: return 1 gene_rows = _resolve_genes(session, pyr, args.genes) - cells, cells_info = _resolve_cells(session, pyr, args.cells) + import contextlib + + reading_cells = stage("reading the cell table") if args.cells else contextlib.nullcontext() + with reading_cells: + cells, cells_info = _resolve_cells(session, pyr, args.cells) density = not args.no_density if args.report: @@ -359,7 +367,11 @@ def main(argv=None) -> int: polys = None if args.outlines: - polys = _resolve_outlines(session, pyr, args.cells, cells_info) + with stage("reading the cell boundaries"): + # The centroids are handed over rather than re-read: the + # boundaries are stored relative to them, and cells.tsv is + # the slower of the two files to parse. + polys = _resolve_outlines(session, pyr, args.cells, cells_info, cells) if polys is None: return 1 diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index b46e84e1..f234abbf 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -256,7 +256,8 @@ def addDisplayPanel(viewer, session, pyr_doc, image_layer, density: bool = True) Two exclusive states, so radio buttons rather than a checkbox: the label then says what each choice IS, instead of naming one and - leaving the other implied. + leaving the other implied. Why it matters is hover text on the two + buttons rather than a paragraph in the panel: it is read once. Switching only swaps the layer's data. A level is summed over the selected genes into a 2D array, so the shape does not change and the @@ -279,11 +280,16 @@ def addDisplayPanel(viewer, session, pyr_doc, image_layer, density: bool = True) group.addButton(r_density) group.addButton(r_counts) - note = QLabel( - "Binning sums, so raw counts make a coarse level binSize^2 brighter " - "than a fine one and the picture jumps at each level change." + # Why the choice exists, as hover text rather than a paragraph: it is + # read once and then never again, and a standing explanation of a + # two-button control is mostly in the way. + why = ( + "Binning sums, so raw counts make a coarse level binSize^2 brighter\n" + "than a fine one and the picture jumps at each level change.\n" + "Density divides that out, so one contrast range serves every level." ) - note.setWordWrap(True) + for w in (r_density, r_counts): + w.setToolTip(why) status = QLabel("") status.setWordWrap(True) @@ -304,7 +310,6 @@ def _switch(want_density: bool): outer.addWidget(r_density) outer.addWidget(r_counts) - outer.addWidget(note) outer.addWidget(status) outer.addStretch() diff --git a/src/ndi/gui/app/genepyramid/progress.py b/src/ndi/gui/app/genepyramid/progress.py new file mode 100644 index 00000000..32144eb1 --- /dev/null +++ b/src/ndi/gui/app/genepyramid/progress.py @@ -0,0 +1,55 @@ +"""Say what the launch is doing, and how long each part took. + +Opening a large section takes tens of seconds before anything is drawn, +and every one of those seconds is spent somewhere specific: reading a +26 MB cell table, placing twelve million contour vertices, handing half +a million polygons to napari. Without this the whole wait is one opaque +block and the only available theory is "NDI is slow". + +Timings go to STDERR so they interleave with the other launch notes and +stay out of anything a caller pipes. On by default: a program that makes +you wait owes you an account of the wait, and these lines cost one print +each. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +import time + +__all__ = ["stage", "note"] + +#: Set NDI_GENEPYRAMID_QUIET=1 to silence the launch narration. +_QUIET_ENV = "NDI_GENEPYRAMID_QUIET" + + +def _quiet() -> bool: + return bool(os.environ.get(_QUIET_ENV, "").strip()) + + +def note(text: str) -> None: + """One launch line, no timing.""" + if not _quiet(): + print(f"[genepyramid] {text}", file=sys.stderr, flush=True) + + +@contextlib.contextmanager +def stage(label: str): + """Time a step of the launch and report it when it finishes. + + The label is printed BEFORE the work, so a step that hangs is named + on screen rather than leaving the user with the previous step's line + and no idea which one is stuck. + """ + if _quiet(): + yield + return + print(f"[genepyramid] {label} ...", file=sys.stderr, flush=True) + t0 = time.perf_counter() + try: + yield + finally: + dt = time.perf_counter() - t0 + print(f"[genepyramid] {label}: {dt:.1f}s", file=sys.stderr, flush=True) diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index cd06a765..b19a3327 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -9,7 +9,9 @@ from typing import Any -from .multiscale import layerSpec, sourceToWorld +import numpy as np + +from .multiscale import layerSpec, sourceToWorld, worldTransform __all__ = ["require_napari", "openPyramid"] @@ -93,40 +95,60 @@ def openPyramid( Returns: The napari Viewer. """ - napari = require_napari() + from .progress import note, stage + + with stage("importing napari"): + napari = require_napari() viewer = napari.Viewer() - image = viewer.add_image(**layerSpec(session, pyr_doc, gene_rows, density, name)) + # The image ladder is LAZY: layerSpec resolves tile paths and the + # level table, and reads no tile bytes. Nothing here is the wait. + with stage("building the pyramid ladder (lazy)"): + image = viewer.add_image(**layerSpec(session, pyr_doc, gene_rows, density, name)) shapes = None if outlines is not None: keep = [p for p in outlines if len(p)] if keep: - paths = [] - for p in keep: - row, col = sourceToWorld(session, pyr_doc, p[:, 0], p[:, 1]) - paths.append(list(zip(row, col))) + # np.column_stack per polygon rather than list(zip(row, col)): + # the tuple form built twelve million Python tuples for the + # opossum section and cost 13.8s, against 1.6s for this. + with stage(f"placing {len(keep):,} outlines"): + (sy, sx), _t = worldTransform(session, pyr_doc) + paths = [np.column_stack([p[:, 1] * sy, p[:, 0] * sx]) for p in keep] # shape_type polygon closes the ring itself, which matches the # format: writeContourFile does not repeat the first vertex. - shapes = viewer.add_shapes( - paths, - shape_type="polygon", - name="cell outlines", - face_color="transparent", - edge_color="cyan", - edge_width=1, + # + # THIS IS THE SLOW ONE and nothing here can make it fast: + # napari triangulates every polygon as it takes them. + note( + f"handing {len(keep):,} polygons to napari -- it triangulates " + f"each one, so this is the long part of the launch. Drop " + f"--outlines to skip it." ) + with stage("napari add_shapes"): + shapes = viewer.add_shapes( + paths, + shape_type="polygon", + name="cell outlines", + face_color="transparent", + edge_color="cyan", + edge_width=1, + ) points = None if cells is not None: - row, col = sourceToWorld(session, pyr_doc, cells["x"], cells["y"]) - points = viewer.add_points( - list(zip(row, col)), - name="cell centroids", - size=cells.get("size", 8), - face_color=cells.get("face_color", "red"), - border_width=0, - ) + with stage(f"placing {len(cells['x']):,} centroids"): + row, col = sourceToWorld(session, pyr_doc, cells["x"], cells["y"]) + coords = np.column_stack([row, col]) + with stage("napari add_points"): + points = viewer.add_points( + coords, + name="cell centroids", + size=cells.get("size", 8), + face_color=cells.get("face_color", "red"), + border_width=0, + ) if controls: from .controls import addAllPanels @@ -136,17 +158,18 @@ def openPyramid( # density and counts; recomputing with all genes would silently # widen what is being shown. image._ndi_gene_rows = gene_rows - addAllPanels( - viewer, - session, - pyr_doc, - image, - density, - cells_doc=cells_doc, - points_layer=points, - shapes_layer=shapes, - labelings=labelings, - ) + with stage("building the control panels"): + addAllPanels( + viewer, + session, + pyr_doc, + image, + density, + cells_doc=cells_doc, + points_layer=points, + shapes_layer=shapes, + labelings=labelings, + ) if show: napari.run() diff --git a/tests/test_doc_gene_contours.py b/tests/test_doc_gene_contours.py index f1088ea5..f7b508ea 100644 --- a/tests/test_doc_gene_contours.py +++ b/tests/test_doc_gene_contours.py @@ -169,3 +169,29 @@ def test_an_unknown_reference_is_refused(centroids): raw = _binary([SQUARE]) with pytest.raises(ValueError, match="expected 'centroid' or 'absolute'"): doc_gene.readContours(_Session(raw), _Doc("sideways", n_cells=1)) + + +def test_passed_in_centroids_are_used_instead_of_a_second_read(monkeypatch): + """cells.tsv is the slower file, and the caller has already parsed it.""" + reads = [] + monkeypatch.setattr(doc_gene, "readCells", lambda s, d: (reads.append(1), (CENTROIDS, {}))[1]) + raw = _binary([SQUARE, EMPTY, TRIANGLE]) + polys, _ = doc_gene.readContours(_Session(raw), _Doc("centroid"), cells=CENTROIDS) + assert reads == [] + np.testing.assert_allclose(polys[0], [[98, 8], [102, 8], [102, 12], [98, 12]]) + + +def test_the_flat_vertex_array_comes_back_for_callers_that_need_it(centroids): + """One array plus offsets is what makes placement a single addition.""" + raw = _binary([SQUARE, EMPTY, TRIANGLE]) + _polys, info = doc_gene.readContourFile(_FH(raw)) + assert info["vertices"].shape == (7, 2) + np.testing.assert_array_equal(info["offsets"], [0, 4, 4, 7]) + + +def test_a_file_with_no_cells_gives_no_polygons(centroids): + """np.split of an empty array would otherwise hand back one of them.""" + raw = _binary([]) + polys, info = doc_gene.readContourFile(_FH(raw)) + assert polys == [] + assert info["nCells"] == 0 From 32ee01e0017600969f3386b7cc7c7ee6db416ed7 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 19:58:46 +0000 Subject: [PATCH 07/52] Show counts in the gene list, and filter it by abundance band The list said which genes exist and nothing about which ones carry the section. Each row now shows the reads that gene contributes, summed over the same variant rows the layer sums, so the number in the list is the number the picture is drawn from. readGeneTotals reads gene_totals.tsv off the pyramid, which had a writer and no reader. Older pyramids do not carry that file: it comes back as None rather than as zeros, because a gene with no reads and a gene whose reads were never recorded are different things, and the column and the band are dropped rather than invented. A totals file whose length disagrees with the gene list is refused for the same reason -- indexing it by gene row would put one gene's reads on another. The two-ended band is ported from the FICTURE viewer's abundance panel, including the parts that make it correct rather than merely plausible. It is a percentile of the READS, not of the list, and on a real section those are wildly different: expression is heavy-tailed, so the top half of the reads is a percent or so of the genes. An entry occupies an interval of the cumulative sweep and sits at its MIDPOINT, since either endpoint would let one dominant gene straddle a cut on the strength of its own bulk; tied totals share one midpoint computed over the whole group, or ties break on sort order and two genes with identical counts land on opposite sides of the same threshold. That arithmetic has a consequence which reads as a bug the first time: a gene carrying 30% of all reads has the interval [70, 100], so a 1% top cut cannot reach it -- correctly, since dropping it would remove 30% of the reads. The readout says so, and names the cut that would work, rather than letting the control look inert. It also names what a top cut did drop, because "you just dropped MBP" is the useful feedback and "1,204 genes hidden" is not. The band filters the LIST; it composes with the text box rather than replacing it, since narrowing to the loud genes and then searching within them is how both get used. superqt's range slider when it is there, two spin boxes when it is not, and the same presets as before. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/doc_gene_export.py | 56 +++- src/ndi/gui/app/genepyramid/controls.py | 323 ++++++++++++++++++++++-- tests/test_doc_gene_export.py | 28 ++ tests/test_gui_genepyramid_controls.py | 60 +++++ 4 files changed, 445 insertions(+), 22 deletions(-) diff --git a/src/ndi/fun/doc_gene_export.py b/src/ndi/fun/doc_gene_export.py index 28adf928..d6b1fad7 100644 --- a/src/ndi/fun/doc_gene_export.py +++ b/src/ndi/fun/doc_gene_export.py @@ -30,7 +30,7 @@ from .doc_gene import _find_level, exportRegion, readTileFile -__all__ = ["exportGef", "exportHdf5", "readGeneList"] +__all__ = ["exportGef", "exportHdf5", "readGeneList", "readGeneTotals"] def readGeneList(session, pyr_doc): @@ -77,6 +77,60 @@ def readGeneList(session, pyr_doc): return ids, names +def readGeneTotals(session, pyr_doc): + """Reads per gene row for THIS pyramid, from ``gene_totals.tsv``. + + Totals belong to the pyramid rather than to the geneList because they + are a property of the counts: several datasets may share one gene + list, and they do not share its totals. + + OLDER PYRAMIDS DO NOT HAVE THIS FILE. It is returned as None rather + than as zeros, because a gene with no reads and a gene whose reads + were never recorded are different things and a caller that cannot + tell them apart would draw an abundance distribution out of nothing. + + Args: + session: an ndi.session or ndi.dataset. + pyr_doc: a spatialGeneExpressionPyramid document. + + Returns: + ``(totals, nRecords)``, both int64 arrays with one entry per gene + row in zero-based order, or ``(None, None)`` when the pyramid + carries no totals. *nRecords* is how many stored records the gene + appears in, which is not the same as its reads. + """ + try: + stored = set(pyr_doc.current_file_list()) + except Exception: + stored = set() + if "gene_totals.tsv" not in stored: + return None, None + + fh = session.database_openbinarydoc(pyr_doc, "gene_totals.tsv") + if fh is None: + return None, None + try: + text = fh.read().decode("utf-8") + finally: + session.database_closebinarydoc(fh) + + rows = [ln for ln in text.splitlines() if ln.strip()] + totals, records = [], [] + for line in rows[1:]: + f = line.split("\t") + # gene_index is written explicitly and is zero-based; asserting it + # is what stops a reordered file from putting one gene's totals on + # another gene, which every later number would inherit silently. + if int(f[0]) != len(totals): + raise ValueError( + f"gene_totals.tsv row {len(totals)} declares gene_index " + f"{f[0]}; the file is out of order or not zero-based" + ) + totals.append(int(f[1])) + records.append(int(f[2]) if len(f) > 2 else 0) + return np.asarray(totals, np.int64), np.asarray(records, np.int64) + + def _gather(session, pyr_doc, bin_size): """Every stored record of one level, in SOURCE coordinates. diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index f234abbf..680dd914 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -22,6 +22,7 @@ __all__ = [ "geneIndex", + "abundanceBand", "labelAgreement", "selectLabelings", "addGenePanel", @@ -66,12 +67,118 @@ def geneIndex(ids, names) -> dict: return dict(sorted(by.items(), key=lambda kv: kv[0].lower())) +def abundanceBand(totals, lo: float = 0.0, hi: float = 100.0): + """Which entries fall in a percentile band OF THE READS. + + NOT a percentile of the list, and the difference is the whole point. + Rank least to most abundant and accumulate the totals; an entry's + position is where its own reads sit in that cumulative sweep. So + ``lo=90`` keeps the handful of genes that between them carry the top + 10% OF ALL READS. Ranking by list position instead would keep 10% of + the genes, which in a real section is a completely different and much + larger set: expression is heavy-tailed, and a few hundred genes out + of thirty thousand carry most of the signal. + + An entry occupies an INTERVAL of that sweep -- the mass before it to + the mass after -- and sits at the interval's midpoint. Using either + endpoint would let one dominant gene straddle a cut and be kept or + dropped on the strength of its own bulk. + + TIES SHARE ONE MIDPOINT, computed over the whole tied group. + Otherwise ties break on sort order, which is arbitrary, and two genes + with identical totals land on opposite sides of the same threshold. + It matters most at the bottom, where thousands of genes tie near zero. + + A consequence that looks like a bug the first time: an entry whose + own reads are a large share of the total cannot be removed by a small + top cut. One gene carrying 30% of all reads has the interval + [70, 100] and the midpoint 85, so ``hi=99`` keeps it -- correctly, + since dropping it would remove 30% of the reads, not 1%. *info* + reports that gene's share and the cut that would drop it, so a panel + can say so rather than appear inert. + + Args: + totals: reads per entry, array-like. + lo, hi: band edges in percent. + + Returns: + ``(keep, info)``. *keep* is a boolean array over *totals*. + """ + import numpy as np + + tot = np.asarray(totals, dtype=np.float64).ravel() + n = len(tot) + grand = float(tot.sum()) + if n == 0 or grand <= 0: + return np.ones(n, bool), {"available": False, "nTotal": n} + + loudest = int(np.argmax(tot)) + top_share = float(tot[loudest] / grand * 100.0) + info = { + "available": True, + "nTotal": n, + "lo": float(lo), + "hi": float(hi), + "topRow": loudest, + "topShare": top_share, + # Where the loudest entry sits, so a panel can name the cut that + # would actually drop it. Its interval is [100 - share, 100]. + "topMid": 100.0 - top_share / 2.0, + } + if lo <= 0.0 and hi >= 100.0: + return ( + np.ones(n, bool), + {**info, "nKept": n, "pctReads": 100.0, "droppedTop": []}, + ) + + order = np.argsort(tot, kind="stable") + srt = tot[order] + cum = np.cumsum(srt) + before = cum - srt + start = np.flatnonzero(np.r_[True, srt[1:] != srt[:-1]]) + stop = np.r_[start[1:], n] - 1 + group_mid = (before[start] + cum[stop]) / 2.0 / grand * 100.0 + mid = np.repeat(group_mid, np.r_[start[1:], n] - start) + + in_band = (mid >= lo) & (mid <= hi) + keep = np.zeros(n, bool) + keep[order[in_band]] = True + + # Which loud entries the top cut removed. At hi=99 that is a handful + # of names, and naming them is the useful feedback -- "you just + # dropped MBP" rather than "1,204 genes hidden". + dropped = [] + if hi < 100.0: + for i in order[::-1][:40]: + if not keep[i]: + dropped.append(int(i)) + if len(dropped) == 8: + break + return keep, { + **info, + "nKept": int(keep.sum()), + "pctReads": float(tot[keep].sum() / grand * 100.0) if keep.any() else 0.0, + "droppedTop": dropped, + } + + # Distinct colours rather than one map applied to everything: the layers # blend additively, so two genes in the same colormap make one picture # that neither of them is. _GENE_COLORMAPS = ("magenta", "green", "cyan", "yellow", "red", "blue") +# The useful band settings are not obvious from a bare slider and are +# nearly the same on every section, so they are offered as presets. +_BAND_PRESETS = ( + ("all", 0.0, 100.0), + ("drop top 1%", 0.0, 99.0), + ("drop top 5%", 0.0, 95.0), + ("middle 5-95%", 5.0, 95.0), + ("top 10% only", 90.0, 100.0), +) + + def addGenePanel(viewer, session, pyr_doc) -> Any: """A filterable list of gene symbols, one checkbox each. @@ -82,10 +189,19 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: layer in napari's own list unticks the box, so the two never disagree. Variants are already combined by :func:`geneIndex`; every row a - symbol names is summed into its layer. + symbol names is summed into its layer, and its counts are the sum + over those rows too. + + TWO FILTERS, applied together. The text box answers "where is this + gene"; the abundance band answers "which genes carry the signal". + They compose, because narrowing to the loud genes and then searching + within them is the normal way to use both. """ - from qtpy.QtCore import Qt + import numpy as np + from qtpy.QtCore import Qt, QTimer from qtpy.QtWidgets import ( + QDoubleSpinBox, + QFormLayout, QHBoxLayout, QLabel, QLineEdit, @@ -96,13 +212,32 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: QWidget, ) - from ....fun.doc_gene_export import readGeneList + from ....fun.doc_gene_export import readGeneList, readGeneTotals from .multiscale import layerSpec ids, names = readGeneList(session, pyr_doc) index = geneIndex(ids, names) n_variants = sum(1 for e in index.values() if len(e["rows"]) > 1) + # Counts per SYMBOL, summed over its variant rows -- the same rows the + # layer sums, so the number in the list is the number the picture is + # drawn from. + totals, _records = readGeneTotals(session, pyr_doc) + if totals is not None and len(totals) != len(ids): + # Indexing a short totals array by gene row would either raise or, + # worse, put one gene's reads on another. Neither is worth risking + # for a column. + print( + f"[genepyramid] gene_totals.tsv has {len(totals)} rows but the " + f"gene list has {len(ids)}; counts and the abundance band are " + f"omitted rather than guessed." + ) + totals = None + if totals is None: + symbol_totals = None + else: + symbol_totals = np.array([int(totals[e["rows"]].sum()) for e in index.values()], np.int64) + box = QWidget() outer = QVBoxLayout(box) head = f"{len(index):,} genes" @@ -119,9 +254,13 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: # Thirty thousand items go in as one batch: each add would otherwise # relayout the list. listw.setUpdatesEnabled(False) - for symbol, entry in index.items(): + for i, (symbol, entry) in enumerate(index.items()): rows, accs = entry["rows"], entry["accessions"] - text = symbol if len(rows) == 1 else f"{symbol} ({len(rows)} variants)" + text = symbol + if symbol_totals is not None: + text += f" \u2014 {int(symbol_totals[i]):,}" + if len(rows) > 1: + text += f" ({len(rows)} variants)" item = QListWidgetItem(text) item.setFlags(item.flags() | Qt.ItemIsUserCheckable) item.setCheckState(Qt.Unchecked) @@ -129,17 +268,71 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: # Accessions are searchable too: a symbol is what people usually # type, but not everything here has one. item.setData(Qt.UserRole + 1, " ".join([symbol] + accs).lower()) - item.setToolTip( - f"{symbol}\n{len(rows)} pyramid column(s): {', '.join(accs) or '(no accession)'}" - ) + tip = [ + symbol, + f"{len(rows)} pyramid column(s): {', '.join(accs) or '(no accession)'}", + ] + if totals is not None: + # By ROW, not zipped against the accessions: duplicates are + # dropped from that list, so position there is not position + # here and pairing them would attach the wrong number. + tip.append("counts per column: " + ", ".join(f"{int(totals[r]):,}" for r in rows)) + item.setToolTip("\n".join(tip)) listw.addItem(item) items[symbol] = item listw.setUpdatesEnabled(True) + + state = {"busy": False, "colour": 0, "keep": None} + + # ---- the abundance band ------------------------------------------ + band_readout = QLabel() + band_readout.setWordWrap(True) + if symbol_totals is None: + band_readout.setText( + "This pyramid has no gene_totals.tsv, so abundance is unknown " + "and the band cannot be applied. Counts are omitted above for " + "the same reason." + ) + outer.addWidget(band_readout) + slider = lo_box = hi_box = None + else: + outer.addWidget(QLabel("Abundance band (percentile of total reads)")) + slider = lo_box = hi_box = None + try: + # napari depends on superqt, so the two-handle widget is + # normally there. Still optional: the spin boxes below are + # fully functional, and a missing optional import should not + # cost the control. + from superqt import QLabeledDoubleRangeSlider + + slider = QLabeledDoubleRangeSlider(Qt.Horizontal) + slider.setRange(0.0, 100.0) + slider.setValue((0.0, 100.0)) + slider.setDecimals(1) + outer.addWidget(slider) + except Exception as e: # pragma: no cover - depends on the install + print(f"[genepyramid] no superqt range slider ({e}); using spin boxes") + form = QFormLayout() + lo_box = QDoubleSpinBox(minimum=0.0, maximum=100.0, decimals=1, singleStep=0.5) + hi_box = QDoubleSpinBox(minimum=0.0, maximum=100.0, decimals=1, singleStep=0.5) + hi_box.setValue(100.0) + form.addRow("low %", lo_box) + form.addRow("high %", hi_box) + outer.addLayout(form) + + presets = QHBoxLayout() + for label, a, b in _BAND_PRESETS: + btn = QPushButton(label) + btn.setFlat(True) + btn.clicked.connect(lambda _=False, a=a, b=b: _setBand(a, b)) + presets.addWidget(btn) + outer.addLayout(presets) + outer.addWidget(band_readout) + outer.addWidget(listw, stretch=1) status = QLabel("") status.setWordWrap(True) - clear = QPushButton("Remove all gene layers") row = QHBoxLayout() row.addStretch() @@ -147,7 +340,72 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: outer.addLayout(row) outer.addWidget(status) - state = {"busy": False, "colour": 0} + # ---- filtering ---------------------------------------------------- + + def _band(): + if slider is not None: + v = slider.value() + return float(v[0]), float(v[1]) + if lo_box is not None: + lo, hi = lo_box.value(), hi_box.value() + # Keep the handles ordered rather than rejecting the edit: a + # user dragging the low box past the high one means to move + # the band, and a silent no-op reads as a broken widget. + return (hi, lo) if lo > hi else (lo, hi) + return 0.0, 100.0 + + def _setBand(lo, hi): + if slider is not None: + slider.blockSignals(True) + slider.setValue((lo, hi)) + slider.blockSignals(False) + elif lo_box is not None: + for w, v in ((lo_box, lo), (hi_box, hi)): + w.blockSignals(True) + w.setValue(v) + w.blockSignals(False) + _applyBand() + + def _applyBand(): + if symbol_totals is None: + state["keep"] = None + _refilter() + return + lo, hi = _band() + keep, info = abundanceBand(symbol_totals, lo, hi) + state["keep"] = keep + band_readout.setText(_bandText(info, list(index))) + _refilter() + + def _refilter(): + needle = search.text().strip().lower() + keep = state["keep"] + listw.setUpdatesEnabled(False) + try: + for i, item in enumerate(items.values()): + ok = True if keep is None else bool(keep[i]) + if ok and needle: + ok = needle in item.data(Qt.UserRole + 1) + item.setHidden(not ok) + finally: + listw.setUpdatesEnabled(True) + + # A slider emits continuously while dragged and the band is arithmetic + # over thirty thousand entries plus a pass over the list, so it is + # applied once the handle settles rather than on every tick. + debounce = QTimer(box) + debounce.setSingleShot(True) + debounce.setInterval(180) + debounce.timeout.connect(_applyBand) + if slider is not None: + slider.valueChanged.connect(lambda *_: debounce.start()) + elif lo_box is not None: + lo_box.valueChanged.connect(lambda *_: debounce.start()) + hi_box.valueChanged.connect(lambda *_: debounce.start()) + + search.textChanged.connect(lambda *_: _refilter()) + + # ---- layers ------------------------------------------------------- def _shown() -> int: return sum(1 for it in items.values() if it.checkState() == Qt.Checked) @@ -156,17 +414,6 @@ def _report(extra: str = ""): n = _shown() status.setText(f"{n} gene layer(s) shown" + (f" -- {extra}" if extra else "")) - def _filter(text: str): - needle = text.strip().lower() - listw.setUpdatesEnabled(False) - try: - for it in items.values(): - it.setHidden(bool(needle) and needle not in it.data(Qt.UserRole + 1)) - finally: - listw.setUpdatesEnabled(True) - - search.textChanged.connect(_filter) - def _add(symbol: str): name = f"gene: {symbol}" if name in viewer.layers: @@ -243,11 +490,45 @@ def _on_layer_removed(event): except Exception: # pragma: no cover - depends on the napari build pass + _applyBand() _report() viewer.window.add_dock_widget(box, name="Genes", area="right") return box +def _bandText(info, symbols) -> str: + """What the band actually selected, in words. + + A percentile band is not self-explanatory: 5-95% sounds like it keeps + most genes and in fact keeps a small minority, since the bottom 5% of + READS is spread over many thousands of barely detected ones. Showing + both numbers, and naming what the top cut removed, is what makes the + control readable. + """ + if not info.get("available"): + return "no abundance information for this pyramid." + parts = [ + f"{info['nKept']:,} of {info['nTotal']:,} genes · " + f"{info['pctReads']:.1f}% of all reads" + ] + dropped = info.get("droppedTop") or [] + if dropped: + parts.append("dropped from the top: " + ", ".join(symbols[i] for i in dropped)) + # The confusing case: a top cut that removes nothing because one gene + # is too large to fit inside it. Say why, and say what cut would work, + # rather than letting the control look broken. + elif info["hi"] < 100.0 and info.get("topShare"): + parts.append( + f"nothing dropped at the top: {symbols[info['topRow']]} alone is " + f"{info['topShare']:.1f}% of all reads, so no gene fits in the top " + f"{100 - info['hi']:.1f}%. Set the high edge below " + f"{info['topMid']:.1f} to drop it." + ) + if info["nKept"] == 0: + parts.append("empty band -- nothing listed.") + return "\n".join(parts) + + # ------------------------------------------------------------- display diff --git a/tests/test_doc_gene_export.py b/tests/test_doc_gene_export.py index ee21f985..a67effc9 100644 --- a/tests/test_doc_gene_export.py +++ b/tests/test_doc_gene_export.py @@ -163,3 +163,31 @@ def test_readGeneList_asserts_zero_based_order(built): assert ids[0] == "ENS00000" assert names[3] == "Gene3" assert len(ids) == len(names) + + +def test_gene_totals_read_back_the_reads_that_were_written(built): + """The counts the panel shows must be the counts the pyramid holds.""" + S, pyr, _x, _y, g, c, ng = built + totals, records = doc_gene_export.readGeneTotals(S, pyr) + assert totals is not None + assert len(totals) == ng + expected = np.bincount(g, weights=c, minlength=ng).astype(np.int64) + np.testing.assert_array_equal(totals, expected) + # n_records is how many stored records a gene appears in, which is a + # different number from its reads and must not be confused with it. + np.testing.assert_array_equal(records, np.bincount(g, minlength=ng)) + + +def test_a_pyramid_without_totals_says_so_rather_than_returning_zeros(built): + """A gene with no reads and a gene never counted are different things.""" + + class _NoFiles: + id = "pyr" + document_properties = {"spatialGeneExpressionPyramid": {}} + + def current_file_list(self): + return ["something_else.tsv"] + + S, _pyr, *_rest = built + totals, records = doc_gene_export.readGeneTotals(S, _NoFiles()) + assert totals is None and records is None diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 0d08e54d..509c573c 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -130,3 +130,63 @@ def test_an_empty_selection_shows_nothing_rather_than_everything(): found = [(["a", "b"], _info("real"))] kept, _redundant, missing = controls.selectLabelings(found, []) assert kept == [] and missing == [] + + +# -------------------------------------------------------- abundanceBand + + +def test_the_full_band_keeps_everything(): + keep, info = controls.abundanceBand([1, 2, 3], 0.0, 100.0) + assert keep.all() + assert info["nKept"] == 3 + assert info["pctReads"] == 100.0 + assert info["droppedTop"] == [] + + +def test_the_band_is_a_percentile_of_READS_not_of_the_list(): + """The distinction is the whole point of the control.""" + # 900 genes with one read each, 10 with a hundred. The top half of + # the READS is those 10 genes -- barely 1% of the list. + totals = [1] * 900 + [100] * 10 + keep, info = controls.abundanceBand(totals, 50.0, 100.0) + assert info["nKept"] == 10 + assert keep[900:].all() and not keep[:900].any() + assert round(info["pctReads"], 1) == 52.6 + + +def test_dropping_the_top_names_what_it_dropped(): + # The loud gene's interval is [19.8, 100], so its midpoint is 59.9 + # and only a cut below that reaches it. + totals = [1] * 99 + [400] + _keep, info = controls.abundanceBand(totals, 0.0, 50.0) + assert info["droppedTop"] == [99] + + +def test_a_gene_too_loud_for_a_small_top_cut_survives_it(): + """Looks like a bug the first time; it is arithmetic, and reported.""" + totals = [10] * 70 + [300] + keep, info = controls.abundanceBand(totals, 0.0, 99.0) + assert keep[-1] + assert info["droppedTop"] == [] + assert info["topShare"] == 30.0 + # The cut that WOULD drop it: its interval is [70, 100], midpoint 85. + assert info["topMid"] == 85.0 + + +def test_ties_land_on_the_same_side_of_a_cut(): + """Otherwise identical totals split on sort order, which is arbitrary.""" + totals = [5] * 10 + [1000] + keep, _info = controls.abundanceBand(totals, 0.0, 50.0) + assert len({bool(v) for v in keep[:10]}) == 1 + + +def test_a_pyramid_with_no_reads_is_not_filtered_into_nothing(): + keep, info = controls.abundanceBand([0, 0, 0], 10.0, 90.0) + assert keep.all() + assert info["available"] is False + + +def test_an_empty_list_is_handled_rather_than_dividing_by_zero(): + keep, info = controls.abundanceBand([], 0.0, 100.0) + assert len(keep) == 0 + assert info["available"] is False From 491df29b6c3fbd5ecd042cefff1c1ba13f0ea619 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 20:40:53 +0000 Subject: [PATCH 08/52] Sort the gene list by name or by counts The abundance band leaves you wanting this. Narrow to the top 10% of reads and you get a few hundred genes in alphabetical order, which hides the one thing you narrowed down to see -- which of them are loud. countsOrder is stable, and that is its whole content: the list arrives alphabetical, so a stable sort leaves equal counts in alphabetical order instead of an arbitrary one, and at the bottom of a real section thousands of genes tie. It is pure and tested for exactly that, because a later edit dropping kind="stable" would look harmless and read as a list that reshuffles itself. The list is REBUILT rather than reordered. Moving thirty thousand items one at a time is quadratic and sorting them through a Python comparison is no cheaper than making them again, while a rebuild is the same single batched pass the panel already pays once. Two things follow from that and both are now explicit: the abundance mask is computed once in the original order, so each item carries its original position and the filter reads that rather than its row in the widget; and the ticks come back from the LAYERS rather than from remembered state, since a drawn layer is what a tick means and reading it is what stops the boxes and the picture from disagreeing across a sort. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 99 ++++++++++++++++++++++--- tests/test_gui_genepyramid_controls.py | 17 +++++ 2 files changed, 105 insertions(+), 11 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 680dd914..853475e9 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -23,6 +23,7 @@ __all__ = [ "geneIndex", "abundanceBand", + "countsOrder", "labelAgreement", "selectLabelings", "addGenePanel", @@ -168,6 +169,26 @@ def abundanceBand(totals, lo: float = 0.0, hi: float = 100.0): _GENE_COLORMAPS = ("magenta", "green", "cyan", "yellow", "red", "blue") +def countsOrder(totals): + """Positions ordered by count, loudest first, ties left alone. + + STABLE, and that is the whole content of this function. The list + arrives alphabetical, so a stable sort leaves genes with equal counts + in alphabetical order instead of an arbitrary one -- and at the + bottom of a real section thousands of them tie, which is precisely + where an arbitrary order is most annoying to read. + + Args: + totals: counts per entry, array-like. + + Returns: + An int array of positions into *totals*. + """ + import numpy as np + + return np.argsort(-np.asarray(totals), kind="stable") + + # The useful band settings are not obvious from a bare slider and are # nearly the same on every section, so they are offered as presets. _BAND_PRESETS = ( @@ -195,11 +216,16 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: TWO FILTERS, applied together. The text box answers "where is this gene"; the abundance band answers "which genes carry the signal". They compose, because narrowing to the loud genes and then searching - within them is the normal way to use both. + within them is the normal way to use both. Sorting is separate from + both and survives them: alphabetical to find a gene you can name, by + counts to see which ones there are -- which is what the band leaves + you wanting, since a few hundred loud genes in alphabetical order + still hide their own ranking. """ import numpy as np from qtpy.QtCore import Qt, QTimer from qtpy.QtWidgets import ( + QComboBox, QDoubleSpinBox, QFormLayout, QHBoxLayout, @@ -249,13 +275,24 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: search.setPlaceholderText("filter by symbol or accession") outer.addWidget(search) + sort_row = QHBoxLayout() + sort_row.addWidget(QLabel("Sort")) + sort_box = QComboBox() + sort_box.addItem("name (A-Z)") + if symbol_totals is not None: + sort_box.addItem("counts (high to low)") + sort_row.addWidget(sort_box, stretch=1) + outer.addLayout(sort_row) + listw = QListWidget() items: dict[str, Any] = {} - # Thirty thousand items go in as one batch: each add would otherwise - # relayout the list. - listw.setUpdatesEnabled(False) - for i, (symbol, entry) in enumerate(index.items()): - rows, accs = entry["rows"], entry["accessions"] + symbols = list(index) + entries = list(index.values()) + state = {"busy": False, "colour": 0, "keep": None} + + def _makeItem(i: int): + symbol = symbols[i] + rows, accs = entries[i]["rows"], entries[i]["accessions"] text = symbol if symbol_totals is not None: text += f" \u2014 {int(symbol_totals[i]):,}" @@ -268,6 +305,10 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: # Accessions are searchable too: a symbol is what people usually # type, but not everything here has one. item.setData(Qt.UserRole + 1, " ".join([symbol] + accs).lower()) + # The symbol's ORIGINAL position, carried on the item so that the + # abundance mask -- computed once, in that order -- is still + # readable after the list has been re-sorted into another one. + item.setData(Qt.UserRole + 2, i) tip = [ symbol, f"{len(rows)} pyramid column(s): {', '.join(accs) or '(no accession)'}", @@ -278,11 +319,45 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: # here and pairing them would attach the wrong number. tip.append("counts per column: " + ", ".join(f"{int(totals[r]):,}" for r in rows)) item.setToolTip("\n".join(tip)) - listw.addItem(item) - items[symbol] = item - listw.setUpdatesEnabled(True) + return item - state = {"busy": False, "colour": 0, "keep": None} + def _populate(order): + """Fill the list in ORDER, a sequence of original positions. + + Rebuilt rather than reordered in place. Moving thirty thousand + items one at a time is quadratic, and sorting them through a + Python comparison is no cheaper than making them again -- while + this is the same single batched pass the panel already pays once. + + THE TICKS COME BACK FROM THE LAYERS, not from remembered state: + the drawn layers are what a tick means, so reading them is what + stops the boxes and the picture from disagreeing after a sort. + """ + was, state["busy"] = state["busy"], True + listw.setUpdatesEnabled(False) + try: + listw.clear() + items.clear() + for i in order: + item = _makeItem(int(i)) + symbol = symbols[int(i)] + if f"gene: {symbol}" in viewer.layers: + item.setCheckState(Qt.Checked) + listw.addItem(item) + items[symbol] = item + finally: + listw.setUpdatesEnabled(True) + state["busy"] = was + _refilter() + + def _onSort(*_): + if sort_box.currentIndex() == 1 and symbol_totals is not None: + order = countsOrder(symbol_totals) + else: + order = range(len(symbols)) # geneIndex already sorted by name + _populate(order) + + sort_box.currentIndexChanged.connect(_onSort) # ---- the abundance band ------------------------------------------ band_readout = QLabel() @@ -382,7 +457,8 @@ def _refilter(): keep = state["keep"] listw.setUpdatesEnabled(False) try: - for i, item in enumerate(items.values()): + for item in items.values(): + i = item.data(Qt.UserRole + 2) ok = True if keep is None else bool(keep[i]) if ok and needle: ok = needle in item.data(Qt.UserRole + 1) @@ -490,6 +566,7 @@ def _on_layer_removed(event): except Exception: # pragma: no cover - depends on the napari build pass + _populate(range(len(symbols))) _applyBand() _report() viewer.window.add_dock_widget(box, name="Genes", area="right") diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 509c573c..81a8310b 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -190,3 +190,20 @@ def test_an_empty_list_is_handled_rather_than_dividing_by_zero(): keep, info = controls.abundanceBand([], 0.0, 100.0) assert len(keep) == 0 assert info["available"] is False + + +# ---------------------------------------------------------- countsOrder + + +def test_the_loudest_gene_comes_first(): + assert list(controls.countsOrder([3, 100, 7])) == [1, 2, 0] + + +def test_equal_counts_keep_the_order_they_arrived_in(): + """The list arrives alphabetical, and thousands of genes tie at the + bottom of a real section; an unstable sort would shuffle them.""" + assert list(controls.countsOrder([5, 9, 5, 5, 9])) == [1, 4, 0, 2, 3] + + +def test_an_empty_list_sorts_to_nothing(): + assert list(controls.countsOrder([])) == [] From 7c1f095bc409c5af04ca739f102e2a057076783b Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 20:48:14 +0000 Subject: [PATCH 09/52] Give the band one row, and the gene list four orders THE BAND IS ONE ROW NOW: a label, the slider, nothing else. The header, the preset buttons and the readout paragraph were four lines of chrome above the thing the panel is for. What the band is, and what it just did -- how many genes it kept, what share of the reads, which loud genes a top cut dropped, and the case where one gene is too loud for a small cut to reach -- is hover text on the slider. The explanation is read once; the list is read every time. The one number worth keeping in sight goes into the header line that was already there, since "1,204 of 30,434 genes" is the same fact that line already states, narrowed. SORTING IS A HEADER ROW: Gene and Counts as flat buttons that read like a table's header and behave like one. Click to sort by that column, click again to reverse. Four orders, which is what a ranked list needs; a menu would have listed all four and made the reader pick wording rather than direction. countsOrder gains a direction, and ascending is a separate sort rather than the descending one reversed -- reversing would reverse the ties with it, so a tied group would read alphabetically one way and backwards the other. At the bottom of a real section thousands of genes tie, so that is most of the list. Asserted in both directions, including that ascending is NOT the reverse of descending. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 149 +++++++++++++----------- tests/test_gui_genepyramid_controls.py | 15 +++ 2 files changed, 98 insertions(+), 66 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 853475e9..9c037fa0 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -169,8 +169,8 @@ def abundanceBand(totals, lo: float = 0.0, hi: float = 100.0): _GENE_COLORMAPS = ("magenta", "green", "cyan", "yellow", "red", "blue") -def countsOrder(totals): - """Positions ordered by count, loudest first, ties left alone. +def countsOrder(totals, ascending: bool = False): + """Positions ordered by count, ties left alone. STABLE, and that is the whole content of this function. The list arrives alphabetical, so a stable sort leaves genes with equal counts @@ -178,26 +178,22 @@ def countsOrder(totals): bottom of a real section thousands of them tie, which is precisely where an arbitrary order is most annoying to read. + Sorting ASCENDING is a separate sort rather than the descending one + reversed: reversing would reverse the ties too, so the alphabetical + order inside each tied group would come back backwards in one + direction and forwards in the other. + Args: totals: counts per entry, array-like. + ascending: quietest first rather than loudest first. Returns: An int array of positions into *totals*. """ import numpy as np - return np.argsort(-np.asarray(totals), kind="stable") - - -# The useful band settings are not obvious from a bare slider and are -# nearly the same on every section, so they are offered as presets. -_BAND_PRESETS = ( - ("all", 0.0, 100.0), - ("drop top 1%", 0.0, 99.0), - ("drop top 5%", 0.0, 95.0), - ("middle 5-95%", 5.0, 95.0), - ("top 10% only", 90.0, 100.0), -) + t = np.asarray(totals) + return np.argsort(t if ascending else -t, kind="stable") def addGenePanel(viewer, session, pyr_doc) -> Any: @@ -225,9 +221,7 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: import numpy as np from qtpy.QtCore import Qt, QTimer from qtpy.QtWidgets import ( - QComboBox, QDoubleSpinBox, - QFormLayout, QHBoxLayout, QLabel, QLineEdit, @@ -266,22 +260,40 @@ def addGenePanel(viewer, session, pyr_doc) -> Any: box = QWidget() outer = QVBoxLayout(box) - head = f"{len(index):,} genes" + base_head = f"{len(index):,} genes" if n_variants: - head += f" ({n_variants:,} with variants, summed)" - outer.addWidget(QLabel(head)) + base_head += f" ({n_variants:,} with variants, summed)" + header = QLabel(base_head) + + def _headText(info=None) -> str: + # The band's own count lives here rather than in a line of its + # own: it is the same fact the header already states, narrowed. + if not info or not info.get("available") or info["nKept"] == info["nTotal"]: + return base_head + return ( + f"{info['nKept']:,} of {info['nTotal']:,} genes · " + f"{info['pctReads']:.1f}% of reads" + ) + + outer.addWidget(header) search = QLineEdit() search.setPlaceholderText("filter by symbol or accession") outer.addWidget(search) + # Column headers, not a menu: two buttons that read like a table's + # header row and behave like one -- click to sort by that column, + # click again to reverse it. Four orders, which is what looking at a + # ranked list actually needs; a menu would have listed all four and + # made the reader pick the wording rather than the direction. sort_row = QHBoxLayout() - sort_row.addWidget(QLabel("Sort")) - sort_box = QComboBox() - sort_box.addItem("name (A-Z)") - if symbol_totals is not None: - sort_box.addItem("counts (high to low)") - sort_row.addWidget(sort_box, stretch=1) + name_btn = QPushButton("Gene") + count_btn = QPushButton("Counts") + for b in (name_btn, count_btn): + b.setFlat(True) + sort_row.addWidget(b) + count_btn.setEnabled(symbol_totals is not None) + sort_row.addStretch() outer.addLayout(sort_row) listw = QListWidget() @@ -350,28 +362,52 @@ def _populate(order): state["busy"] = was _refilter() - def _onSort(*_): - if sort_box.currentIndex() == 1 and symbol_totals is not None: - order = countsOrder(symbol_totals) + # (column, ascending). Names start A-Z; counts, when asked for, start + # loudest-first, because that is the question a counts column is + # being clicked to answer. + sort_state = {"by": "name", "asc": True} + + def _sortBy(column: str): + if sort_state["by"] == column: + sort_state["asc"] = not sort_state["asc"] + else: + sort_state["by"] = column + sort_state["asc"] = column == "name" + _resort() + + def _resort(): + by, asc = sort_state["by"], sort_state["asc"] + if by == "counts" and symbol_totals is not None: + order = countsOrder(symbol_totals, ascending=asc) else: - order = range(len(symbols)) # geneIndex already sorted by name + # geneIndex already sorted by name, and symbols are unique, so + # Z-A is that order backwards -- no tie to preserve. + order = range(len(symbols)) if asc else range(len(symbols) - 1, -1, -1) + arrow = " \u25b2" if asc else " \u25bc" + name_btn.setText("Gene" + (arrow if by == "name" else "")) + count_btn.setText("Counts" + (arrow if by == "counts" else "")) _populate(order) - sort_box.currentIndexChanged.connect(_onSort) + name_btn.clicked.connect(lambda *_: _sortBy("name")) + count_btn.clicked.connect(lambda *_: _sortBy("counts")) # ---- the abundance band ------------------------------------------ - band_readout = QLabel() - band_readout.setWordWrap(True) + # ONE ROW. What the band is and what it just did are hover text on the + # slider rather than a paragraph under it: the explanation is read + # once and the panel is read every time, and the list underneath is + # what the space is for. if symbol_totals is None: - band_readout.setText( - "This pyramid has no gene_totals.tsv, so abundance is unknown " - "and the band cannot be applied. Counts are omitted above for " - "the same reason." + note = QLabel("no gene_totals.tsv: no counts, no band") + note.setToolTip( + "This pyramid was written without gene_totals.tsv, so per-gene " + "abundance is unknown. The counts column and the abundance " + "band are both omitted rather than invented." ) - outer.addWidget(band_readout) + outer.addWidget(note) slider = lo_box = hi_box = None else: - outer.addWidget(QLabel("Abundance band (percentile of total reads)")) + band_row = QHBoxLayout() + band_row.addWidget(QLabel("Abundance band")) slider = lo_box = hi_box = None try: # napari depends on superqt, so the two-handle widget is @@ -384,25 +420,15 @@ def _onSort(*_): slider.setRange(0.0, 100.0) slider.setValue((0.0, 100.0)) slider.setDecimals(1) - outer.addWidget(slider) + band_row.addWidget(slider, stretch=1) except Exception as e: # pragma: no cover - depends on the install print(f"[genepyramid] no superqt range slider ({e}); using spin boxes") - form = QFormLayout() lo_box = QDoubleSpinBox(minimum=0.0, maximum=100.0, decimals=1, singleStep=0.5) hi_box = QDoubleSpinBox(minimum=0.0, maximum=100.0, decimals=1, singleStep=0.5) hi_box.setValue(100.0) - form.addRow("low %", lo_box) - form.addRow("high %", hi_box) - outer.addLayout(form) - - presets = QHBoxLayout() - for label, a, b in _BAND_PRESETS: - btn = QPushButton(label) - btn.setFlat(True) - btn.clicked.connect(lambda _=False, a=a, b=b: _setBand(a, b)) - presets.addWidget(btn) - outer.addLayout(presets) - outer.addWidget(band_readout) + band_row.addWidget(lo_box) + band_row.addWidget(hi_box) + outer.addLayout(band_row) outer.addWidget(listw, stretch=1) @@ -429,18 +455,6 @@ def _band(): return (hi, lo) if lo > hi else (lo, hi) return 0.0, 100.0 - def _setBand(lo, hi): - if slider is not None: - slider.blockSignals(True) - slider.setValue((lo, hi)) - slider.blockSignals(False) - elif lo_box is not None: - for w, v in ((lo_box, lo), (hi_box, hi)): - w.blockSignals(True) - w.setValue(v) - w.blockSignals(False) - _applyBand() - def _applyBand(): if symbol_totals is None: state["keep"] = None @@ -449,7 +463,10 @@ def _applyBand(): lo, hi = _band() keep, info = abundanceBand(symbol_totals, lo, hi) state["keep"] = keep - band_readout.setText(_bandText(info, list(index))) + for w in (slider, lo_box, hi_box): + if w is not None: + w.setToolTip(_bandText(info, symbols)) + header.setText(_headText(info)) _refilter() def _refilter(): @@ -566,7 +583,7 @@ def _on_layer_removed(event): except Exception: # pragma: no cover - depends on the napari build pass - _populate(range(len(symbols))) + _resort() _applyBand() _report() viewer.window.add_dock_widget(box, name="Genes", area="right") diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 81a8310b..03ca3de3 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -199,11 +199,26 @@ def test_the_loudest_gene_comes_first(): assert list(controls.countsOrder([3, 100, 7])) == [1, 2, 0] +def test_ascending_puts_the_quietest_first(): + assert list(controls.countsOrder([3, 100, 7], ascending=True)) == [0, 2, 1] + + def test_equal_counts_keep_the_order_they_arrived_in(): """The list arrives alphabetical, and thousands of genes tie at the bottom of a real section; an unstable sort would shuffle them.""" assert list(controls.countsOrder([5, 9, 5, 5, 9])) == [1, 4, 0, 2, 3] +def test_ties_stay_alphabetical_in_BOTH_directions(): + """Reversing the descending order would reverse the ties with it, so + the same tied group would read forwards one way and backwards the + other. Ascending is its own sort for that reason.""" + totals = [5, 9, 5, 5, 9] + assert list(controls.countsOrder(totals, ascending=True)) == [0, 2, 3, 1, 4] + assert list(controls.countsOrder(totals, ascending=True)) != list( + reversed(controls.countsOrder(totals)) + ) + + def test_an_empty_list_sorts_to_nothing(): assert list(controls.countsOrder([])) == [] From b5d504eebfd51376532f494c211b580626ea4e85 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 8 Sep 2026 20:53:22 +0000 Subject: [PATCH 10/52] A switch per labeling, and the classes behind a toggle Dropping a redundant labeling automatically was wrong in both directions. Below the agreement threshold it did nothing and the reader still saw two of them -- which is what happened here. Above it, the labeling vanished with no way to bring it back. So nothing is dropped now: every labeling the cells document has is shown, the pair that says the same thing is NAMED with the number, and the reader unticks one. That is the same information with the decision left where it belongs. Each labeling is ONE ROW: its own checkbox, the class count, and a toggle that opens its classes. A cellbin carries a couple of labelings with dozens of classes between them, and listing every class of every one filled the dock before anyone had asked to see them. The unlabelled count and the agreement number move to the row's hover text. Switching a labeling off DROPS ITS CONTRIBUTION TO THE FILTER rather than hiding the cells it named. That is the whole point of the switch: an unwanted second opinion should stop having an opinion, not start hiding everything it had named. The joint filter still holds across the labelings that are still on, so a cell is dimmed when any switched-on labeling has its class unticked. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/cli.py | 8 +- src/ndi/gui/app/genepyramid/controls.py | 115 ++++++++++++++++++------ tests/test_gui_genepyramid_controls.py | 10 ++- 3 files changed, 97 insertions(+), 36 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index 4bdcc6f9..62ad3c4b 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -79,10 +79,10 @@ def build_parser() -> argparse.ArgumentParser: default="", metavar="A,B", help="which cell type labelings to show, by name; 'none' shows no " - "labeling panel. Default lets the panel decide, which means two " - "labelings that say the same thing about the section -- a subclass " - "call and the clustering it was transferred onto -- are shown once " - "and the panel says which it kept.", + "labeling panel. Default shows every one the cells document has, " + "each with its own switch, and names any pair that says the same " + "thing about the section -- a subclass call and the clustering it " + "was transferred onto -- so you can switch one off.", ) p.add_argument( "--no-controls", diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 9c037fa0..0c27949c 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -761,14 +761,20 @@ def selectLabelings(found, wanted=None): typo that silently showed everything would look like the flag not working. - Without it, supervised calls are considered first and any later - labeling that agrees with a kept one above :data:`REDUNDANT_AT` is - set aside, so the labeling carrying biological names is the one that - stays. + Without it, everything found is kept and the redundancy is REPORTED + rather than acted on. Dropping one automatically was wrong twice + over: below the threshold it did nothing and the reader still saw + two, and above it the labeling vanished with no way to bring it + back. Naming the pair and letting the reader switch one off is the + same information with the decision left where it belongs. + + Supervised calls are ordered first, so a redundant pair is measured + against the labeling that carries biological names rather than + against a cluster index. Returns: ``(kept, redundant, missing)``. *redundant* holds - ``(name, kept_name, forward, reverse)``, the two agreements + ``(name, other_name, forward, reverse)``, the two agreements measured in both directions so the note can quote a number. """ found = sorted(found, key=lambda t: (bool(t[1]["isUnsupervised"]), t[1]["labelName"] or "")) @@ -782,17 +788,13 @@ def selectLabelings(found, wanted=None): kept, redundant = [], [] for labels, info in found: name = info["labelName"] or "(unnamed)" - dup = None for kept_labels, kept_info in kept: forward = labelAgreement(labels, kept_labels) reverse = labelAgreement(kept_labels, labels) if max(forward, reverse) >= REDUNDANT_AT: - dup = (name, kept_info["labelName"] or "(unnamed)", forward, reverse) + redundant.append((name, kept_info["labelName"] or "(unnamed)", forward, reverse)) break - if dup is None: - kept.append((labels, info)) - else: - redundant.append(dup) + kept.append((labels, info)) return kept, redundant, [] @@ -806,10 +808,17 @@ def addCellTypePanel( rather than being merged. They are not interchangeable: a clustering carries no biological identity. - A labeling that says the same thing as one already shown is named - and then set aside rather than drawn twice; see - :func:`selectLabelings`. Pass *labelings* -- a list of label names -- - to choose outright instead, in which case nothing is collapsed. + ONE ROW EACH, with the classes behind a toggle. A cellbin carries a + couple of labelings with dozens of classes between them, and listing + every class of every one fills the dock before anyone has asked to + see them. + + The row's own checkbox switches the whole labeling off, which drops + its contribution to the filter rather than hiding the cells it + named: an unwanted second opinion should stop having an opinion. + Everything starts on, and a labeling that says the same thing as + another is NAMED rather than removed, so the reader is told which + one to untick instead of finding one of them gone. Hiding works by ALPHA, not by removing points. The row order of the points layer is the row order of cells.tsv and of every labels.tsv @@ -818,7 +827,9 @@ def addCellTypePanel( """ from qtpy.QtWidgets import ( QCheckBox, + QHBoxLayout, QLabel, + QPushButton, QScrollArea, QVBoxLayout, QWidget, @@ -854,11 +865,13 @@ def _note(msg): w.setWordWrap(True) outer.addWidget(w) + # Named, not acted on: it says WHICH of the two to switch off. + agrees = {} for name, other, forward, reverse in redundant: + agrees[name] = (other, max(forward, reverse)) _note( - f"{name} and {other} agree about {100 * max(forward, reverse):.1f}% " - f"of cells -- one grouping under two sets of names -- so only " - f"{other} is shown. Use --labels to choose for yourself." + f"{name} agrees with {other} on {100 * max(forward, reverse):.1f}% " + f"of cells -- untick one of them." ) if missing: have = ", ".join((i["labelName"] or "(unnamed)") for _lb, i in found) @@ -867,15 +880,18 @@ def _note(msg): _note("no labelings to show.") n_points = len(points_layer.data) - # visible[i] is False when ANY ticked-off class covers cell i, so two - # labelings filter jointly rather than the last one clicked winning. + # visible[i] is False when ANY ticked-off class of a labeling that is + # still switched ON covers cell i, so two labelings filter jointly + # rather than the last one clicked winning. hidden_by = {} + switched_off = set() def _refresh(): hide = np.zeros(n_points, dtype=bool) - for mask in hidden_by.values(): - if mask is not None: - hide |= mask + for (label_name, _category), mask in hidden_by.items(): + if mask is None or label_name in switched_off: + continue + hide |= mask for layer in (points_layer, shapes_layer): if layer is None: continue @@ -893,13 +909,28 @@ def _refresh(): pass for labels, info in kept: + name = info["labelName"] or "(unnamed)" kind = "clustering" if info["isUnsupervised"] else "cell type call" - title = QLabel(f"{info['labelName'] or '(unnamed)'} -- {kind}") - outer.addWidget(title) + + # ONE ROW PER LABELING, and the classes are behind it. A cellbin + # carries a couple of labelings with dozens of classes between + # them, and listing every class of every one of them fills the + # dock with checkboxes nobody asked to see yet. + head = QHBoxLayout() + master = QCheckBox(f"{name} ({len(info['categories'])} {kind})") + master.setChecked(True) + tip = [f"{name} -- {kind}, {len(info['categories'])} classes"] if info["nUnlabeled"]: - note = QLabel(f" {info['nUnlabeled']:,} cells unlabelled by this") - note.setWordWrap(True) - outer.addWidget(note) + tip.append(f"{info['nUnlabeled']:,} cells are unlabelled by this") + if name in agrees: + other, share = agrees[name] + tip.append(f"agrees with {other} on {100 * share:.1f}% of cells") + master.setToolTip("\n".join(tip)) + expand = QPushButton("classes \u25b8") + expand.setFlat(True) + head.addWidget(master, stretch=1) + head.addWidget(expand) + outer.addLayout(head) arr = np.asarray(labels, dtype=object) inner = QWidget() @@ -908,7 +939,7 @@ def _refresh(): mask = arr == category cb = QCheckBox(f"{category} ({int(mask.sum()):,})") cb.setChecked(True) - key = (info["labelName"], category) + key = (name, category) def _toggled(on, key=key, mask=mask): hidden_by[key] = None if on else mask[:n_points] @@ -919,8 +950,34 @@ def _toggled(on, key=key, mask=mask): scroll = QScrollArea() scroll.setWidgetResizable(True) scroll.setWidget(inner) + scroll.setMaximumHeight(180) + scroll.setVisible(False) outer.addWidget(scroll) + def _onExpand(*_, scroll=scroll, expand=expand): + shown = not scroll.isVisible() + scroll.setVisible(shown) + expand.setText("classes \u25be" if shown else "classes \u25b8") + + def _onMaster(on, name=name, scroll=scroll, expand=expand): + # Switching a labeling OFF drops its whole contribution to the + # filter rather than hiding its cells: an unwanted second + # opinion should stop having an opinion, not start hiding + # everything it named. + if on: + switched_off.discard(name) + else: + switched_off.add(name) + scroll.setVisible(False) + expand.setText("classes \u25b8") + expand.setEnabled(on) + _refresh() + + expand.clicked.connect(_onExpand) + master.toggled.connect(_onMaster) + + outer.addStretch() + viewer.window.add_dock_widget(box, name="Cell types", area="right") return box diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 03ca3de3..8abc0bd9 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -85,7 +85,10 @@ def test_labelings_of_different_lengths_are_refused(): # ------------------------------------------------------ selectLabelings -def test_the_named_call_is_what_survives_a_redundant_pair(): +def test_a_redundant_pair_is_reported_but_both_are_kept(): + """Dropping one was wrong twice: below the threshold it did nothing + and the reader still saw two, and above it the labeling vanished + with no way back. The panel names the pair and offers a switch.""" n = 200 leiden = ["0"] * (n // 2) + ["1"] * (n // 2) subclass = ["Pvalb"] * (n // 2) + ["Sst"] * (n // 2) @@ -94,13 +97,14 @@ def test_the_named_call_is_what_survives_a_redundant_pair(): (subclass, _info("subclass_nn_column")), ] kept, redundant, missing = controls.selectLabelings(found) - assert [i["labelName"] for _lb, i in kept] == ["subclass_nn_column"] + # Supervised first, so the pair is measured against the named call. + assert [i["labelName"] for _lb, i in kept] == ["subclass_nn_column", "leiden"] assert redundant[0][0] == "leiden" assert redundant[0][1] == "subclass_nn_column" assert missing == [] -def test_two_labelings_that_really_differ_are_both_kept(): +def test_two_labelings_that_really_differ_raise_no_note(): a = ["x"] * 50 + ["y"] * 50 b = ["p", "q"] * 50 found = [(a, _info("a")), (b, _info("b"))] From ed98c018006a953e07a32390d3335f081f50b0b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 01:22:43 +0000 Subject: [PATCH 11/52] A panel that throws must not take the window with it By the time addAllPanels runs the viewer is on screen and the image is drawn. A panel that raised unwound out of openPyramid before napari.run(), so the window appeared and the process exited -- which is seen as napari opening briefly and closing, with the reason lost unless somebody was watching stderr. The docstring already stated the principle for a missing Qt and then applied it only to that import. Each panel now builds independently and a failure costs that panel, named, with its traceback. This is not hypothetical: every panel reads files out of the database to build itself -- genes.tsv and gene_totals.tsv for the genes, one labels.tsv per labeling for the cell types -- and those reads fail for reasons that have nothing to do with the image, a cloud-backed session among them. None of them is a reason to refuse to draw the section. AND --labels NEVER WORKED. addAllPanels called addCellTypePanel without forwarding labelings, in every commit since the flag was added. The flag was parsed, resolved, threaded through openPyramid and then dropped at the last hop, so choosing labelings in the CLI or in the GEF Manager's dialog silently did nothing and the panel always took its default. It went unnoticed because every other layer mentions the argument, so grepping for it finds it everywhere it is not being used. Both that forwarding and the panel isolation are now pinned by tests. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 47 +++++++++++++++---- tests/test_gui_genepyramid_controls.py | 60 +++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 8 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 0c27949c..7d2edc74 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1002,15 +1002,27 @@ def addAllPanels( display mode, the cell types, then the genes. napari stacks docks in the order they arrive, so this order is the layout. - A missing Qt costs the panels rather than the picture: the viewer is - already on screen by the time this runs, and taking it down over a - convenience would be worse than doing without. + A PANEL MUST NOT COST THE PICTURE. By the time this runs the viewer is + on screen and the image layer is drawn; a panel that throws would + unwind out of openPyramid before napari.run(), so the window appears + and the process exits -- a viewer that "opens briefly and closes", + with the reason lost unless someone was watching stderr. + + That is not hypothetical. Each panel reads files out of the database + to build itself: genes.tsv and gene_totals.tsv for the genes, one + labels.tsv per labeling for the cell types. Those reads can fail for + reasons that have nothing to do with the image -- a document whose + binary is not local, a cloud-backed session, a file an older ingest + never wrote -- and none of them is a reason to refuse to draw the + section. So each panel is built independently and a failure costs + that panel, named, with its traceback. """ + import sys + import traceback + try: import qtpy.QtWidgets # noqa: F401 except ImportError as e: # pragma: no cover - depends on the install - import sys - print( f"[genepyramid] control panels unavailable ({e}). The image is " f"unaffected; --genes and --no-density still work at launch.", @@ -1018,9 +1030,28 @@ def addAllPanels( ) return {} + def _build(name, fn): + try: + return fn() + except Exception as e: # noqa: BLE001 - a panel is never worth the window + print( + f"[genepyramid] the {name} panel could not be built " + f"({type(e).__name__}: {e}). The image is unaffected.", + file=sys.stderr, + ) + traceback.print_exc() + return None + made = {} - made["display"] = addDisplayPanel(viewer, session, pyr_doc, image_layer, density) + made["display"] = _build( + "display", lambda: addDisplayPanel(viewer, session, pyr_doc, image_layer, density) + ) if cells_doc is not None and points_layer is not None: - made["cellTypes"] = addCellTypePanel(viewer, session, cells_doc, points_layer, shapes_layer) - made["genes"] = addGenePanel(viewer, session, pyr_doc) + made["cellTypes"] = _build( + "cell types", + lambda: addCellTypePanel( + viewer, session, cells_doc, points_layer, shapes_layer, labelings + ), + ) + made["genes"] = _build("genes", lambda: addGenePanel(viewer, session, pyr_doc)) return made diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 8abc0bd9..2d8f1ae4 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -226,3 +226,63 @@ def test_ties_stay_alphabetical_in_BOTH_directions(): def test_an_empty_list_sorts_to_nothing(): assert list(controls.countsOrder([])) == [] + + +# ------------------------------------------------------- addAllPanels + + +class _Recorder: + """Stands in for a panel builder and remembers how it was called.""" + + def __init__(self, boom=False): + self.calls = [] + self.boom = boom + + def __call__(self, *args, **kwargs): + self.calls.append((args, kwargs)) + if self.boom: + raise RuntimeError("no such file") + return object() + + +def test_the_chosen_labelings_reach_the_cell_type_panel(monkeypatch): + """--labels was accepted, parsed, threaded through openPyramid, and + then dropped at the last hop, so it silently did nothing. Nothing + caught it because every other layer mentioned the argument.""" + cells = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellTypePanel", cells) + monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + + controls.addAllPanels( + object(), + object(), + object(), + object(), + True, + cells_doc=object(), + points_layer=object(), + labelings=["subclass_nn_column"], + ) + assert cells.calls, "the cell type panel was never built" + args, _kwargs = cells.calls[0] + assert args[-1] == ["subclass_nn_column"] + + +def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsys): + """The viewer is already on screen when the panels are built. A panel + that raised unwound out of openPyramid before napari.run(), so the + window appeared and the process exited -- which reads as napari + opening briefly and closing, with the reason lost.""" + genes = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder(boom=True)) + monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) + monkeypatch.setattr(controls, "addGenePanel", genes) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + + assert made["display"] is None + # The panels after the failing one must still be built. + assert genes.calls, "a failing panel stopped the ones after it" + err = capsys.readouterr().err + assert "display" in err and "no such file" in err From a0134f7e6e12c38152e65358f9c900144b501cff Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 01:30:20 +0000 Subject: [PATCH 12/52] Declare the dependency the default secrets backend needs ndi.cloud.profile documents the AES-128/CBC file backend as its DEFAULT and nothing installed it. cryptography was in neither the dependencies nor any extra, so on a clean install _detect_backend fell through to the in-memory backend -- which persists nothing and reads nothing -- and said so only through a logger warning. A saved cloud password could therefore never be loaded, which is not a degraded mode so much as the feature being absent. Not an extra. ndi.cloud is part of the package rather than an opt-in surface like gui or napari, and a default backend that nothing installs is not a default. keyring stays optional: it is preferred when present, but it is the OS-keychain path, and cryptography is the portable one that MATLAB's AES file interoperates with. Worth being clear about what this does NOT fix: the profiles list itself is plain JSON and _load_from_disk reads it regardless of backend, so an empty list_profiles() is a question about where ~/.ndi is, not about this. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- pyproject.toml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index fb381fb3..22e7b807 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,16 @@ dependencies = [ "pydantic>=2.0", "openMINDS>=0.2.0", "scipy>=1.9.0", + # ndi.cloud.profile documents the AES-128/CBC file backend as its + # DEFAULT, and that backend needs cryptography. Without it the + # detector falls through to the in-memory backend, which persists + # nothing and reads nothing -- so a clean install could never load a + # saved cloud password, and said so only through a logger warning. + # Not an extra: ndi.cloud is part of the package rather than an + # opt-in surface, and a default that nothing installs is not a + # default. keyring is preferred when present but stays optional, + # since it is the OS-keychain path rather than the portable one. + "cryptography>=41.0", ] [project.optional-dependencies] From 08800ea7668490e003eeb904ea44d2112392c838 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 01:34:37 +0000 Subject: [PATCH 13/52] Read the profiles file MATLAB actually writes when there is one profile jsonencode of a 1x1 struct produces an object; of a 1xN struct array, an array. So NDI-matlab's Profiles field has a different SHAPE depending on how many profiles exist, and the two languages share this file by design. MATLAB's reader has normalizeProfiles for exactly that. Python had no counterpart: it iterated the object as a list, got the dict's KEYS, and skipped every one as "not a dict". The result was zero profiles and no error -- the file was found, parsed, and silently read as empty -- for the single-profile case, which is what every new user has after making their first. That is the worst possible case to get wrong, and it is the one that was. _as_profile_list normalises both shapes and refuses the rest. Tests cover one, several, absent, null and nonsense, and were checked to fail against the old reader before the fix rather than merely passing after. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/profile.py | 24 +++++- tests/test_cloud_profile_matlab_shape.py | 93 ++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 tests/test_cloud_profile_matlab_shape.py diff --git a/src/ndi/cloud/profile.py b/src/ndi/cloud/profile.py index 4faebc20..854fa47a 100644 --- a/src/ndi/cloud/profile.py +++ b/src/ndi/cloud/profile.py @@ -184,6 +184,28 @@ def _write_secrets_file(filename: Path, payload: dict) -> None: pass +def _as_profile_list(raw) -> list: + """MATLAB writes ONE profile as an object, not an array of one. + + ``jsonencode`` of a 1x1 struct produces ``{...}`` and of a 1xN struct + array produces ``[{...}, ...]``, so NDI-matlab's profiles file has a + different shape for its `Profiles` field depending on how many + profiles the user has. MATLAB's own reader has normalizeProfiles for + exactly this; this is its counterpart. + + Without it, a single-profile file -- which is what every new user + has, having just made their first -- was iterated as a list, yielding + the dict's KEYS, every one of which failed the isinstance(dict) check + and was skipped. The result was zero profiles and no error: the file + was found, parsed, and silently read as empty. + """ + if isinstance(raw, dict): + return [raw] + if isinstance(raw, list): + return raw + return [] + + def _safe_field(name: str) -> str: """Map a secret key to a JSON-safe field name.""" return name.replace(" ", "_").replace(":", "_") @@ -271,7 +293,7 @@ def _load_from_disk(self) -> None: except (ValueError, OSError) as exc: logger.warning("Could not load cloud profiles from %s: %s", self.filename, exc) return - raw = data.get("Profiles") or [] + raw = _as_profile_list(data.get("Profiles")) self.profiles = [] for item in raw: if not isinstance(item, dict): diff --git a/tests/test_cloud_profile_matlab_shape.py b/tests/test_cloud_profile_matlab_shape.py new file mode 100644 index 00000000..6aae151d --- /dev/null +++ b/tests/test_cloud_profile_matlab_shape.py @@ -0,0 +1,93 @@ +"""Reading a profiles file that NDI-matlab wrote. + +The two languages deliberately share ~/.ndi/NDI_Cloud_Profiles.json, so +what MATLAB's jsonencode produces is what this reader has to accept. It +produces a different SHAPE depending on how many profiles there are: a +1x1 struct becomes a JSON object, a 1xN struct array becomes a JSON +array. MATLAB's own reader has normalizeProfiles for that; these tests +pin the Python counterpart. + +The single-profile case is the one that matters most, because it is +what every new user has after making their first. +""" + +from __future__ import annotations + +import json + +from ndi.cloud import profile +from ndi.cloud.profile import _as_profile_list, _ProfileSingleton + +ONE = { + "UID": "abc123", + "Nickname": "steve", + "Email": "steve@walthamdatascience.com", + "Stage": "prod", + "PasswordSecret": "NDI Cloud abc123", +} + + +def _write(tmp_path, profiles_value, default_uid="abc123"): + f = tmp_path / "NDI_Cloud_Profiles.json" + f.write_text(json.dumps({"Profiles": profiles_value, "DefaultUID": default_uid})) + return f + + +def _load(tmp_path, monkeypatch): + monkeypatch.setenv("NDI_PREFDIR", str(tmp_path)) + obj = _ProfileSingleton(backend="memory") + obj._load_from_disk() + obj._adopt_default_as_current() + return obj + + +def test_a_single_profile_written_as_an_object_is_read(tmp_path, monkeypatch): + """MATLAB writes one profile as {...}, not [{...}]. Iterating that as + a list yields its keys, and every key was skipped as 'not a dict' -- + so the file was found, parsed, and silently read as empty.""" + _write(tmp_path, ONE) + obj = _load(tmp_path, monkeypatch) + assert [p.Nickname for p in obj.profiles] == ["steve"] + assert obj.default_uid == "abc123" + assert obj.current_uid == "abc123", "a valid default must become current" + + +def test_several_profiles_written_as_an_array_are_read(tmp_path, monkeypatch): + two = [ONE, {**ONE, "UID": "def456", "Nickname": "other"}] + _write(tmp_path, two) + obj = _load(tmp_path, monkeypatch) + assert [p.Nickname for p in obj.profiles] == ["steve", "other"] + + +def test_a_missing_or_odd_profiles_field_reads_as_none(tmp_path, monkeypatch): + """Absent, null and a scalar all mean 'no profiles here' -- none of + them should raise, and none should invent one.""" + for value in (None, [], "not profiles", 7): + _write(tmp_path, value) + obj = _load(tmp_path, monkeypatch) + assert obj.profiles == [] + + +def test_the_password_secret_is_filled_in_when_matlab_omitted_it(tmp_path, monkeypatch): + """MATLAB derives it from the UID when absent; so must this, or the + secret would be looked up under an empty name.""" + raw = dict(ONE) + del raw["PasswordSecret"] + _write(tmp_path, raw) + obj = _load(tmp_path, monkeypatch) + assert [p.PasswordSecret for p in obj.profiles] == ["NDI Cloud abc123"] + + +def test_the_normaliser_takes_both_shapes_and_refuses_the_rest(): + assert _as_profile_list({"UID": "x"}) == [{"UID": "x"}] + assert _as_profile_list([{"UID": "x"}]) == [{"UID": "x"}] + assert _as_profile_list(None) == [] + assert _as_profile_list("nope") == [] + + +def test_the_public_api_sees_it_too(tmp_path, monkeypatch): + monkeypatch.setenv("NDI_PREFDIR", str(tmp_path)) + monkeypatch.setattr(profile, "_singleton", None) + _write(tmp_path, ONE) + assert [p.Nickname for p in profile.list_profiles()] == ["steve"] + assert profile.get_default() is not None From b18599a014060019ce6544ac59578aaf7bb8dc13 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 01:49:30 +0000 Subject: [PATCH 14/52] Let authenticate() fall back to the saved cloud profile ndi.cloud.profile is the documented place to keep an NDI Cloud password, but nothing in the authentication path ever read it. CloudConfig.from_env and authenticate() were environment-only, so every implicit login went token -> NDI_CLOUD_USERNAME/PASSWORD -> raise, with no step in between. The visible symptom: opening a downloaded cloud dataset in the pyramid viewer dies inside the ndic:// file handler with FileAccessError: ... No valid token and no credentials available. Set NDI_CLOUD_TOKEN or NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD. while a working default profile sat on disk. Every @_auto_client API call had the same hole. MATLAB's authenticate.m has had the equivalent step all along -- authenticatedWithSecret, reading the MATLAB Vault -- so this is a parity gap rather than a new feature. It goes after the environment step, not before: an explicitly exported NDI_CLOUD_USERNAME is a deliberate override and should keep winning over what is on disk. The profile's Stage picks the API URL, but only when neither NDI_CLOUD_URL nor CLOUD_API_ENVIRONMENT is set -- either of those means the caller has already chosen. _profile_credentials() swallows its own failures (no profile file, no default, a locked keyring) so a missing credential surfaces as the caller's own error rather than a traceback from an unrelated call site. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/auth.py | 63 +++++++- tests/test_cloud_auth_profile_fallback.py | 170 ++++++++++++++++++++++ 2 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 tests/test_cloud_auth_profile_fallback.py diff --git a/src/ndi/cloud/auth.py b/src/ndi/cloud/auth.py index 676343d9..c5df4daa 100644 --- a/src/ndi/cloud/auth.py +++ b/src/ndi/cloud/auth.py @@ -17,7 +17,7 @@ import os from datetime import datetime, timezone -from .config import CloudConfig +from .config import _API_URLS, CloudConfig from .exceptions import CloudAuthError logger = logging.getLogger(__name__) @@ -293,12 +293,40 @@ def logout(config: CloudConfig | None = None) -> None: config.org_id = "" +def _profile_credentials() -> tuple[str, str, str]: + """Return ``(email, password, stage)`` from the saved cloud profile. + + The session's current profile wins over the persisted default, matching + :func:`ndi.cloud.profile.get_current` / :func:`~ndi.cloud.profile.get_default`. + + Returns three empty strings when there is no usable profile. Every + failure mode here is benign -- no profile file, no default set, a + secrets backend that cannot decrypt -- and none of them should turn a + missing credential into a traceback from an unrelated call site, so they + all fall through to the caller's own "no credentials" error. + """ + try: + from . import profile as _profile + + entry = _profile.get_current() or _profile.get_default() + if entry is None: + return "", "", "" + password = _profile.get_password(entry.UID) + if not entry.Email or not password: + return "", "", "" + return entry.Email, password, entry.Stage or "" + except Exception as exc: # noqa: BLE001 - see docstring + logger.debug("No usable cloud profile: %s", exc) + return "", "", "" + + def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: """Return an active token and organization ID, attempting login if needed. Priority (matching MATLAB ``authenticate.m``): 1. Existing valid token in config/env (local JWT exp pre-check). 2. Username + password from env → login. + 3. The saved cloud profile (current, else default) → login. Args: config: Optional config. @@ -326,9 +354,38 @@ def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: updated = login(email, password, config) return updated.token, updated.org_id + # 3. Fall back to the saved cloud profile. + # + # MATLAB's authenticate.m has this step -- authenticatedWithSecret, + # which reads the MATLAB Vault -- ahead of the environment one. Python + # had no equivalent at all, so a user who had set up a profile (the + # documented way to keep a cloud password) still got "no credentials + # available" from anything that authenticated implicitly: every + # @_auto_client API call, and, most visibly, the ndic:// file handler + # that fetches pyramid tiles for a downloaded dataset. + # + # It goes last rather than first because an explicitly exported + # NDI_CLOUD_USERNAME is a deliberate override and should keep winning + # over whatever is on disk. + email, password, stage = _profile_credentials() + if email and password: + # Honour the profile's stage, but never over an explicit request: + # either environment variable means the caller has already chosen. + if ( + stage + and not os.environ.get("NDI_CLOUD_URL") + and not os.environ.get("CLOUD_API_ENVIRONMENT") + ): + url = _API_URLS.get(stage) + if url: + config.api_url = url + updated = login(email, password, config) + return updated.token, updated.org_id + raise CloudAuthError( - "No valid token and no credentials available. " - "Set NDI_CLOUD_TOKEN or NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD." + "No valid token and no credentials available. Set NDI_CLOUD_TOKEN or " + "NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD, or save a default cloud " + "profile (ndi.cloud.profile.add(...) then set_default(...))." ) diff --git a/tests/test_cloud_auth_profile_fallback.py b/tests/test_cloud_auth_profile_fallback.py new file mode 100644 index 00000000..cfe91271 --- /dev/null +++ b/tests/test_cloud_auth_profile_fallback.py @@ -0,0 +1,170 @@ +"""authenticate() must reach the saved cloud profile, not just the environment. + +The gap this pins closed: ndi.cloud.profile is the documented place to keep a +cloud password, but nothing in the authentication path ever read it. Every +implicit login -- each @_auto_client API call, and the ndic:// file handler +that fetches pyramid tiles for a downloaded dataset -- went environment-only +and raised "no credentials available" while a perfectly good default profile +sat on disk. MATLAB's authenticate.m has had the equivalent step +(authenticatedWithSecret) all along. +""" + +import os +import unittest +from unittest import mock + +from ndi.cloud import auth +from ndi.cloud.config import CloudConfig +from ndi.cloud.exceptions import CloudAuthError +from ndi.cloud.profile import ProfileEntry + +_ENV_KEYS = ( + "NDI_CLOUD_TOKEN", + "NDI_CLOUD_ORGANIZATION_ID", + "NDI_CLOUD_USERNAME", + "NDI_CLOUD_PASSWORD", + "NDI_CLOUD_URL", + "CLOUD_API_ENVIRONMENT", +) + + +class _NoEnvMixin(unittest.TestCase): + def setUp(self): + patcher = mock.patch.dict(os.environ, {}, clear=False) + patcher.start() + self.addCleanup(patcher.stop) + for key in _ENV_KEYS: + os.environ.pop(key, None) + + +def _login_spy(token="tok", org="org-1"): + """Stand in for auth.login, recording what it was handed.""" + calls = [] + + def fake(email=None, password=None, config=None): + calls.append((email, password, config.api_url if config else None)) + config = config or CloudConfig() + config.token = token + config.org_id = org + return config + + return fake, calls + + +class TestProfileCredentials(_NoEnvMixin): + def test_a_current_profile_is_preferred_over_the_default(self): + current = ProfileEntry(UID="u1", Email="current@example.com", Stage="prod") + default = ProfileEntry(UID="u2", Email="default@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=current), + mock.patch("ndi.cloud.profile.get_default", return_value=default), + mock.patch("ndi.cloud.profile.get_password", return_value="pw"), + ): + self.assertEqual(auth._profile_credentials(), ("current@example.com", "pw", "prod")) + + def test_the_default_profile_is_used_when_no_current_one_is_set(self): + default = ProfileEntry(UID="u2", Email="default@example.com", Stage="dev") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=default), + mock.patch("ndi.cloud.profile.get_password", return_value="pw"), + ): + self.assertEqual(auth._profile_credentials(), ("default@example.com", "pw", "dev")) + + def test_no_profile_at_all_reports_emptiness_rather_than_raising(self): + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + ): + self.assertEqual(auth._profile_credentials(), ("", "", "")) + + def test_a_secrets_backend_that_throws_is_not_allowed_to_escape(self): + entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=entry), + mock.patch( + "ndi.cloud.profile.get_password", side_effect=RuntimeError("locked keyring") + ), + ): + self.assertEqual(auth._profile_credentials(), ("", "", "")) + + def test_a_profile_with_no_stored_password_counts_as_no_credentials(self): + entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=entry), + mock.patch("ndi.cloud.profile.get_password", return_value=""), + ): + self.assertEqual(auth._profile_credentials(), ("", "", "")) + + +class TestAuthenticateReachesTheProfile(_NoEnvMixin): + def test_the_profile_logs_in_when_the_environment_is_empty(self): + fake, calls = _login_spy() + with ( + mock.patch.object(auth, "login", fake), + mock.patch.object( + auth, "_profile_credentials", return_value=("a@example.com", "pw", "prod") + ), + ): + self.assertEqual(auth.authenticate(), ("tok", "org-1")) + self.assertEqual(calls[0][:2], ("a@example.com", "pw")) + + def test_an_exported_username_still_wins_over_the_profile(self): + os.environ["NDI_CLOUD_USERNAME"] = "env@example.com" + os.environ["NDI_CLOUD_PASSWORD"] = "envpw" + fake, calls = _login_spy() + with ( + mock.patch.object(auth, "login", fake), + mock.patch.object( + auth, "_profile_credentials", return_value=("a@example.com", "pw", "prod") + ), + ): + auth.authenticate() + self.assertEqual(calls[0][:2], ("env@example.com", "envpw")) + + def test_a_dev_profile_points_the_config_at_the_dev_api(self): + fake, calls = _login_spy() + with ( + mock.patch.object(auth, "login", fake), + mock.patch.object( + auth, "_profile_credentials", return_value=("a@example.com", "pw", "dev") + ), + ): + auth.authenticate() + self.assertEqual(calls[0][2], "https://dev-api.ndi-cloud.com/v1") + + def test_an_explicit_url_is_not_overridden_by_the_profile_stage(self): + os.environ["NDI_CLOUD_URL"] = "https://example.invalid/v1" + fake, calls = _login_spy() + with ( + mock.patch.object(auth, "login", fake), + mock.patch.object( + auth, "_profile_credentials", return_value=("a@example.com", "pw", "dev") + ), + ): + auth.authenticate() + self.assertEqual(calls[0][2], "https://example.invalid/v1") + + def test_an_explicit_environment_choice_is_not_overridden_either(self): + os.environ["CLOUD_API_ENVIRONMENT"] = "prod" + fake, calls = _login_spy() + with ( + mock.patch.object(auth, "login", fake), + mock.patch.object( + auth, "_profile_credentials", return_value=("a@example.com", "pw", "dev") + ), + ): + auth.authenticate() + self.assertEqual(calls[0][2], "https://api.ndi-cloud.com/v1") + + def test_with_neither_environment_nor_profile_the_error_names_both(self): + with mock.patch.object(auth, "_profile_credentials", return_value=("", "", "")): + with self.assertRaises(CloudAuthError) as ctx: + auth.authenticate() + message = str(ctx.exception) + self.assertIn("NDI_CLOUD_USERNAME", message) + self.assertIn("profile", message) + + +if __name__ == "__main__": + unittest.main() From 14addc280ed60b7024dd6eb24a3c3a7e32e4997a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 02:02:02 +0000 Subject: [PATCH 15/52] Name secrets the way MATLAB names them, not the way that looked right NDI-matlab runs every secret key through matlab.lang.makeValidName before using it as a struct field. It has to: reading the secrets file back through jsondecode requires each JSON key to be a valid MATLAB identifier. Python has no such constraint, so _safe_field substituted underscores for spaces -- the reasonable guess, and the wrong one. makeValidName DELETES whitespace and capitalises the following letter. So MATLAB wrote NDICloud41269713c8f30937_40d3dfe383e33674 and Python looked for NDI_Cloud_41269713c8f30937_40d3dfe383e33674 Neither implementation could read the other's secrets, in either direction, which is the entire point of a shared ~/.ndi store. The symptom was a KeyError from get_password on a profile that had just listed correctly, with its password sitting in the file two lines away. _make_valid_name is the port: whitespace deleted and the next letter capitalised, remaining non-alphanumerics to underscores, a leading 'x' when the result would not start with a letter, keywords suffixed, truncation at namelengthmax. The keyword and length rules are unreachable for NDI's own keys; a port that implements four of five rules is a trap for whoever needs the fifth. Reads fall back to the old name, so a password saved by an older NDI-python still opens; the next write moves it across and drops the stale copy, and remove takes both. Nothing writes the old name again. Note the capitalisation applies to UIDs too: "NDI Cloud f68d..." becomes NDICloudF68d..., and half of Python's uuid4 UIDs begin with a-f. MATLAB does the same thing, so tidying it up would break the interop rather than improve it. There is a test saying so. The last test builds a MATLAB-shaped store from the outside -- a single profile as a JSON object, a <16hex>_<16hex> UID, a MATLAB field name -- and reads the password back through the public API, which covers the whole handoff rather than this one field. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/profile.py | 97 ++++++++- .../test_cloud_profile_secret_field_names.py | 201 ++++++++++++++++++ 2 files changed, 297 insertions(+), 1 deletion(-) create mode 100644 tests/test_cloud_profile_secret_field_names.py diff --git a/src/ndi/cloud/profile.py b/src/ndi/cloud/profile.py index 854fa47a..20423f74 100644 --- a/src/ndi/cloud/profile.py +++ b/src/ndi/cloud/profile.py @@ -206,8 +206,95 @@ def _as_profile_list(raw) -> list: return [] +# MATLAB's reserved words, for the last rule of makeValidName. NDI's own +# secret keys ("NDI Cloud ") can never collide with one, but a port that +# implements four of five rules is a trap for whoever ports the fifth. +_MATLAB_KEYWORDS = frozenset( + { + "break", + "case", + "catch", + "classdef", + "continue", + "else", + "elseif", + "end", + "for", + "function", + "global", + "if", + "otherwise", + "parfor", + "persistent", + "return", + "spmd", + "switch", + "try", + "while", + } +) + +# MATLAB's namelengthmax. +_MATLAB_NAMELENGTHMAX = 63 + + +def _make_valid_name(name: str) -> str: + """Port of MATLAB ``makeValidName(name, 'ReplacementStyle', 'underscore')``. + + NDI-matlab's profile.m runs every secret key through this before using + it as a struct field, because that is what reading the secrets file + back through ``jsondecode`` requires -- a JSON key has to become a + valid MATLAB identifier. Python has no such constraint, but it has to + produce the SAME name or it cannot read the file MATLAB wrote. + + The rule that matters, and the one a reasonable guess gets wrong: + whitespace is **deleted** and the following letter capitalised, not + replaced. So ``"NDI Cloud 41269..."`` becomes ``NDICloud41269...``, + not ``NDI_Cloud_41269...``. + """ + if not name: + return "x" + + # 1. Whitespace is deleted; the next alphabetic character is capitalised. + chars: list[str] = [] + capitalize_next = False + for ch in name: + if ch.isspace(): + capitalize_next = True + continue + if capitalize_next and ch.isalpha(): + ch = ch.upper() + capitalize_next = False + chars.append(ch) + out = "".join(chars) + + # 2. Anything left that is not an ASCII alphanumeric or an underscore + # becomes one underscore each (ReplacementStyle 'underscore'). + out = "".join(c if (c.isascii() and c.isalnum()) or c == "_" else "_" for c in out) + + # 3. A name must begin with a letter. + if not out or not (out[0].isascii() and out[0].isalpha()): + out = "x" + out + + # 4. A name must not BE a keyword. + if out in _MATLAB_KEYWORDS: + out += "_" + + # 5. namelengthmax. + return out[:_MATLAB_NAMELENGTHMAX] + + def _safe_field(name: str) -> str: - """Map a secret key to a JSON-safe field name.""" + """Map a secret key to the field name NDI-matlab uses for it.""" + return _make_valid_name(name) + + +def _legacy_safe_field(name: str) -> str: + """The field name THIS module used to write, before the MATLAB port. + + Reads fall back to it so a password saved by an older NDI-python is + still found. Nothing writes it any more. + """ return name.replace(" ", "_").replace(":", "_") @@ -369,6 +456,9 @@ def _set_secret(self, key: str, value: str) -> None: elif self.backend == "aes": store = _read_secrets_file(self.secrets_filename) store[_safe_field(key)] = _aes_encrypt(value) + # Drop any entry under the name this module used to write, so a + # stale copy of a changed password cannot be read back later. + store.pop(_legacy_safe_field(key), None) _write_secrets_file(self.secrets_filename, store) else: # memory self._memory_store[key] = value @@ -384,6 +474,10 @@ def _get_secret(self, key: str) -> str: if self.backend == "aes": store = _read_secrets_file(self.secrets_filename) entry = store.get(_safe_field(key)) + if entry is None: + # A password written by an older NDI-python is under the old + # name. Still readable; the next write moves it across. + entry = store.get(_legacy_safe_field(key)) if entry is None: raise KeyError(f'No secret stored for "{key}".') return _aes_decrypt(entry) @@ -402,6 +496,7 @@ def _remove_secret(self, key: str) -> None: elif self.backend == "aes": store = _read_secrets_file(self.secrets_filename) store.pop(_safe_field(key), None) + store.pop(_legacy_safe_field(key), None) _write_secrets_file(self.secrets_filename, store) else: self._memory_store.pop(key, None) diff --git a/tests/test_cloud_profile_secret_field_names.py b/tests/test_cloud_profile_secret_field_names.py new file mode 100644 index 00000000..9f6a330a --- /dev/null +++ b/tests/test_cloud_profile_secret_field_names.py @@ -0,0 +1,201 @@ +"""A secret's field name has to be the one NDI-matlab writes, character for character. + +NDI-matlab runs every secret key through matlab.lang.makeValidName before +using it as a struct field -- it has to, because reading the secrets file +back through jsondecode requires each JSON key to be a valid MATLAB +identifier. Python has no such constraint, and had substituted underscores +for spaces, which is the reasonable guess and the wrong one: makeValidName +DELETES whitespace and capitalises the following letter. + +So MATLAB wrote NDICloud and Python looked for NDI_Cloud_. Both +files parsed, both profiles listed, and the password was reported missing: + + KeyError: 'No secret stored for "NDI Cloud 41269713c8f30937_...".' + +The two implementations could not read each other's secrets in either +direction, which is the whole point of the shared ~/.ndi store. +""" + +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from ndi.cloud import profile +from ndi.cloud.profile import _legacy_safe_field, _make_valid_name, _safe_field + + +class TestMakeValidNamePort(unittest.TestCase): + """Cases checked against MATLAB's documented makeValidName behaviour.""" + + def test_the_key_shape_ndi_actually_uses(self): + # Verbatim from a ~/.ndi/NDI_Cloud_Secrets.json written by NDI-matlab. + self.assertEqual( + _safe_field("NDI Cloud 41269713c8f30937_40d3dfe383e33674"), + "NDICloud41269713c8f30937_40d3dfe383e33674", + ) + + def test_whitespace_is_deleted_not_replaced(self): + self.assertEqual(_make_valid_name("a b"), "aB") + + def test_the_character_after_whitespace_is_capitalised(self): + self.assertEqual(_make_valid_name("hello world"), "helloWorld") + + def test_a_digit_after_whitespace_is_not_capitalisable_and_just_joins(self): + self.assertEqual(_make_valid_name("NDI Cloud 4abc"), "NDICloud4abc") + + def test_a_uid_beginning_with_a_letter_is_capitalised_by_the_same_rule(self): + # Not a wart -- MATLAB does this too, and half of NDI-python's uuid4 + # UIDs begin with a-f, so getting it "tidier" than MATLAB would break + # exactly the interop this exists for. + self.assertEqual( + _safe_field("NDI Cloud f68d6370c3f34c9cb4935916fb42ae19"), + "NDICloudF68d6370c3f34c9cb4935916fb42ae19", + ) + + def test_other_invalid_characters_each_become_one_underscore(self): + self.assertEqual(_make_valid_name("a:b-c"), "a_b_c") + + def test_a_name_that_would_start_with_a_digit_gets_a_leading_x(self): + self.assertEqual(_make_valid_name("9lives"), "x9lives") + + def test_a_name_that_would_start_with_an_underscore_gets_one_too(self): + self.assertEqual(_make_valid_name("_leading"), "x_leading") + + def test_a_keyword_is_not_left_as_a_keyword(self): + self.assertEqual(_make_valid_name("end"), "end_") + + def test_a_word_merely_containing_a_keyword_is_untouched(self): + self.assertEqual(_make_valid_name("endpoint"), "endpoint") + + def test_an_empty_key_still_yields_a_usable_name(self): + self.assertEqual(_make_valid_name(""), "x") + + def test_the_name_is_capped_at_namelengthmax(self): + self.assertEqual(len(_make_valid_name("a" * 200)), 63) + + def test_non_ascii_letters_are_replaced_rather_than_kept(self): + # MATLAB identifiers are ASCII; Python's str.isalnum() is not, so a + # naive port would keep these and diverge. + self.assertEqual(_make_valid_name("café"), "caf_") + + def test_the_legacy_name_is_the_one_we_used_to_write(self): + self.assertEqual( + _legacy_safe_field("NDI Cloud abc"), + "NDI_Cloud_abc", + ) + self.assertNotEqual(_legacy_safe_field("NDI Cloud abc"), _safe_field("NDI Cloud abc")) + + +def _matlab_field(uid: str) -> str: + """The field name MATLAB writes for "NDI Cloud ", spelled out here. + + Deliberately NOT _safe_field: an assertion that calls the function under + test to compute what it expects proves nothing. + """ + if uid[:1].isalpha(): + uid = uid[0].upper() + uid[1:] + return "NDICloud" + uid + + +class _AesStoreCase(unittest.TestCase): + """A real AES-backed store in a scratch prefdir.""" + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True)) + patcher = mock.patch.dict(os.environ, {"NDI_PREFDIR": self.dir}) + patcher.start() + self.addCleanup(patcher.stop) + profile.reset() + self.addCleanup(profile.reset) + try: + profile.use_backend("aes") + except Exception as exc: # pragma: no cover - cryptography is a dependency + self.skipTest(f"aes backend unavailable: {exc}") + + @property + def secrets(self) -> Path: + return Path(self.dir) / "NDI_Cloud_Secrets.json" + + def read_secrets(self) -> dict: + return json.loads(self.secrets.read_text()) + + +class TestSecretsAreWrittenWhereMatlabLooks(_AesStoreCase): + def test_a_saved_password_lands_under_the_matlab_field_name(self): + uid = profile.add("work", "a@example.com", "hunter2") + stored = self.read_secrets() + self.assertIn(_matlab_field(uid), stored) + self.assertNotIn(_legacy_safe_field("NDI Cloud " + uid), stored) + + def test_a_saved_password_reads_back(self): + uid = profile.add("work", "a@example.com", "hunter2") + self.assertEqual(profile.get_password(uid), "hunter2") + + def test_a_password_written_by_matlab_is_found(self): + # The file MATLAB leaves, built from the outside: a MATLAB-shaped UID + # (<16hex>_<16hex>, unlike Python's uuid4), a MATLAB-shaped field + # name, and a ciphertext under the shared key derivation. + from ndi.cloud.profile import _aes_encrypt + + uid = "41269713c8f30937_40d3dfe383e33674" + profiles = { + "Profiles": { + "UID": uid, + "Nickname": "work", + "Email": "a@example.com", + "Stage": "prod", + "PasswordSecret": "NDI Cloud " + uid, + }, + "DefaultUID": uid, + } + (Path(self.dir) / "NDI_Cloud_Profiles.json").write_text(json.dumps(profiles)) + self.secrets.write_text( + json.dumps({"NDICloud41269713c8f30937_40d3dfe383e33674": _aes_encrypt("frommatlab")}) + ) + profile.reload() + self.assertEqual(profile.get_password(uid), "frommatlab") + + +class TestOlderPythonSecretsStillOpen(_AesStoreCase): + def _demote_to_legacy(self, uid: str) -> None: + stored = self.read_secrets() + stored[_legacy_safe_field("NDI Cloud " + uid)] = stored.pop(_matlab_field(uid)) + self.secrets.write_text(json.dumps(stored)) + + def test_a_password_saved_by_an_older_ndi_python_is_still_readable(self): + uid = profile.add("work", "a@example.com", "olddog") + self._demote_to_legacy(uid) + self.assertEqual(profile.get_password(uid), "olddog") + + def test_rewriting_it_moves_it_to_the_matlab_name_and_leaves_no_copy(self): + uid = profile.add("work", "a@example.com", "olddog") + self._demote_to_legacy(uid) + profile.set_password(uid, "newtrick") + stored = self.read_secrets() + self.assertIn(_matlab_field(uid), stored) + self.assertNotIn(_legacy_safe_field("NDI Cloud " + uid), stored) + self.assertEqual(profile.get_password(uid), "newtrick") + + def test_removing_a_profile_takes_the_legacy_copy_with_it(self): + uid = profile.add("work", "a@example.com", "olddog") + self._demote_to_legacy(uid) + profile.remove(uid) + stored = self.read_secrets() + self.assertNotIn(_matlab_field(uid), stored) + self.assertNotIn(_legacy_safe_field("NDI Cloud " + uid), stored) + + def test_a_genuinely_absent_secret_still_reports_absence(self): + uid = profile.add("work", "a@example.com", "olddog") + stored = self.read_secrets() + stored.clear() + self.secrets.write_text(json.dumps(stored)) + with self.assertRaises(KeyError): + profile.get_password(uid) + + +if __name__ == "__main__": + unittest.main() From e91bcfcace4adf02a2c7d1687907011df5a212d7 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 02:10:53 +0000 Subject: [PATCH 16/52] Resolve prod vs dev where the config is built, not where the login is A profile carries a Stage, prod or dev, and it decides which API the requests go to. NDI-matlab settles this in profile.switchProfile -- setenv('CLOUD_API_ENVIRONMENT', prof.Stage) -- which api/url.m then reads for every call. Python applied the stage only on the credential path in authenticate(), and that is not the same thing. login() exports NDI_CLOUD_TOKEN. So the SECOND CloudConfig.from_env() in a process carries a valid token, authenticate() returns at its first step and never reaches the credential path, and the config keeps the prod default URL it was built with. A dev profile therefore logged in against dev and issued everything afterwards against prod. Each on-demand pyramid tile builds its own client, so a viewer session failed from the second request onwards with Not found (HTTP 404): {"error":"Dataset ... does not exist or user does not have access"} which reads as a permissions problem rather than a wrong-server one -- the two are indistinguishable from the response, since a dataset that is not on this server is exactly as absent as one you cannot see. The stage now resolves in CloudConfig.from_env, which is the factory for "build from ambient state" and so the right place for it. Precedence: NDI_CLOUD_URL, then CLOUD_API_ENVIRONMENT, then the profile's Stage, then prod -- so both environment variables still override, and an explicitly constructed CloudConfig is untouched by construction. An unrecognised stage falls back to prod rather than building a URL nothing serves. authenticate() keeps only the credential half. Nineteen tests, including the short-circuit itself: a config built with a token already exported still points at dev. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/auth.py | 29 ++-- src/ndi/cloud/config.py | 32 +++- tests/test_cloud_auth_profile_fallback.py | 177 ++++++++++++++++------ 3 files changed, 168 insertions(+), 70 deletions(-) diff --git a/src/ndi/cloud/auth.py b/src/ndi/cloud/auth.py index c5df4daa..531ee470 100644 --- a/src/ndi/cloud/auth.py +++ b/src/ndi/cloud/auth.py @@ -17,7 +17,7 @@ import os from datetime import datetime, timezone -from .config import _API_URLS, CloudConfig +from .config import CloudConfig from .exceptions import CloudAuthError logger = logging.getLogger(__name__) @@ -293,8 +293,8 @@ def logout(config: CloudConfig | None = None) -> None: config.org_id = "" -def _profile_credentials() -> tuple[str, str, str]: - """Return ``(email, password, stage)`` from the saved cloud profile. +def _profile_credentials() -> tuple[str, str]: + """Return ``(email, password)`` from the saved cloud profile. The session's current profile wins over the persisted default, matching :func:`ndi.cloud.profile.get_current` / :func:`~ndi.cloud.profile.get_default`. @@ -310,14 +310,14 @@ def _profile_credentials() -> tuple[str, str, str]: entry = _profile.get_current() or _profile.get_default() if entry is None: - return "", "", "" + return "", "" password = _profile.get_password(entry.UID) if not entry.Email or not password: - return "", "", "" - return entry.Email, password, entry.Stage or "" + return "", "" + return entry.Email, password except Exception as exc: # noqa: BLE001 - see docstring logger.debug("No usable cloud profile: %s", exc) - return "", "", "" + return "", "" def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: @@ -367,18 +367,11 @@ def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: # It goes last rather than first because an explicitly exported # NDI_CLOUD_USERNAME is a deliberate override and should keep winning # over whatever is on disk. - email, password, stage = _profile_credentials() + # The profile's Stage picks prod vs dev, but that is resolved in + # CloudConfig.from_env rather than here, because it has to apply to a + # config that short-circuits at step 1 on an already-exported token. + email, password = _profile_credentials() if email and password: - # Honour the profile's stage, but never over an explicit request: - # either environment variable means the caller has already chosen. - if ( - stage - and not os.environ.get("NDI_CLOUD_URL") - and not os.environ.get("CLOUD_API_ENVIRONMENT") - ): - url = _API_URLS.get(stage) - if url: - config.api_url = url updated = login(email, password, config) return updated.token, updated.org_id diff --git a/src/ndi/cloud/config.py b/src/ndi/cloud/config.py index a5f4d6a8..9fed1b83 100644 --- a/src/ndi/cloud/config.py +++ b/src/ndi/cloud/config.py @@ -26,6 +26,26 @@ } +def _profile_stage() -> str: + """The Stage ('prod'/'dev') of the cloud profile that would be used. + + NDI-matlab's profile.switchProfile does ``setenv('CLOUD_API_ENVIRONMENT', + prof.Stage)`` and api/url.m reads it back, so Stage naming the API + environment is the established contract, not an invention here. + + Empty string when there is no usable profile. Every failure mode is + benign -- no profile file, no default set, a secrets backend that will + not load -- and none should stop a config from being built. + """ + try: + from . import profile as _profile + + entry = _profile.get_current() or _profile.get_default() + return entry.Stage if entry is not None else "" + except Exception: # noqa: BLE001 - see docstring + return "" + + @dataclass class CloudConfig: """NDI Cloud connection configuration. @@ -51,10 +71,18 @@ class CloudConfig: @classmethod def from_env(cls) -> CloudConfig: """Create a CloudConfig from environment variables.""" - # Determine API URL + # Determine API URL. Precedence: an explicit URL, then an explicit + # environment, then the saved profile's Stage, then prod. + # + # The profile step has to be HERE rather than at the point of login. + # login() exports NDI_CLOUD_TOKEN, so every later from_env() carries a + # valid token and authenticate() returns at its first step without + # ever reaching the credential path -- which meant a dev profile + # logged in against dev and then issued every subsequent request + # against prod, where the dataset 404s. api_url = os.environ.get("NDI_CLOUD_URL", "") if not api_url: - env = os.environ.get("CLOUD_API_ENVIRONMENT", "prod") + env = os.environ.get("CLOUD_API_ENVIRONMENT", "") or _profile_stage() or "prod" api_url = _API_URLS.get(env, _API_URLS["prod"]) upload_no_zip_raw = os.environ.get("NDI_CLOUD_UPLOAD_NO_ZIP", "") diff --git a/tests/test_cloud_auth_profile_fallback.py b/tests/test_cloud_auth_profile_fallback.py index cfe91271..cc065785 100644 --- a/tests/test_cloud_auth_profile_fallback.py +++ b/tests/test_cloud_auth_profile_fallback.py @@ -1,15 +1,31 @@ -"""authenticate() must reach the saved cloud profile, not just the environment. - -The gap this pins closed: ndi.cloud.profile is the documented place to keep a -cloud password, but nothing in the authentication path ever read it. Every -implicit login -- each @_auto_client API call, and the ndic:// file handler -that fetches pyramid tiles for a downloaded dataset -- went environment-only -and raised "no credentials available" while a perfectly good default profile -sat on disk. MATLAB's authenticate.m has had the equivalent step -(authenticatedWithSecret) all along. +"""Cloud credentials and the API they are sent to both come from the profile. + +Two gaps, one after the other. + +FIRST: ndi.cloud.profile is the documented place to keep a cloud password, +but nothing in the authentication path read it. Every implicit login -- each +@_auto_client API call, and the ndic:// file handler that fetches pyramid +tiles for a downloaded dataset -- went token -> environment -> raise, and +said "no credentials available" with a working default profile on disk. +MATLAB's authenticate.m has had the equivalent step (authenticatedWithSecret) +all along. + +SECOND, once that worked: the profile's Stage picks prod vs dev, and applying +it only on the login path is not enough. login() exports NDI_CLOUD_TOKEN, so +every config built afterwards short-circuits at authenticate()'s first step +and never reaches the credential path -- a dev profile logged in against dev +and then issued every subsequent request against prod. NDI-matlab settles +this the same way, in profile.switchProfile: + + setenv('CLOUD_API_ENVIRONMENT', prof.Stage) + +so the stage is resolved where the config is built, not where the login is. """ +import base64 +import json import os +import time import unittest from unittest import mock @@ -27,6 +43,9 @@ "CLOUD_API_ENVIRONMENT", ) +_PROD = "https://api.ndi-cloud.com/v1" +_DEV = "https://dev-api.ndi-cloud.com/v1" + class _NoEnvMixin(unittest.TestCase): def setUp(self): @@ -37,6 +56,15 @@ def setUp(self): os.environ.pop(key, None) +def _unexpired_token() -> str: + """A JWT whose exp claim is far enough out that isTokenExpired says no.""" + + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + return f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) + 3600})}.sig" + + def _login_spy(token="tok", org="org-1"): """Stand in for auth.login, recording what it was handed.""" calls = [] @@ -60,7 +88,7 @@ def test_a_current_profile_is_preferred_over_the_default(self): mock.patch("ndi.cloud.profile.get_default", return_value=default), mock.patch("ndi.cloud.profile.get_password", return_value="pw"), ): - self.assertEqual(auth._profile_credentials(), ("current@example.com", "pw", "prod")) + self.assertEqual(auth._profile_credentials(), ("current@example.com", "pw")) def test_the_default_profile_is_used_when_no_current_one_is_set(self): default = ProfileEntry(UID="u2", Email="default@example.com", Stage="dev") @@ -69,14 +97,14 @@ def test_the_default_profile_is_used_when_no_current_one_is_set(self): mock.patch("ndi.cloud.profile.get_default", return_value=default), mock.patch("ndi.cloud.profile.get_password", return_value="pw"), ): - self.assertEqual(auth._profile_credentials(), ("default@example.com", "pw", "dev")) + self.assertEqual(auth._profile_credentials(), ("default@example.com", "pw")) def test_no_profile_at_all_reports_emptiness_rather_than_raising(self): with ( mock.patch("ndi.cloud.profile.get_current", return_value=None), mock.patch("ndi.cloud.profile.get_default", return_value=None), ): - self.assertEqual(auth._profile_credentials(), ("", "", "")) + self.assertEqual(auth._profile_credentials(), ("", "")) def test_a_secrets_backend_that_throws_is_not_allowed_to_escape(self): entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") @@ -86,7 +114,7 @@ def test_a_secrets_backend_that_throws_is_not_allowed_to_escape(self): "ndi.cloud.profile.get_password", side_effect=RuntimeError("locked keyring") ), ): - self.assertEqual(auth._profile_credentials(), ("", "", "")) + self.assertEqual(auth._profile_credentials(), ("", "")) def test_a_profile_with_no_stored_password_counts_as_no_credentials(self): entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") @@ -94,7 +122,7 @@ def test_a_profile_with_no_stored_password_counts_as_no_credentials(self): mock.patch("ndi.cloud.profile.get_current", return_value=entry), mock.patch("ndi.cloud.profile.get_password", return_value=""), ): - self.assertEqual(auth._profile_credentials(), ("", "", "")) + self.assertEqual(auth._profile_credentials(), ("", "")) class TestAuthenticateReachesTheProfile(_NoEnvMixin): @@ -102,9 +130,7 @@ def test_the_profile_logs_in_when_the_environment_is_empty(self): fake, calls = _login_spy() with ( mock.patch.object(auth, "login", fake), - mock.patch.object( - auth, "_profile_credentials", return_value=("a@example.com", "pw", "prod") - ), + mock.patch.object(auth, "_profile_credentials", return_value=("a@example.com", "pw")), ): self.assertEqual(auth.authenticate(), ("tok", "org-1")) self.assertEqual(calls[0][:2], ("a@example.com", "pw")) @@ -115,50 +141,23 @@ def test_an_exported_username_still_wins_over_the_profile(self): fake, calls = _login_spy() with ( mock.patch.object(auth, "login", fake), - mock.patch.object( - auth, "_profile_credentials", return_value=("a@example.com", "pw", "prod") - ), + mock.patch.object(auth, "_profile_credentials", return_value=("a@example.com", "pw")), ): auth.authenticate() self.assertEqual(calls[0][:2], ("env@example.com", "envpw")) - def test_a_dev_profile_points_the_config_at_the_dev_api(self): + def test_a_dev_profiles_login_is_issued_against_dev(self): fake, calls = _login_spy() with ( + mock.patch("ndi.cloud.config._profile_stage", return_value="dev"), mock.patch.object(auth, "login", fake), - mock.patch.object( - auth, "_profile_credentials", return_value=("a@example.com", "pw", "dev") - ), - ): - auth.authenticate() - self.assertEqual(calls[0][2], "https://dev-api.ndi-cloud.com/v1") - - def test_an_explicit_url_is_not_overridden_by_the_profile_stage(self): - os.environ["NDI_CLOUD_URL"] = "https://example.invalid/v1" - fake, calls = _login_spy() - with ( - mock.patch.object(auth, "login", fake), - mock.patch.object( - auth, "_profile_credentials", return_value=("a@example.com", "pw", "dev") - ), - ): - auth.authenticate() - self.assertEqual(calls[0][2], "https://example.invalid/v1") - - def test_an_explicit_environment_choice_is_not_overridden_either(self): - os.environ["CLOUD_API_ENVIRONMENT"] = "prod" - fake, calls = _login_spy() - with ( - mock.patch.object(auth, "login", fake), - mock.patch.object( - auth, "_profile_credentials", return_value=("a@example.com", "pw", "dev") - ), + mock.patch.object(auth, "_profile_credentials", return_value=("a@example.com", "pw")), ): auth.authenticate() - self.assertEqual(calls[0][2], "https://api.ndi-cloud.com/v1") + self.assertEqual(calls[0][2], _DEV) def test_with_neither_environment_nor_profile_the_error_names_both(self): - with mock.patch.object(auth, "_profile_credentials", return_value=("", "", "")): + with mock.patch.object(auth, "_profile_credentials", return_value=("", "")): with self.assertRaises(CloudAuthError) as ctx: auth.authenticate() message = str(ctx.exception) @@ -166,5 +165,83 @@ def test_with_neither_environment_nor_profile_the_error_names_both(self): self.assertIn("profile", message) +class TestTheStageSurvivesTheTokenShortCircuit(_NoEnvMixin): + """The bug a dev user actually hits. + + Each on-demand pyramid tile builds its own client, so this struck from + the second request onwards, after a login that looked fine, as: + + Not found (HTTP 404): {"error":"Dataset ... does not exist or user + does not have access"} + + which reads as a permissions problem rather than a wrong-server one. + """ + + def test_a_config_built_after_login_still_points_at_dev(self): + os.environ["NDI_CLOUD_TOKEN"] = "already-have-one" + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, _DEV) + + def test_authenticate_short_circuits_without_losing_the_dev_url(self): + os.environ["NDI_CLOUD_TOKEN"] = _unexpired_token() + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + config = CloudConfig.from_env() + _, org = auth.authenticate(config) + self.assertEqual(org, "org-1") + self.assertEqual(config.api_url, _DEV) + + +class TestApiUrlPrecedence(_NoEnvMixin): + def test_an_explicit_url_beats_the_profile_stage(self): + os.environ["NDI_CLOUD_URL"] = "https://example.invalid/v1" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, "https://example.invalid/v1") + + def test_an_explicit_environment_beats_the_profile_stage(self): + os.environ["CLOUD_API_ENVIRONMENT"] = "prod" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + def test_the_profile_stage_beats_the_prod_default(self): + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, _DEV) + + def test_with_no_profile_at_all_it_is_prod(self): + with mock.patch("ndi.cloud.config._profile_stage", return_value=""): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + def test_an_unrecognised_stage_falls_back_to_prod_rather_than_a_bad_url(self): + with mock.patch("ndi.cloud.config._profile_stage", return_value="staging"): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + def test_a_profile_store_that_will_not_load_does_not_stop_a_config(self): + with mock.patch("ndi.cloud.profile.get_current", side_effect=RuntimeError("locked")): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + +class TestProfileStage(_NoEnvMixin): + def test_the_current_profile_is_preferred_over_the_default(self): + from ndi.cloud.config import _profile_stage + + current = ProfileEntry(UID="u1", Email="a@example.com", Stage="dev") + default = ProfileEntry(UID="u2", Email="b@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=current), + mock.patch("ndi.cloud.profile.get_default", return_value=default), + ): + self.assertEqual(_profile_stage(), "dev") + + def test_no_profile_yields_no_stage_rather_than_a_guess(self): + from ndi.cloud.config import _profile_stage + + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + ): + self.assertEqual(_profile_stage(), "") + + if __name__ == "__main__": unittest.main() From 106ae7c167aef1b13a28078abdc0191bd79909bf Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 02:43:08 +0000 Subject: [PATCH 17/52] Name tile files from one, because a DID file series is one-based did.document/addFileSeries takes "ONE-BASED member numbers" and refuses anything else -- "Member indices must be positive integers (one-based)". The pyramid writer named its first tile tile.bin_0, putting every pyramid one step outside the convention its own storage layer assumes. That is not cosmetic. NDI-matlab's uploader walks a legacy NAME# series as NAME1, NAME2, ... and treats the first absent name as the end of the series, because that gap was the only termination signal it had. A zero-based pyramid hands it a series beginning at a name it never probes. The tile INDEX stays zero-based: it is a grid position, and index_order defines it as row*tile_columns + column. Only the file SUFFIX moves, so the file holding tile N is tile.bin_. The origin is RECORDED on the document, not inferred. Pyramids already built name their first tile tile.bin_0 and carry no value, which reads as 0 and keeps them opening -- which matters, since the sessions being demonstrated were built before this. Inferring it from the stored names is not possible: a sparse pyramid whose first tile is empty has no _0 under either convention, and a wrong guess shifts every tile one cell along its row. An image that is quietly wrong is worse than one that is missing. Both directions are covered. tileFileName maps a grid index to a name; tileIndexFromName maps back, for the two readers that walk the stored names and work out (row, column) from the suffix -- doc_gene.exportRegion and doc_gene_export.readAsRecords. Reading the suffix AS the index there would have moved every tile one cell right, and the last tile of a row into the next row. The round-trip test builds a pyramid whose 2x2 grid stores tiles 1 and 4 with nothing between them, then walks it the way the uploader does and asserts the walk stops at the hole after one file. That sparseness is the normal case, not an exotic one: neither writer emits a tile with no data. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/doc_gene.py | 63 +++++- src/ndi/fun/doc_gene_export.py | 5 +- src/ndi/gui/app/genepyramid/multiscale.py | 6 +- .../data/spatialGeneExpressionTiles.json | 3 +- .../data/spatialGeneExpressionTiles.md | 23 ++- .../spatialGeneExpressionPyramid_schema.json | 2 +- .../spatialGeneExpressionTiles_schema.json | 13 ++ tests/test_doc_gene_tile_naming.py | 194 ++++++++++++++++++ 8 files changed, 292 insertions(+), 17 deletions(-) create mode 100644 tests/test_doc_gene_tile_naming.py diff --git a/src/ndi/fun/doc_gene.py b/src/ndi/fun/doc_gene.py index 867dda4a..1ef2bc27 100644 --- a/src/ndi/fun/doc_gene.py +++ b/src/ndi/fun/doc_gene.py @@ -615,9 +615,10 @@ def _make_level( lo, hi = int(bounds[t]), int(bounds[t + 1]) if lo == hi: continue # tiles with no data are not written - p = os.path.join(tmpdir, f"tile.bin_{t}") + name = f"tile.bin_{t + TILE_INDEX_ORIGIN}" + p = os.path.join(tmpdir, name) writeTileFile(p, xl[lo:hi], yl[lo:hi], g[lo:hi], cc[lo:hi]) - names.append(f"tile.bin_{t}") + names.append(name) paths.append(p) doc = ( @@ -647,6 +648,7 @@ def _make_level( "data_type_coordinate": "uint16", "tile_compression": "none", "tile_format_version": 1, + "tile_index_origin": TILE_INDEX_ORIGIN, }, ) + session.newdocument() @@ -684,6 +686,58 @@ def _write_gene_totals(gene_index, count, n_genes): return path +# The file suffix is ONE-BASED. A DID file series names its first member +# NAME_1, not NAME_0 -- did.document/addFileSeries takes "ONE-BASED member +# numbers" and refuses anything else ("Member indices must be positive +# integers (one-based)"). The tile INDEX stays zero-based, because it is a +# grid position and index_order defines it as row*tile_columns + column, so +# the suffix is that index plus this origin. +# +# Getting this wrong is not cosmetic: NDI-matlab's uploader walks a legacy +# NAME# series as NAME1, NAME2, ... and stops at the first name that is not +# there, because that gap was how it learned where the series ended. A +# zero-based pyramid therefore hands it a series starting at a name it never +# probes. +TILE_INDEX_ORIGIN = 1 + + +def tileFileName(tiles_props, index): + """Name of the file holding tile *index* of a tiles document. + + *index* is the zero-based grid index (``row * tile_columns + column`` + for row-major); the returned name carries the document's own origin. + + Pyramids written before the one-based convention was honoured name + their first tile ``tile.bin_0`` and record no ``tile_index_origin``, + which reads as 0 and keeps them readable. The origin has to be recorded + rather than detected: a sparse pyramid whose first tile happens to be + empty has no ``_0`` under either convention, so a guess from the stored + names would shift every tile by one on the documents it got wrong -- + a picture that is quietly wrong rather than one that is missing. + """ + try: + origin = int(tiles_props.get("tile_index_origin", 0) or 0) + except (TypeError, ValueError): + origin = 0 + return f"tile.bin_{int(index) + origin}" + + +def tileIndexFromName(tiles_props, name): + """The zero-based grid index of the tile stored under *name*. + + The inverse of :func:`tileFileName`. Readers that walk a document's + stored names and work backwards to (row, column) need this: with a + one-based origin the suffix is one MORE than the grid index, and + reading the suffix as the index puts every tile one cell along its + row -- and, for the last tile of a row, into the next row. + """ + try: + origin = int(tiles_props.get("tile_index_origin", 0) or 0) + except (TypeError, ValueError): + origin = 0 + return int(str(name).rsplit("_", 1)[1]) - origin + + def _find_level(session, pyr_doc, bin_size): """The tiles document for one bin size, and its property struct.""" q = ndi_query("").isa("spatialGeneExpressionTiles") & ndi_query("").depends_on( @@ -743,7 +797,7 @@ def readViewport(session, pyr_doc, bin_size, rect=None, gene_rows=None, density= for r in range(y0 // th, min(max(y1 - 1, y0) // th, rows - 1) + 1): for c in range(x0 // tw, min(max(x1 - 1, x0) // tw, cols - 1) + 1): - name = f"tile.bin_{r * cols + c}" + name = tileFileName(lv, r * cols + c) if name not in stored: info["tiles_empty"] += 1 continue @@ -800,8 +854,7 @@ def exportRegion(session, pyr_doc, bin_size, rect=None): rows_idx, cols_idx, vals, coords = [], [], [], [] n_px = 0 for name in sorted(tile_doc.current_file_list()): - t_id = int(name.rsplit("_", 1)[1]) - tr, tc = divmod(t_id, cols) + tr, tc = divmod(tileIndexFromName(lv, name), cols) tx0, ty0 = tc * tw, tr * th if tx0 >= x1 or tx0 + tw <= x0 or ty0 >= y1 or ty0 + th <= y0: continue diff --git a/src/ndi/fun/doc_gene_export.py b/src/ndi/fun/doc_gene_export.py index d6b1fad7..e2d4d550 100644 --- a/src/ndi/fun/doc_gene_export.py +++ b/src/ndi/fun/doc_gene_export.py @@ -28,7 +28,7 @@ import numpy as np -from .doc_gene import _find_level, exportRegion, readTileFile +from .doc_gene import _find_level, exportRegion, readTileFile, tileIndexFromName __all__ = ["exportGef", "exportHdf5", "readGeneList", "readGeneTotals"] @@ -146,8 +146,7 @@ def _gather(session, pyr_doc, bin_size): xs, ys, gs, cs = [], [], [], [] for name in sorted(tile_doc.current_file_list(), key=lambda s: int(s.rsplit("_", 1)[1])): - t_id = int(name.rsplit("_", 1)[1]) - tr, tc = divmod(t_id, cols) + tr, tc = divmod(tileIndexFromName(lv, name), cols) fh = session.database_openbinarydoc(tile_doc, name) try: t = readTileFile(fh) diff --git a/src/ndi/gui/app/genepyramid/multiscale.py b/src/ndi/gui/app/genepyramid/multiscale.py index c4b3f3a1..05cccfb1 100644 --- a/src/ndi/gui/app/genepyramid/multiscale.py +++ b/src/ndi/gui/app/genepyramid/multiscale.py @@ -111,7 +111,7 @@ def levelArrays(session, pyr_doc, gene_rows=None, density: bool = True) -> list[ dask, da = _require_dask() levels, _frame = levelTable(session, pyr_doc) - from ....fun.doc_gene import _find_level + from ....fun.doc_gene import _find_level, tileFileName out = [] for lv in levels: @@ -120,7 +120,7 @@ def levelArrays(session, pyr_doc, gene_rows=None, density: bool = True) -> list[ rows, cols = lv["tileRows"], lv["tileColumns"] lh, lw = lv["levelHeight"], lv["levelWidth"] - tile_doc, _props = _find_level(session, pyr_doc, b) + tile_doc, lv_props = _find_level(session, pyr_doc, b) stored = set(tile_doc.current_file_list()) scale = b if density else 1 @@ -145,7 +145,7 @@ def levelArrays(session, pyr_doc, gene_rows=None, density: bool = True) -> list[ paths = {} for r in range(rows): for c in range(cols): - name = f"tile.bin_{r * cols + c}" + name = tileFileName(lv_props, r * cols + c) if name not in stored: continue fh = session.database_openbinarydoc(tile_doc, name) diff --git a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json index c0a7ce78..b925c9c8 100644 --- a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json +++ b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json @@ -49,6 +49,7 @@ "data_type_offset": "uint32", "data_type_coordinate": "uint16", "tile_compression": "none", - "tile_format_version": 1 + "tile_format_version": 1, + "tile_index_origin": 1 } } diff --git a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md index 9bdab14c..7b063ecd 100644 --- a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md +++ b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md @@ -116,11 +116,26 @@ The gene dictionary is **not** a file of this document. It lives in the `geneList` document that the parent pyramid depends on, so that a dissociated RNA-seq dataset built on the same annotation can share it. -`#` is a non-negative integer computed from the tile's (row, column) by +`#` is the tile's INDEX plus the document's `tile_index_origin`. The index +is a non-negative grid position computed from the tile's (row, column) by the parent's `index_order`; for `row-major` it is -`row * tile_columns + column`. Tiles containing no data are not written. -Use `ndi.document/current_file_list` to discover which tiles exist — -there is no separate index file. +`row * tile_columns + column`. + +`tile_index_origin` is **1**, because a DID file series is one-based: its +first member is `NAME_1`, and `did.document/addFileSeries` refuses +anything else ("Member indices must be positive integers (one-based)"). +So the tile at (0,0) is stored as `tile.bin_1`. Pyramids written before +this convention was honoured record no `tile_index_origin`, which reads as +0; they name that tile `tile.bin_0` and stay readable. Read the origin +from the document rather than inferring it from the names — a sparse +pyramid whose first tile is empty has no `_0` under either convention, so +a guess would shift every tile by one on the documents it got wrong. + +Tiles containing no data are not written, so the series is SPARSE: an +absent `#` is normal and means that tile held nothing. Use +`ndi.document/current_file_list` to discover which tiles exist — there is +no separate index file, and walking the numbers until one is missing +finds the first hole rather than the end. ### tile.bin_N layout (tile_format_version 1) diff --git a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json index e3412b17..4152c944 100644 --- a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json +++ b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json @@ -181,7 +181,7 @@ "default_value": "row-major", "parameters": "", "queryable": 1, - "documentation": "How (row,column) maps to the suffix N in tile.bin_N. 'row-major': N = row*tile_columns + column. 'column-major': N = column*tile_rows + row." + "documentation": "How (row,column) maps to the tile INDEX N. 'row-major': N = row*tile_columns + column. 'column-major': N = column*tile_rows + row. The FILE holding tile N is named tile.bin_, where the origin is a field of the tiles document; a DID file series is one-based, so it is 1 for documents written since that was honoured and absent (0) for older ones." }, { "name": "origin_corner", diff --git a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json index 6d9257bf..58543e24 100644 --- a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json +++ b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json @@ -216,6 +216,19 @@ ], "queryable": 1, "documentation": "The version of the tile binary layout. See spatialGeneExpressionTiles.md." + }, + { + "name": "tile_index_origin", + "type": "integer", + "default_value": 1, + "parameters": [ + 0, + 1, + 0, + 0 + ], + "queryable": 1, + "documentation": "The suffix of the file holding tile N is N plus this. A DID file series is ONE-BASED -- its first member is NAME_1 -- so this is 1 for every document written since that was honoured. Pyramids written before it name their first tile tile.bin_0 and carry no value here, which reads as 0 and keeps them readable. The tile INDEX itself stays zero-based in either case; it is a grid position, defined by index_order as row*tile_columns + column." } ] } diff --git a/tests/test_doc_gene_tile_naming.py b/tests/test_doc_gene_tile_naming.py new file mode 100644 index 00000000..cbc5a09f --- /dev/null +++ b/tests/test_doc_gene_tile_naming.py @@ -0,0 +1,194 @@ +"""Tile files are named from ONE, because a DID file series is one-based. + +did.document/addFileSeries takes "ONE-BASED member numbers" and refuses +anything else -- "Member indices must be positive integers (one-based)" -- +so the first member of a series is NAME_1. The pyramid writer named its +first tile tile.bin_0, which put every pyramid a step outside the +convention its storage layer assumes. + +That is not cosmetic. NDI-matlab's uploader walks a legacy NAME# series as +NAME1, NAME2, ... and treats the first name that is not there as the end of +the series, because that gap was the only termination signal available. A +zero-based pyramid therefore begins at a name the walk never probes. + +The tile INDEX stays zero-based: it is a grid position, and index_order +defines it as row*tile_columns + column. Only the file SUFFIX moves, and +the document records which origin it used so that pyramids written before +this keep opening. +""" + +import unittest + +from ndi.fun.doc_gene import TILE_INDEX_ORIGIN, tileFileName + + +class TestTheOriginIsOne(unittest.TestCase): + def test_the_convention_is_one_based(self): + self.assertEqual(TILE_INDEX_ORIGIN, 1) + + def test_the_first_tile_of_a_new_document_is_file_one(self): + self.assertEqual(tileFileName({"tile_index_origin": 1}, 0), "tile.bin_1") + + def test_no_new_document_ever_names_a_file_zero(self): + names = {tileFileName({"tile_index_origin": 1}, i) for i in range(200)} + self.assertNotIn("tile.bin_0", names) + + def test_the_index_is_still_the_grid_position(self): + # row 3, column 4 of a 10-wide level is index 34, file 35. + self.assertEqual(tileFileName({"tile_index_origin": 1}, 3 * 10 + 4), "tile.bin_35") + + def test_names_stay_dense_in_the_suffix(self): + self.assertEqual( + [tileFileName({"tile_index_origin": 1}, i) for i in range(3)], + ["tile.bin_1", "tile.bin_2", "tile.bin_3"], + ) + + +class TestOlderPyramidsStillOpen(unittest.TestCase): + """The origin is READ, never guessed. + + A sparse pyramid whose first tile is empty has no tile.bin_0 under + either convention, so the stored names cannot distinguish the two. A + guess that got it wrong would shift every tile by one -- an image that + is quietly wrong rather than one that is missing, which is the worse + of the two failures. + """ + + def test_a_document_with_no_origin_recorded_is_read_as_zero_based(self): + self.assertEqual(tileFileName({}, 0), "tile.bin_0") + self.assertEqual(tileFileName({}, 70), "tile.bin_70") + + def test_an_explicit_zero_is_honoured(self): + self.assertEqual(tileFileName({"tile_index_origin": 0}, 12), "tile.bin_12") + + def test_the_two_conventions_disagree_by_exactly_one(self): + for i in (0, 1, 70, 4999): + old = tileFileName({}, i) + new = tileFileName({"tile_index_origin": 1}, i) + self.assertEqual(int(new.rsplit("_", 1)[1]) - int(old.rsplit("_", 1)[1]), 1) + + def test_an_unusable_value_does_not_raise_mid_render(self): + # A malformed document should not take down a viewer that is part + # way through building a layer; it reads as the legacy origin. + for junk in (None, "", "banana", [1]): + self.assertEqual(tileFileName({"tile_index_origin": junk}, 4), "tile.bin_4") + + +class TestTheDocumentRecordsIt(unittest.TestCase): + def test_the_definition_carries_the_origin(self): + import json + from pathlib import Path + + import ndi + + root = Path(ndi.__file__).parent + d = json.loads( + ( + root / "ndi_common/database_documents/data/spatialGeneExpressionTiles.json" + ).read_text() + ) + self.assertEqual(d["spatialGeneExpressionTiles"]["tile_index_origin"], 1) + + def test_the_schema_declares_it(self): + import json + from pathlib import Path + + import ndi + + root = Path(ndi.__file__).parent + s = json.loads( + ( + root / "ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json" + ).read_text() + ) + names = [f["name"] for f in s["spatialGeneExpressionTiles"]] + self.assertIn("tile_index_origin", names) + + +if __name__ == "__main__": + unittest.main() + + +class TestARealPyramidRoundTrips(unittest.TestCase): + """Build one, then read it back through the two readers that exist.""" + + def setUp(self): + import tempfile + + from ndi.fun.doc_gene import makeGeneList, makePyramid + from ndi.session.dir import ndi_session_dir + + self.dir = tempfile.mkdtemp() + self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True)) + S = ndi_session_dir("tile_naming", self.dir) + sub = S.newdocument("subject", **{"subject.local_identifier": "tn@vhlab"}) + S.database_add(sub) + gl = makeGeneList(S, ["ENSL0001", "ENSL0002", "ENSL0003", "ENSL0004"], ["a", "b", "c", "d"]) + # Coordinates chosen so the grid has HOLES: every point sits in the + # upper-left tile, so the other three are never written. A sparse + # series is the normal case, not the exotic one. + x = [1000, 1001, 1002, 1003] + y = [2000, 2001, 2002, 2003] + self.pyr, self.tiles = makePyramid( + S, + x, + y, + [0, 1, 2, 3], + [2, 3, 5, 7], + gl, + subjectID=sub.id, + binSizes=[1], + grid=2, + basePixelSize=(0.5, 0.5), + ) + self.S = S + + def test_the_first_written_tile_is_file_one(self): + stored = self.tiles[0].current_file_list() + self.assertIn("tile.bin_1", stored) + self.assertNotIn("tile.bin_0", stored) + + def test_the_document_records_the_origin_it_used(self): + props = self.tiles[0].document_properties["spatialGeneExpressionTiles"] + self.assertEqual(props["tile_index_origin"], 1) + + def test_the_series_is_sparse_which_is_why_walking_it_cannot_terminate(self): + # This 2x2 grid stores tiles 1 and 4 and nothing between them. A + # walk that counts 1, 2, ... and treats the first absent name as the + # end of the series finds the HOLE at 2 and never reaches 4 -- which + # is precisely how NDI-matlab's uploader lost 426 of a pyramid's 450 + # tile files while reporting success. + stored = sorted(n for n in self.tiles[0].current_file_list() if n.startswith("tile.bin_")) + self.assertEqual(stored, ["tile.bin_1", "tile.bin_4"]) + + walked = [] + j = 1 + while f"tile.bin_{j}" in stored: + walked.append(f"tile.bin_{j}") + j += 1 + self.assertEqual(walked, ["tile.bin_1"]) # stops one in, at the hole + + def test_current_file_list_finds_every_member_including_past_the_hole(self): + # The replacement for walking: ask the document, which records the + # names that were actually added. + stored = sorted(n for n in self.tiles[0].current_file_list() if n.startswith("tile.bin_")) + self.assertEqual(len(stored), 2) + props = self.tiles[0].document_properties["spatialGeneExpressionTiles"] + self.assertEqual(props["n_tiles_stored"], len(stored)) + + def test_the_counts_come_back_through_readViewport(self): + from ndi.fun.doc_gene import readViewport + + img, info = readViewport(self.S, self.pyr, 1, density=False) + self.assertEqual(info["tiles_read"], 2) + self.assertEqual(float(img.sum()), float(2 + 3 + 5 + 7)) + + def test_the_same_pyramid_reads_wrong_if_the_origin_is_ignored(self): + # Pin the reason the origin is recorded: pretend the document said + # nothing, and the reader looks for a name that is not there. + from ndi.fun.doc_gene import tileFileName + + props = self.tiles[0].document_properties["spatialGeneExpressionTiles"] + stored = set(self.tiles[0].current_file_list()) + self.assertIn(tileFileName(props, 0), stored) + self.assertNotIn(tileFileName({}, 0), stored) From 34a410e3fd6781600925b176ae4b1cfa10a3f1e1 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 02:49:48 +0000 Subject: [PATCH 18/52] Record the new MATLAB functions in the bridge contracts The completeness test flagged four MATLAB functions with no entry. Three are this branch's tile-naming helpers; the fourth, hdf5Contents, has been unrecorded since it was added, and went unnoticed because NDI-python's CI runs on main pushes and pull requests and this branch has had neither. tileFileName and tileIndexFromName are ported and recorded as ordinary entries. The indexing note is the part worth having written down: the tile INDEX is zero-based on both sides, because it is a grid position and index_order defines it as row*tile_columns + column, while the file SUFFIX is one-based because a DID file series names its first member NAME_1. Those are two different numbers and the contract now says so. tileIndexOrigin is marked not_applicable. It exists in MATLAB only because reading an optional numeric field there takes isfield, isnumeric and isscalar guards, where Python has one .get with a default; a Python function wrapping that would be a wrapper around nothing. Its BEHAVIOUR is mirrored and tested on both sides -- absent, empty, non-numeric or non-scalar all read as 0 -- which is what the contract is for. hdf5Contents is marked not_yet_ported with a reason rather than back-filled: it answers a question asked interactively, at a MATLAB prompt, before committing to an ingest that NDI-matlab drives. Nothing in the Python package calls it. h5py would make the port easy; it is the verdict wording that would need mirroring, not the walk. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/ndi_matlab_python_bridge.yaml | 44 ++++++++++++++++++++++ src/ndi/util/ndi_matlab_python_bridge.yaml | 26 +++++++++++++ 2 files changed, 70 insertions(+) diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index 340072b0..d4ea02a7 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -2011,6 +2011,50 @@ functions: becomes a Python keyword of the same name. See readViewportBase for the source-coordinate entry point. + - name: tileFileName + matlab_path: "+ndi/+fun/+doc/+gene/tileFileName.m" + matlab_last_sync_hash: "1e06bc79" + python_path: "ndi/fun/doc_gene.py" + input_arguments: + - name: tiles_props + type_matlab: "struct" + type_python: "dict" + - name: index + type_python: "int" + output_arguments: + - name: name + type_python: "str" + decision_log: > + Exact match. The tile INDEX is zero-based on both sides -- it is a + grid position, and index_order defines it as row*tile_columns + + column, so it is internal data per the indexing_policy. The FILE + SUFFIX is one-based, because a DID file series names its first + member NAME_1 and addFileSeries rejects anything else. The + difference between the two is the document's tile_index_origin, + which this function adds. + + - name: tileIndexFromName + matlab_path: "+ndi/+fun/+doc/+gene/tileIndexFromName.m" + matlab_last_sync_hash: "9920b8f0" + python_path: "ndi/fun/doc_gene.py" + decision_log: > + Exact match; the inverse of tileFileName, for readers that walk a + document's stored names and work back to (row, column). + + - name: tileIndexOrigin + matlab_path: "+ndi/+fun/+doc/+gene/tileIndexOrigin.m" + matlab_last_sync_hash: "6aa994e3" + status: not_applicable + decision_log: > + MATLAB-only helper. It exists because MATLAB has no equivalent of + dict.get with a default and needs isfield/isnumeric/isscalar guards + to read an optional numeric field; in Python that is one .get call + inline in tileFileName and tileIndexFromName, so a separate + function would be a wrapper around nothing. The BEHAVIOUR is + mirrored and tested on both sides: an absent, empty, non-numeric or + non-scalar tile_index_origin reads as 0, which is the legacy + zero-based naming. + - name: exportRegion matlab_path: "+ndi/+fun/+doc/+gene/exportRegion.m" matlab_last_sync_hash: "96f10959" diff --git a/src/ndi/util/ndi_matlab_python_bridge.yaml b/src/ndi/util/ndi_matlab_python_bridge.yaml index a971e217..3f2c1fdf 100644 --- a/src/ndi/util/ndi_matlab_python_bridge.yaml +++ b/src/ndi/util/ndi_matlab_python_bridge.yaml @@ -516,3 +516,29 @@ not_applicable: MATLAB sub-package with find_instance_name.m and find_techniques_names.m. Ported to ndi.openminds_convert module in Python (different namespace). + +# ============================================================================= +# Not-yet-ported MATLAB functions +# ============================================================================= +not_yet_ported: + + - name: hdf5Contents + matlab_path: "+ndi/+util/hdf5Contents.m" + matlab_last_sync_hash: "b745ee1d" + status: not_yet_ported + decision_log: > + Prints a table of contents for an HDF5 container -- groups, + datasets, attributes -- and a verdict naming which layout it is: + square-bin GEF, CellBin GEF, AnnData, or none of those. It exists + because a Stereo-seq run leaves several files with the same stem and + different middles (Y00709C1E2.tissue.gef beside + Y00709C1E2.adjusted.cellbin.gef), all of them HDF5, and the + extension does not say which one ndi.fun.doc.gene.fromGEF can read. + + Not ported yet because it is a MATLAB-console convenience: the + question it answers is asked interactively, before committing to an + ingest, and NDI-matlab is where that ingest is driven from + (ndi.gui.app.GEFManager). Nothing in the Python package calls it. + h5py makes the port straightforward whenever a Python-side ingest + GUI wants the same answer; it is the printing and the verdict + wording that would need mirroring, not the walk. From d00a3d3026e019a171b1f64bf1d3e82b8bdce242 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 03:08:17 +0000 Subject: [PATCH 19/52] Use the bridge status vocabulary main settled on main retired both spellings these entries used, with the replacement named in the failure text: not_applicable -> matlab_only / ported_differently / retired not_yet_ported -> porting_deferred / ported_differently tileIndexOrigin is matlab_only. The documented line is whether the replacement lives in this repo or the language provides it: reading an optional numeric field with a default takes isfield, isnumeric and isscalar guards in MATLAB and so earns a name, but in Python it is one dict.get from the standard library, written inline in tileFileName and tileIndexFromName. That is the "use tempfile" case, not the "use our SyncIndex class" one. hdf5Contents is porting_deferred: a counterpart is wanted, nobody has written it. The section banner carried the retired spelling too. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/ndi_matlab_python_bridge.yaml | 24 +++++++++++++--------- src/ndi/util/ndi_matlab_python_bridge.yaml | 6 +++--- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index 86176baa..ed4452f1 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -2787,16 +2787,20 @@ functions: - name: tileIndexOrigin matlab_path: "+ndi/+fun/+doc/+gene/tileIndexOrigin.m" matlab_last_sync_hash: "6aa994e3" - status: not_applicable - decision_log: > - MATLAB-only helper. It exists because MATLAB has no equivalent of - dict.get with a default and needs isfield/isnumeric/isscalar guards - to read an optional numeric field; in Python that is one .get call - inline in tileFileName and tileIndexFromName, so a separate - function would be a wrapper around nothing. The BEHAVIOUR is - mirrored and tested on both sides: an absent, empty, non-numeric or - non-scalar tile_index_origin reads as 0, which is the legacy - zero-based naming. + status: matlab_only + decision_log: > + Python needs nothing here, by design. The function reads an + optional numeric document field with a default, which in MATLAB + takes isfield, isnumeric and isscalar guards and so earns a name; + in Python it is one dict.get call from the standard library, + written inline in tileFileName and tileIndexFromName. That puts it + on the matlab_only side of the line rather than + ported_differently: the replacement is the language, not something + in this repo. + + The BEHAVIOUR is mirrored and tested on both sides -- an absent, + empty, non-numeric or non-scalar tile_index_origin reads as 0, + which is the legacy zero-based tile naming. - name: exportRegion matlab_path: "+ndi/+fun/+doc/+gene/exportRegion.m" diff --git a/src/ndi/util/ndi_matlab_python_bridge.yaml b/src/ndi/util/ndi_matlab_python_bridge.yaml index 5d150ab7..fe1c3c16 100644 --- a/src/ndi/util/ndi_matlab_python_bridge.yaml +++ b/src/ndi/util/ndi_matlab_python_bridge.yaml @@ -727,14 +727,14 @@ not_applicable: to port. # ============================================================================= -# Not-yet-ported MATLAB functions +# Deferred MATLAB functions # ============================================================================= -not_yet_ported: +porting_deferred: - name: hdf5Contents matlab_path: "+ndi/+util/hdf5Contents.m" matlab_last_sync_hash: "b745ee1d" - status: not_yet_ported + status: porting_deferred decision_log: > Prints a table of contents for an HDF5 container -- groups, datasets, attributes -- and a verdict naming which layout it is: From 0dabe60b839246694aa8088778198949cef86146 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 03:08:55 +0000 Subject: [PATCH 20/52] Record commits in matlab_last_sync_hash, not blobs git hash-object returns the hash of a file's CONTENTS. The field wants a COMMIT -- the point in NDI-matlab's history the entry was examined against -- because the question it exists to answer is git log ..HEAD -- and a blob is not a point in history to walk from. This is the mistake tests/test_matlab_bridge_hashes.py was written for after 35 entries carried blob hashes; the check could not run here, because the local NDI-matlab checkout is shallow and it skips rather than reporting every commit as missing. The five entries now name each file's last-touching commit: 10687267a for the three tile helpers, 742ac0d81 for hdf5Contents, and a328f306e for GEFManager, whose recorded 8edda6d was a commit but a stale one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/ndi_matlab_python_bridge.yaml | 6 +++--- src/ndi/gui/ndi_matlab_python_bridge.yaml | 2 +- src/ndi/util/ndi_matlab_python_bridge.yaml | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index ed4452f1..63ae187f 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -2756,7 +2756,7 @@ functions: - name: tileFileName matlab_path: "+ndi/+fun/+doc/+gene/tileFileName.m" - matlab_last_sync_hash: "1e06bc79" + matlab_last_sync_hash: "10687267a" python_path: "ndi/fun/doc_gene.py" input_arguments: - name: tiles_props @@ -2778,7 +2778,7 @@ functions: - name: tileIndexFromName matlab_path: "+ndi/+fun/+doc/+gene/tileIndexFromName.m" - matlab_last_sync_hash: "9920b8f0" + matlab_last_sync_hash: "10687267a" python_path: "ndi/fun/doc_gene.py" decision_log: > Exact match; the inverse of tileFileName, for readers that walk a @@ -2786,7 +2786,7 @@ functions: - name: tileIndexOrigin matlab_path: "+ndi/+fun/+doc/+gene/tileIndexOrigin.m" - matlab_last_sync_hash: "6aa994e3" + matlab_last_sync_hash: "10687267a" status: matlab_only decision_log: > Python needs nothing here, by design. The function reads an diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index d71c2d3a..56c118a2 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -822,7 +822,7 @@ ported_late: - name: GEFManager matlab_path: "+ndi/+gui/+app/GEFManager.m" - matlab_last_sync_hash: "8edda6d" + matlab_last_sync_hash: "a328f306e" status: porting_deferred decision_log: > MATLAB App Designer window that lists, views, adds and removes diff --git a/src/ndi/util/ndi_matlab_python_bridge.yaml b/src/ndi/util/ndi_matlab_python_bridge.yaml index fe1c3c16..3efcdeb3 100644 --- a/src/ndi/util/ndi_matlab_python_bridge.yaml +++ b/src/ndi/util/ndi_matlab_python_bridge.yaml @@ -733,7 +733,7 @@ porting_deferred: - name: hdf5Contents matlab_path: "+ndi/+util/hdf5Contents.m" - matlab_last_sync_hash: "b745ee1d" + matlab_last_sync_hash: "742ac0d81" status: porting_deferred decision_log: > Prints a table of contents for an HDF5 container -- groups, From 546e68ff9dc5f8042be3815585b6ba6d297030de Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 03:19:53 +0000 Subject: [PATCH 21/52] Clear the bridge drift this PR pair's own MATLAB changes created The bridge CI job pairs the two repos by branch name, so this is the first run that has actually compared these YAMLs against the NDI-matlab branch they describe. Nine entries drift, all of them because THIS PR pair moved the MATLAB file. Two are real ports, not bumps: ndi.cloud.profile.reset now restores the detected backend. Python had the same hole NDI-matlab did: use_backend is a test hook, the override it sets is singleton state that lives for the whole process, and reset cleared everything except that. One test selecting the memory backend left every later set_password writing to a dict discarded at exit -- no error, and get_password returned what it had just stored, so a password looked saved when it was not. That is not a hypothetical; it cost an hour of debugging on the MATLAB side tonight before the cause was found. ndi.preferences gains GUI/GEFManager/ViewerLauncher, mirroring NDI-matlab 362060795, so the shared preference store round-trips a value MATLAB writes. The rest are examined-and-bumped, and the guard is right to demand a reason for each: fromCellBin and fromGEF changed help text only (the GeneIngest -> GEFManager rename, and recordSource now naming fileReference), so their decision_logs say so and name b589db9bb. makeSourceFile's says what actually changed -- describing a file the database does not hold is now a fileReference, a class with no files section, and only attachFile=true still produces a generic_file. The old shape was a generic_file with nothing attached, which is invalid; nothing enforced it until DID-matlab #182 closed a fail-open. A Python port must follow that, so the entry says it. Also fixed from the same run: the hashes for the three tile helpers, hdf5Contents and GEFManager were git hash-object output, which is a BLOB. The field names a commit, or `git log ..HEAD -- ` has nothing to walk from. Full suite 32 failed / 5229 passed, against 40 / 5106 for main in the same working tree. The failure-list diff has eight this branch fixes and none it adds. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/ndi_matlab_python_bridge.yaml | 2 +- src/ndi/cloud/profile.py | 16 ++++++- src/ndi/fun/ndi_matlab_python_bridge.yaml | 44 ++++++++++++++----- src/ndi/ndi_matlab_python_bridge.yaml | 2 +- src/ndi/preferences.py | 13 ++++++ .../test_cloud_profile_secret_field_names.py | 29 ++++++++++++ 6 files changed, 91 insertions(+), 15 deletions(-) diff --git a/src/ndi/cloud/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/ndi_matlab_python_bridge.yaml index 0119aa74..05db356f 100644 --- a/src/ndi/cloud/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/ndi_matlab_python_bridge.yaml @@ -1292,7 +1292,7 @@ functions: # --- profile.py (singleton manager for cloud login profiles) --- - name: profile matlab_path: "+ndi/+cloud/profile.m" - matlab_last_sync_hash: "921a02244" + matlab_last_sync_hash: "e6a580b34" python_path: "ndi/cloud/profile.py" functions: - matlab: "ndi.cloud.profile.list()" diff --git a/src/ndi/cloud/profile.py b/src/ndi/cloud/profile.py index 1c2c385e..bcbab321 100644 --- a/src/ndi/cloud/profile.py +++ b/src/ndi/cloud/profile.py @@ -714,9 +714,23 @@ def reload() -> None: def reset() -> None: - """Clear the in-memory singleton state. Does NOT touch disk.""" + """Clear the in-memory singleton state. Does NOT touch disk. + + THIS INCLUDES A FORCED BACKEND. :func:`use_backend` is a test hook and + the override it sets is in-memory singleton state like any other, so + reset returns it to :func:`_detect_backend`. + + Leaving it out let the memory backend outlive the tests that selected + it: the singleton lives for the whole process, so a test run left + every later :func:`set_password` writing to a dict that is discarded + at exit -- with no error, and with :func:`get_password` returning the + value it had just stored, so it looked as though the password had + been saved. NDI-matlab had the same hole and the same symptom; see + ndi.cloud.profile.reset there. + """ obj = _get_singleton() obj.profiles = [] obj.current_uid = "" obj.default_uid = "" obj._memory_store = {} + obj.backend = _detect_backend() diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index 63ae187f..ce0a0f6c 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -2690,7 +2690,7 @@ functions: - name: makePyramid matlab_path: "+ndi/+fun/+doc/+gene/makePyramid.m" - matlab_last_sync_hash: "6ae508708" + matlab_last_sync_hash: "10687267a" python_path: "ndi/fun/doc_gene.py" decision_log: > Exact match. Options keep their MATLAB spelling (binSizes, grid, @@ -2701,7 +2701,7 @@ functions: - name: readViewport matlab_path: "+ndi/+fun/+doc/+gene/readViewport.m" - matlab_last_sync_hash: "b9972abf1" + matlab_last_sync_hash: "10687267a" python_path: "ndi/fun/doc_gene.py" input_arguments: - name: bin_size @@ -2804,7 +2804,7 @@ functions: - name: exportRegion matlab_path: "+ndi/+fun/+doc/+gene/exportRegion.m" - matlab_last_sync_hash: "b9972abf1" + matlab_last_sync_hash: "10687267a" python_path: "ndi/fun/doc_gene.py" decision_log: > Exact match. Returns raw counts, never density. @@ -2967,7 +2967,7 @@ functions: - name: makeCells matlab_path: "+ndi/+fun/+doc/+gene/makeCells.m" - matlab_last_sync_hash: "6ae508708" + matlab_last_sync_hash: "b589db9bb" python_path: "ndi/fun/doc_gene.py" decision_log: > Writes cells.tsv, optionally contours.bin, and stores the document. @@ -3042,7 +3042,7 @@ functions: - name: fromCellBin matlab_path: "+ndi/+fun/+doc/+gene/fromCellBin.m" - matlab_last_sync_hash: "6ae5087" + matlab_last_sync_hash: "b589db9bb" status: porting_deferred decision_log: > MATLAB reads a SAW cellbin .h5ad with @@ -3053,9 +3053,13 @@ functions: about row order. Ports as a group with the other three entries under this heading; no current Python caller. + NDI-matlab b589db9bb changed one line of the help text -- + recordSource now says it creates a fileReference rather than a + generic_file, following makeSourceFile. Comment only; no + behavioural change, and nothing to port here. - name: fromGEF matlab_path: "+ndi/+fun/+doc/+gene/fromGEF.m" - matlab_last_sync_hash: "a178e46" + matlab_last_sync_hash: "b589db9bb" status: porting_deferred decision_log: > MATLAB reads a Stereo-seq .gef with ndr.format.stereoseq.readGEF @@ -3064,16 +3068,32 @@ functions: path in one call. Ports with the other three entries under this heading. + NDI-matlab b589db9bb changed three lines of the help text: two + naming ndi.gui.app.GeneIngest, which was renamed GEFManager, and + one saying recordSource creates a fileReference rather than a + generic_file, following makeSourceFile. Comments only; no + behavioural change, and nothing to port here. - name: makeSourceFile matlab_path: "+ndi/+fun/+doc/+gene/makeSourceFile.m" - matlab_last_sync_hash: "6ae5087" + matlab_last_sync_hash: "b589db9bb" status: porting_deferred decision_log: > - MATLAB creates the generic_file document that the spatial gene - expression documents' source_file_id dependency points at, so a - pyramid can name the .gef it was built from. Describes by default - (name + checksum + dates), attaches only when attachFile=true. - Ports with the other three entries under this heading. + MATLAB creates the document that the spatial gene expression + documents' source_file_id dependency points at, so a pyramid can + name the .gef it was built from. + + WHICH document changed, in NDI-matlab b589db9bb. Describing a file the + database does not hold is now a fileReference -- a new class with no + files section at all -- and only attachFile=true produces a + generic_file with the bytes attached. The old shape was a + generic_file with nothing attached, which is invalid, since + generic_file requires generic_file.ext; nothing enforced it until + DID-matlab #182 closed a fail-open, at which point adding such a + session to a dataset failed at the add rather than at the write + that made it. + + Ports with the other three entries under this heading. A Python + port must create a fileReference, not a generic_file. - name: readNotes matlab_path: "+ndi/+fun/+doc/+gene/readNotes.m" diff --git a/src/ndi/ndi_matlab_python_bridge.yaml b/src/ndi/ndi_matlab_python_bridge.yaml index 313d873b..8c868d0b 100644 --- a/src/ndi/ndi_matlab_python_bridge.yaml +++ b/src/ndi/ndi_matlab_python_bridge.yaml @@ -1155,7 +1155,7 @@ classes: - name: preferences type: class matlab_path: "+ndi/preferences.m" - matlab_last_sync_hash: "9a9150e7c" + matlab_last_sync_hash: "362060795" python_path: "ndi/preferences.py" python_class: "Preferences" inherits: "" diff --git a/src/ndi/preferences.py b/src/ndi/preferences.py index e70d3534..7d7f67c6 100644 --- a/src/ndi/preferences.py +++ b/src/ndi/preferences.py @@ -211,6 +211,19 @@ def _register_defaults(self) -> None: "subclass ndi.gui.app.SessionApp are added to the built-in " "ndi.gui.app and ndi.app packages.", ) + self._add_item( + "GUI", + "GEFManager", + "ViewerLauncher", + "/usr/local/bin/napariViewGEF", + "str", + "Program the GEF Manager's View button runs to open a pyramid in " + "napari. Normally a small shell wrapper rather than the " + "napariViewGEF console script itself: MATLAB exports library " + "paths of its own, and a Python process started from MATLAB loads " + "MATLAB's copies of libraries it must not use, so the wrapper " + "scrubs the environment before running the real entry point.", + ) def _add_item( self, diff --git a/tests/test_cloud_profile_secret_field_names.py b/tests/test_cloud_profile_secret_field_names.py index 9f6a330a..75218db4 100644 --- a/tests/test_cloud_profile_secret_field_names.py +++ b/tests/test_cloud_profile_secret_field_names.py @@ -199,3 +199,32 @@ def test_a_genuinely_absent_secret_still_reports_absence(self): if __name__ == "__main__": unittest.main() + + +class TestResetRestoresTheDetectedBackend(unittest.TestCase): + """A forced backend must not outlive the test that forced it. + + use_backend is a test hook, and the override it sets is in-memory + singleton state like any other -- the singleton lives for the whole + process. Before reset cleared it, one test selecting the memory + backend left every later set_password writing to a dict discarded at + exit: no error, and get_password returned the value it had just + stored, so it looked as though the password had been saved. NDI-matlab + had the same hole; this is the port of its fix. + """ + + def setUp(self): + profile.reset() + self.addCleanup(profile.reset) + + def test_a_forced_backend_does_not_survive_a_reset(self): + detected = profile.backend() + profile.use_backend("memory") + self.assertEqual(profile.backend(), "memory") + profile.reset() + self.assertEqual(profile.backend(), detected) + + def test_the_restored_backend_is_a_real_one(self): + profile.use_backend("memory") + profile.reset() + self.assertIn(profile.backend(), ("keyring", "aes", "memory")) From e38f945e03464d1730deb3e1d0f8f3c7f005a1e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 03:23:51 +0000 Subject: [PATCH 22/52] Test the panel orchestration without needing a Qt binding test (3.10/3.11/3.12) failed on two of this branch's own tests, and both passed here. The difference is the install: addAllPanels refuses to build anything when qtpy.QtWidgets will not import, and CI's ndi_install.py --dev brings no Qt binding, while this container happens to have PyQt5. So on CI the function returned {} before touching a panel, and the two tests got an empty dict -- "the cell type panel was never built" and KeyError: 'display'. The tests were wrong, not the guard. What they check is the ORCHESTRATION around the panels: that --labels reaches the cell type panel, and that one panel throwing neither stops the ones after it nor unwinds into openPyramid after the viewer is on screen. None of that is about Qt. The probe is now _importQtWidgets rather than an inline import, so a test can stand in for it. Worth noting what the old shape cost: those two tests asserted their behaviour anywhere a binding happened to be installed and silently asserted nothing everywhere else -- passing either way, which is the failure mode a test is supposed to prevent. The guard's own path was marked no-cover because an inline import cannot be reached from a test. It can now, so it has one: no binding builds nothing, attempts nothing, and prints a line naming what still works. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 18 ++++++++++++-- tests/test_gui_genepyramid_controls.py | 31 +++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 7d2edc74..16b3d357 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -985,6 +985,20 @@ def _onMaster(on, name=name, scroll=scroll, expand=expand): # ----------------------------------------------------------------- all +def _importQtWidgets() -> None: + """Import qtpy.QtWidgets, or raise ImportError. + + A named function rather than an inline import inside addAllPanels so a + test can stand in for it. What addAllPanels does AROUND the panels -- + the order they are built in, and that one throwing does not stop the + rest or take the window down -- is not about Qt, and a CI job that + installs no Qt binding should still hold that behaviour to account. + Without this seam those tests passed anywhere a binding happened to be + installed and silently asserted nothing anywhere else. + """ + import qtpy.QtWidgets # noqa: F401 + + def addAllPanels( viewer, session, @@ -1021,8 +1035,8 @@ def addAllPanels( import traceback try: - import qtpy.QtWidgets # noqa: F401 - except ImportError as e: # pragma: no cover - depends on the install + _importQtWidgets() + except ImportError as e: print( f"[genepyramid] control panels unavailable ({e}). The image is " f"unaffected; --genes and --no-density still work at launch.", diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 2d8f1ae4..abf9a2f5 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -249,6 +249,9 @@ def test_the_chosen_labelings_reach_the_cell_type_panel(monkeypatch): """--labels was accepted, parsed, threaded through openPyramid, and then dropped at the last hop, so it silently did nothing. Nothing caught it because every other layer mentioned the argument.""" + # addAllPanels refuses to build anything without a Qt binding, and CI + # installs none. What is under test here is the orchestration, not Qt. + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) cells = _Recorder() monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) monkeypatch.setattr(controls, "addCellTypePanel", cells) @@ -274,6 +277,9 @@ def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsy that raised unwound out of openPyramid before napari.run(), so the window appeared and the process exited -- which reads as napari opening briefly and closing, with the reason lost.""" + # addAllPanels refuses to build anything without a Qt binding, and CI + # installs none. What is under test here is the orchestration, not Qt. + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) genes = _Recorder() monkeypatch.setattr(controls, "addDisplayPanel", _Recorder(boom=True)) monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) @@ -286,3 +292,28 @@ def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsy assert genes.calls, "a failing panel stopped the ones after it" err = capsys.readouterr().err assert "display" in err and "no such file" in err + + +def test_no_qt_binding_costs_the_panels_and_not_the_picture(monkeypatch, capsys): + """A headless-ish install has no Qt binding, and CI is one of them. + + addAllPanels must then build nothing and say so, rather than raising + into openPyramid after the viewer is already on screen. The message + names what still works, because the image and --genes do. + """ + + def _no_qt(): + raise ImportError("No module named 'qtpy.QtWidgets'") + + monkeypatch.setattr(controls, "_importQtWidgets", _no_qt) + built = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", built) + monkeypatch.setattr(controls, "addGenePanel", built) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + + assert made == {} + assert not built.calls, "no panel should be attempted without a binding" + err = capsys.readouterr().err + assert "control panels unavailable" in err + assert "image is unaffected" in err From fe5b0ec7743b4928fef512034fc9dadfcacb5a07 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 03:27:24 +0000 Subject: [PATCH 23/52] Vendor fileReference into ndi_common, where Python can read it The bridge job's ndi_common sync check runs with NDI_COMMON_SYNC_STRICT=1 against the paired NDI-matlab checkout, and found three files this PR pair added on the MATLAB side only: database_documents/data/fileReference.json database_documents/data/fileReference.md schema_documents/data/fileReference_schema.json Vendoring is the right half of that choice, not MATLAB_ONLY. A fileReference is what makeSourceFile now writes to describe the .gef a pyramid was built from, so it lands in any session ingested from NDI-matlab. Python opening that session needs the definition and schema to validate the document -- without them a session that MATLAB wrote would fail to read for want of a file that was never copied across, which is exactly the class of break this check exists to catch before it reaches a user. Byte-identical to the MATLAB copies, so the divergence check stays quiet. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- .../data/fileReference.json | 25 +++++++ .../database_documents/data/fileReference.md | 66 +++++++++++++++++ .../data/fileReference_schema.json | 73 +++++++++++++++++++ 3 files changed, 164 insertions(+) create mode 100644 src/ndi/ndi_common/database_documents/data/fileReference.json create mode 100644 src/ndi/ndi_common/database_documents/data/fileReference.md create mode 100644 src/ndi/ndi_common/schema_documents/data/fileReference_schema.json diff --git a/src/ndi/ndi_common/database_documents/data/fileReference.json b/src/ndi/ndi_common/database_documents/data/fileReference.json new file mode 100644 index 00000000..845ea9a8 --- /dev/null +++ b/src/ndi/ndi_common/database_documents/data/fileReference.json @@ -0,0 +1,25 @@ +{ + "document_class": { + "definition": "$NDIDOCUMENTPATH\/data\/fileReference.json", + "validation": "$NDISCHEMAPATH\/data\/fileReference_schema.json", + "class_name": "fileReference", + "property_list_name": "fileReference", + "class_version": 1, + "superclasses": [ + { "definition": "$NDIDOCUMENTPATH\/base.json"} + ] + }, + "depends_on": [ + { "name": "document_id", "value": "" } + ], + "fileReference": { + "filename": "", + "originalPath": "", + "formatOntology": "", + "dateCreated": 0, + "dateUpdated": 0, + "fileSize": 0, + "checksum": "", + "checksumAlgorithm": "MD5" + } +} diff --git a/src/ndi/ndi_common/database_documents/data/fileReference.md b/src/ndi/ndi_common/database_documents/data/fileReference.md new file mode 100644 index 00000000..10746a64 --- /dev/null +++ b/src/ndi/ndi_common/database_documents/data/fileReference.md @@ -0,0 +1,66 @@ +# fileReference + +The `fileReference` document class records a file that exists **outside** +the database: its name, its size, its dates and its checksum. It holds no +file of its own. + +## Why it is not generic_file + +`generic_file` is for a file the database **holds**. Its class declares +`generic_file.ext` with `mustbenotempty: 1`, and the cloud download path +(`ndi.cloud.download.downloadGenericFiles`) exists to fetch those bytes +back out. A `generic_file` document without its file is not a lighter +version of one — it is an invalid one. + +That distinction was learned the hard way. `ndi.fun.doc.gene.makeSourceFile` +used to describe a source `.gef` by creating a `generic_file` document and +deliberately not attaching the file, on the reasoning that ingesting a copy +of a 9.4 GB container beside the pyramid derived from it doubles the +storage to hold bytes the pyramid already summarises. The reasoning was +right; the class was wrong. The documents were schema-invalid from the +moment they were made, and stayed invisible only because DID's file +validation used to fail open — it fell through to "valid" when a required +file was absent (fixed in did-matlab #182). When that was tightened, every +affected session became uncopyable into a dataset, with an error naming a +file_list that was in fact perfectly correct. + +So the two cases now have two classes, and the difference is in the name: + +| | holds the bytes | records the identity | +| --- | --- | --- | +| `generic_file` | yes, `generic_file.ext` required | — | +| `fileReference` | no files at all | filename, size, dates, checksum | + +## What identifies the file + +**The checksum, not the path.** A filename tells you what somebody called +a file; a checksum tells you whether the file you have now is the file the +document was made from. Two SAW runs of the same chip produce files with +the same name and different contents, which is exactly the confusion this +is meant to survive. `originalPath` is recorded as a hint about where the +file was, not a promise about where it is — paths do not survive moving +between machines. + +`checksumAlgorithm` is stored alongside the checksum so a later reader can +verify it without guessing. An empty `checksum` means *not computed*; it +never means the file was empty. + +`fileSize` is a cheap second field: two files of different sizes are +certainly not the same file, and the size is readable without opening it. + +## Pointing at one + +`fileReference` carries the same optional `document_id` dependency +`generic_file` does, so it can name a document it is about. In the other +direction, documents point at it through their own dependencies — +`spatialGeneExpressionPyramid`, `spatialGeneExpressionTiles` and +`spatialGeneExpressionCells` all declare an optional `source_file_id`, +which may name either class. Those dependencies are untyped, so a session +built before this class existed, pointing `source_file_id` at a +`generic_file`, keeps working. + +## See also + +`ndi.fun.doc.gene.makeSourceFile`, which creates a `fileReference` by +default and a `generic_file` when asked to attach the file; +`ndi.cloud.download.downloadGenericFiles`, which is for the other class. diff --git a/src/ndi/ndi_common/schema_documents/data/fileReference_schema.json b/src/ndi/ndi_common/schema_documents/data/fileReference_schema.json new file mode 100644 index 00000000..b894efdf --- /dev/null +++ b/src/ndi/ndi_common/schema_documents/data/fileReference_schema.json @@ -0,0 +1,73 @@ +{ + "classname": "fileReference", + "superclasses": [ "base" ], + "depends_on": [ + { "name": "document_id", "mustbenotempty": 0} + ], + "fileReference": [ + { + "name": "filename", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The file's name with extension, without a directory (e.g. 'Opossum.tissue.gef')." + }, + { + "name": "originalPath", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The full path the file was read from, if it is worth recording. It is a hint about where the file was, not a promise about where it is: paths do not survive being moved between machines, which is why the checksum rather than this field identifies the file." + }, + { + "name": "formatOntology", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The ontology node describing the file format in the form of ontology:nodeID (e.g 'EMPTY:0000002'). Empty when not known; a wrong term is worse than none because it is machine-readable and will be believed." + }, + { + "name": "dateCreated", + "type": "double", + "default_value": [0], + "parameters": [0,10000000,1], + "queryable": 1, + "documentation": "The time the file was created, as a datenum. The file's own date, not the date the document was made." + }, + { + "name": "dateUpdated", + "type": "double", + "default_value": [0], + "parameters": [0,10000000,1], + "queryable": 1, + "documentation": "The time the file was last updated, as a datenum." + }, + { + "name": "fileSize", + "type": "double", + "default_value": [0], + "parameters": [0,1e18,1], + "queryable": 1, + "documentation": "The file's size in bytes, or 0 if it was not recorded. A cheap second identifying field: two files that differ in size are certainly not the same file, and it is readable without opening the file." + }, + { + "name": "checksum", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The checksum of the file's contents, in the algorithm named by checksumAlgorithm. Empty when it was not computed; empty means unknown and never means the file had no content." + }, + { + "name": "checksumAlgorithm", + "type": "string", + "default_value": "MD5", + "parameters": "", + "queryable": 1, + "documentation": "The algorithm the checksum was computed with, so a future reader can verify it without guessing. 'MD5' is what ndi.fun.file.MD5 produces." + } + ] +} From 3e1ad5d60860a0af0bd97fca5b2317b7b66474ad Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 10:53:13 +0000 Subject: [PATCH 24/52] Show how far along a cloud fetch is, not just that one started stage() printed a label and then nothing moved until the step finished. On a directory-backed session that is fine: the reads are milliseconds. On a cloud-backed one the cell table, the contour file and the gene list are each a whole download before the window can appear, and a static label for a minute is indistinguishable from a hang -- which is what a first open of a cloud dataset actually looks like. getFile takes an optional progress callback, called per chunk with the running total and the Content-Length. That is the one place every on-demand fetch passes through -- cell table, contours, gene list and every tile all arrive by fetch_cloud_file -> getFile -- so reporting anywhere else would cover part of the wait and not the rest. It reaches the caller through a module-level observer rather than a parameter, because there is nowhere to thread one through: DID fixes the file handler's signature, counting positional parameters to choose between the two- and three-argument forms, so a fourth would be read as the context dict. stage() installs the renderer itself, so every step gets the bar with no change at the call sites and the label reads as one line: [genepyramid] reading the cell table: 12.3 MB of 26.0 MB (47%) Three things it deliberately does not do. It does not draw when stderr is not a terminal, because a \r-redrawn line becomes thousands of lines in a log or a CI capture; the stage timings still say what happened. It does not report a total the server did not send, rather than inventing a denominator for a chunked response. And a callback that throws is dropped and the download continues -- reporting is never worth the bytes. The renderer locks its own state: tiles are fetched from _TileFetcher worker threads, so the observer is called from more than one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/api/files.py | 29 +++++ src/ndi/cloud/filehandler.py | 47 +++++++- src/ndi/gui/app/genepyramid/progress.py | 92 ++++++++++++++- tests/test_gui_genepyramid_progress.py | 149 ++++++++++++++++++++++++ 4 files changed, 314 insertions(+), 3 deletions(-) create mode 100644 tests/test_gui_genepyramid_progress.py diff --git a/src/ndi/cloud/api/files.py b/src/ndi/cloud/api/files.py index 680f8178..47c1c549 100644 --- a/src/ndi/cloud/api/files.py +++ b/src/ndi/cloud/api/files.py @@ -180,6 +180,8 @@ def getFile( url: NonEmptyStr, target_path: str | Path, timeout: int = 120, + *, + progress=None, ) -> bool: """Download a file from a presigned URL. @@ -187,6 +189,17 @@ def getFile( without a backup. Callers that must not clobber an existing file have to check for it themselves. + ``progress``, if given, is called as ``progress(done, total)`` after each + chunk, in bytes; *total* is the ``Content-Length`` the server sent, or + None when it sent none. It is keyword-only so it cannot be mistaken for + ``timeout``, and it is called from whichever thread is downloading -- + a caller that renders it is responsible for its own locking. + + WHY HERE. This is the one place every on-demand fetch passes through: + the cell table, the contour file, the gene list and every pyramid tile + all arrive by fetch_cloud_file -> getFile. Reporting anywhere else + would cover some of the wait and not the rest. + MATLAB equivalent: +cloud/+api/+files/getFile.m """ import logging @@ -201,9 +214,25 @@ def getFile( resp = requests.get(url, timeout=timeout, stream=True) if resp.status_code == 200: + total = None + if progress is not None: + # Absent on a chunked response, and a caller must cope with that + # rather than the bar guessing a denominator. + try: + total = int(resp.headers.get("Content-Length", "") or 0) or None + except (TypeError, ValueError): + total = None + done = 0 with open(target_path, "wb") as fh: for chunk in resp.iter_content(chunk_size=8192): fh.write(chunk) + if progress is not None: + done += len(chunk) + # A reporting callback must never cost the download. + try: + progress(done, total) + except Exception: # noqa: BLE001 + progress = None return True # Log the failure with S3 error details when available diff --git a/src/ndi/cloud/filehandler.py b/src/ndi/cloud/filehandler.py index 2b65caf4..0b1b9f8c 100644 --- a/src/ndi/cloud/filehandler.py +++ b/src/ndi/cloud/filehandler.py @@ -15,6 +15,7 @@ from __future__ import annotations +import contextlib import logging from pathlib import Path from typing import TYPE_CHECKING @@ -118,6 +119,37 @@ def updateFileInfoForRemoteFiles(doc_props: dict, cloud_dataset_id: str) -> None files["file_info"] = fi_list[0] +#: Installed by :func:`watchFetches`. Module-level rather than a parameter +#: because DID fixes the file handler's signature -- it counts positional +#: parameters to decide between the two- and three-argument forms -- so +#: there is nowhere to thread one through from a caller. +_fetch_observer = None + + +@contextlib.contextmanager +def watchFetches(observer): + """Report every on-demand cloud fetch to *observer* for the duration. + + *observer* is called as ``observer(event, uri, done, total)``, where + *event* is ``"start"``, ``"chunk"`` or ``"done"``, sizes are bytes, and + *total* is None when the server sent no Content-Length. + + Restores the previous observer on exit, including on an exception, so + nested use and a failed launch both leave the hook as they found it. + + Fetches happen on whichever thread asked for the file -- the launch + thread for the cell table and the gene list, a _TileFetcher worker for + a tile -- so an observer that renders anything owns its own locking. + """ + global _fetch_observer + previous = _fetch_observer + _fetch_observer = observer + try: + yield + finally: + _fetch_observer = previous + + def fetch_cloud_file( ndic_uri: str, target_path: str | Path, @@ -163,7 +195,20 @@ def fetch_cloud_file( tmp_path = target.with_suffix(target.suffix + ".tmp") logger.debug("Fetching cloud file %s -> %s", ndic_uri, target) - success = getFile(download_url, tmp_path, timeout=300) + observer = _fetch_observer + if observer is None: + success = getFile(download_url, tmp_path, timeout=300) + else: + observer("start", ndic_uri, 0, None) + try: + success = getFile( + download_url, + tmp_path, + timeout=300, + progress=lambda done, total: observer("chunk", ndic_uri, done, total), + ) + finally: + observer("done", ndic_uri, 0, None) if success: tmp_path.rename(target) diff --git a/src/ndi/gui/app/genepyramid/progress.py b/src/ndi/gui/app/genepyramid/progress.py index 32144eb1..479b6ff1 100644 --- a/src/ndi/gui/app/genepyramid/progress.py +++ b/src/ndi/gui/app/genepyramid/progress.py @@ -17,9 +17,10 @@ import contextlib import os import sys +import threading import time -__all__ = ["stage", "note"] +__all__ = ["stage", "note", "cloudFetches", "humanBytes"] #: Set NDI_GENEPYRAMID_QUIET=1 to silence the launch narration. _QUIET_ENV = "NDI_GENEPYRAMID_QUIET" @@ -49,7 +50,94 @@ def stage(label: str): print(f"[genepyramid] {label} ...", file=sys.stderr, flush=True) t0 = time.perf_counter() try: - yield + # Every stage gets the byte bar. On a directory-backed session + # nothing is fetched and this costs one hook install; on a + # cloud-backed one it is the difference between a label that sits + # there for a minute and a line that moves. + with cloudFetches(label): + yield finally: dt = time.perf_counter() - t0 print(f"[genepyramid] {label}: {dt:.1f}s", file=sys.stderr, flush=True) + + +def humanBytes(n: int) -> str: + """A byte count at a glance. 26214400 is not a number anyone reads.""" + step = 1024.0 + value = float(n) + for unit in ("B", "KB", "MB", "GB"): + if value < step or unit == "GB": + return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}" + value /= step + return f"{value:.1f} GB" + + +@contextlib.contextmanager +def cloudFetches(label: str = "fetching"): + """Draw a progress line for every cloud file fetched inside this block. + + A session opened from a local directory fetches nothing and this costs + one hook install; there is no output and no reason to ask the caller + which kind of session they have. + + A CLOUD session is the case this exists for. The cell table, the + contour file and the gene list are each a whole download before the + window can appear, and stage() alone prints a label and then nothing + moves for however long that takes -- which is indistinguishable from a + hang. This redraws one line in place with bytes and, when the server + sent a Content-Length, a percentage. + + ONE LINE, AND ONLY FOR THE FILE BEING WAITED ON. Tiles are fetched + from _TileFetcher threads once the viewer is up, several possibly at + once; interleaving a line each would scroll the launch notes away. So + the renderer tracks a count of files and shows the newest, and every + write is under a lock because the callers are threads. + """ + if _quiet() or not sys.stderr.isatty(): + # Not a terminal: a redrawn line becomes thousands of lines in a + # log or a CI capture. The stage timings still say what happened. + yield + return + + try: + from ....cloud.filehandler import watchFetches + except Exception: # noqa: BLE001 - reporting is never worth the launch + yield + return + + state = {"files": 0, "done": 0, "total": None, "shown": False} + lock = threading.Lock() + + def render(): + n, done, total = state["files"], state["done"], state["total"] + if total: + pct = min(100.0, 100.0 * done / total) + body = f"{humanBytes(done)} of {humanBytes(total)} ({pct:.0f}%)" + else: + body = humanBytes(done) + suffix = f", file {n}" if n > 1 else "" + sys.stderr.write(f"\r[genepyramid] {label}: {body}{suffix} ") + sys.stderr.flush() + state["shown"] = True + + def observer(event, _uri, done, total): + with lock: + if event == "start": + state["files"] += 1 + state["done"] = 0 + state["total"] = None + elif event == "chunk": + state["done"] = done + state["total"] = total + render() + + try: + with watchFetches(observer): + yield + finally: + with lock: + if state["shown"]: + # Leave the cursor on a clean line, so the next stage note + # does not land on top of half a progress bar. + sys.stderr.write("\r" + " " * 78 + "\r") + sys.stderr.flush() diff --git a/tests/test_gui_genepyramid_progress.py b/tests/test_gui_genepyramid_progress.py new file mode 100644 index 00000000..9cf9f787 --- /dev/null +++ b/tests/test_gui_genepyramid_progress.py @@ -0,0 +1,149 @@ +"""The launch says how far along a cloud fetch is, not just that one started. + +stage() printed a label and then nothing moved until the step finished. On +a directory-backed session that is fine -- the reads are milliseconds. On a +cloud-backed one the cell table, the contour file and the gene list are +each a whole download before the window can appear, and a static label for +a minute is indistinguishable from a hang. + +getFile is the one place every on-demand fetch passes through, so the byte +counts are taken there and surfaced through a module-level observer: DID +fixes the file handler's signature (it counts positional parameters to +choose between the two- and three-argument forms), so there is nowhere to +thread a callback through from a caller. +""" + +import io +import sys +import unittest +from unittest import mock + +from ndi.cloud import filehandler +from ndi.gui.app.genepyramid import progress + + +class TestHumanBytes(unittest.TestCase): + def test_it_reads_at_a_glance(self): + self.assertEqual(progress.humanBytes(0), "0 B") + self.assertEqual(progress.humanBytes(900), "900 B") + self.assertEqual(progress.humanBytes(26214400), "25.0 MB") + + def test_a_big_file_does_not_become_a_wall_of_digits(self): + self.assertEqual(progress.humanBytes(3 * 1024**3), "3.0 GB") + + +class TestTheObserverHook(unittest.TestCase): + def test_it_is_absent_by_default(self): + self.assertIsNone(filehandler._fetch_observer) + + def test_it_is_installed_for_the_block_and_removed_after(self): + seen = [] + with filehandler.watchFetches(lambda *a: seen.append(a)): + self.assertIsNotNone(filehandler._fetch_observer) + self.assertIsNone(filehandler._fetch_observer) + + def test_a_raise_inside_the_block_still_removes_it(self): + with self.assertRaises(ValueError): + with filehandler.watchFetches(lambda *a: None): + raise ValueError("boom") + self.assertIsNone(filehandler._fetch_observer) + + def test_nesting_restores_the_outer_one_rather_than_clearing(self): + outer = lambda *a: None # noqa: E731 + with filehandler.watchFetches(outer): + with filehandler.watchFetches(lambda *a: None): + pass + self.assertIs(filehandler._fetch_observer, outer) + self.assertIsNone(filehandler._fetch_observer) + + +class _FakeResponse: + """Enough of requests.Response for getFile's streaming branch.""" + + status_code = 200 + + def __init__(self, chunks, length=None): + self._chunks = chunks + self.headers = {} if length is None else {"Content-Length": str(length)} + + def iter_content(self, chunk_size=8192): # noqa: ARG002 + yield from self._chunks + + +class TestGetFileReportsBytes(unittest.TestCase): + def _run(self, chunks, length, progress_fn, tmp): + from ndi.cloud.api import files as files_api + + with mock.patch("requests.get", return_value=_FakeResponse(chunks, length)): + return files_api.getFile("https://example.invalid/x", tmp, progress=progress_fn) + + def setUp(self): + import tempfile + from pathlib import Path + + self.dir = tempfile.mkdtemp() + self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True)) + self.tmp = Path(self.dir) / "out.bin" + + def test_the_running_total_is_cumulative_not_per_chunk(self): + seen = [] + self._run([b"a" * 10, b"b" * 5], 15, lambda d, t: seen.append((d, t)), self.tmp) + self.assertEqual(seen, [(10, 15), (15, 15)]) + + def test_the_total_comes_from_content_length(self): + seen = [] + self._run([b"x" * 4], 4, lambda d, t: seen.append((d, t)), self.tmp) + self.assertEqual(seen[-1][1], 4) + + def test_a_chunked_response_reports_no_total_rather_than_guessing(self): + seen = [] + self._run([b"x" * 4], None, lambda d, t: seen.append((d, t)), self.tmp) + self.assertIsNone(seen[-1][1]) + + def test_a_nonsense_content_length_is_no_total_rather_than_a_crash(self): + from ndi.cloud.api import files as files_api + + resp = _FakeResponse([b"x"], None) + resp.headers = {"Content-Length": "banana"} + seen = [] + with mock.patch("requests.get", return_value=resp): + files_api.getFile( + "https://example.invalid/x", self.tmp, progress=lambda d, t: seen.append((d, t)) + ) + self.assertIsNone(seen[-1][1]) + + def test_the_file_still_lands_when_the_callback_throws(self): + # A reporting callback must never cost the download. + def boom(done, total): + raise RuntimeError("renderer died") + + ok = self._run([b"a" * 8, b"b" * 8], 16, boom, self.tmp) + self.assertTrue(ok) + self.assertEqual(self.tmp.read_bytes(), b"a" * 8 + b"b" * 8) + + def test_no_callback_is_the_untouched_path(self): + ok = self._run([b"z" * 3], 3, None, self.tmp) + self.assertTrue(ok) + self.assertEqual(self.tmp.read_bytes(), b"z" * 3) + + +class TestTheRendererStaysOutOfLogs(unittest.TestCase): + def test_a_non_tty_gets_no_redrawn_line(self): + # A \r-redrawn bar becomes thousands of lines in a CI capture, so + # it is suppressed there; the stage timings still say what happened. + buf = io.StringIO() # StringIO.isatty() is False + with mock.patch.object(sys, "stderr", buf): + with progress.cloudFetches("reading"): + self.assertIsNone(filehandler._fetch_observer) + self.assertEqual(buf.getvalue(), "") + + def test_quiet_suppresses_it_too(self): + import os + + with mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_QUIET": "1"}): + with progress.cloudFetches("reading"): + self.assertIsNone(filehandler._fetch_observer) + + +if __name__ == "__main__": + unittest.main() From 32fa8516c9fb8374a24c23a516bce8bebebfeb24 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 11:19:21 +0000 Subject: [PATCH 25/52] Put the launch progress in a window, since that is where it is watched The stderr bar was the wrong medium. The viewer is normally started from NDI-matlab's GEF Manager View button, which runs it as a child process with no terminal attached: the bar rendered to a stderr nobody was looking at, and reported nothing while the Dock icon bounced. So the indicator is now a small Qt window -- a stage label and a bar -- built BEFORE napari is imported, because the wait starts before there is a viewer to host it. Opening the session, the cell table and the contour file are all spent first, and on a cold cache each is a whole download. napari reuses the QApplication this creates; the high-DPI attributes are set here for that reason, since napari sets them before creating its own and an app made without them renders at the wrong scale on a Retina display. NDI_GENEPYRAMID_PROGRESS forces gui, text or off. Unset, it takes the window when one can be built, the stderr bar when stderr is a terminal, and neither otherwise. The stderr narration itself stays on in every mode: it is the record of where the time went. THE DISPLAY GUARD IS THE PART THAT MATTERS. Qt does not raise when it cannot reach a display -- it prints a line and calls abort() -- so a try/except around QApplication() catches nothing, and the first run of the test suite took pytest down with it rather than falling back. The guard checks the platform and the environment before touching Qt at all: macOS and Windows have a window server, X11 and Wayland say so in the environment, and an explicit QT_QPA_PLATFORM counts (which is how this is tested, offscreen). A headless launch now gets text or nothing, and never a dead process. Byte updates from a _TileFetcher worker are dropped rather than drawn: touching a widget off the GUI thread is a crash, not a wrong number. The window closes when the viewer is built and about to be handed over -- an indicator that outlives what it reports on makes a finished launch look stuck, and it would sit over the picture it exists to reach. 25 tests, all headless. The display guard is covered directly, because the failure it prevents cannot be caught after the fact. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/progress.py | 183 ++++++++++++++++++++++-- src/ndi/gui/app/genepyramid/viewer.py | 8 +- tests/test_gui_genepyramid_progress.py | 111 +++++++++++++- 3 files changed, 287 insertions(+), 15 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/progress.py b/src/ndi/gui/app/genepyramid/progress.py index 479b6ff1..7b428dd8 100644 --- a/src/ndi/gui/app/genepyramid/progress.py +++ b/src/ndi/gui/app/genepyramid/progress.py @@ -20,11 +20,15 @@ import threading import time -__all__ = ["stage", "note", "cloudFetches", "humanBytes"] +__all__ = ["stage", "note", "cloudFetches", "humanBytes", "closeLaunchWindow"] #: Set NDI_GENEPYRAMID_QUIET=1 to silence the launch narration. _QUIET_ENV = "NDI_GENEPYRAMID_QUIET" +#: "gui", "text" or "off". Unset picks gui when a Qt binding loads and a +#: window can be created, text when stderr is a terminal, off otherwise. +_MODE_ENV = "NDI_GENEPYRAMID_PROGRESS" + def _quiet() -> bool: return bool(os.environ.get(_QUIET_ENV, "").strip()) @@ -47,13 +51,18 @@ def stage(label: str): if _quiet(): yield return + # The stderr narration stays on in every mode. It is the record of + # where the time went, and it costs one print per step whether or not + # anyone is watching a window. print(f"[genepyramid] {label} ...", file=sys.stderr, flush=True) + if _mode() == "gui": + _guiStage(label) t0 = time.perf_counter() try: - # Every stage gets the byte bar. On a directory-backed session - # nothing is fetched and this costs one hook install; on a + # Every stage gets the byte reporting. On a directory-backed + # session nothing is fetched and this costs one hook install; on a # cloud-backed one it is the difference between a label that sits - # there for a minute and a line that moves. + # there for a minute and something that moves. with cloudFetches(label): yield finally: @@ -93,9 +102,8 @@ def cloudFetches(label: str = "fetching"): the renderer tracks a count of files and shows the newest, and every write is under a lock because the callers are threads. """ - if _quiet() or not sys.stderr.isatty(): - # Not a terminal: a redrawn line becomes thousands of lines in a - # log or a CI capture. The stage timings still say what happened. + mode = _mode() + if mode == "off": yield return @@ -107,6 +115,7 @@ def cloudFetches(label: str = "fetching"): state = {"files": 0, "done": 0, "total": None, "shown": False} lock = threading.Lock() + launch_thread = threading.get_ident() def render(): n, done, total = state["files"], state["done"], state["total"] @@ -129,7 +138,13 @@ def observer(event, _uri, done, total): elif event == "chunk": state["done"] = done state["total"] = total - render() + if mode == "gui": + # Only from the launch thread. Touching a widget from + # a _TileFetcher worker is a crash, not a wrong number. + if threading.get_ident() == launch_thread: + _guiBytes(done, total) + else: + render() try: with watchFetches(observer): @@ -141,3 +156,155 @@ def observer(event, _uri, done, total): # does not land on top of half a progress bar. sys.stderr.write("\r" + " " * 78 + "\r") sys.stderr.flush() + + +# ------------------------------------------------------------------ gui + +#: The launch window, once built. One per process: the launch is one +#: sequence on one thread, and a second window would be a second claim +#: about what is happening. +_window = None +_window_failed = False + + +def _mode() -> str: + """Which indicator to use: ``"gui"``, ``"text"`` or ``"off"``.""" + asked = os.environ.get(_MODE_ENV, "").strip().lower() + if asked in ("gui", "text", "off"): + return asked + if _quiet(): + return "off" + # GUI first: the viewer is normally started from NDI-matlab's View + # button, where there is no terminal at all and a stderr bar is + # written to nobody. Falling back to text keeps a terminal launch + # informative. + if _launchWindow() is not None: + return "gui" + return "text" if sys.stderr.isatty() else "off" + + +def _displayLikely() -> bool: + """Whether building a QApplication is safe to attempt. + + THIS GUARD IS NOT OPTIONAL. Qt does not raise when it cannot reach a + display: it prints "could not connect to display" and calls abort(). + That kills the process, so try/except around QApplication() catches + nothing -- the launch would die instead of falling back to text, which + is a worse failure than having no progress window at all. + + macOS and Windows always have a window server, and they are where the + viewer is started from NDI-matlab's View button. On X11/Wayland the + environment says: DISPLAY or WAYLAND_DISPLAY, or an explicit + QT_QPA_PLATFORM (offscreen included, which is how this is tested). + """ + if sys.platform in ("darwin", "win32"): + return True + return bool( + os.environ.get("DISPLAY") + or os.environ.get("WAYLAND_DISPLAY") + or os.environ.get("QT_QPA_PLATFORM") + ) + + +def _launchWindow(): + """The launch window, built on first use, or None if Qt will not. + + Built BEFORE napari is imported, because the wait this reports on + starts before there is a viewer to host it -- opening the session, + the cell table and the contour file are all spent before napari is + even imported. That means creating the QApplication ourselves, which + napari then reuses. + + The high-DPI attributes are set here for that reason: napari sets + them before creating its own app, and on a Retina display an app + created without them renders at the wrong scale. Setting them is + harmless where they are already the default (Qt6 deprecates both). + """ + global _window, _window_failed + if _window is not None or _window_failed: + return _window + try: + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QApplication, + QLabel, + QProgressBar, + QVBoxLayout, + QWidget, + ) + + app = QApplication.instance() + if app is None and not _displayLikely(): + # No app to borrow and no display to make one against. See + # _displayLikely: attempting it anyway aborts the process. + _window_failed = True + return None + if app is None: + for attr in ("AA_EnableHighDpiScaling", "AA_UseHighDpiPixmaps"): + with contextlib.suppress(AttributeError): + QApplication.setAttribute(getattr(Qt, attr), True) + app = QApplication([]) + + w = QWidget() + w.setWindowTitle("Opening pyramid") + layout = QVBoxLayout(w) + title = QLabel("Opening pyramid") + font = title.font() + font.setBold(True) + title.setFont(font) + step = QLabel("starting ...") + bar = QProgressBar() + bar.setRange(0, 0) # busy until a byte count gives it a range + layout.addWidget(title) + layout.addWidget(step) + layout.addWidget(bar) + w.resize(420, 110) + w.show() + app.processEvents() + + _window = {"app": app, "widget": w, "step": step, "bar": bar} + except Exception: # noqa: BLE001 - no display, no binding, no window + _window_failed = True + _window = None + return _window + + +def _guiStage(label: str) -> None: + win = _launchWindow() + if win is None: + return + with contextlib.suppress(Exception): + win["step"].setText(label) + win["bar"].setRange(0, 0) # unknown length again + win["app"].processEvents() + + +def _guiBytes(done: int, total) -> None: + win = _launchWindow() + if win is None: + return + with contextlib.suppress(Exception): + if total: + win["bar"].setRange(0, 100) + win["bar"].setValue(int(100 * done / total)) + win["bar"].setFormat(f"{humanBytes(done)} of {humanBytes(total)} (%p%)") + else: + win["bar"].setRange(0, 0) + win["bar"].setFormat(humanBytes(done)) + win["app"].processEvents() + + +def closeLaunchWindow() -> None: + """Take the launch window down. Safe to call when there never was one. + + Called once the viewer is built and about to be handed to the user: + a progress window that outlives the thing it was reporting on is + worse than none, because it makes a finished launch look stuck. + """ + global _window + win, _window = _window, None + if win is None: + return + with contextlib.suppress(Exception): + win["widget"].close() + win["app"].processEvents() diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index b19a3327..02f12840 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -95,7 +95,7 @@ def openPyramid( Returns: The napari Viewer. """ - from .progress import note, stage + from .progress import closeLaunchWindow, note, stage with stage("importing napari"): napari = require_napari() @@ -171,6 +171,12 @@ def openPyramid( labelings=labelings, ) + # The launch window has reported everything it can: the viewer is + # built and the panels are on it. Leaving it up past this point would + # make a finished launch look stuck, and it would sit over the picture + # it exists to get you to. + closeLaunchWindow() + if show: napari.run() return viewer diff --git a/tests/test_gui_genepyramid_progress.py b/tests/test_gui_genepyramid_progress.py index 9cf9f787..fc2af06a 100644 --- a/tests/test_gui_genepyramid_progress.py +++ b/tests/test_gui_genepyramid_progress.py @@ -128,14 +128,13 @@ def test_no_callback_is_the_untouched_path(self): class TestTheRendererStaysOutOfLogs(unittest.TestCase): - def test_a_non_tty_gets_no_redrawn_line(self): - # A \r-redrawn bar becomes thousands of lines in a CI capture, so - # it is suppressed there; the stage timings still say what happened. - buf = io.StringIO() # StringIO.isatty() is False - with mock.patch.object(sys, "stderr", buf): + def test_off_installs_no_observer_at_all(self): + # A \r-redrawn bar becomes thousands of lines in a CI capture. + import os + + with mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_PROGRESS": "off"}): with progress.cloudFetches("reading"): self.assertIsNone(filehandler._fetch_observer) - self.assertEqual(buf.getvalue(), "") def test_quiet_suppresses_it_too(self): import os @@ -144,6 +143,106 @@ def test_quiet_suppresses_it_too(self): with progress.cloudFetches("reading"): self.assertIsNone(filehandler._fetch_observer) + def test_text_mode_does_install_one(self): + import os + + with mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_PROGRESS": "text"}): + with progress.cloudFetches("reading"): + self.assertIsNotNone(filehandler._fetch_observer) + + +class TestTheDisplayGuard(unittest.TestCase): + """Qt aborts the PROCESS when it cannot reach a display. + + It does not raise, so try/except around QApplication() catches + nothing: without this guard a headless launch would die instead of + falling back to text. Found by running the test suite, which took + pytest down with it. + """ + + def test_a_mac_always_has_a_window_server(self): + with mock.patch.object(sys, "platform", "darwin"): + self.assertTrue(progress._displayLikely()) + + def test_windows_too(self): + with mock.patch.object(sys, "platform", "win32"): + self.assertTrue(progress._displayLikely()) + + def test_bare_linux_says_no(self): + import os + + with ( + mock.patch.object(sys, "platform", "linux"), + mock.patch.dict(os.environ, {}, clear=True), + ): + self.assertFalse(progress._displayLikely()) + + def test_x11_wayland_or_an_explicit_platform_say_yes(self): + import os + + for var in ("DISPLAY", "WAYLAND_DISPLAY", "QT_QPA_PLATFORM"): + with ( + mock.patch.object(sys, "platform", "linux"), + mock.patch.dict(os.environ, {var: ":0"}, clear=True), + ): + self.assertTrue(progress._displayLikely(), var) + + +class TestModePrecedence(unittest.TestCase): + def setUp(self): + # _mode consults the window, which is a process-wide singleton. + self.addCleanup(setattr, progress, "_window", progress._window) + self.addCleanup(setattr, progress, "_window_failed", progress._window_failed) + + def _mode_with(self, env): + import os + + with mock.patch.dict(os.environ, env, clear=False): + return progress._mode() + + def test_the_env_var_wins_over_everything(self): + for asked in ("gui", "text", "off"): + self.assertEqual(self._mode_with({"NDI_GENEPYRAMID_PROGRESS": asked}), asked) + + def test_quiet_means_off_whatever_else_is_true(self): + self.assertEqual(self._mode_with({"NDI_GENEPYRAMID_QUIET": "1"}), "off") + + def test_no_window_and_no_terminal_is_off_not_a_crash(self): + import os + + progress._window, progress._window_failed = None, True + with ( + mock.patch.object(sys, "stderr", io.StringIO()), + mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_PROGRESS": ""}, clear=False), + ): + self.assertEqual(progress._mode(), "off") + + +class TestClosingIsAlwaysSafe(unittest.TestCase): + def test_closing_when_there_never_was_a_window(self): + self.addCleanup(setattr, progress, "_window", progress._window) + progress._window = None + progress.closeLaunchWindow() # must not raise + + def test_closing_twice(self): + self.addCleanup(setattr, progress, "_window", progress._window) + progress._window = None + progress.closeLaunchWindow() + progress.closeLaunchWindow() + + def test_a_widget_that_throws_on_close_does_not_escape(self): + # The viewer is built by this point; a failing indicator must not + # take the window with it. + self.addCleanup(setattr, progress, "_window", progress._window) + + class _Boom: + def close(self): + raise RuntimeError("gone") + + progress._window = {"widget": _Boom(), "app": _Boom()} + progress.closeLaunchWindow() + self.assertIsNone(progress._window) + if __name__ == "__main__": unittest.main() From aba4abfc1ae262435d3cf5ad80e94ca860e4bdd5 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 12:20:19 +0000 Subject: [PATCH 26/52] Size the tile grid from the data, and build a level a band at a time Follows NDI-matlab 38aa4e8fb. A grid is a fixed FRACTION of the extent, so the same 9x9 that gives a mouse section 20 MB tiles gave a real ferret hemisphere 257 MB ones -- five times the budget the grid was sized for. grid now defaults to None and is chosen from where the records actually are, against tileBudgetBytes (50 MB by default). The budget is on the BIGGEST tile at the finest level, not the average one: tissue does not fill its bounding box, and on that ferret the densest tile ran 2.5x the median, so budgeting the median misses by that factor. gridRange (3..64) bounds the search, its upper bound a file COUNT limit rather than a geometric one -- grid**2 files per level, all of them written, stored and uploaded, which is a cost the density estimate cannot see and which is therefore reported through progressFcn. The two languages reached this from different sides. Python already sorted on a tile-major key; MATLAB scanned every record against every grid cell, which a data-sized grid makes untenable, and has now adopted the Python key. Both now go further in the same direction: - the tile, the tile-local pixel and the gene are DECODED back out of the key rather than carried alongside it through the sort. Two full-length arrays go through the sort where six used to. - a level is built one band of tile rows at a time. A tile spans th level rows and the full width of its row, so records in different tile rows never share a tile and can be sorted and written apart. The working set becomes the section divided by the grid -- and because the grid is now chosen from the data, a section big enough to need the chunking is the one that gets a fine grid to chunk with. Neither changes a byte of what is written: a tile is a contiguous rectangle, so global row-major order inside it IS local row-major order. The shared conformance tile still holds both languages to it. progressFcn(fraction, text) is new here and reports the grid it chose, then every level and every band within a level. The pyramid is the slow half of an ingest -- 951 s of a 971 s ferret build -- and reported only at its ends, so minutes of it were indistinguishable from a hang. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/fun/doc_gene.py | 366 +++++++++++++++++----- src/ndi/fun/ndi_matlab_python_bridge.yaml | 45 ++- src/ndi/gui/ndi_matlab_python_bridge.yaml | 2 +- tests/test_doc_gene_pyramid_grid.py | 207 ++++++++++++ 4 files changed, 533 insertions(+), 87 deletions(-) create mode 100644 tests/test_doc_gene_pyramid_grid.py diff --git a/src/ndi/fun/doc_gene.py b/src/ndi/fun/doc_gene.py index 91df9d63..223221df 100644 --- a/src/ndi/fun/doc_gene.py +++ b/src/ndi/fun/doc_gene.py @@ -399,7 +399,9 @@ def makePyramid( gene_list_doc, subjectID: str, binSizes=(1, 2, 4, 8, 16, 32), - grid: int = 9, + grid: int | None = None, + tileBudgetBytes: int = 50 * 2**20, + gridRange=(3, 64), basePixelSize=(0.5, 0.5), pixelSizeUnits: str = "micrometer", label: str = "", @@ -407,6 +409,7 @@ def makePyramid( chipSerial: str = "", pipelineVersion: str = "", origin=None, + progressFcn=None, ): """Bin flat spatial records into a tiled pyramid and store its documents. @@ -432,9 +435,28 @@ def makePyramid( the step's AREA factor while leaving the mean unchanged, so uniform small ratios zoom most smoothly. grid: tile grid, ``grid`` by ``grid`` at EVERY level. Constant - because nonzeros barely fall with bin size, so every level - wants comparable tiling and a viewport maps to tile indices - once, independent of zoom. + across levels because nonzeros barely fall with bin size, so + every level wants comparable tiling and a viewport maps to + tile indices once, independent of zoom. ``None``, the default, + SIZES IT FROM THE DATA: a grid is a fixed fraction of the + extent, so the same 9x9 that gives a mouse section 20 MB tiles + gave a real ferret hemisphere 257 MB ones. + tileBudgetBytes: what the automatic grid aims for in the BIGGEST + tile at the finest level, not the average one. Tissue does not + spread evenly over its bounding box, so the densest tile of a + real section runs a few times the median (2.5x, measured) and + budgeting the median misses by that factor. Bigger tiles mean + fewer files to store and upload; smaller tiles mean less to + fetch before a viewer can draw. Ignored when ``grid`` is set. + gridRange: bounds on the automatic grid. The upper bound is a file + COUNT limit rather than a geometric one -- ``grid**2`` files + per level, every one of them written, stored and uploaded. + progressFcn: called as ``progressFcn(fraction, text)`` with + ``fraction`` in [0, 1] across this call. This is the slow half + of an ingest and reporting only at its ends makes minutes of + it indistinguishable from a hang, so the grid choice, every + level and every band within a level report. ``None`` is + silent. basePixelSize: (x, y) physical size of one base pixel. origin: ``(min_x, min_y)`` of the tiled region, or None to take it from the data. Prefer the acquisition's own bounding box when @@ -482,6 +504,26 @@ def makePyramid( extent_y = int(y.max()) - min_y + 1 bins = sorted(int(b) for b in binSizes) + if grid is None: + _tick(progressFcn, 0.0, "Choosing the tile grid...") + grid, est_bytes = _choose_grid( + x, y, min_x, min_y, extent_x, extent_y, bins[0], tileBudgetBytes, gridRange + ) + # Said out loud, because the file COUNT is a cost this choice + # cannot see: grid**2 files per level, all of which have to be + # written, stored and uploaded. A caller who cares more about that + # than about fetch latency raises tileBudgetBytes or fixes grid. + _tick( + progressFcn, + 0.0, + f"Tile grid {grid}x{grid}: {grid * grid:,} files per level, " + f"biggest tile about {_human_bytes(est_bytes)}.", + ) + else: + grid = int(grid) + if grid < 1: + raise ValueError("grid must be a positive integer, or None to size it from the data") + pyr_doc = ( _blank( "spatialGeneExpressionPyramid", @@ -510,6 +552,7 @@ def makePyramid( pyr_doc = pyr_doc.set_dependency_value("geneList_id", gene_list_doc.id) pyr_doc = pyr_doc.set_dependency_value("subject_id", subjectID) + _tick(progressFcn, 0.0, "Summing per-gene totals...") totals_path = _write_gene_totals(gene_index, count, n_genes) try: pyr_doc = _store_doc(session, pyr_doc, ["gene_totals.tsv"], [totals_path]) @@ -517,27 +560,36 @@ def makePyramid( if os.path.exists(totals_path): os.unlink(totals_path) - tile_docs = [ - _make_level( - session, - x, - y, - gene_index, - count, - min_x, - min_y, - extent_x, - extent_y, - b, - grid, - n_genes, - pyr_doc, - basePixelSize, - pixelSizeUnits, - subjectID, + # Equal shares per level. Every level sorts the SAME record array -- + # coarsening merges pixels but leaves the genes distinct, so the record + # count barely falls with bin size and bin32 costs about what bin1 + # costs. Weighting the shares by output size would report a lie. + tile_docs = [] + for k, b in enumerate(bins): + lo = 0.05 + 0.95 * k / len(bins) + hi = 0.05 + 0.95 * (k + 1) / len(bins) + tile_docs.append( + _make_level( + session, + x, + y, + gene_index, + count, + min_x, + min_y, + extent_x, + extent_y, + b, + grid, + n_genes, + pyr_doc, + basePixelSize, + pixelSizeUnits, + subjectID, + _level_progress(progressFcn, lo, hi, k + 1, len(bins), b), + ) ) - for b in bins - ] + _tick(progressFcn, 1.0, "Pyramid complete.") return pyr_doc, tile_docs @@ -558,71 +610,51 @@ def _make_level( base_pixel_size, pixel_size_units, subject_id, + progress=None, ): - """Build and store one resolution level.""" + """Build and store one resolution level. + + ONE BAND OF TILE ROWS AT A TIME, and this is what decides how much + memory a section needs. A tile spans ``th`` level rows and the full + width of its row, so records in different tile rows never share a tile + and can be sorted, collapsed and written entirely apart. Doing the + whole level at once means the sort's working set scales with the + section; doing it a band at a time means it scales with the section + DIVIDED BY THE GRID -- and because the grid is chosen from the data, a + section big enough to need the chunking is exactly the one that gets a + fine grid to chunk with. + + The cost is ``grid`` passes over the coordinates to select the bands, + cheap next to ``grid`` sorts, and a build that is somewhat slower and + fits. Nothing about what is written changes: a band holds whole tiles. + """ lw = -(-extent_x // b) # ceil lh = -(-extent_y // b) tw = -(-lw // grid) th = -(-lh // grid) - n_tiles = grid * grid - - px = (x - min_x) // b - py = (y - min_y) // b - tcol = px // tw - trow = py // th - xl = px - tcol * tw - yl = py - trow * th - tid = trow * grid + tcol - - # ONE sort, on a TILE-MAJOR key, so tile boundaries fall out of the - # sorted order with a searchsorted instead of needing a second sort. - # - # int64 from the first term. This product reaches ~1e13 on a real - # section; under NumPy's weak promotion an int32 array times a Python - # int stays int32, which wraps SILENTLY at 2.15e9. Distinct - # (pixel, gene) pairs then collide and the dedup below merges them -- - # about 1.7M spurious merges on one measured section, matching the - # n**2/2**33 collision estimate. Assert rather than trust: no fixture - # small enough to run quickly can reach the overflow. - key = ((tid.astype(np.int64) * th + yl) * tw + xl) * n_genes + gi - if key.dtype != np.int64: - raise TypeError(f"sort key must be int64, got {key.dtype}") - span = np.int64(th) * tw * n_genes - if n_tiles * span >= 2**63: - raise OverflowError(f"sort key would reach {n_tiles * span:,}, past int64") - - order = np.argsort(key, kind="stable") - key = key[order] - xl = xl[order] - yl = yl[order] - g = gi[order] - # int32 is enough: a group sums at most b*b base records. - cc = c[order].astype(np.int64) - - # Collapse duplicate (pixel, gene) pairs created by binning. - new = np.empty(len(key), bool) - new[0] = True - np.not_equal(key[1:], key[:-1], out=new[1:]) - starts = np.flatnonzero(new) - cc = np.add.reduceat(cc, starts) - xl, yl, g, key = xl[starts], yl[starts], g[starts], key[starts] - cc = np.minimum(cc, 65535) # data_type_count is uint16 - - tid_sorted = key // span - bounds = np.searchsorted(tid_sorted, np.arange(n_tiles + 1)) names, paths = [], [] tmpdir = tempfile.mkdtemp() try: - for t in range(n_tiles): - lo, hi = int(bounds[t]), int(bounds[t + 1]) - if lo == hi: - continue # tiles with no data are not written - name = f"tile.bin_{t + TILE_INDEX_ORIGIN}" - p = os.path.join(tmpdir, name) - writeTileFile(p, xl[lo:hi], yl[lo:hi], g[lo:hi], cc[lo:hi]) - names.append(name) - paths.append(p) + for trow in range(grid): + _tick(progress, 0.05 + 0.85 * trow / grid, f"tile row {trow + 1} of {grid}...") + # The band's bounds in SOURCE units. Computing each record's + # level row to compare against would allocate the full-length + # array this loop exists not to allocate; the bounds map back + # into source units exactly, so the comparison happens on the + # coordinates as they are. + y_lo = min_y + trow * th * b + y_hi = y_lo + th * b + sel = (y >= y_lo) & (y < y_hi) + if not sel.any(): + continue + for name, data in _make_band( + x[sel], y[sel], gi[sel], c[sel], min_x, min_y, b, grid, tw, th, n_genes + ): + p = os.path.join(tmpdir, name) + writeTileFile(p, *data) + names.append(name) + paths.append(p) doc = ( _blank( @@ -658,6 +690,7 @@ def _make_level( ) doc = doc.set_dependency_value("spatialGeneExpressionPyramid_id", pyr_doc.id) doc = doc.set_dependency_value("subject_id", subject_id) + _tick(progress, 0.92, f"storing {len(names)} tiles...") return _store_doc(session, doc, names, paths) finally: for p in paths: @@ -667,6 +700,179 @@ def _make_level( os.rmdir(tmpdir) +def _make_band(x, y, gi, c, min_x, min_y, b, grid, tw, th, n_genes): + """Collapse one band of tile rows, yielding ``(name, arrays)`` per tile. + + THE KEY SORTS TILE-MAJOR: tile index, then the pixel within the tile, + then the gene. Two things follow. + + Tile boundaries fall out of the sorted order, so no second sort and no + pass per grid cell is needed to find them -- which matters once the + grid is sized from the data and can reach 64x64, i.e. 4096 cells. + + And the key already encodes the tile, the tile-local pixel AND the + gene, so none of them has to survive the sort: they are DECODED back + out of it afterwards. Two full-length arrays go through the sort where + six used to. The decode is exact for as long as the int64 guard below + holds, which is the condition the key already needed. + + int64 from the first term. This product reaches ~1e13 on a real + section; under NumPy's weak promotion an int32 array times a Python int + stays int32, which wraps SILENTLY at 2.15e9. Distinct (pixel, gene) + pairs then collide and the dedup merges them -- about 1.7M spurious + merges on one measured section, matching the n**2/2**33 collision + estimate. Assert rather than trust: no fixture small enough to run + quickly can reach the overflow. + """ + py = (y - min_y) // b + trow = py // th + yl = py - trow * th + del py + px = (x - min_x) // b + tcol = px // tw + xl = px - tcol * tw + del px + tid = trow * grid + tcol + del trow, tcol + + key = ((tid.astype(np.int64) * th + yl) * tw + xl) * n_genes + gi + if key.dtype != np.int64: + raise TypeError(f"sort key must be int64, got {key.dtype}") + span = np.int64(th) * tw * n_genes + if grid * grid * span >= 2**63: + raise OverflowError(f"sort key would reach {grid * grid * span:,}, past int64") + del tid, xl, yl + + order = np.argsort(key, kind="stable") + key = key[order] + # int64 because a group sums at most b*b base records. + cc = c[order].astype(np.int64) + del order + + # Collapse duplicate (pixel, gene) pairs created by binning. + new = np.empty(len(key), bool) + new[0] = True + np.not_equal(key[1:], key[:-1], out=new[1:]) + starts = np.flatnonzero(new) + del new + cc = np.minimum(np.add.reduceat(cc, starts), 65535) # data_type_count is uint16 + key = key[starts] + del starts + + # Decode the survivors back out of the key. xl and yl come out already + # tile-local, which is the form writeTileFile wants. + g, key = key % n_genes, key // n_genes + xl, key = key % tw, key // tw + yl, tid = key % th, key // th + del key + + # One contiguous slice per occupied tile. Tiles with no data are not + # written at all, so the stored series has holes in it and a + # document's file list, not a walk, is what reports which exist. + t_start = np.flatnonzero(np.r_[True, tid[1:] != tid[:-1]]) + t_end = np.r_[t_start[1:], len(tid)] + for lo, hi in zip(t_start, t_end): + name = f"tile.bin_{int(tid[lo]) + TILE_INDEX_ORIGIN}" + yield name, (xl[lo:hi], yl[lo:hi], g[lo:hi], cc[lo:hi]) + + +def _tick(fcn, frac, txt): + """Silent when no handle was given, so every phase can report progress + without each call site testing for a display first.""" + if fcn is None: + return + fcn(float(frac), txt) + + +def _level_progress(fcn, lo, hi, k, n_levels, b): + """Re-scale a caller's [0, 1] handle onto the ``[lo, hi]`` slice of it, + so a level reports its own progress in its own terms without knowing + where in the whole build it sits. ``None`` stays ``None``.""" + if fcn is None: + return None + + def sub(frac, txt): + fcn(lo + (hi - lo) * frac, f"Level {k} of {n_levels} (bin {b}): {txt}") + + return sub + + +def _human_bytes(n): + """Progress text, so a rounded human unit rather than a byte count.""" + units = ("B", "KB", "MB", "GB", "TB") + k = 0 + n = float(n) + while n >= 1024 and k < len(units) - 1: + n /= 1024 + k += 1 + return f"{n:.1f} {units[k]}" + + +def _choose_grid(x, y, min_x, min_y, extent_x, extent_y, b1, budget, g_range): + """Size the tile grid from where the data actually is. + + The thing being budgeted is the LARGEST tile at the FINEST level: the + one a viewer waits on and the one that has to be uploaded. Two facts + make the obvious estimate -- total bytes over ``grid**2`` -- wrong by a + factor of a few: + + 1. Tissue does not fill its bounding box, and where it is present it + is not uniform. On a real ferret hemisphere the densest tile ran + 2.5x the median, so budgeting the mean tile overshoots by 2.5x. + 2. Coarser levels are cheaper per record but not by much -- binning + merges pixels while the genes in them stay distinct -- so bin1 + binds and the other levels come along. + + So this histograms the records over the extent and budgets the + heaviest cell, rather than assuming they are spread evenly. + + Returns: + ``(grid, estimated_biggest_tile_bytes)``. + """ + H = 256 # histogram cells per side + + # Subsample. Choosing a grid does not need every record, and a full + # pass would cost two more full-length arrays at exactly the moment + # this is trying to save them. Records in a .gef are ordered by GENE, + # so a fixed stride is spatially unbiased. + n = len(x) + step = max(1, -(-n // 20_000_000)) + cx = np.minimum(((x[::step] - min_x) * H) // extent_x, H - 1).astype(np.int64) + cy = np.minimum(((y[::step] - min_y) * H) // extent_y, H - 1).astype(np.int64) + h = np.bincount(cy * H + cx, minlength=H * H).reshape(H, H) + frac = h / h.sum() # share of all records, per histogram cell + del cx, cy, h + + # Bytes a record costs in tile format version 1: 4 for the gene index, + # 2 for the count, plus the per-pixel row header amortised over the + # genes detected in that pixel. Measured at ~10.6 on a full section; + # 11 is used so the estimate errs towards a smaller tile. + bytes_per_record = 11 + + g_lo = max(int(g_range[0]), 1) + g_hi = max(int(g_range[1]), g_lo) + # A tile may not exceed 65536 level-pixels on a side: tile-local + # coordinates are uint16. This is a floor on the search, so the + # automatic grid never proposes a geometry the writer would refuse. + side = max(-(-extent_x // b1), -(-extent_y // b1)) + g_lo = max(g_lo, -(-side // 65536)) + g_hi = max(g_hi, g_lo) + + est = float("inf") + for g in range(g_lo, g_hi + 1): + # Both partitions are uniform over the same extent, so a histogram + # cell belongs to the tile containing its centre. The map is the + # same along both axes and separable, so aggregating H x H down to + # g x g is one indicator matrix applied on each side. + m = np.minimum(((np.arange(H) * 2 + 1) * g) // (2 * H), g - 1) + a = np.zeros((H, g)) + a[np.arange(H), m] = 1.0 + est = float((a.T @ frac @ a).max()) * n * bytes_per_record + if est <= budget: + return g, est + return g_hi, est + + def _write_gene_totals(gene_index, count, n_genes): """Per-gene totals for THIS dataset. diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index ce0a0f6c..9f352e72 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -2690,14 +2690,41 @@ functions: - name: makePyramid matlab_path: "+ndi/+fun/+doc/+gene/makePyramid.m" - matlab_last_sync_hash: "10687267a" + matlab_last_sync_hash: "38aa4e8fb" python_path: "ndi/fun/doc_gene.py" decision_log: > Exact match. Options keep their MATLAB spelling (binSizes, grid, - basePixelSize, subjectID). subjectID is REQUIRED on both sides: the - pyramid document declares subject_id mustbenotempty, because a - section is measured from an animal and NDI records that on the - measurement rather than leaving it to be inferred. + tileBudgetBytes, gridRange, basePixelSize, subjectID). subjectID is + REQUIRED on both sides: the pyramid document declares subject_id + mustbenotempty, because a section is measured from an animal and NDI + records that on the measurement rather than leaving it to be + inferred. + + NDI-matlab 38aa4e8fb sized the tile grid from the data and built a level + one band of tile rows at a time; both are ported here. grid now + defaults to [] / None on both sides, because a grid is a fixed + FRACTION of the extent and the 9x9 that gives a mouse section 20 MB + tiles gave a real ferret hemisphere 257 MB ones. tileBudgetBytes + (50 MB) budgets the BIGGEST tile at the finest level rather than the + average one: tissue does not fill its bounding box and the densest + tile of a real section ran 2.5x the median. gridRange (3..64) bounds + it, the upper bound being a file COUNT limit -- grid**2 files per + level, all of them written, stored and uploaded. + + The two languages reached the same place from different sides. + Python already sorted on a TILE-MAJOR key, so tile boundaries fell + out of the sorted order; MATLAB scanned every record against every + grid cell, which a data-sized grid makes untenable, so MATLAB + adopted the Python key. Both now also DECODE the tile, the pixel and + the gene back out of that key instead of carrying them alongside it, + and both build a level one band of tile rows at a time. Neither + changes a byte of what is written -- a tile is a contiguous + rectangle, so global row-major order inside it IS local row-major + order -- which the shared conformance tile still holds them to. + + progressFcn(fraction, text) is on both, silent when not given. The + pyramid is the slow half of an ingest (951 s of a 971 s ferret + build) and reported only at its ends. - name: readViewport matlab_path: "+ndi/+fun/+doc/+gene/readViewport.m" @@ -3059,7 +3086,7 @@ functions: behavioural change, and nothing to port here. - name: fromGEF matlab_path: "+ndi/+fun/+doc/+gene/fromGEF.m" - matlab_last_sync_hash: "b589db9bb" + matlab_last_sync_hash: "38aa4e8fb" status: porting_deferred decision_log: > MATLAB reads a Stereo-seq .gef with ndr.format.stereoseq.readGEF @@ -3073,6 +3100,12 @@ functions: one saying recordSource creates a fileReference rather than a generic_file, following makeSourceFile. Comments only; no behavioural change, and nothing to port here. + + NDI-matlab 38aa4e8fb stopped promoting the reader's int32/uint16 records + to double before calling makePyramid -- 32 bytes a record held for + the whole build where 14 does, 14 GB on a 7.7e8-record section -- + and hands makePyramid a progressFcn scaled onto the last 30% of its + own. Both belong to the deferred port, not to doc_gene.py. - name: makeSourceFile matlab_path: "+ndi/+fun/+doc/+gene/makeSourceFile.m" matlab_last_sync_hash: "b589db9bb" diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index 56c118a2..8a4b4d38 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -822,7 +822,7 @@ ported_late: - name: GEFManager matlab_path: "+ndi/+gui/+app/GEFManager.m" - matlab_last_sync_hash: "a328f306e" + matlab_last_sync_hash: "38aa4e8fb" status: porting_deferred decision_log: > MATLAB App Designer window that lists, views, adds and removes diff --git a/tests/test_doc_gene_pyramid_grid.py b/tests/test_doc_gene_pyramid_grid.py new file mode 100644 index 00000000..96991183 --- /dev/null +++ b/tests/test_doc_gene_pyramid_grid.py @@ -0,0 +1,207 @@ +"""The tile grid is sized from the data, and the build reports as it goes. + +``makePyramid`` used to tile every section 9x9. A grid is a fixed FRACTION +of the extent, so the same 9x9 that gives a mouse section 20 MB tiles gave +a real ferret hemisphere 257 MB ones -- five times the budget the grid was +sized for. ``grid`` now defaults to ``None``, which picks the grid from +where the records actually are. + +What is held here is the CONTRACT, not the particular number: a denser +section gets a finer grid than a sparse one of the same extent, the bounds +are respected, an explicit grid still wins, and the document records what +was chosen so a reader never has to guess. The exact grid depends on an +estimate of tile bytes and is allowed to move. + +The second half is about the BANDS. A level is now built one band of tile +rows at a time, so the sort's working set is the section divided by the +grid rather than the whole section. Bands that overlapped would +double-count and bands that left a gap would drop records, and neither +would raise anything -- so the totals are checked. +""" + +import numpy as np +import pytest + +from ndi.document import ndi_document +from ndi.fun.doc_gene import makeGeneList, makePyramid, readTileFile +from ndi.session.dir import ndi_session_dir + + +def records(n_pix, span, n_genes=8): + """``n_pix`` x ``n_pix`` lattice points over a span x span box, all genes.""" + px, py = np.meshgrid( + np.round(np.linspace(0, span - 1, n_pix)).astype(np.int64), + np.round(np.linspace(0, span - 1, n_pix)).astype(np.int64), + ) + x = np.tile(px.ravel(), n_genes) + y = np.tile(py.ravel(), n_genes) + gi = np.repeat(np.arange(n_genes, dtype=np.int64), px.size) + c = np.ones(x.size, dtype=np.int64) + return x, y, gi, c + + +@pytest.fixture +def fixture(tmp_path): + # ndi_session_dir requires the directory to exist already. + d = tmp_path / "sess" + d.mkdir(parents=True, exist_ok=True) + session = ndi_session_dir("gene_pg", str(d)) + sub = ndi_document( + "subject", + # session.id is a METHOD here, while document.id is a property. + **{"base.session_id": session.id(), "subject.local_identifier": "pg@vhlab"}, + ) + session.database_add(sub) + gl = makeGeneList( + session, + [f"ENSG{k:04d}" for k in range(8)], + [f"g{k}" for k in range(8)], + ) + return session, sub.id, gl + + +def pyr_props(doc): + return doc.document_properties["spatialGeneExpressionPyramid"] + + +class TestGridIsSizedFromTheData: + def test_auto_grid_is_used_by_default(self, fixture): + session, subject, gl = fixture + x, y, gi, c = records(40, 4000) + pyr, _ = makePyramid(session, x, y, gi, c, gl, subject, binSizes=(1,)) + p = pyr_props(pyr) + assert 3 <= p["tile_rows"] <= 64 + # readViewport reads tile_rows and tile_columns separately. + assert p["tile_rows"] == p["tile_columns"] + + def test_denser_section_gets_a_finer_grid(self, fixture): + """THE POINT OF THE CHANGE. Same extent, more records, so the tiles + a fixed grid would produce are bigger and the grid has to be finer + to hold the same byte budget. A fixed 9x9 gives these two the same + answer, which is the bug.""" + session, subject, gl = fixture + budget = 12 * 1024 + sparse, _ = makePyramid( + session, *records(20, 4000), gl, subject, binSizes=(1,), tileBudgetBytes=budget + ) + dense, _ = makePyramid( + session, *records(120, 4000), gl, subject, binSizes=(1,), tileBudgetBytes=budget + ) + assert pyr_props(dense)["tile_rows"] > pyr_props(sparse)["tile_rows"] + + def test_grid_range_is_respected(self, fixture): + """An impossible budget clamps at the top of the range rather than + running the grid away: every extra row is another 2*grid+1 files + per level to write and upload.""" + session, subject, gl = fixture + pyr, _ = makePyramid( + session, + *records(60, 4000), + gl, + subject, + binSizes=(1,), + tileBudgetBytes=1, + gridRange=(2, 5), + ) + assert pyr_props(pyr)["tile_rows"] == 5 + + def test_explicit_grid_still_wins(self, fixture): + session, subject, gl = fixture + pyr, _ = makePyramid(session, *records(40, 4000), gl, subject, binSizes=(1,), grid=7) + assert pyr_props(pyr)["tile_rows"] == 7 + + def test_bad_explicit_grid_is_rejected(self, fixture): + session, subject, gl = fixture + with pytest.raises(ValueError, match="positive integer"): + makePyramid(session, *records(10, 1000), gl, subject, binSizes=(1,), grid=0) + + +class TestBandsPartitionTheRecords: + def test_counts_are_conserved_across_bands(self, fixture, tmp_path): + """A level is built one band of tile rows at a time. 8 genes over a + 37x37 lattice tiled 5x5, so the band bounds land mid-lattice rather + than on it.""" + session, subject, gl = fixture + x, y, gi, _ = records(37, 4000) + c = np.arange(1, x.size + 1, dtype=np.int64) + _, tiles = makePyramid(session, x, y, gi, c, gl, subject, binSizes=(1, 8), grid=5) + for k, d in enumerate(tiles): + total = 0 + for name in d.current_file_list(): + if not name.startswith("tile.bin_"): + continue + fh = session.database_openbinarydoc(d, name) + try: + t = readTileFile(fh) + finally: + session.database_closebinarydoc(fh) + total += int(np.sum(t["count"].astype(np.int64))) + assert total == int(c.sum()), f"level {k} lost or duplicated counts across bands" + + def test_every_record_reaches_a_tile(self, fixture): + """A gap between bands would silently drop records rather than + raise, so the record count is checked as well as the totals.""" + session, subject, gl = fixture + x, y, gi, c = records(23, 3000) + _, tiles = makePyramid(session, x, y, gi, c, gl, subject, binSizes=(1,), grid=4) + d = tiles[0] + n = 0 + for name in d.current_file_list(): + if not name.startswith("tile.bin_"): + continue + fh = session.database_openbinarydoc(d, name) + try: + n += int(readTileFile(fh)["count"].size) + finally: + session.database_closebinarydoc(fh) + # bin1 over a lattice: no two records share a (pixel, gene) pair, so + # nothing collapses and the count must come back whole. + assert n == x.size + + +class TestProgressReporting: + def test_reports_within_the_build(self, fixture): + """The pyramid is the slow half of an ingest -- 951 s of a 971 s + ferret build -- and reported only at its ends, so minutes of it + looked like a hang.""" + session, subject, gl = fixture + seen = [] + makePyramid( + session, + *records(20, 4000), + gl, + subject, + binSizes=(1, 2), + grid=2, + progressFcn=lambda f, t: seen.append((f, t)), + ) + fracs = [f for f, _ in seen] + texts = [t for _, t in seen] + assert len(seen) > 4, "a two-level build must report more than its two endpoints" + assert 0.0 <= min(fracs) and max(fracs) <= 1.0 + # A progress bar that jumps back reads as a bug. + assert fracs == sorted(fracs) + assert any("Level 1 of 2" in t for t in texts) + assert any("Level 2 of 2" in t for t in texts) + assert any("tile row" in t for t in texts) + + def test_reports_the_grid_it_chose(self, fixture): + """The file COUNT is a cost the choice cannot see, so it is said + out loud: a caller who minds it raises the budget or fixes grid.""" + session, subject, gl = fixture + seen = [] + makePyramid( + session, + *records(20, 4000), + gl, + subject, + binSizes=(1,), + progressFcn=lambda f, t: seen.append(t), + ) + assert any("files per level" in t for t in seen) + + def test_silent_with_no_handle(self, fixture): + """Most callers have no display and should not have to pass a + do-nothing handle.""" + session, subject, gl = fixture + makePyramid(session, *records(10, 1000), gl, subject, binSizes=(1,), grid=2) From 8a86d37252992b1f90c08785f1df6daf4b889fb8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 12:35:57 +0000 Subject: [PATCH 27/52] Turn the whole picture: an arbitrary rotation about the view centre Sections are not mounted square. Lining one up against an atlas plate or a companion section wants an arbitrary angle, and napari offers only 90-degree steps on the canvas. A dock control -- slider, spin box, reset -- puts ONE shared affine on the image ladder, the centroids and the outlines. One matrix rather than a rotation per layer is the point: napari applies affine after each layer's own scale and translate, so the same matrix is the same world-space rotation even though the three arrive in different units, and a control that could slide them apart is a control that can produce a wrong picture. Three things decide whether the picture is right, and all three are pure functions tested with no display and no napari, because CI installs no Qt binding and the Qt panel around them cannot be exercised there: - THE SIGN. napari rows increase downward -- the pyramid records origin_corner: upper-left. The two flips that come from that cancel, so the matrix is the ordinary one, but that is checked rather than reasoned about: a point to the right of the pivot must land above it at +90 degrees. - THE PIVOT, which is the one point the rotation leaves alone, and is taken from viewer.camera.center so that "rotate about what I am looking at" is what happens. It is captured when the interaction STARTS, not per tick: rotating does not move the camera, so re-reading it mid-drag would usually give the same answer, and "usually" is the problem -- pan during a drag and the picture would jump as the pivot moved under it. - THAT THE LAYERS MOVE TOGETHER, including that None entries are skipped rather than raising, since outlines and centroids are optional and a session without them is the normal case. The angle is ABSOLUTE. Every change rebuilds the affine from zero, so the picture cannot drift out of square through repeated small adjustments the way an incremental compose would. The orchestration tests in test_gui_genepyramid_controls stubbed the three panel builders and would have run this one for real -- building Qt widgets with no display, which aborts the process rather than failing a test. They stub it too now, and two new cases hold what the stubbing would otherwise hide: that the panel is built at all, and that it is handed all three layers. One known cost, not fixed here: the lazy tile fetcher asks for the axis-aligned bounding box of the visible region, so a rotated view pulls up to twice the tiles at 45 degrees. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 203 ++++++++++++++++++++++++ tests/test_gui_genepyramid_controls.py | 48 ++++++ tests/test_gui_genepyramid_rotation.py | 135 ++++++++++++++++ 3 files changed, 386 insertions(+) create mode 100644 tests/test_gui_genepyramid_rotation.py diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 16b3d357..a0cf6057 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -985,6 +985,202 @@ def _onMaster(on, name=name, scroll=scroll, expand=expand): # ----------------------------------------------------------------- all +def rotationAffine(angle_deg, center): + """A 3x3 homogeneous affine rotating ``(row, col)`` about ``center``. + + Pure, and separate from the panel, because the thing most likely to be + wrong here is the SIGN and the PIVOT, and neither needs a display to + check. + + napari layer coordinates are ``(row, col)`` with row increasing + DOWNWARD -- the pyramid records ``origin_corner: upper-left`` and the + viewer honours it. The two sign flips that come from that cancel, so + the matrix is the ordinary one and a positive angle turns the section + ANTICLOCKWISE on screen. Checked rather than reasoned about: a point + to the right of the pivot lands above it at +90 degrees. + + Args: + angle_deg: rotation in degrees, positive anticlockwise on screen. + center: ``(row, col)`` pivot in WORLD coordinates -- the same space + ``viewer.camera.center`` reports, which is what makes "rotate + about what I am looking at" expressible. + + Returns: + ``numpy.ndarray`` of shape (3, 3), suitable for ``layer.affine``. + """ + import numpy as np + + t = np.deg2rad(float(angle_deg)) + c, s = np.cos(t), np.sin(t) + r = np.array([[c, -s], [s, c]], float) + # A = T(center) . R . T(-center), so the offset is center - R @ center: + # the pivot is the one point the rotation leaves alone. + cy, cx = float(center[0]), float(center[1]) + offset = np.array([cy, cx]) - r @ np.array([cy, cx]) + a = np.eye(3) + a[:2, :2] = r + a[:2, 2] = offset + return a + + +def rotationCenter(viewer): + """The current view centre as ``(row, col)`` in world coordinates. + + ``viewer.camera.center`` is reported as ``(z, y, x)`` even for 2D data, + where z is 0, so the last two entries are the ones that mean anything + here. Taking the LAST two rather than indexing 1 and 2 keeps this + right if a 2-tuple is ever handed back instead. + """ + c = tuple(float(v) for v in viewer.camera.center) + if len(c) < 2: + raise ValueError(f"camera centre has no row/column to read: {c!r}") + return (c[-2], c[-1]) + + +def applyRotation(layers, angle_deg, center): + """Put ONE shared affine on every layer, so they turn together. + + napari applies ``affine`` after each layer's own scale and translate, + so the same matrix means the same world-space rotation on all of them + even though the image ladder, the centroids and the outlines arrive in + different units. That is the whole reason this is one matrix rather + than a rotation per layer: the section, the cells and their outlines + are one picture, and a control that could slide them apart would be a + control that can produce a wrong picture. + + ``None`` entries are skipped -- outlines and centroids are optional and + a session without them is the normal case, not an error. + + Returns: + The affine that was applied. + """ + a = rotationAffine(angle_deg, center) + for layer in layers: + if layer is None: + continue + layer.affine = a + return a + + +def addRotationPanel(viewer, layers) -> Any: + """A slider and a box for turning the whole picture. + + Sections are not mounted square, and a reader lining one up against an + atlas plate or a companion section wants an arbitrary angle rather than + the 90-degree steps napari offers on the canvas. + + THE PIVOT IS TAKEN WHEN THE INTERACTION STARTS, not on every tick. + Rotating the layers does not move the camera, so re-reading the view + centre mid-drag would usually give the same answer -- but "usually" is + the problem: pan during a drag and the picture would jump as the pivot + moved under it. Captured once per interaction, a drag is a drag. + + The angle is ABSOLUTE, not accumulated: every change rebuilds the + affine from zero, so the picture cannot drift out of square through + repeated small adjustments the way an incremental compose would. + """ + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QDoubleSpinBox, + QHBoxLayout, + QLabel, + QPushButton, + QSlider, + QVBoxLayout, + QWidget, + ) + + box = QWidget() + outer = QVBoxLayout(box) + outer.addWidget(QLabel("Rotation")) + + slider = QSlider(Qt.Horizontal) + slider.setMinimum(-180) + slider.setMaximum(180) + slider.setValue(0) + spin = QDoubleSpinBox() + spin.setRange(-180.0, 180.0) + spin.setDecimals(1) + spin.setSingleStep(0.5) + spin.setSuffix(" deg") + reset = QPushButton("Reset") + + why = ( + "Turns the image, the centroids and the outlines together, about\n" + "the centre of the current view. Positive is anticlockwise.\n" + "The pivot is taken when you start moving the control, so panning\n" + "between adjustments re-centres it." + ) + for w in (slider, spin): + w.setToolTip(why) + + status = QLabel("0.0 deg") + status.setWordWrap(True) + + # One-element lists rather than nonlocal: these are read and written + # from several nested callbacks and a mutable holder keeps them in one + # place instead of scattering `nonlocal` declarations. + pivot = [None] + guard = [False] + + def _pivot(): + if pivot[0] is None: + pivot[0] = rotationCenter(viewer) + return pivot[0] + + def _apply(angle): + try: + applyRotation(layers, angle, _pivot()) + except Exception as e: # noqa: BLE001 - a control never costs the picture + status.setText(f"failed: {e}") + return + status.setText(f"{angle:.1f} deg") + + def _sync(angle, source): + # The slider and the box show the same number, so each has to move + # the other -- and setting a widget's value emits its own signal, + # which would come straight back here. The guard makes the pair one + # control rather than two that argue. + if guard[0]: + return + guard[0] = True + try: + if source is not slider: + slider.setValue(int(round(angle))) + if source is not spin: + spin.setValue(float(angle)) + _apply(float(angle)) + finally: + guard[0] = False + + def _release(): + # Interaction over: the next one re-reads the view centre. + pivot[0] = None + + slider.sliderPressed.connect(lambda: pivot[0] or _pivot()) + slider.sliderReleased.connect(_release) + slider.valueChanged.connect(lambda v: _sync(v, slider)) + spin.editingFinished.connect(lambda: (_sync(spin.value(), spin), _release())) + + def _reset(): + pivot[0] = None + _sync(0.0, None) + pivot[0] = None + + reset.clicked.connect(_reset) + + row = QHBoxLayout() + row.addWidget(slider, 1) + row.addWidget(spin) + row.addWidget(reset) + outer.addLayout(row) + outer.addWidget(status) + outer.addStretch() + + viewer.window.add_dock_widget(box, name="Rotation", area="right") + return box + + def _importQtWidgets() -> None: """Import qtpy.QtWidgets, or raise ImportError. @@ -1060,6 +1256,13 @@ def _build(name, fn): made["display"] = _build( "display", lambda: addDisplayPanel(viewer, session, pyr_doc, image_layer, density) ) + # Rotation sits with the other whole-picture controls, above the ones + # that choose WHAT is drawn: it changes how the section is presented, + # not which genes or cells are in it. + made["rotation"] = _build( + "rotation", + lambda: addRotationPanel(viewer, [image_layer, shapes_layer, points_layer]), + ) if cells_doc is not None and points_layer is not None: made["cellTypes"] = _build( "cell types", diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index abf9a2f5..c119f6c8 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -256,6 +256,7 @@ def test_the_chosen_labelings_reach_the_cell_type_panel(monkeypatch): monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) monkeypatch.setattr(controls, "addCellTypePanel", cells) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) controls.addAllPanels( object(), @@ -284,6 +285,7 @@ def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsy monkeypatch.setattr(controls, "addDisplayPanel", _Recorder(boom=True)) monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", genes) + monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -309,6 +311,7 @@ def _no_qt(): built = _Recorder() monkeypatch.setattr(controls, "addDisplayPanel", built) monkeypatch.setattr(controls, "addGenePanel", built) + monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -317,3 +320,48 @@ def _no_qt(): err = capsys.readouterr().err assert "control panels unavailable" in err assert "image is unaffected" in err + + +def test_the_rotation_panel_gets_all_three_layers(monkeypatch): + """Rotation turns the section, the centroids and the outlines + TOGETHER, so it has to be handed all three. Handing it fewer would + leave the cells behind when the image turned -- a wrong picture that + still looks like a picture.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + rotation = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) + monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + monkeypatch.setattr(controls, "addRotationPanel", rotation) + + image, points, shapes = object(), object(), object() + controls.addAllPanels( + object(), + object(), + object(), + image, + True, + cells_doc=object(), + points_layer=points, + shapes_layer=shapes, + ) + assert rotation.calls, "the rotation panel was never built" + args, _kwargs = rotation.calls[0] + assert set(map(id, args[1])) == {id(image), id(shapes), id(points)} + + +def test_a_session_with_no_cells_still_gets_rotation(monkeypatch): + """Outlines and centroids are optional. The panel takes None for the + ones that are absent rather than being skipped, because the image + alone is still worth being able to turn.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + rotation = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) + monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + monkeypatch.setattr(controls, "addRotationPanel", rotation) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert made["rotation"] is not None + args, _kwargs = rotation.calls[0] + assert args[1][1] is None and args[1][2] is None diff --git a/tests/test_gui_genepyramid_rotation.py b/tests/test_gui_genepyramid_rotation.py new file mode 100644 index 00000000..72d4f6f7 --- /dev/null +++ b/tests/test_gui_genepyramid_rotation.py @@ -0,0 +1,135 @@ +"""Turning the whole picture: the affine, the pivot, and the layers together. + +Sections are not mounted square, and lining one up against an atlas plate +or a companion section wants an arbitrary angle rather than the 90-degree +steps napari offers on the canvas. + +Everything worth holding here is testable with no display and no napari: +the SIGN of the rotation, the PIVOT it turns about, and that every layer +gets the SAME matrix. The Qt panel around them is not tested -- CI +installs no Qt binding -- but the three things that could produce a +visibly wrong picture are. +""" + +import numpy as np +import pytest + +from ndi.gui.app.genepyramid.controls import ( + applyRotation, + rotationAffine, + rotationCenter, +) + + +def apply(a, point): + """Push a ``(row, col)`` point through a 3x3 homogeneous affine.""" + v = a @ np.array([point[0], point[1], 1.0]) + return (v[0], v[1]) + + +class TestTheMatrix: + def test_zero_is_the_identity(self): + assert np.allclose(rotationAffine(0, (100.0, 200.0)), np.eye(3)) + + def test_a_full_turn_is_the_identity(self): + assert np.allclose(rotationAffine(360, (100.0, 200.0)), np.eye(3), atol=1e-9) + + def test_the_pivot_is_the_one_point_that_does_not_move(self): + """The whole promise of 'rotate about the view centre'.""" + c = (137.0, -41.5) + for angle in (-180, -33, 0, 17, 90, 180): + assert np.allclose(apply(rotationAffine(angle, c), c), c, atol=1e-9) + + def test_positive_is_anticlockwise_on_screen(self): + """THE SIGN, checked rather than reasoned about. + + napari rows increase DOWNWARD (the pyramid records + ``origin_corner: upper-left``). A point to the RIGHT of the pivot + must land ABOVE it -- smaller row, same column -- at +90 degrees. + """ + c = (0.0, 0.0) + row, col = apply(rotationAffine(90, c), (0.0, 1.0)) + assert row == pytest.approx(-1.0, abs=1e-9), "right of pivot should go ABOVE it" + assert col == pytest.approx(0.0, abs=1e-9) + + def test_it_is_a_rotation_not_a_scale_or_a_flip(self): + """A determinant of -1 would mirror the section, which would be a + wrong picture that still looks like a picture.""" + for angle in (13, 47, 90, 180, -75): + m = rotationAffine(angle, (10.0, 20.0))[:2, :2] + assert np.linalg.det(m) == pytest.approx(1.0, abs=1e-12) + assert np.allclose(m @ m.T, np.eye(2), atol=1e-12) + + def test_distance_from_the_pivot_is_preserved(self): + c = (50.0, 60.0) + p = (110.0, -20.0) + before = np.hypot(p[0] - c[0], p[1] - c[1]) + got = apply(rotationAffine(29, c), p) + after = np.hypot(got[0] - c[0], got[1] - c[1]) + assert after == pytest.approx(before, rel=1e-12) + + def test_opposite_angles_undo_each_other(self): + """The control is ABSOLUTE, not accumulated, but the matrix still + has to be a proper inverse or repeated adjustment would drift.""" + c = (7.0, 3.0) + a = rotationAffine(38, c) + b = rotationAffine(-38, c) + assert np.allclose(a @ b, np.eye(3), atol=1e-12) + + +class TestTheCentre: + class FakeCamera: + def __init__(self, center): + self.center = center + + class FakeViewer: + def __init__(self, center): + self.camera = TestTheCentre.FakeCamera(center) + + def test_a_3d_camera_centre_gives_row_and_column(self): + """napari reports ``(z, y, x)`` even for 2D data, with z = 0.""" + assert rotationCenter(self.FakeViewer((0.0, 250.0, 400.0))) == (250.0, 400.0) + + def test_a_2_tuple_still_works(self): + """Taking the LAST two rather than indexing 1 and 2.""" + assert rotationCenter(self.FakeViewer((250.0, 400.0))) == (250.0, 400.0) + + def test_a_centre_with_nothing_to_read_raises(self): + with pytest.raises(ValueError, match="no row/column"): + rotationCenter(self.FakeViewer((5.0,))) + + +class FakeLayer: + def __init__(self): + self.affine = None + + +class TestTheLayersMoveTogether: + def test_every_layer_gets_the_same_matrix(self): + """The section, the cells and their outlines are ONE picture. A + control that could slide them apart is one that can produce a + wrong picture.""" + layers = [FakeLayer(), FakeLayer(), FakeLayer()] + applied = applyRotation(layers, 23.5, (100.0, 100.0)) + for layer in layers: + assert np.allclose(layer.affine, applied) + + def test_missing_layers_are_skipped_not_an_error(self): + """Outlines and centroids are optional; a session without them is + the normal case.""" + image = FakeLayer() + applyRotation([image, None, None], 10.0, (0.0, 0.0)) + assert image.affine is not None + + def test_it_returns_what_it_applied(self): + image = FakeLayer() + applied = applyRotation([image], 45.0, (1.0, 2.0)) + assert np.allclose(applied, rotationAffine(45.0, (1.0, 2.0))) + + def test_a_later_angle_replaces_rather_than_compounds(self): + """Absolute, not accumulated: two 30-degree settings leave the + picture at 30 degrees, not 60.""" + image = FakeLayer() + applyRotation([image], 30.0, (0.0, 0.0)) + applyRotation([image], 30.0, (0.0, 0.0)) + assert np.allclose(image.affine, rotationAffine(30.0, (0.0, 0.0))) From 78a2d03e4467cac4328830781477e2d09dfe386a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:15:47 +0000 Subject: [PATCH 28/52] Drop the two lines each panel was saying twice The dock already carries the panel's name, so "Base layer values" above the radio buttons said it again; and the status line under them restated which button had just been clicked, which the buttons themselves show. Two rows of vertical space per panel, spent on nothing. The status labels are HIDDEN rather than deleted. Both are also the only place a failure surfaces -- a level rebuild that throws, a rotation that cannot be applied -- and a control that can fail without saying so is worse than a redundant line. They appear when there is something to say and go away again when there is not. The rotation panel had picked up the same two redundancies, so it loses them too. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index a0cf6057..45045201 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -644,7 +644,6 @@ def addDisplayPanel(viewer, session, pyr_doc, image_layer, density: bool = True) box = QWidget() outer = QVBoxLayout(box) - outer.addWidget(QLabel("Base layer values")) r_density = QRadioButton("density (counts per base pixel)") r_counts = QRadioButton("raw counts (summed per bin)") @@ -666,8 +665,14 @@ def addDisplayPanel(viewer, session, pyr_doc, image_layer, density: bool = True) for w in (r_density, r_counts): w.setToolTip(why) + # Kept, but HIDDEN until something goes wrong. It used to sit under the + # buttons restating which one was just clicked, which the buttons + # already show. Deleting it outright would make a failed switch silent, + # and a control that can fail without saying so is worse than a + # redundant line -- so it appears only when there is something to say. status = QLabel("") status.setWordWrap(True) + status.hide() def _switch(want_density: bool): try: @@ -677,8 +682,9 @@ def _switch(want_density: bool): image_layer.reset_contrast_limits() except Exception as e: status.setText(f"failed: {e}") + status.show() return - status.setText("density" if want_density else "raw counts") + status.hide() r_density.toggled.connect(lambda on: _switch(True) if on else None) r_counts.toggled.connect(lambda on: _switch(False) if on else None) @@ -1092,7 +1098,6 @@ def addRotationPanel(viewer, layers) -> Any: box = QWidget() outer = QVBoxLayout(box) - outer.addWidget(QLabel("Rotation")) slider = QSlider(Qt.Horizontal) slider.setMinimum(-180) @@ -1114,8 +1119,11 @@ def addRotationPanel(viewer, layers) -> Any: for w in (slider, spin): w.setToolTip(why) - status = QLabel("0.0 deg") + # Hidden until it has something to report, for the same reason as the + # display panel's: the spin box already shows the angle. + status = QLabel("") status.setWordWrap(True) + status.hide() # One-element lists rather than nonlocal: these are read and written # from several nested callbacks and a mutable holder keeps them in one @@ -1133,8 +1141,9 @@ def _apply(angle): applyRotation(layers, angle, _pivot()) except Exception as e: # noqa: BLE001 - a control never costs the picture status.setText(f"failed: {e}") + status.show() return - status.setText(f"{angle:.1f} deg") + status.hide() def _sync(angle, source): # The slider and the box show the same number, so each has to move From 991439dfdb4b25391fad4e7c57c40ed86f04392c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:22:27 +0000 Subject: [PATCH 29/52] Blur the cells into a field, with width and brightness as separate knobs Sparse centroids are single marks and hard to pick out. napari draws a Points layer as discrete marks and offers no kernel to widen, so "broader dots" means rendering the cells into an IMAGE and blurring that -- a second layer under the points, with the points still drawn on top. THE KERNEL IS UNITY. Blurring spreads each cell's contribution rather than adding to it, so widening does not also brighten and the contrast limits set at one width still mean something at another. That is what keeps width and brightness separate knobs, and it is why this panel has only a width control: the density layer is an ordinary Image layer, so contrast, gamma and colormap stay in napari's own layer controls where a reader already looks for them. Two things the raster has to get right, both tested without a display: - IT LANDS ON THE CELLS. scale and translate come from the same bounds the histogram was binned over, so the blur sits under the centroids rather than beside them. - IT IS COARSE ON PURPOSE. A ferret section is ~40,000 x 59,000 base pixels; a full-resolution density image would be 2.4 billion pixels to hold a few hundred thousand cells. The longest side is capped and the layer scaled to match. sigma is in WORLD units rather than raster pixels, so the knob means the same thing whatever the section's extent instead of drifting with it. mode="constant" rather than the default reflect: a section is not periodic, and folding mass back would put a bright rim of doubled cells around the tissue. The layer is built on FIRST USE. Most sessions never touch this, and rasterising several hundred thousand centroids is not worth paying for unasked on a launch that is already the thing people complain about. The orchestration tests in test_gui_genepyramid_controls stub the panel builders; left unstubbed this one builds Qt widgets for real, which aborts the process rather than failing a test. It is stubbed too now, and a new case holds what the stubbing would otherwise hide: the panel is built only where there is a points layer to rasterise, and is handed it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 209 ++++++++++++++++++++++++ tests/test_gui_genepyramid_controls.py | 30 ++++ tests/test_gui_genepyramid_density.py | 111 +++++++++++++ 3 files changed, 350 insertions(+) create mode 100644 tests/test_gui_genepyramid_density.py diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 45045201..7b4077c7 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1068,6 +1068,214 @@ def applyRotation(layers, angle_deg, center): return a +def densityRaster(row, col, max_side: int = 2048): + """Bin centroids onto a raster, coarse enough to blur cheaply. + + Points cannot be blurred. napari draws a Points layer as discrete + marks, and there is no kernel to widen -- so making sparse cells + easier to see means rendering them into an IMAGE and blurring that. + + The raster is deliberately coarse. A ferret section is ~40,000 x + 59,000 base pixels, and a full-resolution density image would be 2.4 + billion pixels to hold a few hundred thousand cells. The longest side + is capped instead, and the layer carries a matching ``scale`` and + ``translate`` so it still lands on top of the cells it was made from. + + Args: + row, col: centroid coordinates in WORLD units, as the points layer + holds them. + max_side: longest raster side in pixels. + + Returns: + ``(counts, step, origin)`` -- the 2D histogram, the world units one + raster pixel spans, and the ``(row, col)`` world position of its + top-left corner. + """ + import numpy as np + + row = np.asarray(row, float).ravel() + col = np.asarray(col, float).ravel() + if row.size == 0: + return np.zeros((1, 1)), 1.0, (0.0, 0.0) + + r0, r1 = float(row.min()), float(row.max()) + c0, c1 = float(col.min()), float(col.max()) + # A degenerate extent -- one cell, or a row of them -- still has to + # produce a raster with area, or the histogram below has no bins. + span = max(r1 - r0, c1 - c0, 1.0) + step = span / max_side + + nr = max(int(np.ceil((r1 - r0) / step)) + 1, 1) + nc = max(int(np.ceil((c1 - c0) / step)) + 1, 1) + ri = np.clip(((row - r0) / step).astype(np.int64), 0, nr - 1) + ci = np.clip(((col - c0) / step).astype(np.int64), 0, nc - 1) + counts = np.bincount(ri * nc + ci, minlength=nr * nc).reshape(nr, nc) + return counts.astype(np.float32), step, (r0, c0) + + +def blurRaster(counts, sigma_world: float, step: float): + """Gaussian-blur a density raster with a UNITY kernel. + + Unity meaning the kernel sums to one, so blurring SPREADS the signal + rather than adding to it: a cell contributes the same total whatever + the width. That is what keeps brightness a separate knob from width -- + widen the dots and the picture does not also get brighter, so the + contrast limits set for one sigma still mean something at another. + + ``mode="constant"`` rather than the default reflect, so mass leaves at + the edges instead of being folded back and piling up there. A section + is not periodic and its border cells are not doubled. + + Args: + counts: raster from :func:`densityRaster`. + sigma_world: blur width in WORLD units -- the same units the + centroids are in, so the knob means the same thing at every + zoom rather than drifting with the raster size. + step: world units per raster pixel, from :func:`densityRaster`. + + Returns: + The blurred raster. sigma <= 0 returns the input unchanged, which + is what "no blur" should cost. + """ + import numpy as np + from scipy.ndimage import gaussian_filter + + counts = np.asarray(counts, np.float32) + if sigma_world is None or sigma_world <= 0 or step <= 0: + return counts + return gaussian_filter(counts, sigma=float(sigma_world) / float(step), mode="constant") + + +def addCellBlurPanel(viewer, points_layer, max_side: int = 2048) -> Any: + """A width knob for a blurred density image under the cell centroids. + + Sparse centroids are single marks and hard to pick out. This adds a + second layer -- the same cells, binned and Gaussian-blurred -- that can + be widened until they read as a field rather than dust, with the + original points still drawn on top. + + THE LAYER IS BUILT ON FIRST USE, not at launch. Most sessions never + touch this, and rasterising several hundred thousand centroids is not + worth paying for unasked on a launch that is already the thing people + complain about. + + Brightness stays napari's own: the density layer is an ordinary Image + layer, so its contrast limits, gamma and colormap are in the layer + controls where a reader already looks for them. This knob does width + and nothing else, which is why the kernel is unity. + """ + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QCheckBox, + QDoubleSpinBox, + QHBoxLayout, + QLabel, + QSlider, + QVBoxLayout, + QWidget, + ) + + box = QWidget() + outer = QVBoxLayout(box) + + show = QCheckBox("blurred cell density") + show.setToolTip( + "Bins the centroids and blurs them with a unity Gaussian, so widening\n" + "spreads each cell rather than brightening it. Contrast and colormap\n" + "stay in the layer controls." + ) + slider = QSlider(Qt.Horizontal) + spin = QDoubleSpinBox() + spin.setRange(0.0, 500.0) + spin.setDecimals(1) + spin.setSuffix(" um") + + status = QLabel("") + status.setWordWrap(True) + status.hide() + + state = {"layer": None, "counts": None, "step": 1.0, "origin": (0.0, 0.0)} + + def _raster(): + if state["counts"] is None: + data = points_layer.data + counts, step, origin = densityRaster(data[:, 0], data[:, 1], max_side) + state.update(counts=counts, step=step, origin=origin) + # The slider is in WORLD units and its useful range depends on + # how big the section is, so it is set from the data rather + # than guessed: a tenth of the raster is a wide blur anywhere. + widest = max(step * max_side / 10.0, 1.0) + slider.setRange(0, 100) + spin.setRange(0.0, widest) + spin.setSingleStep(widest / 100.0) + return state["counts"], state["step"], state["origin"] + + def _apply(sigma): + try: + counts, step, origin = _raster() + img = blurRaster(counts, sigma, step) + if state["layer"] is None: + state["layer"] = viewer.add_image( + img, + name="cell density", + colormap="magma", + blending="additive", + scale=(step, step), + translate=origin, + ) + # Under the points, not over them: the blur is context for + # the centroids, not a replacement. + viewer.layers.move(len(viewer.layers) - 1, 0) + else: + state["layer"].data = img + state["layer"].visible = show.isChecked() + state["layer"].reset_contrast_limits() + except Exception as e: # noqa: BLE001 - a control never costs the picture + status.setText(f"failed: {e}") + status.show() + return + status.hide() + + def _sigma(): + return spin.value() + + guard = [False] + + def _sync(value, source): + if guard[0]: + return + guard[0] = True + try: + if source is not spin: + spin.setValue(spin.maximum() * value / 100.0) + if source is not slider and spin.maximum() > 0: + slider.setValue(int(round(100.0 * spin.value() / spin.maximum()))) + if show.isChecked(): + _apply(_sigma()) + finally: + guard[0] = False + + slider.valueChanged.connect(lambda v: _sync(v, slider)) + spin.valueChanged.connect(lambda v: _sync(v, spin)) + show.toggled.connect(lambda on: _apply(_sigma()) if on else _hide()) + + def _hide(): + if state["layer"] is not None: + state["layer"].visible = False + + row = QHBoxLayout() + row.addWidget(QLabel("width")) + row.addWidget(slider, 1) + row.addWidget(spin) + outer.addWidget(show) + outer.addLayout(row) + outer.addWidget(status) + outer.addStretch() + + viewer.window.add_dock_widget(box, name="Cell density", area="right") + return box + + def addRotationPanel(viewer, layers) -> Any: """A slider and a box for turning the whole picture. @@ -1279,5 +1487,6 @@ def _build(name, fn): viewer, session, cells_doc, points_layer, shapes_layer, labelings ), ) + made["cellDensity"] = _build("cell density", lambda: addCellBlurPanel(viewer, points_layer)) made["genes"] = _build("genes", lambda: addGenePanel(viewer, session, pyr_doc)) return made diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index c119f6c8..5d7f2a00 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -257,6 +257,7 @@ def test_the_chosen_labelings_reach_the_cell_type_panel(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", cells) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) controls.addAllPanels( object(), @@ -286,6 +287,7 @@ def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsy monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", genes) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -312,6 +314,7 @@ def _no_qt(): monkeypatch.setattr(controls, "addDisplayPanel", built) monkeypatch.setattr(controls, "addGenePanel", built) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -333,6 +336,7 @@ def test_the_rotation_panel_gets_all_three_layers(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", rotation) + monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) image, points, shapes = object(), object(), object() controls.addAllPanels( @@ -360,8 +364,34 @@ def test_a_session_with_no_cells_still_gets_rotation(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", rotation) + monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) assert made["rotation"] is not None args, _kwargs = rotation.calls[0] assert args[1][1] is None and args[1][2] is None + + +def test_the_density_panel_is_built_only_where_there_are_cells(monkeypatch): + """It rasterises the centroids, so with no points layer there is + nothing to rasterise -- and a panel offering to blur nothing is worse + than no panel.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + density = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) + monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellBlurPanel", density) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert "cellDensity" not in made + assert not density.calls + + points = object() + made = controls.addAllPanels( + object(), object(), object(), object(), True, cells_doc=object(), points_layer=points + ) + assert made["cellDensity"] is not None + args, _kwargs = density.calls[0] + assert args[1] is points, "the panel needs the layer it will rasterise" diff --git a/tests/test_gui_genepyramid_density.py b/tests/test_gui_genepyramid_density.py new file mode 100644 index 00000000..2ff2667d --- /dev/null +++ b/tests/test_gui_genepyramid_density.py @@ -0,0 +1,111 @@ +"""Blurring sparse centroids so they read as a field rather than dust. + +napari draws a Points layer as discrete marks and offers no kernel to +widen, so "broader dots" means rendering the cells into an image and +blurring THAT. The two things worth holding are the raster (does it land +where the cells are, at a size that can be blurred cheaply) and the +kernel's UNITY -- widening must spread each cell's contribution, not add +to it, or width and brightness stop being separate knobs. + +No display and no napari needed for either. +""" + +import numpy as np +import pytest + +from ndi.gui.app.genepyramid.controls import blurRaster, densityRaster + + +class TestTheRaster: + def test_it_lands_where_the_cells_are(self): + """scale and translate have to put the raster back on top of the + points it was built from, or the blur sits beside the cells.""" + row = np.array([100.0, 300.0]) + col = np.array([50.0, 250.0]) + counts, step, origin = densityRaster(row, col, max_side=64) + assert origin == (100.0, 50.0) + # Each cell must map back into the raster at its own corner. + for r, c in zip(row, col): + ri = int((r - origin[0]) / step) + ci = int((c - origin[1]) / step) + assert counts[ri, ci] >= 1 + + def test_every_cell_is_counted_exactly_once(self): + rng = np.random.default_rng(0) + row = rng.uniform(0, 5000, 5000) + col = rng.uniform(0, 9000, 5000) + counts, _, _ = densityRaster(row, col, max_side=256) + assert counts.sum() == 5000 + + def test_the_long_side_is_capped(self): + """A ferret section is ~40,000 x 59,000 base pixels; a full-res + density image would be 2.4 billion pixels for a few hundred + thousand cells.""" + row = np.array([0.0, 59000.0]) + col = np.array([0.0, 40000.0]) + counts, _, _ = densityRaster(row, col, max_side=512) + assert max(counts.shape) <= 513 + assert counts.shape[0] >= counts.shape[1], "the taller axis stays taller" + + def test_no_cells_is_not_a_crash(self): + counts, step, origin = densityRaster([], [], max_side=64) + assert counts.shape == (1, 1) + assert step > 0 + + def test_a_single_cell_still_makes_a_raster_with_area(self): + """A degenerate extent would otherwise give a histogram no bins.""" + counts, step, _ = densityRaster([7.0], [9.0], max_side=64) + assert counts.sum() == 1 + assert step > 0 + + def test_cells_in_a_line_still_work(self): + counts, _, _ = densityRaster([5.0, 5.0, 5.0], [1.0, 2.0, 3.0], max_side=32) + assert counts.sum() == 3 + + +class TestTheKernelIsUnity: + def test_blurring_spreads_rather_than_adds(self): + """THE POINT. Widening must not brighten, or the contrast limits + set at one width stop meaning anything at another. + + The frame is big enough for the kernel to fit inside it -- scipy + truncates at 4 sigma, so a wide blur on a small raster loses mass + off the edge, which is the deliberate behaviour the edge test + below pins and not a failure of unity. + """ + counts = np.zeros((128, 128), np.float32) + counts[64, 64] = 1.0 + for sigma in (1.0, 3.0, 6.0): + out = blurRaster(counts, sigma_world=sigma, step=1.0) + assert out.sum() == pytest.approx(1.0, rel=1e-4) + + def test_a_wider_blur_lowers_the_peak(self): + counts = np.zeros((64, 64), np.float32) + counts[32, 32] = 1.0 + narrow = blurRaster(counts, 1.0, 1.0).max() + wide = blurRaster(counts, 6.0, 1.0).max() + assert wide < narrow + + def test_zero_sigma_is_the_input_unchanged(self): + counts = np.arange(16, dtype=np.float32).reshape(4, 4) + assert np.array_equal(blurRaster(counts, 0.0, 1.0), counts) + assert np.array_equal(blurRaster(counts, -1.0, 1.0), counts) + + def test_sigma_is_in_world_units_not_raster_pixels(self): + """The knob has to mean the same thing whatever the raster size, + or its useful range would drift with the section's extent.""" + counts = np.zeros((64, 64), np.float32) + counts[32, 32] = 1.0 + # 10 world units at 2 units/pixel is the same blur as 5 at 1. + a = blurRaster(counts, sigma_world=10.0, step=2.0) + b = blurRaster(counts, sigma_world=5.0, step=1.0) + assert np.allclose(a, b) + + def test_edge_mass_leaves_rather_than_folding_back(self): + """A section is not periodic; reflecting would double its border + cells and put a bright rim around the tissue.""" + counts = np.zeros((32, 32), np.float32) + counts[0, 0] = 1.0 + out = blurRaster(counts, 3.0, 1.0) + assert out.sum() < 1.0, "mass at a corner should leave the frame" + assert out.max() < 1.0 From 4ca398de61fa07edc1405fbcde8ae008a5da5e84 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:27:24 +0000 Subject: [PATCH 30/52] Cap the density raster where the blur still keeps up with the slider I had picked 2048 assuming the blur cost about 50 ms. Measured, with scipy's gaussian_filter across the useful sigma range: 1024^2 38 ms (sigma 2px) 126 ms (sigma 20px) 2048^2 177 ms 442 ms 4096^2 757 ms 1815 ms The blur is redrawn on every change of the width knob, so this sits on the interactive path, not the loading one -- 2048 would have made the slider lag by a third to half a second per tick. 1024 is the largest that still redraws inside a drag. Binning is not what costs and never was: 400,000 centroids histogram in 13 ms, once, and the result is kept. The cap is about the blur. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 26 ++++++++++++++++++------- tests/test_gui_genepyramid_density.py | 14 +++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 7b4077c7..15551e06 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1068,18 +1068,30 @@ def applyRotation(layers, angle_deg, center): return a -def densityRaster(row, col, max_side: int = 2048): +def densityRaster(row, col, max_side: int = 1024): """Bin centroids onto a raster, coarse enough to blur cheaply. Points cannot be blurred. napari draws a Points layer as discrete marks, and there is no kernel to widen -- so making sparse cells easier to see means rendering them into an IMAGE and blurring that. - The raster is deliberately coarse. A ferret section is ~40,000 x - 59,000 base pixels, and a full-resolution density image would be 2.4 - billion pixels to hold a few hundred thousand cells. The longest side - is capped instead, and the layer carries a matching ``scale`` and - ``translate`` so it still lands on top of the cells it was made from. + The raster is deliberately coarse, for two reasons. A ferret section + is ~40,000 x 59,000 base pixels, so a full-resolution density image + would be 2.4 billion pixels to hold a few hundred thousand cells. And + the BLUR is redrawn on every change of the width knob, which puts the + cap on the interactive path rather than the loading one -- measured, + on this raster, with scipy's gaussian_filter: + + 1024^2 38 ms at sigma 2px, 126 ms at sigma 20px + 2048^2 177 ms 442 ms + 4096^2 757 ms 1815 ms + + 1024 is the largest that still redraws inside a slider drag. Binning + the points is not what costs -- 400,000 cells histogram in 13 ms, once + -- so the cap is about the blur, not the counting. + + The layer carries a matching ``scale`` and ``translate`` so it still + lands on top of the cells it was made from. Args: row, col: centroid coordinates in WORLD units, as the points layer @@ -1146,7 +1158,7 @@ def blurRaster(counts, sigma_world: float, step: float): return gaussian_filter(counts, sigma=float(sigma_world) / float(step), mode="constant") -def addCellBlurPanel(viewer, points_layer, max_side: int = 2048) -> Any: +def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: """A width knob for a blurred density image under the cell centroids. Sparse centroids are single marks and hard to pick out. This adds a diff --git a/tests/test_gui_genepyramid_density.py b/tests/test_gui_genepyramid_density.py index 2ff2667d..76dcd60e 100644 --- a/tests/test_gui_genepyramid_density.py +++ b/tests/test_gui_genepyramid_density.py @@ -109,3 +109,17 @@ def test_edge_mass_leaves_rather_than_folding_back(self): out = blurRaster(counts, 3.0, 1.0) assert out.sum() < 1.0, "mass at a corner should leave the frame" assert out.max() < 1.0 + + +def test_the_default_raster_stays_inside_a_slider_drag(): + """The blur is redrawn on every change of the width knob, so the cap + is on the interactive path. Measured with scipy's gaussian_filter: + 1024^2 is 38-126 ms across the useful sigma range, 2048^2 is 177-442, + 4096^2 is 757-1815. A default that lags the drag is the wrong default, + and this is the sort of number that drifts back up unnoticed.""" + import numpy as np + + row = np.array([0.0, 59000.0]) + col = np.array([0.0, 40000.0]) + counts, _, _ = densityRaster(row, col) + assert max(counts.shape) <= 1025 From 3cb17c2d5965cc91b4426f0550d08e703ecfcc27 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:32:35 +0000 Subject: [PATCH 31/52] Add "Re-blur here", so the density can be sharp where you are looking The standing raster covers the whole section, so it goes blocky once you zoom past it: 1024 across a 59,000-unit ferret is ~58 base pixels a raster pixel. This re-bins just what is on screen, putting the same number of raster pixels over a much smaller region -- full resolution for that zoom. ON DEMAND RATHER THAN ON CAMERA MOVE, deliberately. The blur is 38-126 ms and camera events fire continuously through a pan, so following the camera would stutter through every drag unless it were debounced, and a debounce is a guess about how long someone has stopped moving. A button makes the cost explicit and puts it where the user already knows they have finished moving. If it turns out to be tiresome, a debounced auto-refresh is a small addition on top of this. Two things it has to get right, both tested with no display: - THE RECTANGLE. napari's camera.zoom is canvas PIXELS PER WORLD UNIT, so the visible extent is the canvas size over the zoom, centred on camera.center. Qt reports (width, height) while napari axes are (row, col), so width spans the columns and height the rows -- the one thing here that is easy to get backwards and invisible whenever the window happens to be square. - THE FALLBACK. The canvas size is reached through private attributes that have moved between napari versions, so it is tried rather than assumed. A viewer that will not say leaves the blur over the whole section and says so, because a blur over the WRONG rectangle would be worse than a coarse one over the right rectangle. A re-blur changes the raster's scale and translate as well as its data, so all three are set now; data alone would leave the image stretched over the rectangle it used to cover. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 110 +++++++++++++++++++++++- tests/test_gui_genepyramid_density.py | 83 ++++++++++++++++++ 2 files changed, 189 insertions(+), 4 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 15551e06..8bd260f6 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1158,6 +1158,54 @@ def blurRaster(counts, sigma_world: float, step: float): return gaussian_filter(counts, sigma=float(sigma_world) / float(step), mode="constant") +def viewportBounds(center, zoom, canvas_px): + """The visible world rectangle, as ``(r0, r1, c0, c1)``. + + napari's ``camera.zoom`` is canvas PIXELS PER WORLD UNIT, so the + visible extent is the canvas size divided by it, centred on + ``camera.center``. Qt reports a canvas as ``(width, height)`` while + napari axes are ``(row, col)``, so width belongs to the column span + and height to the row span -- the one place this is easy to get + backwards, and invisible when the window happens to be square. + + Pure arithmetic, separate from the viewer, because it is the part that + can be wrong and the part that needs no display to check. + + Args: + center: ``camera.center``, ``(z, y, x)`` even in 2D; the last two + are read. + zoom: ``camera.zoom``. + canvas_px: ``(width, height)`` in device pixels. + """ + cy, cx = float(center[-2]), float(center[-1]) + w_px, h_px = float(canvas_px[0]), float(canvas_px[1]) + if zoom <= 0: + raise ValueError(f"camera zoom must be positive, got {zoom!r}") + half_r = (h_px / float(zoom)) / 2.0 + half_c = (w_px / float(zoom)) / 2.0 + return (cy - half_r, cy + half_r, cx - half_c, cx + half_c) + + +def canvasSize(viewer): + """The canvas size in pixels, or None if this napari will not say. + + Reached through private attributes that have moved between napari + versions, so it is tried rather than assumed and the caller falls back + to the whole section rather than failing. A blur over the wrong + rectangle would be worse than a coarse one over the right rectangle. + """ + for path in (("_qt_viewer",), ("qt_viewer",)): + obj = viewer.window + try: + for name in path: + obj = getattr(obj, name) + size = obj.canvas.size + return (float(size[0]), float(size[1])) + except Exception: # noqa: BLE001 - any miss means "cannot tell" + continue + return None + + def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: """A width knob for a blurred density image under the cell centroids. @@ -1182,6 +1230,7 @@ def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: QDoubleSpinBox, QHBoxLayout, QLabel, + QPushButton, QSlider, QVBoxLayout, QWidget, @@ -1191,6 +1240,13 @@ def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: outer = QVBoxLayout(box) show = QCheckBox("blurred cell density") + here = QPushButton("Re-blur here") + here.setToolTip( + "Re-bins just what is on screen, at full resolution for this zoom.\n" + "The standing raster covers the whole section and goes blocky when\n" + "you zoom past it; this trades that for a view that has to be\n" + "refreshed after you move." + ) show.setToolTip( "Bins the centroids and blurs them with a unity Gaussian, so widening\n" "spreads each cell rather than brightening it. Contrast and colormap\n" @@ -1208,20 +1264,59 @@ def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: state = {"layer": None, "counts": None, "step": 1.0, "origin": (0.0, 0.0)} + def _rebin(bounds=None): + """Bin the centroids, optionally only those inside BOUNDS. + + Whole-section by default. Restricted to the viewport, the same + max_side spans a smaller region, so the raster is finer exactly + where it is being looked at -- which is the whole point of the + button, and why this is on demand rather than on every camera + move: the blur is 38-126 ms and a pan would stutter through it. + """ + import numpy as np + + data = points_layer.data + row, col = data[:, 0], data[:, 1] + if bounds is not None: + r0, r1, c0, c1 = bounds + keep = (row >= r0) & (row <= r1) & (col >= c0) & (col <= c1) + if not np.any(keep): + raise ValueError("no cells in view") + row, col = row[keep], col[keep] + counts, step, origin = densityRaster(row, col, max_side) + state.update(counts=counts, step=step, origin=origin) + def _raster(): if state["counts"] is None: - data = points_layer.data - counts, step, origin = densityRaster(data[:, 0], data[:, 1], max_side) - state.update(counts=counts, step=step, origin=origin) + _rebin() # The slider is in WORLD units and its useful range depends on # how big the section is, so it is set from the data rather # than guessed: a tenth of the raster is a wide blur anywhere. - widest = max(step * max_side / 10.0, 1.0) + widest = max(state["step"] * max_side / 10.0, 1.0) slider.setRange(0, 100) spin.setRange(0.0, widest) spin.setSingleStep(widest / 100.0) return state["counts"], state["step"], state["origin"] + def _reblurHere(): + size = canvasSize(viewer) + if size is None: + status.setText( + "this napari will not report its canvas size, so the blur " + "stays over the whole section" + ) + status.show() + return + try: + bounds = viewportBounds(viewer.camera.center, viewer.camera.zoom, size) + _rebin(bounds) + except Exception as e: # noqa: BLE001 - a control never costs the picture + status.setText(f"failed: {e}") + status.show() + return + show.setChecked(True) + _apply(_sigma()) + def _apply(sigma): try: counts, step, origin = _raster() @@ -1239,7 +1334,12 @@ def _apply(sigma): # the centroids, not a replacement. viewer.layers.move(len(viewer.layers) - 1, 0) else: + # scale and translate too: a re-blur over the viewport + # changes all three, and data alone would leave the image + # stretched over the old rectangle. state["layer"].data = img + state["layer"].scale = (step, step) + state["layer"].translate = origin state["layer"].visible = show.isChecked() state["layer"].reset_contrast_limits() except Exception as e: # noqa: BLE001 - a control never costs the picture @@ -1279,6 +1379,8 @@ def _hide(): row.addWidget(QLabel("width")) row.addWidget(slider, 1) row.addWidget(spin) + row.addWidget(here) + here.clicked.connect(_reblurHere) outer.addWidget(show) outer.addLayout(row) outer.addWidget(status) diff --git a/tests/test_gui_genepyramid_density.py b/tests/test_gui_genepyramid_density.py index 76dcd60e..5cca0c81 100644 --- a/tests/test_gui_genepyramid_density.py +++ b/tests/test_gui_genepyramid_density.py @@ -123,3 +123,86 @@ def test_the_default_raster_stays_inside_a_slider_drag(): col = np.array([0.0, 40000.0]) counts, _, _ = densityRaster(row, col) assert max(counts.shape) <= 1025 + + +class TestTheViewportRectangle: + """The standing raster covers the whole section, so it goes blocky + when you zoom past it. "Re-blur here" re-bins just what is on screen, + which puts the same number of raster pixels over a smaller region. + + The arithmetic is where this can be wrong, and it needs no display. + """ + + def test_the_visible_extent_is_the_canvas_over_the_zoom(self): + """napari's zoom is canvas PIXELS PER WORLD UNIT.""" + from ndi.gui.app.genepyramid.controls import viewportBounds + + # 800x600 canvas at 2 px per world unit -> 400 x 300 world units. + r0, r1, c0, c1 = viewportBounds((0.0, 100.0, 200.0), 2.0, (800, 600)) + assert (c1 - c0) == pytest.approx(400.0) + assert (r1 - r0) == pytest.approx(300.0) + + def test_width_belongs_to_columns_and_height_to_rows(self): + """The one thing here that is easy to get backwards, and invisible + whenever the window happens to be square.""" + from ndi.gui.app.genepyramid.controls import viewportBounds + + r0, r1, c0, c1 = viewportBounds((0.0, 0.0, 0.0), 1.0, (1000, 200)) + assert (c1 - c0) == pytest.approx(1000.0), "canvas WIDTH spans columns" + assert (r1 - r0) == pytest.approx(200.0), "canvas HEIGHT spans rows" + + def test_it_is_centred_on_the_camera(self): + from ndi.gui.app.genepyramid.controls import viewportBounds + + r0, r1, c0, c1 = viewportBounds((0.0, 50.0, -20.0), 1.0, (100, 100)) + assert (r0 + r1) / 2 == pytest.approx(50.0) + assert (c0 + c1) / 2 == pytest.approx(-20.0) + + def test_a_zoom_of_zero_is_refused_rather_than_dividing(self): + from ndi.gui.app.genepyramid.controls import viewportBounds + + with pytest.raises(ValueError, match="zoom must be positive"): + viewportBounds((0.0, 0.0, 0.0), 0.0, (100, 100)) + + def test_a_viewport_raster_is_finer_than_a_whole_section_one(self): + """THE POINT OF THE BUTTON: same max_side, smaller region.""" + rng = np.random.default_rng(1) + row = rng.uniform(0, 59000, 20000) + col = rng.uniform(0, 40000, 20000) + _, whole_step, _ = densityRaster(row, col, max_side=256) + + keep = (row > 29000) & (row < 30000) & (col > 20000) & (col < 21000) + _, view_step, _ = densityRaster(row[keep], col[keep], max_side=256) + assert view_step < whole_step / 10 + + +class TestTheCanvasSizeIsOptional: + """It is reached through private napari attributes that have moved + between versions, so a miss falls back to the whole section rather + than failing -- a blur over the wrong rectangle would be worse than a + coarse one over the right rectangle.""" + + def test_a_viewer_that_will_not_say_returns_none(self): + from ndi.gui.app.genepyramid.controls import canvasSize + + class Bare: + window = object() + + assert canvasSize(Bare()) is None + + def test_it_reads_the_canvas_when_one_is_there(self): + from ndi.gui.app.genepyramid.controls import canvasSize + + class Canvas: + size = (1280, 720) + + class QtViewer: + canvas = Canvas() + + class Window: + _qt_viewer = QtViewer() + + class Viewer: + window = Window() + + assert canvasSize(Viewer()) == (1280.0, 720.0) From 9923617074254d1fcd02e52c5e0a0e532de21052 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:41:22 +0000 Subject: [PATCH 32/52] Put the viewer on the cloud palette, and let the blur follow the view TWO GREYS, one per language. napari defaults to its dark theme and the docks inherit it, so the viewer came out grey against every other NDI applet -- white bodies under a navy header. viewer.theme is set to "light", and each control panel gets a stylesheet built from ndi.gui.cloud_colors: the same triplets ndi.gui.cloudColors holds on the MATLAB side, so the two stay one look rather than two approximations of one. The stylesheet is applied in addAllPanels' _build rather than in each builder, so a panel added later cannot forget and come out grey among white ones. On the MATLAB side the figure and its layouts were already on c.offWhite -- but the CONTROLS were not. A uitable, a uitextarea, a uilistbox and a uieditfield each default to MATLAB's own grey chrome, and the table is the largest surface in the window, so the app read as grey with a tinted border. A paper() helper beside accent() puts them on the palette, and Browse... picks up the accent the other buttons already had. Both are defensive in the same way and for the same reason: a control that will not take a colour, or a Qt build that refuses a stylesheet, costs the colour and not the window. Losing a panel over a shade would be a poor trade. FOLLOW THE VIEW is the other half of Re-blur here, and it is DEBOUNCED rather than live. Camera events fire continuously through a pan and each redraw is 38-126 ms, so tracking them directly would stutter the whole drag. A single-shot timer restarts on every camera event and fires once movement has stopped -- the same moment a hand would have reached for the button. Off by default, and disabled outright with a reason in its tooltip if this napari will not report camera movement. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 92 ++++++++++++++++++++++++- src/ndi/gui/app/genepyramid/viewer.py | 11 +++ tests/test_gui_genepyramid_density.py | 47 +++++++++++++ 3 files changed, 148 insertions(+), 2 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 8bd260f6..bb6d1ed2 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1224,7 +1224,7 @@ def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: controls where a reader already looks for them. This knob does width and nothing else, which is why the kernel is unity. """ - from qtpy.QtCore import Qt + from qtpy.QtCore import Qt, QTimer from qtpy.QtWidgets import ( QCheckBox, QDoubleSpinBox, @@ -1240,6 +1240,13 @@ def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: outer = QVBoxLayout(box) show = QCheckBox("blurred cell density") + follow = QCheckBox("follow the view") + follow.setToolTip( + "Re-blurs by itself a moment after you stop moving.\n" + "Off by default: the redraw is 38-126 ms, so following every camera\n" + "event would stutter through a pan -- hence the pause rather than\n" + "live tracking." + ) here = QPushButton("Re-blur here") here.setToolTip( "Re-bins just what is on screen, at full resolution for this zoom.\n" @@ -1381,6 +1388,37 @@ def _hide(): row.addWidget(spin) row.addWidget(here) here.clicked.connect(_reblurHere) + + # DEBOUNCED, not live. Camera events fire continuously through a pan + # and each redraw is 38-126 ms, so following them directly would + # stutter the whole drag. The timer restarts on every event and only + # fires once movement has stopped, which is the same moment a hand + # would have reached for the button. + settle = QTimer(box) + settle.setSingleShot(True) + settle.setInterval(400) + settle.timeout.connect(lambda: _reblurHere() if follow.isChecked() else None) + + def _cameraMoved(_event=None): + if follow.isChecked(): + settle.start() + + def _watchCamera(on): + # Connected once and left connected: _cameraMoved checks the box + # itself, so toggling cannot leak a second connection, and + # disconnecting a napari event is the fiddlier half of the API. + if on: + _reblurHere() + + for _evt in ("zoom", "center"): + try: + getattr(viewer.camera.events, _evt).connect(_cameraMoved) + except Exception: # noqa: BLE001 - following is optional, the panel is not + follow.setEnabled(False) + follow.setToolTip("this napari does not report camera movement") + + follow.toggled.connect(_watchCamera) + outer.addWidget(follow) outer.addWidget(show) outer.addLayout(row) outer.addWidget(status) @@ -1512,6 +1550,51 @@ def _reset(): return box +def cloudPaperStyle(): + """A Qt stylesheet putting a widget on the NDI Cloud palette. + + napari ships a dark theme and its docks inherit it, so the control + panels came out grey against every other NDI applet, which are white + bodies with a navy header. The palette is not invented here -- it is + :mod:`ndi.gui.cloud_colors`, the same triplets ndi.gui.cloudColors + holds on the MATLAB side, so the two stay one look rather than two + approximations of one. + """ + from ndi.gui.cloud_colors import CloudColors + + c = CloudColors() + + def hexOf(triplet): + return "#" + "".join(f"{int(round(v * 255)):02x}" for v in triplet) + + body = hexOf(c.off_white) + ink = hexOf(c.dark_blue) + edge = hexOf(c.neutral_grey) + field = hexOf(c.white) + return ( + f"QWidget {{ background: {body}; color: {ink}; }}" + f"QLabel, QCheckBox, QRadioButton {{ background: transparent; color: {ink}; }}" + f"QLineEdit, QAbstractItemView, QDoubleSpinBox, QSpinBox " + f"{{ background: {field}; color: {ink}; border: 1px solid {edge}; }}" + f"QPushButton {{ background: {field}; color: {ink};" + f" border: 1px solid {edge}; padding: 3px 8px; }}" + ) + + +def applyCloudStyle(widget) -> bool: + """Put one panel on the cloud palette. False if it would not take. + + Styling is never worth the window, and a Qt build that refuses a + stylesheet should cost the colour rather than the panel -- the same + trade ndi.gui.app.GEFManager.paper makes on the MATLAB side. + """ + try: + widget.setStyleSheet(cloudPaperStyle()) + return True + except Exception: # noqa: BLE001 - a colour is never worth the panel + return False + + def _importQtWidgets() -> None: """Import qtpy.QtWidgets, or raise ImportError. @@ -1573,7 +1656,12 @@ def addAllPanels( def _build(name, fn): try: - return fn() + panel = fn() + # Styled here rather than in each builder, so a panel added + # later cannot forget and come out grey among white ones. + if panel is not None: + applyCloudStyle(panel) + return panel except Exception as e: # noqa: BLE001 - a panel is never worth the window print( f"[genepyramid] the {name} panel could not be built " diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index 02f12840..64cf32e6 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -95,12 +95,23 @@ def openPyramid( Returns: The napari Viewer. """ + import sys + from .progress import closeLaunchWindow, note, stage with stage("importing napari"): napari = require_napari() viewer = napari.Viewer() + # THE LIGHT THEME. napari defaults to its dark one, which left the + # window grey against every other NDI applet -- white bodies under a + # navy header. Guarded rather than assumed: theme names are napari's + # and could move, and a viewer that opens in the wrong grey is better + # than one that does not open. + try: + viewer.theme = "light" + except Exception as e: # noqa: BLE001 - a colour is never worth the window + print(f"[genepyramid] could not set the light theme ({e})", file=sys.stderr) # The image ladder is LAZY: layerSpec resolves tile paths and the # level table, and reads no tile bytes. Nothing here is the wait. with stage("building the pyramid ladder (lazy)"): diff --git a/tests/test_gui_genepyramid_density.py b/tests/test_gui_genepyramid_density.py index 5cca0c81..bce1c688 100644 --- a/tests/test_gui_genepyramid_density.py +++ b/tests/test_gui_genepyramid_density.py @@ -206,3 +206,50 @@ class Viewer: window = Window() assert canvasSize(Viewer()) == (1280.0, 720.0) + + +class TestTheCloudPalette: + """napari ships a dark theme and its docks inherit it, so the panels + came out grey against every other NDI applet -- white bodies under a + navy header. The palette is not invented in the stylesheet; it comes + from ndi.gui.cloud_colors, the same triplets ndi.gui.cloudColors holds + on the MATLAB side, so the two stay one look rather than two + approximations of one.""" + + def test_the_stylesheet_carries_the_cloud_colours(self): + from ndi.gui.app.genepyramid.controls import cloudPaperStyle + from ndi.gui.cloud_colors import CloudColors + + css = cloudPaperStyle() + c = CloudColors() + + def hexOf(t): + return "#" + "".join(f"{int(round(v * 255)):02x}" for v in t) + + assert hexOf(c.off_white) in css, "panel bodies are the cloud off-white" + assert hexOf(c.dark_blue) in css, "text is the cloud navy" + assert hexOf(c.white) in css, "fields are white" + + def test_a_widget_that_refuses_a_stylesheet_costs_the_colour_only(self): + """Styling is never worth the panel -- the same trade + ndi.gui.app.GEFManager.paper makes on the MATLAB side.""" + from ndi.gui.app.genepyramid.controls import applyCloudStyle + + class Refuses: + def setStyleSheet(self, _css): + raise RuntimeError("no stylesheets here") + + assert applyCloudStyle(Refuses()) is False + + def test_a_widget_that_takes_it_reports_success(self): + from ndi.gui.app.genepyramid.controls import applyCloudStyle + + class Takes: + css = None + + def setStyleSheet(self, css): + self.css = css + + w = Takes() + assert applyCloudStyle(w) is True + assert w.css From 5531e29f2d867d20b8f5b2fcc4cb68245588142d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 13:44:53 +0000 Subject: [PATCH 33/52] Bump GEFManager's sync hash: the palette fix moved it The bridge completeness gate caught my own drift. NDI-matlab 92ddd79c1 put the GEF Manager's controls on the cloud palette, and this entry still recorded 38aa4e8fb, so the check reported the file as moved -- which it was. Nothing to port. That MATLAB commit is styling only, and the Python side made the same correction independently and in its own idiom: the napari viewer sets the light theme and styles its dock panels from ndi.gui.cloud_colors, which holds the identical triplets. These are two different windows that agreed to look alike, not one window and its port. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/ndi_matlab_python_bridge.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index 8a4b4d38..258a1044 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -822,7 +822,7 @@ ported_late: - name: GEFManager matlab_path: "+ndi/+gui/+app/GEFManager.m" - matlab_last_sync_hash: "38aa4e8fb" + matlab_last_sync_hash: "92ddd79c1" status: porting_deferred decision_log: > MATLAB App Designer window that lists, views, adds and removes @@ -843,3 +843,13 @@ ported_late: is the ingest and the listing; the viewing the MATLAB app offers is this side's, reached across a process boundary rather than reimplemented. + + NDI-matlab 92ddd79c1 is styling only: a paper() helper puts the + uitable, uitextarea, uilistbox and uieditfield on the cloud + palette's white body, since the figure and its layouts were already + on c.offWhite while the controls kept MATLAB's own grey. The Python + side made the SAME correction independently and in its own idiom -- + the napari viewer sets the light theme and styles its dock panels + from ndi.gui.cloud_colors, the identical palette. Nothing to port: + these are two different windows that agreed to look alike, not one + window and its port. From 71df5c9bfbdb1ef706e9842faabb348f0564b22c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 14:05:44 +0000 Subject: [PATCH 34/52] Open a downloaded dataset as a dataset, not as a session napariViewGEF pointed at a downloaded cloud dataset reported "no spatialGeneExpressionPyramid" for a directory that plainly had one -- the same path opened in MATLAB as an ndi.dataset.dir listed the pyramid and its levels. THE TWO LOOK IDENTICAL ON DISK. A dataset keeps its database at /.ndi exactly as a session does, so ndi_session_dir opens a downloaded dataset without complaint and then finds almost nothing in it: a dataset's documents may live in LINKED SESSIONS, and only ndi_dataset.database_search follows those links -- "searches the session's database directly, then also searches linked sessions", as it says. The session reader looks in the dataset's own database and stops. Nothing errors. The path is right, the data is there, and the reader is one level too shallow. The dataset reading is tried first and kept when it finds pyramids. A plain session opened as a dataset has no linked sessions and answers the same as before, but that is not relied on: if the dataset reading raises, or finds nothing where the session reading finds something, the session reading wins. Neither working is the only error, and it names both. _asDataset and _asSession are named functions rather than inline imports so the tests can stand in for them -- what _open_session does AROUND the two readings is not about how either is constructed. For the session one there is a second reason: ndi.session.dir resolves to the CLASS, not the module, because the package attribute shadows the submodule, so patching it where it is imported does not work. That cost a test run to find and is worth a line rather than rediscovering. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/cli.py | 56 +++++++++++++++++++++- tests/test_gui_genepyramid_cli.py | 75 ++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index 62ad3c4b..1d0f0b71 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -98,12 +98,66 @@ def build_parser() -> argparse.ArgumentParser: return p -def _open_session(path): +def _asDataset(path): + """Open PATH as a dataset. A named function, not an inline import, so + a test can stand in for it: what _open_session does AROUND the two + readings -- which it prefers, and that a failure of one does not lose + the other -- is not about how either is constructed.""" + from ndi.dataset import ndi_dataset_dir + + return ndi_dataset_dir(path) + + +def _asSession(path): + """Open PATH as a session. Named for the same reason as _asDataset, + and with more cause: ``ndi.session.dir`` resolves to the CLASS rather + than the module (the package attribute shadows the submodule), so + patching it where it is imported does not work.""" from ndi.session.dir import ndi_session_dir return ndi_session_dir(path) +def _open_session(path): + """Open PATH, whether it holds a session or a downloaded dataset. + + THE TWO LOOK IDENTICAL ON DISK. A dataset keeps its database at + /.ndi exactly as a session does, so ndi_session_dir opens a + downloaded dataset without complaint -- and then finds almost + nothing in it. A dataset's documents may live in LINKED SESSIONS, + and only ndi_dataset.database_search follows those links; the + session reader looks in the dataset's own database and stops there. + + The symptom is a downloaded dataset reporting "no + spatialGeneExpressionPyramid" while the same directory opened in + MATLAB as an ndi.dataset.dir lists the pyramid and its levels. The + path is right, the data is there, and the reader is looking one + level too shallow. + + So the dataset reading is tried first and kept when it finds + pyramids. A plain session opened as a dataset simply has no linked + sessions and answers the same as before, but that is not relied on: + if the dataset reading raises, or finds nothing where the session + reading finds something, the session reading wins. + """ + opened = None + for describe in (_asDataset, _asSession): + try: + candidate = describe(path) + except Exception: # noqa: BLE001 - the other reading may still work + continue + try: + if _pyramids(candidate): + return candidate + except Exception: # noqa: BLE001 - likewise + continue + if opened is None: + opened = candidate + if opened is None: + raise ValueError(f"{path} could not be opened as either an NDI session or a dataset") + return opened + + def _pyramids(session): from ndi.query import ndi_query diff --git a/tests/test_gui_genepyramid_cli.py b/tests/test_gui_genepyramid_cli.py index 204c6dc1..a9830fae 100644 --- a/tests/test_gui_genepyramid_cli.py +++ b/tests/test_gui_genepyramid_cli.py @@ -205,3 +205,78 @@ def test_auto_cells_with_no_document_says_so(one_pyramid): def test_no_cells_flag_means_no_overlay(one_pyramid, capsys): assert main([one_pyramid, "--report"]) == 0 assert "centroids" not in capsys.readouterr().out + + +# ------------------------------------------------------- opening a dataset + + +class TestOpeningADownloadedDataset: + """A downloaded cloud dataset and a session are indistinguishable on + disk -- both keep their database at /.ndi -- so opening one as + the other succeeds and then finds almost nothing. A dataset's + documents may live in LINKED SESSIONS, and only + ndi_dataset.database_search follows those links. + + The symptom was a downloaded ferret dataset reporting "no + spatialGeneExpressionPyramid" while the same directory opened in + MATLAB as an ndi.dataset.dir listed the pyramid and its levels. + """ + + def _install(self, monkeypatch, dataset, session): + from ndi.gui.app.genepyramid import cli + + monkeypatch.setattr(cli, "_pyramids", lambda obj: getattr(obj, "pyramids", [])) + monkeypatch.setattr(cli, "_asDataset", dataset) + monkeypatch.setattr(cli, "_asSession", session) + return cli + + def test_the_dataset_reading_wins_when_it_finds_the_pyramids(self, monkeypatch): + class Dataset: + pyramids = ["pyr"] + + class Session: + pyramids = [] + + cli = self._install(monkeypatch, lambda _p: Dataset(), lambda _p: Session()) + assert isinstance(cli._open_session("/some/download"), Dataset) + + def test_a_plain_session_still_opens_when_the_dataset_reading_is_empty(self, monkeypatch): + class Dataset: + pyramids = [] + + class Session: + pyramids = ["pyr"] + + cli = self._install(monkeypatch, lambda _p: Dataset(), lambda _p: Session()) + assert isinstance(cli._open_session("/some/session"), Session) + + def test_a_dataset_reading_that_raises_does_not_lose_the_session(self, monkeypatch): + def boom(_p): + raise RuntimeError("not a dataset") + + class Session: + pyramids = ["pyr"] + + cli = self._install(monkeypatch, boom, lambda _p: Session()) + assert isinstance(cli._open_session("/some/session"), Session) + + def test_neither_reading_working_says_so(self, monkeypatch): + import pytest + + def boom(_p): + raise RuntimeError("nope") + + cli = self._install(monkeypatch, boom, boom) + with pytest.raises(ValueError, match="either an NDI session or a dataset"): + cli._open_session("/nowhere") + + def test_an_empty_path_still_returns_something_to_report_on(self, monkeypatch): + """Both open, neither has pyramids: the caller wants the object so + it can print its own 'no pyramid here' message naming the path, + not an exception from the opener.""" + + class Empty: + pyramids = [] + + cli = self._install(monkeypatch, lambda _p: Empty(), lambda _p: Empty()) + assert isinstance(cli._open_session("/empty"), Empty) From 20d2a2c17065695bcb2ce085919815b757c4c720 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 14:12:08 +0000 Subject: [PATCH 35/52] Let a dataset say where it lives, so its tiles can be fetched Opening a downloaded dataset in the viewer got past "no spatialGeneExpressionPyramid" and then failed on the first tile: RuntimeError: Cannot fetch 'tile.bin_262' from thread ThreadPoolExecutor-1_0: this session cannot be reopened for another thread ... That is the tile fetcher's own guard, and it was right to fire. NDI's database belongs to the thread that opened it and dask runs blocks on a pool, so the fetcher builds a per-thread handle by reopening whatever it was given AT ITS OWN PATH. ndi_session_dir exposes `path` and could be reopened. ndi_dataset_dir kept the same value privately as `_path`, so _reopener found nothing, _ensurePool returned None, and the fetcher could only explain itself and stop. MATLAB's ndi.dataset.dir has declared a public `path` property since it was written ("the file path of the session"), so this was a parity gap rather than a design choice: this side simply never exposed the _path it was already keeping. A directory-backed dataset that cannot say which directory is a hole regardless of who asks. The reopened handle resolves tiles correctly because ndi_dataset.database_openbinarydoc already dispatches to the doc's owning session, member sessions included -- the case its own docstring says would otherwise raise FileNotFoundError "even when the file is really there". Found by fixing the layer above it. Opening a downloaded dataset AS a dataset (71df5c9) was necessary and not sufficient: it moved the failure from "no pyramid" to the first tile, because it changed the type the fetcher was handed to one that could not be reopened. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/dataset/_dataset.py | 19 ++++++++ tests/test_gui_genepyramid_multiscale.py | 61 ++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/src/ndi/dataset/_dataset.py b/src/ndi/dataset/_dataset.py index 2f834969..ae77333c 100644 --- a/src/ndi/dataset/_dataset.py +++ b/src/ndi/dataset/_dataset.py @@ -1328,6 +1328,25 @@ def __init__( # datasets that don't yet have session_in_a_dataset tracking). self._ensure_session_tracking() + @property + def path(self) -> Path: + """The directory this dataset lives in. + + MATLAB equivalent: ``ndi.dataset.dir``'s ``path`` property, which + it has declared since it was written -- this side simply never + exposed the ``_path`` it was already keeping, which made a + directory-backed dataset unable to say where it was. + + That gap had teeth. Anything asking a session-or-dataset where it + lives got an answer from a session and None from a dataset, and + the failure landed far from the cause: the gene pyramid's tile + fetcher builds a per-thread handle by reopening whatever it was + given AT ITS OWN PATH, and with no path it cannot, so opening a + downloaded dataset in the napari viewer failed on the first tile + with a threading error rather than a missing attribute. + """ + return self._path + @staticmethod def _must_not_be_session(path: Path) -> None: """Raise if *path* holds a standalone session rather than a dataset. diff --git a/tests/test_gui_genepyramid_multiscale.py b/tests/test_gui_genepyramid_multiscale.py index 357c5060..cf99a9bb 100644 --- a/tests/test_gui_genepyramid_multiscale.py +++ b/tests/test_gui_genepyramid_multiscale.py @@ -351,3 +351,64 @@ def test_layer_spec_name_falls_back_when_the_pyramid_has_no_label(tmp_path): grid=1, ) assert layerSpec(S, pyr)["name"] == "genes" + + +class TestADatasetCanBeReopenedPerThread: + """A downloaded cloud dataset opened in the viewer failed on its first + tile with a threading error, not a missing attribute. + + The tile fetcher builds a per-thread session handle by reopening + whatever it was given AT ITS OWN PATH -- NDI's database belongs to the + thread that opened it, and dask runs blocks on a pool. A session + exposes ``path`` and could be reopened; ``ndi_dataset_dir`` kept the + same value privately as ``_path`` and could not, so the fetcher had no + way to build a handle and said so from the worker thread. + + MATLAB's ``ndi.dataset.dir`` has declared a public ``path`` property + since it was written, so this was a parity gap rather than a design + choice. + """ + + def test_a_directory_backed_dataset_reports_its_path(self): + from ndi.dataset import ndi_dataset_dir + + assert isinstance(ndi_dataset_dir.path, property) + + def test_the_fetcher_can_reopen_anything_with_a_path(self): + from ndi.gui.app.genepyramid.multiscale import _TileFetcher + + class Pathed: + def __init__(self, p): + self._p = p + + @property + def path(self): + return self._p + + assert _TileFetcher(Pathed("/tmp/somewhere"))._reopen is not None + + def test_something_with_no_path_is_left_alone(self): + """Anything holding a live client or credentials is not duplicated + on a guess -- reopening one could re-authenticate per thread.""" + from ndi.gui.app.genepyramid.multiscale import _TileFetcher + + class Clientish: + pass + + assert _TileFetcher(Clientish())._reopen is None + + def test_a_reopened_handle_is_the_same_class_at_the_same_path(self): + from ndi.gui.app.genepyramid.multiscale import _TileFetcher + + class Pathed: + def __init__(self, p): + self._p = p + + @property + def path(self): + return self._p + + original = Pathed("/tmp/here") + rebuilt = _TileFetcher(original)._reopen() + assert type(rebuilt) is Pathed + assert rebuilt.path == "/tmp/here" From c6588b9e88b198c3d65c72308256968d182e26be Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 14:19:33 +0000 Subject: [PATCH 36/52] Pair GEFManager's sync hash with the non-blocking dialog NDI-matlab cf0983b5c made the View dialog non-modal and gave the app an onClose. Nothing to port -- the Python side draws no dialog at all; View builds a command line and runs napariViewGEF, which is already its own process and has never blocked anything -- but the hash has to follow the file or the completeness gate reports drift, as it correctly did the last time I moved this file without bumping it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/ndi_matlab_python_bridge.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index 258a1044..022d160a 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -822,7 +822,7 @@ ported_late: - name: GEFManager matlab_path: "+ndi/+gui/+app/GEFManager.m" - matlab_last_sync_hash: "92ddd79c1" + matlab_last_sync_hash: "cf0983b5c" status: porting_deferred decision_log: > MATLAB App Designer window that lists, views, adds and removes @@ -853,3 +853,9 @@ ported_late: from ndi.gui.cloud_colors, the identical palette. Nothing to port: these are two different windows that agreed to look alike, not one window and its port. + + NDI-matlab cf0983b5c made the View dialog non-modal -- onView no + longer ends in uiwait -- and gave the app an onClose that takes the + dialog with it. Nothing to port: the Python side draws no dialog at + all. View builds a command line and runs napariViewGEF, which is + already its own process and has never blocked anything. From 8f73cf067d71d9d7d8aa9846c186115eed20408e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 14:37:15 +0000 Subject: [PATCH 37/52] Give the canvas its dark back, and stop the long step calling itself lazy TWO SURFACES, and I had confused them. The ask was for the control PANELS to be on the NDI Cloud palette like every other applet; I set napari's whole theme to "light", which did that and took the CANVAS with it. Imaging data is read against a dark ground, and the canvas is not a panel. The theme is left alone now; the panels are still styled individually by controls.applyCloudStyle, which does not touch the canvas. And the launch reporting was lying about where the time goes. One stage wrapped both layerSpec and add_image under the label "building the pyramid ladder (lazy)". Only the first of those is lazy: add_image -> Image._post_init -> refresh -> set_view_slice -> _level_materializer(thumbnail_level) -> np.asarray(data[level]) napari builds its thumbnail by materialising the WHOLE COARSEST LEVEL. On a real section that is gigabytes of tiles, and on a downloaded dataset it is gigabytes of FETCHING -- minutes, behind a label claiming nothing was being read. A ferret hemisphere sat there long enough to be reported as a hang, and it was not hung; it was working, silently, on the step the label said was free. Split in two, and the second one named for what it does. That does not make it fast -- the coarsest level really is that big, because binning merges pixels while the genes in them stay distinct -- but it makes the wait legible instead of looking like a stall. Making it actually fast needs a small gene-summed overview raster stored at build time, which is a rebuild and a separate decision. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/viewer.py | 34 +++++++++++++++++---------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index 64cf32e6..f82ef2fd 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -95,27 +95,37 @@ def openPyramid( Returns: The napari Viewer. """ - import sys - from .progress import closeLaunchWindow, note, stage with stage("importing napari"): napari = require_napari() viewer = napari.Viewer() - # THE LIGHT THEME. napari defaults to its dark one, which left the - # window grey against every other NDI applet -- white bodies under a - # navy header. Guarded rather than assumed: theme names are napari's - # and could move, and a viewer that opens in the wrong grey is better - # than one that does not open. - try: - viewer.theme = "light" - except Exception as e: # noqa: BLE001 - a colour is never worth the window - print(f"[genepyramid] could not set the light theme ({e})", file=sys.stderr) + # NAPARI'S OWN THEME IS LEFT ALONE, which means the canvas stays dark. + # Setting it to "light" did put the docks on a light ground, but it + # took the CANVAS with it -- and a dark canvas is what imaging data is + # read against. What was actually wanted was the PANELS on the NDI + # Cloud palette, and those are styled individually in + # ndi.gui.app.genepyramid.controls.applyCloudStyle, which does not + # touch the canvas. Two different surfaces, and only one of them + # should be white. # The image ladder is LAZY: layerSpec resolves tile paths and the # level table, and reads no tile bytes. Nothing here is the wait. with stage("building the pyramid ladder (lazy)"): - image = viewer.add_image(**layerSpec(session, pyr_doc, gene_rows, density, name)) + spec = layerSpec(session, pyr_doc, gene_rows, density, name) + # SPLIT FROM THE LADDER, because only one of these two is lazy and the + # label used to claim both were. Building the ladder reads nothing. + # add_image then makes napari's thumbnail, and napari builds that by + # materialising the WHOLE COARSEST LEVEL -- Image._post_init calls + # refresh, which reaches _level_materializer and computes the dask + # array for the thumbnail level. On a real section that is gigabytes + # of tiles, and on a downloaded dataset it is gigabytes of fetching. + # + # It is the longest step of a cold launch and it used to sit behind a + # label that said "(lazy)", which is why it read as a hang. Naming it + # does not make it fast; it makes it legible. + with stage("drawing the overview (napari reads the coarsest level)"): + image = viewer.add_image(**spec) shapes = None if outlines is not None: From 927116bcd79b559cf53428de2e6ad489d8f807a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 14:38:29 +0000 Subject: [PATCH 38/52] Leave napari alone: the grey windows were the MATLAB ones "The GEF manager window and the viewer window are mostly gray" meant the GEF Manager and the VIEW-IN-NAPARI LAUNCHER DIALOG -- both MATLAB uifigures, and both already on the cloud palette since NDI-matlab 92ddd79c1 put their uitable, uitextarea, uilistbox and uieditfield on c.white beside the layouts that were already c.offWhite. I had read "the viewer window" as napari and changed two things there that nobody asked for. The theme went back in 8f73cf0, which returned the canvas to dark. This takes the other one: the dock panels are no longer restyled. Removed rather than left dormant, because on reflection it was also the wrong look. napari's own layer list and layer controls stay in napari's theme, so white panels beside them would be two palettes in one window -- a worse result than either alone, and one that would read as a bug rather than a decision. The MATLAB side is unaffected: those windows are ours, sit beside the other NDI applets, and should match them. The napari viewer is a third-party window we borrow, and it should look like itself. One line to put back if it turns out to be wanted. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 52 +------------------------ tests/test_gui_genepyramid_density.py | 47 ---------------------- 2 files changed, 1 insertion(+), 98 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index bb6d1ed2..e4a77012 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1550,51 +1550,6 @@ def _reset(): return box -def cloudPaperStyle(): - """A Qt stylesheet putting a widget on the NDI Cloud palette. - - napari ships a dark theme and its docks inherit it, so the control - panels came out grey against every other NDI applet, which are white - bodies with a navy header. The palette is not invented here -- it is - :mod:`ndi.gui.cloud_colors`, the same triplets ndi.gui.cloudColors - holds on the MATLAB side, so the two stay one look rather than two - approximations of one. - """ - from ndi.gui.cloud_colors import CloudColors - - c = CloudColors() - - def hexOf(triplet): - return "#" + "".join(f"{int(round(v * 255)):02x}" for v in triplet) - - body = hexOf(c.off_white) - ink = hexOf(c.dark_blue) - edge = hexOf(c.neutral_grey) - field = hexOf(c.white) - return ( - f"QWidget {{ background: {body}; color: {ink}; }}" - f"QLabel, QCheckBox, QRadioButton {{ background: transparent; color: {ink}; }}" - f"QLineEdit, QAbstractItemView, QDoubleSpinBox, QSpinBox " - f"{{ background: {field}; color: {ink}; border: 1px solid {edge}; }}" - f"QPushButton {{ background: {field}; color: {ink};" - f" border: 1px solid {edge}; padding: 3px 8px; }}" - ) - - -def applyCloudStyle(widget) -> bool: - """Put one panel on the cloud palette. False if it would not take. - - Styling is never worth the window, and a Qt build that refuses a - stylesheet should cost the colour rather than the panel -- the same - trade ndi.gui.app.GEFManager.paper makes on the MATLAB side. - """ - try: - widget.setStyleSheet(cloudPaperStyle()) - return True - except Exception: # noqa: BLE001 - a colour is never worth the panel - return False - - def _importQtWidgets() -> None: """Import qtpy.QtWidgets, or raise ImportError. @@ -1656,12 +1611,7 @@ def addAllPanels( def _build(name, fn): try: - panel = fn() - # Styled here rather than in each builder, so a panel added - # later cannot forget and come out grey among white ones. - if panel is not None: - applyCloudStyle(panel) - return panel + return fn() except Exception as e: # noqa: BLE001 - a panel is never worth the window print( f"[genepyramid] the {name} panel could not be built " diff --git a/tests/test_gui_genepyramid_density.py b/tests/test_gui_genepyramid_density.py index bce1c688..5cca0c81 100644 --- a/tests/test_gui_genepyramid_density.py +++ b/tests/test_gui_genepyramid_density.py @@ -206,50 +206,3 @@ class Viewer: window = Window() assert canvasSize(Viewer()) == (1280.0, 720.0) - - -class TestTheCloudPalette: - """napari ships a dark theme and its docks inherit it, so the panels - came out grey against every other NDI applet -- white bodies under a - navy header. The palette is not invented in the stylesheet; it comes - from ndi.gui.cloud_colors, the same triplets ndi.gui.cloudColors holds - on the MATLAB side, so the two stay one look rather than two - approximations of one.""" - - def test_the_stylesheet_carries_the_cloud_colours(self): - from ndi.gui.app.genepyramid.controls import cloudPaperStyle - from ndi.gui.cloud_colors import CloudColors - - css = cloudPaperStyle() - c = CloudColors() - - def hexOf(t): - return "#" + "".join(f"{int(round(v * 255)):02x}" for v in t) - - assert hexOf(c.off_white) in css, "panel bodies are the cloud off-white" - assert hexOf(c.dark_blue) in css, "text is the cloud navy" - assert hexOf(c.white) in css, "fields are white" - - def test_a_widget_that_refuses_a_stylesheet_costs_the_colour_only(self): - """Styling is never worth the panel -- the same trade - ndi.gui.app.GEFManager.paper makes on the MATLAB side.""" - from ndi.gui.app.genepyramid.controls import applyCloudStyle - - class Refuses: - def setStyleSheet(self, _css): - raise RuntimeError("no stylesheets here") - - assert applyCloudStyle(Refuses()) is False - - def test_a_widget_that_takes_it_reports_success(self): - from ndi.gui.app.genepyramid.controls import applyCloudStyle - - class Takes: - css = None - - def setStyleSheet(self, css): - self.css = css - - w = Takes() - assert applyCloudStyle(w) is True - assert w.css From f14b437eaac97b69cad11efc3f4289bac705e512 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 14:41:51 +0000 Subject: [PATCH 39/52] Paint each launch step before it blocks, instead of queueing it The launch window sat on "importing napari" for minutes while the label underneath had moved on several steps -- so a launch that was working read as one that was stuck, on the wrong step. The label was never wrong. setText only QUEUES a repaint, and processEvents services what is queued at that instant. The step that follows then blocks the thread with no event loop running, so any paint the window needs AFTER that point -- it gets exposed, the compositor asks again, the OS marks the app unresponsive -- is never serviced. The window freezes on whichever label was current when the first slow step began, and on a cold launch that is the napari import. repaint() paints synchronously, before returning, so each label is on the glass before its step blocks. The byte counter gets the same treatment, for the same reason: a byte count that only lands once the transfer has finished has reported nothing. This does not animate, and cannot: without an event loop nothing moves between steps. What it buys is that each step is legible WHILE it runs, rather than one step being legible for all of them. Making the window live through a blocking step would mean running the work off the main thread, which napari's viewer cannot be. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/progress.py | 23 +++++++ tests/test_gui_genepyramid_progress.py | 87 +++++++++++++++++++++++++ 2 files changed, 110 insertions(+) diff --git a/src/ndi/gui/app/genepyramid/progress.py b/src/ndi/gui/app/genepyramid/progress.py index 7b428dd8..492389b5 100644 --- a/src/ndi/gui/app/genepyramid/progress.py +++ b/src/ndi/gui/app/genepyramid/progress.py @@ -270,6 +270,25 @@ def _launchWindow(): def _guiStage(label: str) -> None: + """Show the step that is about to run, and MAKE SURE IT IS PAINTED. + + setText queues a repaint; processEvents services what is queued NOW. + Between them they can still leave the old text on screen, because the + step that follows blocks the thread for as long as it takes and no + event loop runs meanwhile -- so any paint the window needs after that + point (it gets exposed, the compositor asks again, the OS marks the + app unresponsive) is never serviced. + + The effect is a window frozen on whichever label was current when the + FIRST slow step began, which made a launch that was working look + stuck on "importing napari" for minutes while the label underneath + had long since moved on. + + repaint() paints synchronously, before returning, so the new label is + on the glass before the blocking work starts. It does not animate -- + nothing can, without an event loop -- but each step is legible while + it runs instead of one step being legible for all of them. + """ win = _launchWindow() if win is None: return @@ -277,6 +296,7 @@ def _guiStage(label: str) -> None: win["step"].setText(label) win["bar"].setRange(0, 0) # unknown length again win["app"].processEvents() + win["widget"].repaint() def _guiBytes(done: int, total) -> None: @@ -292,6 +312,9 @@ def _guiBytes(done: int, total) -> None: win["bar"].setRange(0, 0) win["bar"].setFormat(humanBytes(done)) win["app"].processEvents() + # Synchronous, for the same reason as _guiStage: a byte count that + # only lands when the transfer finishes has reported nothing. + win["widget"].repaint() def closeLaunchWindow() -> None: diff --git a/tests/test_gui_genepyramid_progress.py b/tests/test_gui_genepyramid_progress.py index fc2af06a..1ab6b748 100644 --- a/tests/test_gui_genepyramid_progress.py +++ b/tests/test_gui_genepyramid_progress.py @@ -246,3 +246,90 @@ def close(self): if __name__ == "__main__": unittest.main() + + +class TestTheLabelIsActuallyPainted: + """The launch window sat on "importing napari" for minutes while the + label underneath had moved on several steps. + + setText only QUEUES a repaint, and processEvents services what is + queued at that instant -- but the step that follows blocks the thread + with no event loop running, so any paint the window needs after that + point is never serviced. The window freezes on whichever label was + current when the first slow step began. repaint() paints + synchronously, before returning, so each label reaches the glass + before its step blocks. + """ + + def _fakeWindow(self): + painted = [] + + class Widget: + def repaint(self): + painted.append("widget") + + class App: + def processEvents(self): + painted.append("events") + + class Label: + text = None + + def setText(self, t): + self.text = t + + class Bar: + def setRange(self, *_a): + pass + + def setValue(self, *_a): + pass + + def setFormat(self, *_a): + pass + + return { + "app": App(), + "widget": Widget(), + "step": Label(), + "bar": Bar(), + }, painted + + def test_a_stage_paints_before_it_returns(self, monkeypatch): + from ndi.gui.app.genepyramid import progress + + win, painted = self._fakeWindow() + monkeypatch.setattr(progress, "_launchWindow", lambda: win) + progress._guiStage("reading the tiles") + assert win["step"].text == "reading the tiles" + assert "widget" in painted, "the label must be on the glass, not just queued" + + def test_a_byte_update_paints_too(self, monkeypatch): + """A byte count that only lands when the transfer finishes has + reported nothing.""" + from ndi.gui.app.genepyramid import progress + + win, painted = self._fakeWindow() + monkeypatch.setattr(progress, "_launchWindow", lambda: win) + progress._guiBytes(50, 100) + assert "widget" in painted + + def test_a_window_that_will_not_paint_costs_nothing(self, monkeypatch): + """Styling and progress are never worth the launch.""" + from ndi.gui.app.genepyramid import progress + + class Hostile: + def repaint(self): + raise RuntimeError("no") + + win, _painted = self._fakeWindow() + win["widget"] = Hostile() + monkeypatch.setattr(progress, "_launchWindow", lambda: win) + progress._guiStage("still fine") # must not raise + + def test_no_window_is_not_an_error(self, monkeypatch): + from ndi.gui.app.genepyramid import progress + + monkeypatch.setattr(progress, "_launchWindow", lambda: None) + progress._guiStage("headless") + progress._guiBytes(1, 2) From 3b3bd2dc44fa29d0de5a6d17651920e29f1cf497 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 14:53:04 +0000 Subject: [PATCH 40/52] Say the launch steps in words a biologist reads, not ours "lazy" is a programmer's word for it, and it was not the only one. The labels named our implementation and napari's API rather than what the user is waiting for: importing napari -> starting napari building the pyramid ladder (lazy) -> preparing the image levels (read on demand) drawing the overview (napari reads -> drawing the overview (reads the coarsest level) the whole lowest-resolution level) napari add_shapes -> drawing the cell outlines napari add_points -> drawing the cell centroids "add_shapes" and "add_points" were method names showing through. "ladder" and "coarsest level" are how the pyramid is built, not what is happening to the picture. The parentheticals stay, because they are the two places where knowing WHY the wait is what it is changes what the user does: "read on demand" says the tiles have not been fetched yet, and "reads the whole lowest-resolution level" says why the step after it is the long one. The outline note loses "triangulates" for "cut into triangles", which is the same fact without the word. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/viewer.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index f82ef2fd..dbbab7ab 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -97,7 +97,7 @@ def openPyramid( """ from .progress import closeLaunchWindow, note, stage - with stage("importing napari"): + with stage("starting napari"): napari = require_napari() viewer = napari.Viewer() @@ -111,7 +111,7 @@ def openPyramid( # should be white. # The image ladder is LAZY: layerSpec resolves tile paths and the # level table, and reads no tile bytes. Nothing here is the wait. - with stage("building the pyramid ladder (lazy)"): + with stage("preparing the image levels (read on demand)"): spec = layerSpec(session, pyr_doc, gene_rows, density, name) # SPLIT FROM THE LADDER, because only one of these two is lazy and the # label used to claim both were. Building the ladder reads nothing. @@ -124,7 +124,7 @@ def openPyramid( # It is the longest step of a cold launch and it used to sit behind a # label that said "(lazy)", which is why it read as a hang. Naming it # does not make it fast; it makes it legible. - with stage("drawing the overview (napari reads the coarsest level)"): + with stage("drawing the overview (reads the whole lowest-resolution level)"): image = viewer.add_image(**spec) shapes = None @@ -143,11 +143,11 @@ def openPyramid( # THIS IS THE SLOW ONE and nothing here can make it fast: # napari triangulates every polygon as it takes them. note( - f"handing {len(keep):,} polygons to napari -- it triangulates " - f"each one, so this is the long part of the launch. Drop " - f"--outlines to skip it." + f"drawing {len(keep):,} cell outlines. Each one has to be cut " + f"into triangles before it can be drawn, which is the long " + f"part of the launch. Leave out --outlines to skip it." ) - with stage("napari add_shapes"): + with stage("drawing the cell outlines"): shapes = viewer.add_shapes( paths, shape_type="polygon", @@ -162,7 +162,7 @@ def openPyramid( with stage(f"placing {len(cells['x']):,} centroids"): row, col = sourceToWorld(session, pyr_doc, cells["x"], cells["y"]) coords = np.column_stack([row, col]) - with stage("napari add_points"): + with stage("drawing the cell centroids"): points = viewer.add_points( coords, name="cell centroids", From 94b7c748d1464db26e013e91709f914c339fc532 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 15:04:23 +0000 Subject: [PATCH 41/52] Blur the gene layers, and give the rotation slider a row of its own A gene's counts are a handful of pixels scattered over a section. Drawn honestly they are dust, and a reader looking for where a gene is expressed cannot see a pattern in it. blurLevels spreads each pixel over its neighbourhood so the pattern reads. Two things make it more than a call to gaussian_filter: THE WIDTH IS A DISTANCE, NOT A PIXEL COUNT. Each level of the ladder is binned differently, so a fixed number of level-pixels would be a different physical distance on each -- the blur would visibly change width as the viewer switched levels while zooming, which is exactly what a multiscale ladder exists to avoid. Dividing by each level's bin size makes one setting mean one distance everywhere. The knob itself is in micrometres, since that is the unit a reader thinks in; basePixelUm reads base_pixel_size_x to convert, falling back to the Stereo-seq 0.5 um pitch rather than dividing by zero on a document that does not say. BLURRED PER TILE, WITH A HALO. Levels are dask arrays whose blocks are tiles, and a plain per-block filter sees each tile's edge as the end of the world -- leaving a seam at every boundary, in a grid, which reads as an artefact of the data rather than of the drawing. map_overlap gives each block truncate*sigma of its neighbours, which is the kernel's reach, so the result matches blurring the level whole: max error 0.0 against scipy over sixteen blocks. The kernel is unity, so widening spreads counts rather than adding to them and width stays independent of brightness. The peak does fall a long way, so each re-blur resets contrast limits; without that a widened layer goes black. The unblurred ladder is kept per layer and every change re-blurs from it, so the knob is absolute rather than compounding. This replaces the centroid-density raster, which blurred the wrong thing: the ask was the gene data, not the cell dots or the contours. Its 360 lines and its tests go. Rotation panel: the slider shared a row with the angle box and the Reset button, which left it a stub too short to aim with -- a dock panel is narrow and those two take a fixed width out of it whatever is left over. It now gets the full width of a row beneath them. The box's own - and + buttons step five degrees, a turn you can see, where half a degree took ten clicks to show anything; finer angles are still typed. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 543 ++++++++---------------- tests/test_gui_genepyramid_blur.py | 144 +++++++ tests/test_gui_genepyramid_controls.py | 30 -- tests/test_gui_genepyramid_density.py | 208 --------- tests/test_gui_genepyramid_rotation.py | 119 +++++- 5 files changed, 439 insertions(+), 605 deletions(-) create mode 100644 tests/test_gui_genepyramid_blur.py delete mode 100644 tests/test_gui_genepyramid_density.py diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index e4a77012..4278b59d 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -30,6 +30,8 @@ "addDisplayPanel", "addCellTypePanel", "addAllPanels", + "blurLevels", + "basePixelUm", ] @@ -300,7 +302,7 @@ def _headText(info=None) -> str: items: dict[str, Any] = {} symbols = list(index) entries = list(index.values()) - state = {"busy": False, "colour": 0, "keep": None} + state = {"busy": False, "colour": 0, "keep": None, "sigma": 0.0, "raw": {}} def _makeItem(i: int): symbol = symbols[i] @@ -432,6 +434,28 @@ def _resort(): outer.addWidget(listw, stretch=1) + # ---- blur --------------------------------------------------------- + # Gene expression is a handful of counts in scattered single pixels. + # Drawn honestly it is dust; blurred it reads as a pattern. On the + # GENE layers only -- the base layer is already dense and the cell + # layers are not what anyone is squinting at. + blur_row = QHBoxLayout() + blur_row.addWidget(QLabel("blur")) + blur = QDoubleSpinBox() + blur.setRange(0.0, 200.0) + blur.setDecimals(1) + blur.setSingleStep(1.0) + blur.setSuffix(" um") + blur.setToolTip( + "Spreads each gene's counts over its neighbourhood, so a sparse\n" + "gene reads as a pattern instead of dust. The width is a DISTANCE,\n" + "so it stays the same as you zoom between levels. 0 is off.\n" + "Spreading lowers the peak a long way -- the contrast limits are\n" + "reset each time so the layer stays visible." + ) + blur_row.addWidget(blur, 1) + outer.addLayout(blur_row) + status = QLabel("") status.setWordWrap(True) clear = QPushButton("Remove all gene layers") @@ -507,12 +531,33 @@ def _report(extra: str = ""): n = _shown() status.setText(f"{n} gene layer(s) shown" + (f" -- {extra}" if extra else "")) + def _binSizes(): + # Cached: levelTable reads the level documents, and the blur knob + # would otherwise re-read them on every tick of the spin box. + if state.get("bins") is None: + from ....fun.doc_gene import levelTable + + levels, _f = levelTable(session, pyr_doc) + state["bins"] = [lv["binSize"] for lv in levels] + return state["bins"] + + def _blurred(data): + """The ladder as it should be drawn at the current width.""" + if not state["sigma"]: + return data + return blurLevels(data, _binSizes(), state["sigma"] / basePixelUm(pyr_doc)) + def _add(symbol: str): name = f"gene: {symbol}" if name in viewer.layers: return rows = index[symbol]["rows"] spec = layerSpec(session, pyr_doc, rows, True, name) + # The UNBLURRED ladder is kept, because the knob is absolute: each + # change re-blurs from the original rather than blurring what was + # already blurred, which would compound and could not be undone. + state["raw"][name] = spec["data"] + spec["data"] = _blurred(spec["data"]) cmap = _GENE_COLORMAPS[state["colour"] % len(_GENE_COLORMAPS)] state["colour"] += 1 try: @@ -531,8 +576,28 @@ def _remove(symbol: str): name = f"gene: {symbol}" if name in viewer.layers: del viewer.layers[name] + state["raw"].pop(name, None) _report() + def _reblur(_value=None): + state["sigma"] = float(blur.value()) + for name, raw in list(state["raw"].items()): + if name not in viewer.layers: + state["raw"].pop(name, None) + continue + layer = viewer.layers[name] + try: + layer.data = _blurred(raw) + # Spreading a sparse signal lowers its peak a long way, so + # limits set for the unblurred layer would show black. + layer.reset_contrast_limits() + except Exception as e: # noqa: BLE001 - a knob never costs a layer + status.setText(f"blur failed: {e}") + return + _report() + + blur.valueChanged.connect(_reblur) + def _on_item_changed(item): if state["busy"]: return @@ -1068,366 +1133,6 @@ def applyRotation(layers, angle_deg, center): return a -def densityRaster(row, col, max_side: int = 1024): - """Bin centroids onto a raster, coarse enough to blur cheaply. - - Points cannot be blurred. napari draws a Points layer as discrete - marks, and there is no kernel to widen -- so making sparse cells - easier to see means rendering them into an IMAGE and blurring that. - - The raster is deliberately coarse, for two reasons. A ferret section - is ~40,000 x 59,000 base pixels, so a full-resolution density image - would be 2.4 billion pixels to hold a few hundred thousand cells. And - the BLUR is redrawn on every change of the width knob, which puts the - cap on the interactive path rather than the loading one -- measured, - on this raster, with scipy's gaussian_filter: - - 1024^2 38 ms at sigma 2px, 126 ms at sigma 20px - 2048^2 177 ms 442 ms - 4096^2 757 ms 1815 ms - - 1024 is the largest that still redraws inside a slider drag. Binning - the points is not what costs -- 400,000 cells histogram in 13 ms, once - -- so the cap is about the blur, not the counting. - - The layer carries a matching ``scale`` and ``translate`` so it still - lands on top of the cells it was made from. - - Args: - row, col: centroid coordinates in WORLD units, as the points layer - holds them. - max_side: longest raster side in pixels. - - Returns: - ``(counts, step, origin)`` -- the 2D histogram, the world units one - raster pixel spans, and the ``(row, col)`` world position of its - top-left corner. - """ - import numpy as np - - row = np.asarray(row, float).ravel() - col = np.asarray(col, float).ravel() - if row.size == 0: - return np.zeros((1, 1)), 1.0, (0.0, 0.0) - - r0, r1 = float(row.min()), float(row.max()) - c0, c1 = float(col.min()), float(col.max()) - # A degenerate extent -- one cell, or a row of them -- still has to - # produce a raster with area, or the histogram below has no bins. - span = max(r1 - r0, c1 - c0, 1.0) - step = span / max_side - - nr = max(int(np.ceil((r1 - r0) / step)) + 1, 1) - nc = max(int(np.ceil((c1 - c0) / step)) + 1, 1) - ri = np.clip(((row - r0) / step).astype(np.int64), 0, nr - 1) - ci = np.clip(((col - c0) / step).astype(np.int64), 0, nc - 1) - counts = np.bincount(ri * nc + ci, minlength=nr * nc).reshape(nr, nc) - return counts.astype(np.float32), step, (r0, c0) - - -def blurRaster(counts, sigma_world: float, step: float): - """Gaussian-blur a density raster with a UNITY kernel. - - Unity meaning the kernel sums to one, so blurring SPREADS the signal - rather than adding to it: a cell contributes the same total whatever - the width. That is what keeps brightness a separate knob from width -- - widen the dots and the picture does not also get brighter, so the - contrast limits set for one sigma still mean something at another. - - ``mode="constant"`` rather than the default reflect, so mass leaves at - the edges instead of being folded back and piling up there. A section - is not periodic and its border cells are not doubled. - - Args: - counts: raster from :func:`densityRaster`. - sigma_world: blur width in WORLD units -- the same units the - centroids are in, so the knob means the same thing at every - zoom rather than drifting with the raster size. - step: world units per raster pixel, from :func:`densityRaster`. - - Returns: - The blurred raster. sigma <= 0 returns the input unchanged, which - is what "no blur" should cost. - """ - import numpy as np - from scipy.ndimage import gaussian_filter - - counts = np.asarray(counts, np.float32) - if sigma_world is None or sigma_world <= 0 or step <= 0: - return counts - return gaussian_filter(counts, sigma=float(sigma_world) / float(step), mode="constant") - - -def viewportBounds(center, zoom, canvas_px): - """The visible world rectangle, as ``(r0, r1, c0, c1)``. - - napari's ``camera.zoom`` is canvas PIXELS PER WORLD UNIT, so the - visible extent is the canvas size divided by it, centred on - ``camera.center``. Qt reports a canvas as ``(width, height)`` while - napari axes are ``(row, col)``, so width belongs to the column span - and height to the row span -- the one place this is easy to get - backwards, and invisible when the window happens to be square. - - Pure arithmetic, separate from the viewer, because it is the part that - can be wrong and the part that needs no display to check. - - Args: - center: ``camera.center``, ``(z, y, x)`` even in 2D; the last two - are read. - zoom: ``camera.zoom``. - canvas_px: ``(width, height)`` in device pixels. - """ - cy, cx = float(center[-2]), float(center[-1]) - w_px, h_px = float(canvas_px[0]), float(canvas_px[1]) - if zoom <= 0: - raise ValueError(f"camera zoom must be positive, got {zoom!r}") - half_r = (h_px / float(zoom)) / 2.0 - half_c = (w_px / float(zoom)) / 2.0 - return (cy - half_r, cy + half_r, cx - half_c, cx + half_c) - - -def canvasSize(viewer): - """The canvas size in pixels, or None if this napari will not say. - - Reached through private attributes that have moved between napari - versions, so it is tried rather than assumed and the caller falls back - to the whole section rather than failing. A blur over the wrong - rectangle would be worse than a coarse one over the right rectangle. - """ - for path in (("_qt_viewer",), ("qt_viewer",)): - obj = viewer.window - try: - for name in path: - obj = getattr(obj, name) - size = obj.canvas.size - return (float(size[0]), float(size[1])) - except Exception: # noqa: BLE001 - any miss means "cannot tell" - continue - return None - - -def addCellBlurPanel(viewer, points_layer, max_side: int = 1024) -> Any: - """A width knob for a blurred density image under the cell centroids. - - Sparse centroids are single marks and hard to pick out. This adds a - second layer -- the same cells, binned and Gaussian-blurred -- that can - be widened until they read as a field rather than dust, with the - original points still drawn on top. - - THE LAYER IS BUILT ON FIRST USE, not at launch. Most sessions never - touch this, and rasterising several hundred thousand centroids is not - worth paying for unasked on a launch that is already the thing people - complain about. - - Brightness stays napari's own: the density layer is an ordinary Image - layer, so its contrast limits, gamma and colormap are in the layer - controls where a reader already looks for them. This knob does width - and nothing else, which is why the kernel is unity. - """ - from qtpy.QtCore import Qt, QTimer - from qtpy.QtWidgets import ( - QCheckBox, - QDoubleSpinBox, - QHBoxLayout, - QLabel, - QPushButton, - QSlider, - QVBoxLayout, - QWidget, - ) - - box = QWidget() - outer = QVBoxLayout(box) - - show = QCheckBox("blurred cell density") - follow = QCheckBox("follow the view") - follow.setToolTip( - "Re-blurs by itself a moment after you stop moving.\n" - "Off by default: the redraw is 38-126 ms, so following every camera\n" - "event would stutter through a pan -- hence the pause rather than\n" - "live tracking." - ) - here = QPushButton("Re-blur here") - here.setToolTip( - "Re-bins just what is on screen, at full resolution for this zoom.\n" - "The standing raster covers the whole section and goes blocky when\n" - "you zoom past it; this trades that for a view that has to be\n" - "refreshed after you move." - ) - show.setToolTip( - "Bins the centroids and blurs them with a unity Gaussian, so widening\n" - "spreads each cell rather than brightening it. Contrast and colormap\n" - "stay in the layer controls." - ) - slider = QSlider(Qt.Horizontal) - spin = QDoubleSpinBox() - spin.setRange(0.0, 500.0) - spin.setDecimals(1) - spin.setSuffix(" um") - - status = QLabel("") - status.setWordWrap(True) - status.hide() - - state = {"layer": None, "counts": None, "step": 1.0, "origin": (0.0, 0.0)} - - def _rebin(bounds=None): - """Bin the centroids, optionally only those inside BOUNDS. - - Whole-section by default. Restricted to the viewport, the same - max_side spans a smaller region, so the raster is finer exactly - where it is being looked at -- which is the whole point of the - button, and why this is on demand rather than on every camera - move: the blur is 38-126 ms and a pan would stutter through it. - """ - import numpy as np - - data = points_layer.data - row, col = data[:, 0], data[:, 1] - if bounds is not None: - r0, r1, c0, c1 = bounds - keep = (row >= r0) & (row <= r1) & (col >= c0) & (col <= c1) - if not np.any(keep): - raise ValueError("no cells in view") - row, col = row[keep], col[keep] - counts, step, origin = densityRaster(row, col, max_side) - state.update(counts=counts, step=step, origin=origin) - - def _raster(): - if state["counts"] is None: - _rebin() - # The slider is in WORLD units and its useful range depends on - # how big the section is, so it is set from the data rather - # than guessed: a tenth of the raster is a wide blur anywhere. - widest = max(state["step"] * max_side / 10.0, 1.0) - slider.setRange(0, 100) - spin.setRange(0.0, widest) - spin.setSingleStep(widest / 100.0) - return state["counts"], state["step"], state["origin"] - - def _reblurHere(): - size = canvasSize(viewer) - if size is None: - status.setText( - "this napari will not report its canvas size, so the blur " - "stays over the whole section" - ) - status.show() - return - try: - bounds = viewportBounds(viewer.camera.center, viewer.camera.zoom, size) - _rebin(bounds) - except Exception as e: # noqa: BLE001 - a control never costs the picture - status.setText(f"failed: {e}") - status.show() - return - show.setChecked(True) - _apply(_sigma()) - - def _apply(sigma): - try: - counts, step, origin = _raster() - img = blurRaster(counts, sigma, step) - if state["layer"] is None: - state["layer"] = viewer.add_image( - img, - name="cell density", - colormap="magma", - blending="additive", - scale=(step, step), - translate=origin, - ) - # Under the points, not over them: the blur is context for - # the centroids, not a replacement. - viewer.layers.move(len(viewer.layers) - 1, 0) - else: - # scale and translate too: a re-blur over the viewport - # changes all three, and data alone would leave the image - # stretched over the old rectangle. - state["layer"].data = img - state["layer"].scale = (step, step) - state["layer"].translate = origin - state["layer"].visible = show.isChecked() - state["layer"].reset_contrast_limits() - except Exception as e: # noqa: BLE001 - a control never costs the picture - status.setText(f"failed: {e}") - status.show() - return - status.hide() - - def _sigma(): - return spin.value() - - guard = [False] - - def _sync(value, source): - if guard[0]: - return - guard[0] = True - try: - if source is not spin: - spin.setValue(spin.maximum() * value / 100.0) - if source is not slider and spin.maximum() > 0: - slider.setValue(int(round(100.0 * spin.value() / spin.maximum()))) - if show.isChecked(): - _apply(_sigma()) - finally: - guard[0] = False - - slider.valueChanged.connect(lambda v: _sync(v, slider)) - spin.valueChanged.connect(lambda v: _sync(v, spin)) - show.toggled.connect(lambda on: _apply(_sigma()) if on else _hide()) - - def _hide(): - if state["layer"] is not None: - state["layer"].visible = False - - row = QHBoxLayout() - row.addWidget(QLabel("width")) - row.addWidget(slider, 1) - row.addWidget(spin) - row.addWidget(here) - here.clicked.connect(_reblurHere) - - # DEBOUNCED, not live. Camera events fire continuously through a pan - # and each redraw is 38-126 ms, so following them directly would - # stutter the whole drag. The timer restarts on every event and only - # fires once movement has stopped, which is the same moment a hand - # would have reached for the button. - settle = QTimer(box) - settle.setSingleShot(True) - settle.setInterval(400) - settle.timeout.connect(lambda: _reblurHere() if follow.isChecked() else None) - - def _cameraMoved(_event=None): - if follow.isChecked(): - settle.start() - - def _watchCamera(on): - # Connected once and left connected: _cameraMoved checks the box - # itself, so toggling cannot leak a second connection, and - # disconnecting a napari event is the fiddlier half of the API. - if on: - _reblurHere() - - for _evt in ("zoom", "center"): - try: - getattr(viewer.camera.events, _evt).connect(_cameraMoved) - except Exception: # noqa: BLE001 - following is optional, the panel is not - follow.setEnabled(False) - follow.setToolTip("this napari does not report camera movement") - - follow.toggled.connect(_watchCamera) - outer.addWidget(follow) - outer.addWidget(show) - outer.addLayout(row) - outer.addWidget(status) - outer.addStretch() - - viewer.window.add_dock_widget(box, name="Cell density", area="right") - return box - - def addRotationPanel(viewer, layers) -> Any: """A slider and a box for turning the whole picture. @@ -1466,7 +1171,11 @@ def addRotationPanel(viewer, layers) -> Any: spin = QDoubleSpinBox() spin.setRange(-180.0, 180.0) spin.setDecimals(1) - spin.setSingleStep(0.5) + # The step is what the box's own - and + buttons move by, and five + # degrees is a visible turn -- half a degree took ten clicks to show + # anything. Anyone wanting finer than five types the number, which the + # one decimal place still accepts. + spin.setSingleStep(5.0) spin.setSuffix(" deg") reset = QPushButton("Reset") @@ -1538,11 +1247,17 @@ def _reset(): reset.clicked.connect(_reset) + # TWO ROWS, NOT ONE. Sharing a row with the box and the button left + # the slider a stub too short to aim with: a dock panel is narrow, and + # the number and the Reset button take a fixed width out of it whatever + # is left over. On its own row the slider gets the panel's full width, + # which is what makes it draggable. row = QHBoxLayout() - row.addWidget(slider, 1) row.addWidget(spin) row.addWidget(reset) + row.addStretch() outer.addLayout(row) + outer.addWidget(slider) outer.addWidget(status) outer.addStretch() @@ -1550,6 +1265,107 @@ def _reset(): return box +def blurLevels(levels, binSizes, sigmaBasePixels: float, truncate: float = 3.0): + """Gaussian-blur a multiscale ladder by the same DISTANCE at every level. + + Gene expression is a handful of counts in scattered single pixels. + Drawn honestly it is dust, and a reader looking for where a gene is + expressed cannot see a pattern in it. Blurring spreads each pixel over + its neighbourhood so the pattern reads, at the cost of no longer being + able to point at one bin. + + THE WIDTH IS IN BASE PIXELS, NOT LEVEL PIXELS, and that is the whole + difficulty. Each level is binned differently, so a fixed number of + level-pixels would be a different physical distance on each one -- the + blur would visibly change width as the viewer switched levels while + zooming, which is precisely what a multiscale ladder exists to avoid. + Dividing by the level's bin size makes one setting mean one distance + everywhere. + + BLURRED PER BLOCK, WITH A HALO. Levels are dask arrays whose blocks + are tiles, and a plain per-block filter would see each tile's edge as + the end of the world -- leaving a seam at every tile boundary, in a + grid, which reads as an artefact of the data rather than of the + drawing. map_overlap gives each block ``truncate * sigma`` of its + neighbours to work from, which is exactly the reach of the kernel, so + the result matches blurring the level whole. + + The kernel is unity: widening spreads each pixel's counts rather than + adding to them, so width and brightness stay separate knobs. Contrast limits will want resetting afterwards + all the same -- spreading a sparse signal lowers its peak a long way. + + Args: + levels: dask arrays, finest first, as ``layerSpec`` returns. + binSizes: each level's bin size in base pixels, same order. + sigmaBasePixels: blur width, in BASE pixels. 0 or less returns the + ladder untouched, which is what "no blur" should cost. + truncate: kernel cutoff in sigmas; also the halo, so the two + cannot disagree. + + Returns: + A new list of dask arrays. The input is not modified. + """ + import numpy as np + from scipy.ndimage import gaussian_filter + + if sigmaBasePixels is None or sigmaBasePixels <= 0: + return list(levels) + + out = [] + for arr, b in zip(levels, binSizes): + sigma = float(sigmaBasePixels) / float(b) + if sigma <= 0: + out.append(arr) + continue + # +1 so integer truncation can never make the halo shorter than + # the kernel's reach, which is the one way a seam gets back in. + depth = int(np.ceil(truncate * sigma)) + 1 + out.append( + arr.map_overlap( + gaussian_filter, + depth=depth, + boundary=0, + sigma=sigma, + truncate=truncate, + mode="constant", + dtype=arr.dtype, + ) + ) + return out + + +def basePixelUm(pyr_doc, fallback: float = 0.5) -> float: + """One base pixel's physical width, so a blur can be set in microns. + + The blur knob is in micrometres because that is the unit a reader + thinks in -- a spread of ten microns means something about tissue, + ten base pixels means something about the chip. :func:`blurLevels` + works in base pixels, and this is the conversion between them. + + Only the X size is used. A pyramid whose pixels are not square would + want an anisotropic sigma, and neither the ladder nor the knob is + built for that; Stereo-seq's DNB grid is square, so the case has not + arisen. If it does, this is where it would be noticed. + + Args: + pyr_doc: a spatialGeneExpressionPyramid document. + fallback: used when the document does not say, or says something + that is not a positive number. 0.5 um is the Stereo-seq DNB + pitch, which is what every pyramid seen so far records. A + wrong-but-plausible scale makes the knob mean the wrong + distance; a zero would make it divide by zero. + + Returns: + Micrometres per base pixel, always positive. + """ + try: + p = pyr_doc.document_properties["spatialGeneExpressionPyramid"] + um = float(p["base_pixel_size_x"]) + except Exception: # noqa: BLE001 - a missing field costs the default, not the knob + return float(fallback) + return um if um > 0 else float(fallback) + + def _importQtWidgets() -> None: """Import qtpy.QtWidgets, or raise ImportError. @@ -1639,6 +1455,5 @@ def _build(name, fn): viewer, session, cells_doc, points_layer, shapes_layer, labelings ), ) - made["cellDensity"] = _build("cell density", lambda: addCellBlurPanel(viewer, points_layer)) made["genes"] = _build("genes", lambda: addGenePanel(viewer, session, pyr_doc)) return made diff --git a/tests/test_gui_genepyramid_blur.py b/tests/test_gui_genepyramid_blur.py new file mode 100644 index 00000000..7b35ce9c --- /dev/null +++ b/tests/test_gui_genepyramid_blur.py @@ -0,0 +1,144 @@ +"""Blurring the gene ladder so sparse expression reads as a pattern. + +A gene's counts are a handful of pixels scattered over a section. Drawn +honestly they are dust. Blurring spreads each one over its neighbourhood +so the pattern shows -- and the things worth holding are that the width +means ONE PHYSICAL DISTANCE at every level of the ladder, that blurring +per tile leaves NO SEAM at the tile boundaries, and that the kernel is +UNITY so width and brightness stay separate knobs. + +Dask arrays and scipy only. No display, no napari, no Qt. +""" + +import numpy as np +import pytest + +from ndi.gui.app.genepyramid.controls import basePixelUm, blurLevels + +da = pytest.importorskip("dask.array") +pytest.importorskip("scipy.ndimage") + + +def _ladder(binSizes=(1, 2, 4), side=64, chunks=16): + """A point of counts in the middle of each level of a ladder.""" + out = [] + for _b in binSizes: + a = np.zeros((side, side), dtype=np.float32) + a[side // 2, side // 2] = 100.0 + out.append(da.from_array(a, chunks=chunks)) + return out + + +class TestTheWidth: + def test_one_setting_is_one_distance_at_every_level(self): + """A level binned 4x is four base pixels to the pixel, so the same + world blur has to be a QUARTER as wide in that level's own pixels. + Otherwise the blur visibly changes width as the viewer switches + levels while zooming -- exactly what a ladder exists to avoid.""" + from scipy.ndimage import gaussian_filter + + bins = [1, 2, 4] + got = blurLevels(_ladder(bins), bins, sigmaBasePixels=8.0) + for arr, b in zip(got, bins): + src = np.zeros((64, 64), dtype=np.float32) + src[32, 32] = 100.0 + want = gaussian_filter(src, sigma=8.0 / b, truncate=3.0, mode="constant") + assert np.allclose(np.asarray(arr), want, atol=1e-4) + + def test_no_blur_costs_nothing(self): + """Zero and below return the ladder untouched -- the SAME arrays, + not equal ones, so switching the knob off cannot cost a recompute.""" + levels = _ladder() + for sigma in (0.0, -1.0, None): + got = blurLevels(levels, [1, 2, 4], sigma) + assert [id(a) for a in got] == [id(a) for a in levels] + + def test_the_input_ladder_is_not_modified(self): + levels = _ladder() + before = [np.asarray(a).copy() for a in levels] + blurLevels(levels, [1, 2, 4], 4.0) + for a, b in zip(levels, before): + assert np.array_equal(np.asarray(a), b) + + +class TestTheSeams: + def test_a_tiled_level_blurs_as_if_it_were_whole(self): + """map_overlap's halo is the point of the whole function. A plain + per-block filter sees each tile's edge as the end of the world and + leaves a seam at every boundary -- in a GRID, which reads as an + artefact of the data rather than of the drawing.""" + from scipy.ndimage import gaussian_filter + + rng = np.random.default_rng(11) + src = rng.random((64, 64)).astype(np.float32) + # 16 blocks, so there are interior boundaries in both directions. + arr = da.from_array(src, chunks=16) + got = np.asarray(blurLevels([arr], [1], 3.0)[0]) + want = gaussian_filter(src, sigma=3.0, truncate=3.0, mode="constant") + assert np.max(np.abs(got - want)) == pytest.approx(0.0, abs=1e-5) + + def test_the_halo_grows_with_the_width(self): + """A halo shorter than the kernel's reach is how a seam gets back + in, so the widest blur has to survive too.""" + from scipy.ndimage import gaussian_filter + + rng = np.random.default_rng(12) + src = rng.random((64, 64)).astype(np.float32) + arr = da.from_array(src, chunks=8) + for sigma in (0.5, 2.0, 7.0): + got = np.asarray(blurLevels([arr], [1], sigma)[0]) + want = gaussian_filter(src, sigma=sigma, truncate=3.0, mode="constant") + assert np.max(np.abs(got - want)) < 1e-5, sigma + + +class TestTheKernel: + def test_it_spreads_rather_than_brightens(self): + """Unity: the counts move outward, they do not multiply. Width and + brightness have to stay separate knobs, because a reader adjusting + one is not asking for the other.""" + levels = _ladder(binSizes=(1,), side=128) + total = float(np.asarray(levels[0]).sum()) + for sigma in (1.0, 4.0, 12.0): + got = np.asarray(blurLevels(levels, [1], sigma)[0]) + # Well inside the frame, so nothing leaves at the edges. + assert float(got.sum()) == pytest.approx(total, rel=1e-4) + assert float(got.max()) < total + + def test_the_peak_falls_as_the_width_grows(self): + """Which is why the caller resets contrast limits afterwards: the + limits that suited the unblurred layer would show black.""" + levels = _ladder(binSizes=(1,), side=128) + peaks = [float(np.asarray(blurLevels(levels, [1], s)[0]).max()) for s in (1.0, 4.0, 12.0)] + assert peaks[0] > peaks[1] > peaks[2] + + +class _Doc: + def __init__(self, props): + self.document_properties = props + + +class TestTheScale: + def test_it_reads_the_pyramid(self): + """The knob is in microns because that is the unit a reader thinks + in; blurLevels works in base pixels. This is the conversion.""" + doc = _Doc({"spatialGeneExpressionPyramid": {"base_pixel_size_x": 0.25}}) + assert basePixelUm(doc) == pytest.approx(0.25) + + @pytest.mark.parametrize( + "props", + [ + {}, + {"spatialGeneExpressionPyramid": {}}, + {"spatialGeneExpressionPyramid": {"base_pixel_size_x": 0}}, + {"spatialGeneExpressionPyramid": {"base_pixel_size_x": -1}}, + {"spatialGeneExpressionPyramid": {"base_pixel_size_x": "wide"}}, + ], + ) + def test_a_document_that_does_not_say_costs_the_default(self, props): + """A zero would make the knob divide by zero and a missing field + would traceback out of a spin box callback. Neither is worth a + control, so the Stereo-seq pitch stands in.""" + assert basePixelUm(_Doc(props)) == pytest.approx(0.5) + + def test_the_fallback_is_the_caller_s_to_choose(self): + assert basePixelUm(_Doc({}), fallback=2.0) == pytest.approx(2.0) diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 5d7f2a00..c119f6c8 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -257,7 +257,6 @@ def test_the_chosen_labelings_reach_the_cell_type_panel(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", cells) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) - monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) controls.addAllPanels( object(), @@ -287,7 +286,6 @@ def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsy monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", genes) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) - monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -314,7 +312,6 @@ def _no_qt(): monkeypatch.setattr(controls, "addDisplayPanel", built) monkeypatch.setattr(controls, "addGenePanel", built) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) - monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -336,7 +333,6 @@ def test_the_rotation_panel_gets_all_three_layers(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", rotation) - monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) image, points, shapes = object(), object(), object() controls.addAllPanels( @@ -364,34 +360,8 @@ def test_a_session_with_no_cells_still_gets_rotation(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", rotation) - monkeypatch.setattr(controls, "addCellBlurPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) assert made["rotation"] is not None args, _kwargs = rotation.calls[0] assert args[1][1] is None and args[1][2] is None - - -def test_the_density_panel_is_built_only_where_there_are_cells(monkeypatch): - """It rasterises the centroids, so with no points layer there is - nothing to rasterise -- and a panel offering to blur nothing is worse - than no panel.""" - monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) - density = _Recorder() - monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) - monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) - monkeypatch.setattr(controls, "addGenePanel", _Recorder()) - monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) - monkeypatch.setattr(controls, "addCellBlurPanel", density) - - made = controls.addAllPanels(object(), object(), object(), object(), True) - assert "cellDensity" not in made - assert not density.calls - - points = object() - made = controls.addAllPanels( - object(), object(), object(), object(), True, cells_doc=object(), points_layer=points - ) - assert made["cellDensity"] is not None - args, _kwargs = density.calls[0] - assert args[1] is points, "the panel needs the layer it will rasterise" diff --git a/tests/test_gui_genepyramid_density.py b/tests/test_gui_genepyramid_density.py deleted file mode 100644 index 5cca0c81..00000000 --- a/tests/test_gui_genepyramid_density.py +++ /dev/null @@ -1,208 +0,0 @@ -"""Blurring sparse centroids so they read as a field rather than dust. - -napari draws a Points layer as discrete marks and offers no kernel to -widen, so "broader dots" means rendering the cells into an image and -blurring THAT. The two things worth holding are the raster (does it land -where the cells are, at a size that can be blurred cheaply) and the -kernel's UNITY -- widening must spread each cell's contribution, not add -to it, or width and brightness stop being separate knobs. - -No display and no napari needed for either. -""" - -import numpy as np -import pytest - -from ndi.gui.app.genepyramid.controls import blurRaster, densityRaster - - -class TestTheRaster: - def test_it_lands_where_the_cells_are(self): - """scale and translate have to put the raster back on top of the - points it was built from, or the blur sits beside the cells.""" - row = np.array([100.0, 300.0]) - col = np.array([50.0, 250.0]) - counts, step, origin = densityRaster(row, col, max_side=64) - assert origin == (100.0, 50.0) - # Each cell must map back into the raster at its own corner. - for r, c in zip(row, col): - ri = int((r - origin[0]) / step) - ci = int((c - origin[1]) / step) - assert counts[ri, ci] >= 1 - - def test_every_cell_is_counted_exactly_once(self): - rng = np.random.default_rng(0) - row = rng.uniform(0, 5000, 5000) - col = rng.uniform(0, 9000, 5000) - counts, _, _ = densityRaster(row, col, max_side=256) - assert counts.sum() == 5000 - - def test_the_long_side_is_capped(self): - """A ferret section is ~40,000 x 59,000 base pixels; a full-res - density image would be 2.4 billion pixels for a few hundred - thousand cells.""" - row = np.array([0.0, 59000.0]) - col = np.array([0.0, 40000.0]) - counts, _, _ = densityRaster(row, col, max_side=512) - assert max(counts.shape) <= 513 - assert counts.shape[0] >= counts.shape[1], "the taller axis stays taller" - - def test_no_cells_is_not_a_crash(self): - counts, step, origin = densityRaster([], [], max_side=64) - assert counts.shape == (1, 1) - assert step > 0 - - def test_a_single_cell_still_makes_a_raster_with_area(self): - """A degenerate extent would otherwise give a histogram no bins.""" - counts, step, _ = densityRaster([7.0], [9.0], max_side=64) - assert counts.sum() == 1 - assert step > 0 - - def test_cells_in_a_line_still_work(self): - counts, _, _ = densityRaster([5.0, 5.0, 5.0], [1.0, 2.0, 3.0], max_side=32) - assert counts.sum() == 3 - - -class TestTheKernelIsUnity: - def test_blurring_spreads_rather_than_adds(self): - """THE POINT. Widening must not brighten, or the contrast limits - set at one width stop meaning anything at another. - - The frame is big enough for the kernel to fit inside it -- scipy - truncates at 4 sigma, so a wide blur on a small raster loses mass - off the edge, which is the deliberate behaviour the edge test - below pins and not a failure of unity. - """ - counts = np.zeros((128, 128), np.float32) - counts[64, 64] = 1.0 - for sigma in (1.0, 3.0, 6.0): - out = blurRaster(counts, sigma_world=sigma, step=1.0) - assert out.sum() == pytest.approx(1.0, rel=1e-4) - - def test_a_wider_blur_lowers_the_peak(self): - counts = np.zeros((64, 64), np.float32) - counts[32, 32] = 1.0 - narrow = blurRaster(counts, 1.0, 1.0).max() - wide = blurRaster(counts, 6.0, 1.0).max() - assert wide < narrow - - def test_zero_sigma_is_the_input_unchanged(self): - counts = np.arange(16, dtype=np.float32).reshape(4, 4) - assert np.array_equal(blurRaster(counts, 0.0, 1.0), counts) - assert np.array_equal(blurRaster(counts, -1.0, 1.0), counts) - - def test_sigma_is_in_world_units_not_raster_pixels(self): - """The knob has to mean the same thing whatever the raster size, - or its useful range would drift with the section's extent.""" - counts = np.zeros((64, 64), np.float32) - counts[32, 32] = 1.0 - # 10 world units at 2 units/pixel is the same blur as 5 at 1. - a = blurRaster(counts, sigma_world=10.0, step=2.0) - b = blurRaster(counts, sigma_world=5.0, step=1.0) - assert np.allclose(a, b) - - def test_edge_mass_leaves_rather_than_folding_back(self): - """A section is not periodic; reflecting would double its border - cells and put a bright rim around the tissue.""" - counts = np.zeros((32, 32), np.float32) - counts[0, 0] = 1.0 - out = blurRaster(counts, 3.0, 1.0) - assert out.sum() < 1.0, "mass at a corner should leave the frame" - assert out.max() < 1.0 - - -def test_the_default_raster_stays_inside_a_slider_drag(): - """The blur is redrawn on every change of the width knob, so the cap - is on the interactive path. Measured with scipy's gaussian_filter: - 1024^2 is 38-126 ms across the useful sigma range, 2048^2 is 177-442, - 4096^2 is 757-1815. A default that lags the drag is the wrong default, - and this is the sort of number that drifts back up unnoticed.""" - import numpy as np - - row = np.array([0.0, 59000.0]) - col = np.array([0.0, 40000.0]) - counts, _, _ = densityRaster(row, col) - assert max(counts.shape) <= 1025 - - -class TestTheViewportRectangle: - """The standing raster covers the whole section, so it goes blocky - when you zoom past it. "Re-blur here" re-bins just what is on screen, - which puts the same number of raster pixels over a smaller region. - - The arithmetic is where this can be wrong, and it needs no display. - """ - - def test_the_visible_extent_is_the_canvas_over_the_zoom(self): - """napari's zoom is canvas PIXELS PER WORLD UNIT.""" - from ndi.gui.app.genepyramid.controls import viewportBounds - - # 800x600 canvas at 2 px per world unit -> 400 x 300 world units. - r0, r1, c0, c1 = viewportBounds((0.0, 100.0, 200.0), 2.0, (800, 600)) - assert (c1 - c0) == pytest.approx(400.0) - assert (r1 - r0) == pytest.approx(300.0) - - def test_width_belongs_to_columns_and_height_to_rows(self): - """The one thing here that is easy to get backwards, and invisible - whenever the window happens to be square.""" - from ndi.gui.app.genepyramid.controls import viewportBounds - - r0, r1, c0, c1 = viewportBounds((0.0, 0.0, 0.0), 1.0, (1000, 200)) - assert (c1 - c0) == pytest.approx(1000.0), "canvas WIDTH spans columns" - assert (r1 - r0) == pytest.approx(200.0), "canvas HEIGHT spans rows" - - def test_it_is_centred_on_the_camera(self): - from ndi.gui.app.genepyramid.controls import viewportBounds - - r0, r1, c0, c1 = viewportBounds((0.0, 50.0, -20.0), 1.0, (100, 100)) - assert (r0 + r1) / 2 == pytest.approx(50.0) - assert (c0 + c1) / 2 == pytest.approx(-20.0) - - def test_a_zoom_of_zero_is_refused_rather_than_dividing(self): - from ndi.gui.app.genepyramid.controls import viewportBounds - - with pytest.raises(ValueError, match="zoom must be positive"): - viewportBounds((0.0, 0.0, 0.0), 0.0, (100, 100)) - - def test_a_viewport_raster_is_finer_than_a_whole_section_one(self): - """THE POINT OF THE BUTTON: same max_side, smaller region.""" - rng = np.random.default_rng(1) - row = rng.uniform(0, 59000, 20000) - col = rng.uniform(0, 40000, 20000) - _, whole_step, _ = densityRaster(row, col, max_side=256) - - keep = (row > 29000) & (row < 30000) & (col > 20000) & (col < 21000) - _, view_step, _ = densityRaster(row[keep], col[keep], max_side=256) - assert view_step < whole_step / 10 - - -class TestTheCanvasSizeIsOptional: - """It is reached through private napari attributes that have moved - between versions, so a miss falls back to the whole section rather - than failing -- a blur over the wrong rectangle would be worse than a - coarse one over the right rectangle.""" - - def test_a_viewer_that_will_not_say_returns_none(self): - from ndi.gui.app.genepyramid.controls import canvasSize - - class Bare: - window = object() - - assert canvasSize(Bare()) is None - - def test_it_reads_the_canvas_when_one_is_there(self): - from ndi.gui.app.genepyramid.controls import canvasSize - - class Canvas: - size = (1280, 720) - - class QtViewer: - canvas = Canvas() - - class Window: - _qt_viewer = QtViewer() - - class Viewer: - window = Window() - - assert canvasSize(Viewer()) == (1280.0, 720.0) diff --git a/tests/test_gui_genepyramid_rotation.py b/tests/test_gui_genepyramid_rotation.py index 72d4f6f7..60971878 100644 --- a/tests/test_gui_genepyramid_rotation.py +++ b/tests/test_gui_genepyramid_rotation.py @@ -6,11 +6,13 @@ Everything worth holding here is testable with no display and no napari: the SIGN of the rotation, the PIVOT it turns about, and that every layer -gets the SAME matrix. The Qt panel around them is not tested -- CI -installs no Qt binding -- but the three things that could produce a -visibly wrong picture are. +gets the SAME matrix. The panel's ARRANGEMENT needs real widgets, so +those tests build it offscreen and skip where there is no Qt binding +-- which is CI today. """ +from types import SimpleNamespace + import numpy as np import pytest @@ -133,3 +135,114 @@ def test_a_later_angle_replaces_rather_than_compounds(self): applyRotation([image], 30.0, (0.0, 0.0)) applyRotation([image], 30.0, (0.0, 0.0)) assert np.allclose(image.affine, rotationAffine(30.0, (0.0, 0.0))) + + +# --------------------------------------------------------------- layout + + +@pytest.fixture(scope="module") +def qt(): + """A real Qt, drawn to nowhere, or a skip. + + The panel's arrangement is the thing being checked here, and an + arrangement is a property of real widgets in a real layout -- stubbing + them out would only test the stubs. The offscreen platform gives real + widgets with no display, which is what CI has. CI does not install Qt + at all today, so this skips there and runs for anyone who does. + """ + import os + + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Viewer: + def __init__(self): + self.window = _Window() + self.camera = SimpleNamespace(center=(0.0, 100.0, 200.0)) + + +def _rotationPanel(qt): + from ndi.gui.app.genepyramid.controls import addRotationPanel + + viewer = _Viewer() + return addRotationPanel(viewer, []) + + +def _rows(qt, box): + """The panel's top-level layout, as a list of rows. + + Each entry is either a widget or the list of widgets on a nested row, + which is exactly the distinction the arrangement turns on. + """ + layout = box.layout() + out = [] + for i in range(layout.count()): + item = layout.itemAt(i) + if item.widget() is not None: + out.append(item.widget()) + elif item.layout() is not None: + sub = item.layout() + out.append( + [sub.itemAt(j).widget() for j in range(sub.count()) if sub.itemAt(j).widget()] + ) + return out + + +class TestTheArrangement: + def test_the_slider_gets_a_row_of_its_own(self, qt): + """Sharing a row with the number and the Reset button left it a + stub too short to aim with: a dock panel is narrow and those two + take a fixed width out of it whatever is left over.""" + box = _rotationPanel(qt) + rows = _rows(qt, box) + sliders = [r for r in rows if isinstance(r, qt.QSlider)] + assert len(sliders) == 1, "the slider is not a row by itself" + + def test_the_slider_sits_under_the_number(self, qt): + """Which is where it was asked for, and reads as belonging to the + angle above it rather than as a third control beside it.""" + box = _rotationPanel(qt) + rows = _rows(qt, box) + spin_row = next( + i + for i, r in enumerate(rows) + if isinstance(r, list) and any(isinstance(w, qt.QDoubleSpinBox) for w in r) + ) + slider_row = next(i for i, r in enumerate(rows) if isinstance(r, qt.QSlider)) + assert slider_row > spin_row + + def test_the_number_keeps_the_reset_button_company(self, qt): + box = _rotationPanel(qt) + rows = _rows(qt, box) + row = next(r for r in rows if isinstance(r, list)) + assert any(isinstance(w, qt.QDoubleSpinBox) for w in row) + assert any(isinstance(w, qt.QPushButton) for w in row) + + +class TestTheStep: + def test_the_buttons_move_five_degrees(self, qt): + """Half a degree took ten clicks to show anything. Five is a turn + you can see, and the box still takes a typed 12.5 for finer.""" + box = _rotationPanel(qt) + spin = box.findChild(qt.QDoubleSpinBox) + assert spin.singleStep() == pytest.approx(5.0) + + def test_finer_can_still_be_typed(self, qt): + box = _rotationPanel(qt) + spin = box.findChild(qt.QDoubleSpinBox) + assert spin.decimals() >= 1 + spin.setValue(12.5) + assert spin.value() == pytest.approx(12.5) From 363620add938e4c10324ecc7d528c58b02781f37 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 15:08:44 +0000 Subject: [PATCH 42/52] Give the abundance band its own line, and the gene list room to be read The band slider shared a row with its label, and it carries its own value labels at each end of the track -- so in a dock this narrow what was left after "Abundance band" was the two numbers and no track between them. Unreachable rather than merely cramped. The label is now the line above and the slider has the row to itself, which is the same lesson the rotation slider taught yesterday. The gene list came up four rows tall, wedged between the band above and the blur below. Finding a gene among 26,444 through a porthole is not a control. A dock panel is only as tall as it is asked to be, so it is now asked: a 260px minimum on the list, which the panel's stretch already hands any further space to. Both are checked on real widgets built offscreen, since an arrangement is a property of a layout and stubbing the widgets would only test the stubs. They fail on the old layout and skip where there is no Qt binding -- which is CI today. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 20 +++- tests/test_gui_genepyramid_genepanel.py | 145 ++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 5 deletions(-) create mode 100644 tests/test_gui_genepyramid_genepanel.py diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 4278b59d..0e953c7f 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -299,6 +299,11 @@ def _headText(info=None) -> str: outer.addLayout(sort_row) listw = QListWidget() + # THE LIST IS WHAT THE PANEL IS FOR. Left to its own size hint it came + # up four rows tall, wedged between the band above and the blur below, + # and finding a gene among 26,444 meant scrolling a porthole. A dock + # panel is only as tall as it is asked to be, so it is asked. + listw.setMinimumHeight(260) items: dict[str, Any] = {} symbols = list(index) entries = list(index.values()) @@ -394,10 +399,15 @@ def _resort(): count_btn.clicked.connect(lambda *_: _sortBy("counts")) # ---- the abundance band ------------------------------------------ - # ONE ROW. What the band is and what it just did are hover text on the - # slider rather than a paragraph under it: the explanation is read - # once and the panel is read every time, and the list underneath is - # what the space is for. + # THE SLIDER GETS ITS OWN LINE. Sharing one with the label left it a + # stub in a narrow dock -- and this slider carries its own value + # labels at each end, so what was left after "Abundance band" was the + # numbers and no track between them: unreachable rather than merely + # cramped. Same lesson as the rotation slider. + # + # What the band is and what it just did stay hover text rather than a + # paragraph: the explanation is read once and the panel is read every + # time, and the list underneath is what the space is for. if symbol_totals is None: note = QLabel("no gene_totals.tsv: no counts, no band") note.setToolTip( @@ -408,8 +418,8 @@ def _resort(): outer.addWidget(note) slider = lo_box = hi_box = None else: + outer.addWidget(QLabel("Abundance band")) band_row = QHBoxLayout() - band_row.addWidget(QLabel("Abundance band")) slider = lo_box = hi_box = None try: # napari depends on superqt, so the two-handle widget is diff --git a/tests/test_gui_genepyramid_genepanel.py b/tests/test_gui_genepyramid_genepanel.py new file mode 100644 index 00000000..258b0375 --- /dev/null +++ b/tests/test_gui_genepyramid_genepanel.py @@ -0,0 +1,145 @@ +"""How the Genes panel is laid out, checked on real widgets. + +A dock panel is narrow, and a wide control that shares its row with a +label ends up a stub. That went wrong twice -- the rotation slider, then +the abundance band, whose two value labels ate what little track was +left and made it unreachable rather than merely cramped. An arrangement +is a property of real widgets in a real layout, so these build the panel +offscreen rather than stubbing it, and skip where there is no Qt. + +What the panel DOES -- the filters, the sorting, the layers -- is tested +in test_gui_genepyramid_controls.py with no widgets at all. +""" + +import os + +import numpy as np +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +@pytest.fixture(scope="module") +def qt(): + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Layers(dict): + events = None + + +class _Viewer: + def __init__(self): + self.window = _Window() + self.layers = _Layers() + + +@pytest.fixture +def panel(qt, monkeypatch): + """The real Genes panel over a stub gene list.""" + from ndi.fun import doc_gene_export + from ndi.gui.app.genepyramid import controls + + symbols = [f"GENE{i:04d}" for i in range(40)] + monkeypatch.setattr( + doc_gene_export, "readGeneList", lambda *a, **k: (list(symbols), list(symbols)) + ) + monkeypatch.setattr( + doc_gene_export, + "readGeneTotals", + lambda *a, **k: (np.arange(len(symbols), dtype=float), None), + ) + viewer = _Viewer() + return controls.addGenePanel(viewer, object(), object()) + + +def _rows(box): + """The panel's top-level layout: widgets, and nested rows as lists.""" + layout = box.layout() + out = [] + for i in range(layout.count()): + item = layout.itemAt(i) + if item.widget() is not None: + out.append(item.widget()) + elif item.layout() is not None: + sub = item.layout() + out.append( + [sub.itemAt(j).widget() for j in range(sub.count()) if sub.itemAt(j).widget()] + ) + return out + + +def _index(rows, predicate): + return next(i for i, r in enumerate(rows) if predicate(r)) + + +class TestTheAbundanceBand: + def test_the_slider_shares_its_row_with_nothing(self, qt, panel): + """Its own value labels sit at each end of the track, so a label + beside it leaves the numbers and no track between them.""" + superqt = pytest.importorskip("superqt") + rows = _rows(panel) + row = _index( + rows, + lambda r: isinstance(r, list) + and any(isinstance(w, superqt.QLabeledDoubleRangeSlider) for w in r), + ) + assert len(rows[row]) == 1 + + def test_its_label_is_the_line_above(self, qt, panel): + superqt = pytest.importorskip("superqt") + rows = _rows(panel) + row = _index( + rows, + lambda r: isinstance(r, list) + and any(isinstance(w, superqt.QLabeledDoubleRangeSlider) for w in r), + ) + above = rows[row - 1] + assert isinstance(above, qt.QLabel) + assert "Abundance band" in above.text() + + def test_the_band_still_spans_the_whole_range(self, qt, panel): + """The move is layout only: a band that came up narrowed would + hide genes before anyone touched it.""" + superqt = pytest.importorskip("superqt") + slider = panel.findChild(superqt.QLabeledDoubleRangeSlider) + assert tuple(slider.value()) == (0.0, 100.0) + + +class TestTheList: + def test_it_opens_tall_enough_to_read(self, qt, panel): + """Four rows of 26,444 genes is a porthole. The dock is only as + tall as it is asked to be, so it is asked.""" + listw = panel.findChild(qt.QListWidget) + assert listw.minimumHeight() >= 200 + + def test_it_takes_the_space_that_is_going(self, qt, panel): + """Everything else on the panel is a fixed row; growth belongs to + the list, not to the gap under it.""" + layout = panel.layout() + listw = panel.findChild(qt.QListWidget) + i = next(i for i in range(layout.count()) if layout.itemAt(i).widget() is listw) + assert layout.stretch(i) >= 1 + + +class TestTheBlur: + def test_the_knob_is_on_the_gene_panel(self, qt, panel): + """Which is the point of it: the gene layers are what is sparse.""" + boxes = panel.findChildren(qt.QDoubleSpinBox) + assert any(b.suffix().strip() == "um" for b in boxes) + + def test_it_starts_off(self, qt, panel): + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + assert blur.value() == pytest.approx(0.0) From c12443f75a766bff36e0b90d6689a9e17d4e7638 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 15:13:57 +0000 Subject: [PATCH 43/52] Say which credential was missing, and add a report that says where from "Binary file 'tile.bin_280' not found for document ..." is what an authentication failure looks like from the viewer: the tile is in the document's file list and has an ndic:// location, so it exists -- NDI simply could not authenticate to fetch it. The message underneath said "No valid token and no credentials available", which is equally true of never having logged in and of a token that lapsed while the window was open, and those want opposite responses. authenticate() now names which of its three steps came up empty. The expired case in particular says so, says when, and says the thing a re-login does not reveal: the token lives in os.environ, so it belongs to ONE PROCESS. Logging in from a shell -- or from MATLAB, whose vault Python cannot read -- leaves a separately launched viewer exactly as unauthenticated as it was. The remedy named in every case is credentials authenticate can reach, because with those it renews the token itself and the first step stops mattering. credentialReport() walks the same three steps and reports each, to be run IN THE ENVIRONMENT THAT FAILED. It reports the token by presence and expiry, never by value, and prints no password, so it can be pasted into an issue. Why this surfaces as scattered tiles rather than a blank picture: tiles already in DID's local cache need no credentials, so a mostly-downloaded section draws and the gaps are wherever the cache is cold. Blur widens that -- map_overlap gives each visible tile its eight neighbours -- so turning it up reaches tiles a still view never asked for. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/__init__.py | 2 + src/ndi/cloud/auth.py | 115 ++++++++++++++++++- tests/test_cloud_auth_profile_fallback.py | 128 ++++++++++++++++++++++ 3 files changed, 240 insertions(+), 5 deletions(-) diff --git a/src/ndi/cloud/__init__.py b/src/ndi/cloud/__init__.py index c36ef984..6862cde1 100644 --- a/src/ndi/cloud/__init__.py +++ b/src/ndi/cloud/__init__.py @@ -29,6 +29,7 @@ from .auth import ( authenticate, changePassword, + credentialReport, isTokenExpired, login, logout, @@ -56,6 +57,7 @@ "CloudSyncError", "CloudUploadError", "authenticate", + "credentialReport", "isTokenExpired", "login", "logout", diff --git a/src/ndi/cloud/auth.py b/src/ndi/cloud/auth.py index 531ee470..210f8b57 100644 --- a/src/ndi/cloud/auth.py +++ b/src/ndi/cloud/auth.py @@ -22,6 +22,108 @@ logger = logging.getLogger(__name__) + +def credentialReport() -> str: + """What this process can and cannot authenticate with, as text. + + An authentication failure reaches most people through something else + -- a tile that will not load, an API call that raises -- and from + there it is hard to tell WHERE the credentials were supposed to come + from. This walks the same three steps :func:`authenticate` walks and + reports each one, so the answer is a line rather than a bisection. + + It answers, in particular, the question that a re-login does not: the + token lives in ``os.environ``, so it belongs to ONE PROCESS. Logging + in from a shell, or from MATLAB, leaves a separately launched viewer + exactly as unauthenticated as it was. Run this IN THE ENVIRONMENT THAT + FAILED, not in the one you logged in from. + + Nothing here prints a password or a token; the token is reported as + present or absent and by its expiry, never by its value. + + Returns: + A short multi-line report, safe to paste. + """ + config = CloudConfig.from_env() + lines = [ + f"api : {config.api_url}", + f"token in env : {'yes' if os.environ.get('NDI_CLOUD_TOKEN') else 'no'}", + ] + if config.token: + try: + expires = getTokenExpiration(config.token) + state = "EXPIRED" if isTokenExpired(config.token) else "valid" + lines.append(f"token : {state}, exp {expires:%Y-%m-%d %H:%M UTC}") + except CloudAuthError as exc: + lines.append(f"token : unreadable ({exc})") + lines.append(f"organization : {config.org_id or '(none)'}") + lines.append(f"username in env: {'yes' if os.environ.get('NDI_CLOUD_USERNAME') else 'no'}") + email, password = _profile_credentials() + if email and password: + lines.append(f"profile : {email}") + else: + lines.append("profile : none usable") + + # The verdict last, because it is what the reader came for, and it is + # the only line that can disagree with the ones above -- a credential + # that is present but wrong fails here and nowhere else. + try: + _token, org = authenticate(config) + except CloudAuthError as exc: + lines.append(f"authenticate : FAILED -- {exc}") + else: + lines.append(f"authenticate : ok (organization {org})") + return "\n".join(lines) + + +def _noCredentialsMessage(config: CloudConfig) -> str: + """Say WHICH step of :func:`authenticate` came up empty. + + Three quite different situations end at the same raise, and the reader + of the message is usually looking at a failed file fetch rather than at + a login: a token that expired while the process ran, a token that was + never set, or credentials that exist but did not resolve. Naming the + one that happened is the difference between "log back in and relaunch" + and "you have not set this up". + + The remedy is the same sentence in every case, because it is the one + that also prevents the next occurrence: with a username and password + reachable, :func:`authenticate` renews an expired token by itself and + step 1 stops mattering. + """ + fix = ( + "Set NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD, or save a default cloud " + "profile (ndi.cloud.profile.add(...) then set_default(...)), so the " + "token can be renewed without you. NDI_CLOUD_TOKEN alone works only " + "until it expires." + ) + if not config.token: + return f"Not logged in to NDI Cloud: no token, no username/password, no profile. {fix}" + + if isTokenExpired(config.token): + when = "" + try: + when = f" (it expired at {getTokenExpiration(config.token):%Y-%m-%d %H:%M UTC})" + except CloudAuthError: + when = " (its expiry could not be read, which counts as expired)" + # The token is held in os.environ, so a login in another shell does + # NOT reach a process that is already running -- which is exactly + # the shape of this failure when a viewer has been open a while. + return ( + f"The NDI Cloud token has expired{when}, and there are no " + f"credentials to renew it with. A login in another shell will " + f"not reach this process: log in and start it again. " + f"{fix}" + ) + + # A token that is neither absent nor expired: step 1 wanted an org id + # too, and did not get one. + return ( + "The NDI Cloud token is present and unexpired but carries no " + f"organization id, so it cannot be used. {fix}" + ) + + # --------------------------------------------------------------------------- # JWT helpers (no cryptographic verification — matches MATLAB behaviour) # --------------------------------------------------------------------------- @@ -375,11 +477,14 @@ def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: updated = login(email, password, config) return updated.token, updated.org_id - raise CloudAuthError( - "No valid token and no credentials available. Set NDI_CLOUD_TOKEN or " - "NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD, or save a default cloud " - "profile (ndi.cloud.profile.add(...) then set_default(...))." - ) + # WHICH OF THE THREE FAILED IS THE WHOLE DIAGNOSIS, and the old message + # ran them together. "No valid token and no credentials available" is + # true whether you never logged in or logged in two hours ago, and + # those want opposite responses -- and it lands where it is least + # legible: on a tile fetch, in a napari toast, one per tile, an hour + # into a session that was working. An EXPIRED token in particular is + # invisible from that sentence, so the tile reads as missing data. + raise CloudAuthError(_noCredentialsMessage(config)) # --------------------------------------------------------------------------- diff --git a/tests/test_cloud_auth_profile_fallback.py b/tests/test_cloud_auth_profile_fallback.py index cc065785..34e9c8a3 100644 --- a/tests/test_cloud_auth_profile_fallback.py +++ b/tests/test_cloud_auth_profile_fallback.py @@ -243,5 +243,133 @@ def test_no_profile_yields_no_stage_rather_than_a_guess(self): self.assertEqual(_profile_stage(), "") +def _expired_token(ago_seconds: int = 3600) -> str: + """A JWT whose exp claim is in the past.""" + + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + return f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) - ago_seconds})}.sig" + + +class TestWhatTheFailureSays(_NoEnvMixin): + """Three quite different situations end at the same raise. + + The reader of this message is usually looking at a FAILED FILE FETCH, + not at a login: a napari toast on a tile, one per tile, an hour into a + session that was working. "No valid token and no credentials + available" is equally true of never having logged in and of a token + that lapsed while the viewer was open, and those want opposite + responses -- so the message has to say which happened. + """ + + def _raised(self, config): + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + ): + with self.assertRaises(CloudAuthError) as caught: + auth.authenticate(config) + return str(caught.exception) + + def test_an_expired_token_is_named_as_expired(self): + """The one that reads as missing data if it is not said out loud: + the tile is there, the token lapsed.""" + message = self._raised(CloudConfig(token=_expired_token(), org_id="org-1")) + self.assertIn("expired", message.lower()) + self.assertNotIn("Not logged in", message) + + def test_an_expired_token_says_when(self): + message = self._raised(CloudConfig(token=_expired_token(), org_id="org-1")) + self.assertIn("UTC", message) + + def test_an_expired_token_says_another_shell_will_not_do(self): + """The token lives in os.environ, so logging in elsewhere does not + reach a process that is already running. Without that sentence the + obvious remedy looks like it did not work.""" + message = self._raised(CloudConfig(token=_expired_token(), org_id="org-1")) + self.assertIn("another shell", message) + self.assertIn("start it again", message) + + def test_no_token_at_all_says_so_instead(self): + message = self._raised(CloudConfig(token="", org_id="")) + self.assertIn("Not logged in", message) + self.assertNotIn("expired", message.lower().split("only until")[0]) + + def test_a_token_with_no_organization_is_its_own_case(self): + """Step 1 wants both. A token that passes the expiry check and is + still refused is otherwise indistinguishable from a bad password.""" + message = self._raised(CloudConfig(token=_unexpired_token(), org_id="")) + self.assertIn("organization id", message) + + def test_every_case_says_how_to_stop_it_happening_again(self): + """Credentials it can reach are the fix for all three: with them, + authenticate renews the token itself and step 1 stops mattering.""" + for config in ( + CloudConfig(token="", org_id=""), + CloudConfig(token=_expired_token(), org_id="org-1"), + CloudConfig(token=_unexpired_token(), org_id=""), + ): + message = self._raised(config) + self.assertIn("NDI_CLOUD_USERNAME", message) + self.assertIn("profile", message) + + +class TestTheCredentialReport(_NoEnvMixin): + """One line instead of a bisection. + + An authentication failure reaches most people through something else + -- a tile that will not load -- and from there it is hard to say where + the credentials were meant to come from. The report walks the same + three steps authenticate walks and names each. + """ + + def _report(self, profile_entry=None, password=""): + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=profile_entry), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + mock.patch("ndi.cloud.profile.get_password", return_value=password), + ): + return auth.credentialReport() + + def test_it_says_every_step_came_up_empty(self): + report = self._report() + self.assertIn("token in env : no", report) + self.assertIn("username in env: no", report) + self.assertIn("profile : none usable", report) + self.assertIn("FAILED", report) + + def test_an_expired_token_is_reported_with_its_expiry(self): + os.environ["NDI_CLOUD_TOKEN"] = _expired_token() + report = self._report() + self.assertIn("token in env : yes", report) + self.assertIn("EXPIRED", report) + self.assertIn("UTC", report) + + def test_a_live_token_reads_as_valid(self): + os.environ["NDI_CLOUD_TOKEN"] = _unexpired_token() + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + report = self._report() + self.assertIn("valid", report) + self.assertIn("authenticate : ok", report) + + def test_a_usable_profile_is_named(self): + entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") + with mock.patch.object(auth, "login", _login_spy()[0]): + report = self._report(profile_entry=entry, password="pw") + self.assertIn("profile : a@example.com", report) + + def test_it_never_prints_the_secret(self): + """It is meant to be pasted into a chat or an issue.""" + token = _unexpired_token() + os.environ["NDI_CLOUD_TOKEN"] = token + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + os.environ["NDI_CLOUD_USERNAME"] = "a@example.com" + os.environ["NDI_CLOUD_PASSWORD"] = "hunter2" + report = self._report() + self.assertNotIn(token, report) + self.assertNotIn("hunter2", report) + + if __name__ == "__main__": unittest.main() From a740eec3a1fb590a087aa49d8b0256b26ba2184c Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 15:20:26 +0000 Subject: [PATCH 44/52] Rotate the layers that arrive later, not just the ones there at launch The rotation panel was built with the layers that existed when the viewer opened and never asked again. Tick a gene while the section is turned and its layer arrives SQUARE: the gene then sits beside the anatomy rather than on it, which is a wrong picture rather than an unrotated one, and nothing on screen says which of the two layers is the one to believe. Two halves, and both were missing. The panel now takes the viewer's own layer list at the moment it applies -- "what is on screen" is a question with a current answer, not one settled at launch -- so a later rotation reaches everything added since. And it connects to the layer list's inserted event, so a layer added while a rotation is in force is turned as it arrives. THE MATRIX IS HANDED OVER, NOT RECOMPUTED. The pivot is re-read between interactions, so rebuilding the affine from the angle would pivot the new layer about wherever the view centre had got to; two layers rotated by equal angles about different pivots do not line up. The one in force is kept and applied verbatim, which is also why a square picture leaves an arriving layer alone rather than stamping an identity over an affine some caller set for its own reasons. Six tests, four of which fail on the old fixed-list behaviour. The insert hook is wrapped because a headless caller composing layers by hand has no emitter, and a control that raises on construction costs the whole picture. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 58 +++++++++- src/ndi/gui/ndi_matlab_python_bridge.yaml | 2 +- tests/test_gui_genepyramid_rotation.py | 133 ++++++++++++++++++++++ 3 files changed, 191 insertions(+), 2 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 0e953c7f..172feb9b 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1209,6 +1209,32 @@ def addRotationPanel(viewer, layers) -> Any: # place instead of scattering `nonlocal` declarations. pivot = [None] guard = [False] + # The affine currently in force, or None while the picture is square. + # Kept rather than recomputed, because a layer added later has to get + # THE SAME matrix -- rebuilding it from the angle would use whatever + # the view centre is by then, and two layers rotated by equal angles + # about different pivots do not line up. + current = [None] + + def _layers(): + """Every layer the rotation governs, as of right now. + + NOT the list this panel was built with. That list is the layers + that existed at launch, and a gene ticked afterwards is not in it + -- it arrived square while the section around it was turned, which + is a picture that is wrong rather than merely unrotated. The + viewer's own list is the answer to "what is on screen", so it is + what gets asked, with the passed-in layers kept as a seed for + anything not registered with the viewer. + """ + seen, out = set(), [] + for group in (layers or (), getattr(viewer, "layers", ()) or ()): + for layer in group: + if layer is None or id(layer) in seen: + continue + seen.add(id(layer)) + out.append(layer) + return out def _pivot(): if pivot[0] is None: @@ -1217,13 +1243,43 @@ def _pivot(): def _apply(angle): try: - applyRotation(layers, angle, _pivot()) + current[0] = applyRotation(_layers(), angle, _pivot()) except Exception as e: # noqa: BLE001 - a control never costs the picture status.setText(f"failed: {e}") status.show() return status.hide() + def _onInserted(event=None): + """Turn a newly added layer to match the ones already turned. + + Ticking a gene while the section is rotated adds its layer square, + and the two then disagree about where the tissue is -- the gene + lands beside the anatomy rather than on it. Since the matrix is + shared and absolute, catching the layer as it arrives is enough: + it gets exactly what everything else has. + """ + a = current[0] + if a is None: + return + layer = getattr(event, "value", None) + # Fall back to sweeping the viewer: napari's event carries the new + # layer, but a caller that inserts by other means may not, and a + # missed layer is the failure this exists to prevent. + targets = [layer] if layer is not None else _layers() + for target in targets: + try: + target.affine = a + except Exception: # noqa: BLE001 - a stray entry never costs the rest + continue + + # Wrapped: the fakes a headless test builds have no event system, and + # neither does a viewer whose layer list is a plain container. + try: + viewer.layers.events.inserted.connect(_onInserted) + except Exception: # noqa: BLE001 - see above + pass + def _sync(angle, source): # The slider and the box show the same number, so each has to move # the other -- and setting a widget's value emits its own signal, diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index 022d160a..213915f3 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -822,7 +822,7 @@ ported_late: - name: GEFManager matlab_path: "+ndi/+gui/+app/GEFManager.m" - matlab_last_sync_hash: "cf0983b5c" + matlab_last_sync_hash: "f99d0ca57" status: porting_deferred decision_log: > MATLAB App Designer window that lists, views, adds and removes diff --git a/tests/test_gui_genepyramid_rotation.py b/tests/test_gui_genepyramid_rotation.py index 60971878..3610c946 100644 --- a/tests/test_gui_genepyramid_rotation.py +++ b/tests/test_gui_genepyramid_rotation.py @@ -246,3 +246,136 @@ def test_finer_can_still_be_typed(self, qt): assert spin.decimals() >= 1 spin.setValue(12.5) assert spin.value() == pytest.approx(12.5) + + +# ------------------------------------------------- layers added later + + +class _Event: + def __init__(self, value): + self.value = value + + +class _Signal: + """Just enough of napari's EventEmitter to connect and emit.""" + + def __init__(self): + self._slots = [] + + def connect(self, fn): + self._slots.append(fn) + + def emit(self, event): + for fn in self._slots: + fn(event) + + +class _LayerList(list): + """A layer list that reports insertions, as napari's does.""" + + def __init__(self, *layers): + super().__init__(layers) + self.events = SimpleNamespace(inserted=_Signal()) + + def add(self, layer): + self.append(layer) + self.events.inserted.emit(_Event(layer)) + return layer + + +class _LiveViewer: + def __init__(self, *layers): + self.window = _Window() + self.camera = SimpleNamespace(center=(0.0, 100.0, 200.0)) + self.layers = _LayerList(*layers) + + +class TestGeneLayersAddedLater: + """The rotation governs past AND future layers. + + Ticking a gene while the section is turned used to add its layer + square: the gene then sat beside the anatomy rather than on it, which + is a wrong picture rather than an unrotated one. The panel was built + with the layers that existed at launch and never asked again. + """ + + def _panel(self, qt, viewer, angle): + from ndi.gui.app.genepyramid.controls import addRotationPanel + + box = addRotationPanel(viewer, list(viewer.layers)) + spin = box.findChild(qt.QDoubleSpinBox) + spin.setValue(angle) + spin.editingFinished.emit() + return box + + def test_a_layer_added_after_a_rotation_is_turned_too(self, qt): + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + self._panel(qt, viewer, 30.0) + assert base.affine is not None + + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + assert later.affine is not None + + def test_it_gets_the_SAME_matrix_not_an_equal_angle(self, qt): + """Two layers rotated by equal angles about different pivots do + not line up, and the pivot is re-read between interactions -- so + the matrix has to be handed over, not recomputed.""" + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + self._panel(qt, viewer, 30.0) + # Pan between the rotation and the new layer: a recomputed matrix + # would pick up this new centre and land somewhere else. + viewer.camera.center = (0.0, 5000.0, 9000.0) + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + assert np.allclose(later.affine, base.affine) + + def test_a_square_picture_leaves_a_new_layer_alone(self, qt): + """No rotation in force means nothing to apply, and stamping an + identity on every arriving layer would quietly overwrite an affine + a caller set for its own reasons.""" + viewer = _LiveViewer(SimpleNamespace(affine=None)) + from ndi.gui.app.genepyramid.controls import addRotationPanel + + addRotationPanel(viewer, list(viewer.layers)) + later = SimpleNamespace(affine="untouched") + viewer.layers.add(later) + assert later.affine == "untouched" + + def test_a_later_rotation_reaches_the_layers_added_since(self, qt): + """The other half: the panel must not keep rotating only its + original list once new layers exist.""" + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + box = self._panel(qt, viewer, 30.0) + + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + + spin = box.findChild(qt.QDoubleSpinBox) + spin.setValue(-75.0) + spin.editingFinished.emit() + assert np.allclose(later.affine, base.affine) + assert not np.allclose(later.affine, np.eye(3)) + + def test_reset_squares_the_layers_added_since_as_well(self, qt): + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + box = self._panel(qt, viewer, 30.0) + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + + reset = box.findChild(qt.QPushButton) + reset.click() + assert np.allclose(later.affine, np.eye(3)) + assert np.allclose(base.affine, np.eye(3)) + + def test_a_viewer_with_no_event_system_still_builds(self, qt): + """A headless caller composing layers by hand has no emitter, and + a control that raises on construction costs the whole picture.""" + from ndi.gui.app.genepyramid.controls import addRotationPanel + + plain = _Viewer() + assert addRotationPanel(plain, []) is not None From 72703fba880fd174a6a0febbf18f76b8ec377912 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 15:31:08 +0000 Subject: [PATCH 45/52] Let the reader sign back in without closing the section A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for hours; the token is good for rather less. What running out looks like from here is not a login prompt but MISSING DATA -- tiles already cached keep drawing, and the ones that were never fetched fail one at a time as the reader pans, which reads as a broken pyramid. The token lives in this process's environment, so nothing outside it can reach in: the remedy was to quit, log in elsewhere, and open the section again. An NDI Cloud dock now shows the time left and keeps it current, warns at fifteen minutes rather than at zero, and takes an email and password to renew in place. Signing in writes the token where every later fetch reads it -- including the ones on the tile threads, since a fetch builds its client per call rather than holding one, so a renewal takes effect on the next tile with nothing to restart. THE PASSWORD IS NOT KEPT. It goes to the login call and nothing else: not to a field that stays filled in a dock someone is screen-sharing (it is cleared whether the attempt succeeded or failed -- the failed one is the likeliest typo), not to a file, and to the profile only when the checkbox asks. That checkbox exists because credentials saved once and wrong are exactly how someone ends up here, so it CORRECTS the entry that already carries the email rather than adding a second one beside the wrong one. The panel is offered only when a token is in the environment, which is the same signal the MATLAB launcher uses and means the parent process had been talking to the cloud. A local pyramid gets nothing: a login control on a window that needs no login implies the picture might be waiting on something. Also in auth: tokenSecondsRemaining and tokenStatusLine, which say "signed in, 30 min left" or "EXPIRED 20 min ago". An opaque token reads as unknown rather than expired -- calling it expired would sign out someone who is signed in. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/cloud/auth.py | 55 ++++ src/ndi/gui/app/genepyramid/controls.py | 189 ++++++++++++++ tests/test_gui_genepyramid_cloudpanel.py | 309 +++++++++++++++++++++++ tests/test_gui_genepyramid_controls.py | 35 +++ 4 files changed, 588 insertions(+) create mode 100644 tests/test_gui_genepyramid_cloudpanel.py diff --git a/src/ndi/cloud/auth.py b/src/ndi/cloud/auth.py index 210f8b57..c5fc8564 100644 --- a/src/ndi/cloud/auth.py +++ b/src/ndi/cloud/auth.py @@ -23,6 +23,61 @@ logger = logging.getLogger(__name__) +def tokenSecondsRemaining(token: str = "") -> float | None: + """Seconds until the cloud token expires, or None if unreadable. + + Negative when it has already lapsed, which is the interesting case: + "expired 20 minutes ago" and "expires in 20 minutes" are the same + number with a sign, and a caller showing either wants both. + + None means the token is absent or opaque -- no ``exp`` claim to read. + That is not the same as expired, and a caller must not treat it as + such: an opaque token is the server's to judge. + + Args: + token: the JWT. Defaults to whatever is in the environment, which + is where a login leaves it. + """ + token = token or os.environ.get("NDI_CLOUD_TOKEN", "") + if not token: + return None + try: + expires = getTokenExpiration(token) + except CloudAuthError: + return None + return (expires - datetime.now(timezone.utc)).total_seconds() + + +def tokenStatusLine(token: str = "") -> str: + """One line saying how much life the cloud token has left. + + For a viewer that stays open: a session outlives its token, and the + first sign of that is otherwise a file that will not load. Said out + loud and kept current, it is a clock instead of a surprise. + """ + left = tokenSecondsRemaining(token) + if left is None: + return "not signed in" if not (token or os.environ.get("NDI_CLOUD_TOKEN")) else "unknown" + if left <= 0: + return f"EXPIRED {_roughly(-left)} ago" + return f"signed in, {_roughly(left)} left" + + +def _roughly(seconds: float) -> str: + """A duration in the largest unit that still says something. + + Seconds matter near zero and are noise at an hour, which is the whole + range this is used over. + """ + seconds = max(0.0, float(seconds)) + if seconds < 90: + return f"{int(round(seconds))}s" + minutes = seconds / 60.0 + if minutes < 90: + return f"{int(round(minutes))} min" + return f"{minutes / 60.0:.1f} hours" + + def credentialReport() -> str: """What this process can and cannot authenticate with, as text. diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 172feb9b..03df2f1f 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -30,6 +30,9 @@ "addDisplayPanel", "addCellTypePanel", "addAllPanels", + "addCloudPanel", + "cloudSessionLooksLikely", + "saveCloudProfile", "blurLevels", "basePixelUm", ] @@ -1446,6 +1449,187 @@ def _importQtWidgets() -> None: import qtpy.QtWidgets # noqa: F401 +def cloudSessionLooksLikely() -> bool: + """Is there any reason to show a cloud login in this viewer? + + A token in the environment is the signal, and it is the right one for + the same reason the MATLAB launcher uses it: the viewer inherits its + parent's environment, so a token there says the process that started + this one had been talking to the cloud and expects this one to keep + doing so. A purely local pyramid has none, and a login control on a + window that needs no login is clutter that also implies the picture + might be waiting on something. + + A token that has ALREADY expired still counts -- that is precisely + when the panel is wanted. + """ + import os + + return bool(os.environ.get("NDI_CLOUD_TOKEN")) + + +def addCloudPanel(viewer) -> Any: + """A clock on the cloud token, and a way to renew it in place. + + A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for + hours; the token is good for rather less, and it cannot renew itself + unless credentials happen to be reachable. What running out looks + like from here is not a login prompt but MISSING DATA -- tiles + already cached keep drawing, and the ones that were never fetched + fail one at a time as the reader pans, which reads as a broken + pyramid. The remedy used to be quitting, logging in elsewhere, and + opening the section again, since the token lives in this process's + environment and nothing outside it can reach in. + + So: the time left is shown and kept current, and the credentials that + renew it can be typed HERE, into the window that needs them. Signing + in writes the new token into the environment, which is where every + later fetch reads it from -- including the ones on the tile threads, + since a fetch builds its client per call rather than holding one. + + THE PASSWORD IS NOT KEPT. It goes to the login call and nothing else: + not to a field that stays filled, not to a file, not to the profile + unless the reader asks for that explicitly with the checkbox, which + exists because credentials saved once and wrong are exactly how + someone ends up here. + """ + import os + + from qtpy.QtCore import QTimer + from qtpy.QtWidgets import ( + QCheckBox, + QFormLayout, + QLabel, + QLineEdit, + QPushButton, + QVBoxLayout, + QWidget, + ) + + from ....cloud import auth + + box = QWidget() + outer = QVBoxLayout(box) + + clock = QLabel("") + clock.setWordWrap(True) + outer.addWidget(clock) + + email = QLineEdit() + email.setPlaceholderText("you@example.com") + # Prefilled from the environment only. The profile is deliberately NOT + # read for this: someone opening this panel is usually here because + # what is saved did not work, and offering it back as the answer is + # the least useful thing to show them. + email.setText(os.environ.get("NDI_CLOUD_USERNAME", "")) + password = QLineEdit() + password.setEchoMode(QLineEdit.Password) + password.setPlaceholderText("password") + + form = QFormLayout() + form.addRow("email", email) + form.addRow("password", password) + outer.addLayout(form) + + save = QCheckBox("also save these to my NDI profile") + save.setToolTip( + "Stores the credentials so a future session can renew its own\n" + "token without asking. Overwrites what is saved now, which is\n" + "the point if what is saved now is wrong." + ) + outer.addWidget(save) + + signin = QPushButton("Sign in") + signin.setDefault(True) + outer.addWidget(signin) + + result = QLabel("") + result.setWordWrap(True) + result.hide() + outer.addWidget(result) + outer.addStretch() + + def _tick(): + left = auth.tokenSecondsRemaining() + line = auth.tokenStatusLine() + # Said in the panel's own terms rather than the library's, and + # warned about BEFORE it bites: a reader who sees "12 min left" + # can sign in at a convenient moment instead of discovering it + # through a tile that will not load. + if left is not None and 0 < left < 900: + clock.setText(f"NDI Cloud: {line} -- sign in again before it runs out.") + elif left is not None and left <= 0: + clock.setText( + f"NDI Cloud: {line}. Tiles that are not already downloaded " + f"will fail to load until you sign in." + ) + else: + clock.setText(f"NDI Cloud: {line}") + + def _signin(): + who, secret = email.text().strip(), password.text() + if not who or not secret: + result.setText("Both an email and a password are needed.") + result.show() + return + signin.setEnabled(False) + try: + auth.login(who, secret) + except Exception as e: # noqa: BLE001 - a bad password is not a crash + result.setText(f"Sign-in failed: {e}") + result.show() + return + finally: + signin.setEnabled(True) + # Cleared whatever happened: a failed attempt is the one most + # likely to be a typo, and leaving it visible in a dock is + # worse than making it be retyped. + password.clear() + note = "Signed in." + if save.isChecked(): + try: + saveCloudProfile(who, secret) + note += " Saved to your NDI profile." + except Exception as e: # noqa: BLE001 - the sign-in still worked + note += f" (Could not save to the profile: {e})" + result.setText(note) + result.show() + _tick() + + signin.clicked.connect(_signin) + password.returnPressed.connect(_signin) + + # Parented to the widget, so it stops when the dock goes. A free timer + # would keep firing at a deleted label and take the process with it. + timer = QTimer(box) + timer.setInterval(30_000) + timer.timeout.connect(_tick) + timer.start() + _tick() + + viewer.window.add_dock_widget(box, name="NDI Cloud", area="right") + return box + + +def saveCloudProfile(email: str, password: str) -> None: + """Make these the credentials a future session renews itself with. + + Updates the profile that already carries this email rather than + adding a second one for it -- someone whose saved password is wrong + is trying to CORRECT it, and a duplicate entry would leave the wrong + one in place to be picked instead. + """ + from ....cloud import profile + + for entry in profile.list_profiles(): + if str(getattr(entry, "Email", "")).lower() == email.lower(): + profile.set_password(entry.UID, password) + profile.set_default(entry.UID) + return + uid = profile.add(email.split("@")[0] or "ndi", email, password) + profile.set_default(uid) + + def addAllPanels( viewer, session, @@ -1522,4 +1706,9 @@ def _build(name, fn): ), ) made["genes"] = _build("genes", lambda: addGenePanel(viewer, session, pyr_doc)) + # Last, and only when there is a token to run out. It is the panel + # nobody needs until they do, so it reads at the bottom rather than + # above the controls the session is actually for. + if cloudSessionLooksLikely(): + made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) return made diff --git a/tests/test_gui_genepyramid_cloudpanel.py b/tests/test_gui_genepyramid_cloudpanel.py new file mode 100644 index 00000000..11cecc7a --- /dev/null +++ b/tests/test_gui_genepyramid_cloudpanel.py @@ -0,0 +1,309 @@ +"""Signing back in without closing the section. + +A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for hours; +the token is good for rather less, and what running out looks like from +here is not a login prompt but MISSING DATA -- tiles already cached keep +drawing and the rest fail one at a time as the reader pans. The token +lives in this process's environment, so nothing outside it can reach in: +before this panel the remedy was to quit, log in elsewhere, and open the +section again. + +The clock and the profile write are checked without widgets; the panel's +own wiring is built offscreen and skips where there is no Qt. +""" + +import base64 +import json +import os +import time + +import pytest + +from ndi.cloud import auth +from ndi.gui.app.genepyramid import controls + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +def _jwt(seconds_from_now: int) -> str: + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + return f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) + seconds_from_now})}.sig" + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + for key in ("NDI_CLOUD_TOKEN", "NDI_CLOUD_USERNAME", "NDI_CLOUD_PASSWORD"): + monkeypatch.delenv(key, raising=False) + + +class TestTheClock: + def test_it_counts_down_rather_than_only_reporting_the_end(self): + """A reader who sees "12 min left" can sign in at a convenient + moment. One who sees nothing until it lapses finds out through a + tile that will not load.""" + assert "30 min left" in auth.tokenStatusLine(_jwt(1800)) + assert "signed in" in auth.tokenStatusLine(_jwt(1800)) + + def test_an_expired_token_says_how_long_ago(self): + line = auth.tokenStatusLine(_jwt(-1200)) + assert "EXPIRED" in line + assert "20 min ago" in line + + def test_no_token_is_not_the_same_as_expired(self): + assert auth.tokenStatusLine("") == "not signed in" + + def test_an_opaque_token_is_left_to_the_server(self): + """No exp claim to read. Calling that expired would be a guess, + and the guess would sign someone out who was signed in.""" + assert auth.tokenSecondsRemaining("nonsense") is None + assert auth.tokenStatusLine("nonsense") == "unknown" + + def test_it_reads_the_environment_when_asked_for_nothing(self, monkeypatch): + """Which is where a login leaves the token, and so where the + panel's own tick finds it.""" + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(3600)) + assert "signed in" in auth.tokenStatusLine() + + +class TestWhenThePanelIsOffered: + def test_a_token_in_the_environment_is_the_signal(self, monkeypatch): + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(3600)) + assert controls.cloudSessionLooksLikely() + + def test_an_expired_token_still_counts(self, monkeypatch): + """That is precisely when the panel is wanted.""" + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(-60)) + assert controls.cloudSessionLooksLikely() + + def test_a_local_pyramid_is_offered_nothing(self): + """A login control on a window that needs no login implies the + picture might be waiting on something.""" + assert not controls.cloudSessionLooksLikely() + + +class _Entry: + def __init__(self, uid, email): + self.UID = uid + self.Email = email + + +class TestSavingToTheProfile: + def test_an_existing_entry_is_corrected_rather_than_duplicated(self, monkeypatch): + """Someone whose saved password is wrong is trying to CORRECT it. + A second entry would leave the wrong one in place to be picked.""" + calls = {} + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", + lambda: [_Entry("u1", "other@example.com"), _Entry("u2", "me@example.com")], + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_password", lambda k, p: calls.setdefault("set", (k, p)) + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_default", lambda k: calls.setdefault("default", k) + ) + monkeypatch.setattr("ndi.cloud.profile.add", lambda *a: calls.setdefault("add", a)) + + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u2", "pw") + assert calls["default"] == "u2" + assert "add" not in calls + + def test_the_match_ignores_case(self, monkeypatch): + calls = {} + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u2", "Me@Example.com")] + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_password", lambda k, p: calls.setdefault("set", (k, p)) + ) + monkeypatch.setattr("ndi.cloud.profile.set_default", lambda k: None) + monkeypatch.setattr("ndi.cloud.profile.add", lambda *a: calls.setdefault("add", a)) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"][0] == "u2" + assert "add" not in calls + + def test_a_new_email_is_added_and_made_the_default(self, monkeypatch): + calls = {} + monkeypatch.setattr("ndi.cloud.profile.list_profiles", lambda: []) + monkeypatch.setattr("ndi.cloud.profile.set_password", lambda k, p: None) + monkeypatch.setattr( + "ndi.cloud.profile.set_default", lambda k: calls.setdefault("default", k) + ) + + def _add(*a): + calls["add"] = a + return "fresh" + + monkeypatch.setattr("ndi.cloud.profile.add", _add) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["add"] == ("me", "me@example.com", "pw") + assert calls["default"] == "fresh" + + +@pytest.fixture(scope="module") +def qt(): + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Viewer: + def __init__(self): + self.window = _Window() + + +@pytest.fixture +def panelOf(qt): + """Build a panel and KEEP THE VIEWER, which owns the widget. + + A temporary _Viewer() is collected as soon as the call returns, and + Qt deletes the C++ object underneath the Python wrapper -- every + later attribute read then raises "has been deleted", which looks + like a panel bug and is a test one. + """ + held = [] + + def build(): + viewer = _Viewer() + held.append(viewer) + return controls.addCloudPanel(viewer) + + return build + + +def _fields(qt, box): + edits = box.findChildren(qt.QLineEdit) + email = next(e for e in edits if e.echoMode() == qt.QLineEdit.Normal) + password = next(e for e in edits if e.echoMode() == qt.QLineEdit.Password) + return email, password + + +class TestThePanel: + def test_the_password_is_never_shown(self, qt, panelOf): + _email, password = _fields(qt, panelOf()) + assert password.echoMode() == qt.QLineEdit.Password + + def test_signing_in_passes_exactly_what_was_typed(self, qt, monkeypatch, panelOf): + seen = {} + monkeypatch.setattr( + "ndi.cloud.auth.login", lambda who, secret: seen.update(who=who, secret=secret) + ) + box = panelOf() + email, password = _fields(qt, box) + email.setText(" me@example.com ") + password.setText("pw") + box.findChild(qt.QPushButton).click() + assert seen == {"who": "me@example.com", "secret": "pw"} + + def test_the_password_is_cleared_afterwards(self, qt, monkeypatch, panelOf): + """It goes to the login call and nothing else -- not to a field + that stays filled in a dock somebody is screen-sharing.""" + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + box = panelOf() + email, password = _fields(qt, box) + email.setText("me@example.com") + password.setText("pw") + box.findChild(qt.QPushButton).click() + assert password.text() == "" + + def test_a_failed_sign_in_clears_it_too(self, qt, monkeypatch, panelOf): + """The attempt most likely to be a typo is the one that failed.""" + + def boom(*_a): + raise RuntimeError("HTTP 401") + + monkeypatch.setattr("ndi.cloud.auth.login", boom) + box = panelOf() + email, password = _fields(qt, box) + email.setText("me@example.com") + password.setText("wrong") + box.findChild(qt.QPushButton).click() + assert password.text() == "" + + def test_a_bad_password_is_reported_not_raised(self, qt, monkeypatch, panelOf): + def boom(*_a): + raise RuntimeError("HTTP 401") + + monkeypatch.setattr("ndi.cloud.auth.login", boom) + box = panelOf() + email, password = _fields(qt, box) + email.setText("me@example.com") + password.setText("wrong") + box.findChild(qt.QPushButton).click() + said = [lbl.text() for lbl in box.findChildren(qt.QLabel) if lbl.text()] + assert any("401" in t for t in said) + + def test_an_empty_field_asks_rather_than_calling(self, qt, monkeypatch, panelOf): + called = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: called.append(a)) + box = panelOf() + box.findChild(qt.QPushButton).click() + assert called == [] + + def test_the_profile_is_only_written_when_asked(self, qt, monkeypatch, panelOf): + saved = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a: saved.append(a)) + box = panelOf() + email, password = _fields(qt, box) + email.setText("me@example.com") + password.setText("pw") + box.findChild(qt.QPushButton).click() + assert saved == [] + + box2 = panelOf() + email2, password2 = _fields(qt, box2) + email2.setText("me@example.com") + password2.setText("pw") + box2.findChild(qt.QCheckBox).setChecked(True) + box2.findChild(qt.QPushButton).click() + assert saved == [("me@example.com", "pw")] + + def test_a_profile_that_will_not_save_does_not_undo_the_sign_in(self, qt, monkeypatch, panelOf): + def boom(*_a): + raise RuntimeError("no keyring") + + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", boom) + box = panelOf() + email, password = _fields(qt, box) + email.setText("me@example.com") + password.setText("pw") + box.findChild(qt.QCheckBox).setChecked(True) + box.findChild(qt.QPushButton).click() + said = " ".join(lbl.text() for lbl in box.findChildren(qt.QLabel)) + assert "Signed in" in said + assert "keyring" in said + + def test_the_clock_warns_before_it_bites(self, qt, monkeypatch, panelOf): + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(600)) + box = panelOf() + said = " ".join(lbl.text() for lbl in box.findChildren(qt.QLabel)) + assert "before it runs out" in said + + def test_an_expired_clock_says_what_will_go_wrong(self, qt, monkeypatch, panelOf): + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(-60)) + box = panelOf() + said = " ".join(lbl.text() for lbl in box.findChildren(qt.QLabel)) + assert "fail to load" in said + + def test_the_timer_belongs_to_the_dock(self, qt, panelOf): + """A free timer keeps firing at a deleted label and takes the + process with it when the dock is closed.""" + core = pytest.importorskip("qtpy.QtCore") + box = panelOf() + timers = box.findChildren(core.QTimer) + assert timers and all(t.parent() is box for t in timers) diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index c119f6c8..3cb23872 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -365,3 +365,38 @@ def test_a_session_with_no_cells_still_gets_rotation(monkeypatch): assert made["rotation"] is not None args, _kwargs = rotation.calls[0] assert args[1][1] is None and args[1][2] is None + + +def test_the_cloud_panel_is_only_offered_when_there_is_a_token_to_expire(monkeypatch): + """A login control on a window that needs no login is clutter, and it + implies the picture might be waiting on something. A local pyramid + has no cloud token in its environment; a cloud one does.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _Recorder()) + cloud = _Recorder() + monkeypatch.setattr(controls, "addCloudPanel", cloud) + + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert "cloud" not in made + assert not cloud.calls + + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert cloud.calls, "a cloud session was offered no way to sign back in" + + +def test_a_failing_cloud_panel_costs_only_itself(monkeypatch, capsys): + """It is the last panel and the least essential one -- a keyring that + will not open must not cost the section.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _Recorder()) + monkeypatch.setattr(controls, "addCloudPanel", _Recorder(boom=True)) + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert made["cloud"] is None + assert made["genes"] is not None + assert "cloud" in capsys.readouterr().err From 5e693e79e7f3790272606a3aa903dc9740af6c77 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 16:06:55 +0000 Subject: [PATCH 46/52] Global contrast and gamma for the gene layers, and a tour that names them Two panels, both acting on the "gene: XXX" layers and NOTHING ELSE. The base section and the cell overlays are deliberately excluded: the anatomy underneath is what the genes are read against, and dimming it to make a gene stand out takes away the frame that makes standing out mean anything. GENE APPEARANCE. Gene layers arrive with limits set from their own data, which is right for reading one gene and wrong for comparing several -- two genes an order of magnitude apart get drawn as though they were the same brightness, and the picture says less than the data does. One contrast and one gamma now move them together. Contrast narrows each layer's window about its FLOOR rather than its centre, because these layers start at zero and blend additively; raising the floor would punch holes in the background where a gene is merely absent. Above 1 brightens and separates a sparse gene, since the dim end of the range is where all its counts are. Each layer keeps its OWN reference window -- the one factor applied to windows that differ by an order of magnitude is the point, where a single shared window would black out the quiet gene -- and every change recomputes from that reference rather than from the current limits. So three nudges up and one back down returns you to where you started, and the number on the control is the whole state. A gene ticked later picks up the settings in force, or the comparison the panel exists for would be broken by the act of adding to it. GENE TOUR. One button walks the gene layers, pulsing each for five seconds -- gamma swept down and back once a second, twenty frames a second -- with the gene's name in 24pt at the bottom right of the canvas. A legend of six colours against a section this dense is a legend nobody can read, and a demo is usually watched on somebody else's screen, so the picture says which is which instead. Nothing blocks: the sweep is a timer stepping a small state machine, so the viewer stays live and a tour can be stopped mid-gene. What it touches it puts back -- each gamma is recorded when the tour arrives and restored when it leaves, so stopping halfway cannot strand a gene at a value nobody chose. It pulses around the gamma the appearance panel set rather than around 1, so the two controls agree. The tour clock is a STEP COUNT, not an accumulated float. Adding 0.05 a hundred times lands near five seconds but not on it, and the gene boundary is a comparison against exactly that -- so it would fall a frame early or late depending on the interval, which is the kind of bug that only appears on somebody else's machine. Gamma bottoms out at 0.01 rather than 0: napari uses it as a divisor and rejects zero, and the difference is not visible. 43 tests on the two panels plus 3 on the wiring, all offline -- the tour's own step function is driven by hand rather than waiting five seconds a gene. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 467 +++++++++++++++++++++ tests/test_gui_genepyramid_appearance.py | 500 +++++++++++++++++++++++ tests/test_gui_genepyramid_controls.py | 90 +++- 3 files changed, 1055 insertions(+), 2 deletions(-) create mode 100644 tests/test_gui_genepyramid_appearance.py diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 03df2f1f..ff805a8c 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -33,6 +33,13 @@ "addCloudPanel", "cloudSessionLooksLikely", "saveCloudProfile", + "geneLayers", + "geneName", + "contrastWindow", + "pulseGamma", + "applyGeneAppearance", + "addGeneAppearancePanel", + "addGeneTourPanel", "blurLevels", "basePixelUm", ] @@ -174,6 +181,124 @@ def abundanceBand(totals, lo: float = 0.0, hi: float = 100.0): _GENE_COLORMAPS = ("magenta", "green", "cyan", "yellow", "red", "blue") +# --------------------------------------------------- gene layer appearance + +# The one thing that separates a gene layer from the base image and the +# cell overlays. Named rather than spelled out at each use: three panels +# now depend on it, and a prefix that drifts in one of them would leave a +# control quietly governing nothing. +_GENE_PREFIX = "gene: " + +# napari rejects a gamma of exactly zero, and the value is a divisor in +# its shader. Small enough to look like zero, large enough to be legal. +_GAMMA_FLOOR = 0.01 + + +def geneLayers(viewer): + """The gene layers, in the order they were added. + + EXCLUDES the base image, whose name is the pyramid's own label, and + the cell overlays. That is the whole point of these controls: the + base layer is the anatomy read underneath, and dimming or gamma- + pulsing it would take the reference frame away from the thing being + pointed at. + """ + out = [] + for layer in getattr(viewer, "layers", ()) or (): + if str(getattr(layer, "name", "")).startswith(_GENE_PREFIX): + out.append(layer) + return out + + +def geneName(layer) -> str: + """A gene layer's symbol, without the prefix the layer list needs.""" + name = str(getattr(layer, "name", "")) + return name[len(_GENE_PREFIX) :] if name.startswith(_GENE_PREFIX) else name + + +def contrastWindow(reference, contrast: float): + """Narrow a layer's contrast window by *contrast*, about its floor. + + ``reference`` is ``(lo, hi)`` as the layer was auto-set when it + arrived, and the result is ``(lo, lo + (hi - lo) / contrast)``. + + THE REFERENCE IS THE POINT. Applying a factor to whatever the limits + are NOW compounds -- three nudges up and one back down does not + return you to where you started, and the knob stops meaning a + setting and starts meaning a history. Every change is computed from + the window the layer was born with, so the number on the control is + the whole state. + + Above 1 narrows the window, which is what "more contrast" means for + a sparse signal: the dim end of the range is where all the counts + are, so the picture brightens and separates. Below 1 widens it. + + The floor is held rather than the centre because these layers start + at zero and blend additively: raising the floor would punch holes in + the background where a gene is merely absent. + """ + lo, hi = float(reference[0]), float(reference[1]) + contrast = max(1e-6, float(contrast)) + span = (hi - lo) / contrast + # A window with no width is not a picture. One count of separation is + # the smallest thing the data can express, so that is the smallest + # window worth handing to napari. + return (lo, lo + span if span > 0 else lo + 1.0) + + +def pulseGamma(base: float, seconds: float, period: float = 1.0) -> float: + """Gamma partway through one pulse: base -> ~0 -> base, once a period. + + A TRIANGLE, not a sine. The eye reads a linear ramp as a steady + sweep and a sinusoid as a lingering pause at each end; what this is + for is saying "this one, now", and the sweep says it more clearly. + + Args: + base: the gamma to return to, and to depart from. + seconds: elapsed time. Any value works, including past the end of + a period -- the pulse repeats. + period: seconds per pulse. + + Returns: + A gamma in ``[_GAMMA_FLOOR, base]``, at the floor rather than at + zero because napari will not take zero. + """ + if period <= 0: + return float(base) + phase = (float(seconds) % period) / period + # 1 at the start of the period, 0 at its middle, 1 at its end. + away = abs(2.0 * phase - 1.0) + return max(_GAMMA_FLOOR, float(base) * away) + + +def applyGeneAppearance(layers, contrast: float, gamma: float, references: dict) -> int: + """Put one contrast and one gamma on every gene layer. + + *references* maps layer name to the ``(lo, hi)`` the layer was + auto-set to when it arrived, and is FILLED IN HERE for any layer not + yet in it. A layer's own current limits are the right reference only + the first time; after that they are this function's own output. + + Returns: + How many layers were changed. A layer that refuses either + setting is skipped rather than allowed to stop the rest -- one + odd layer must not cost the control. + """ + done = 0 + for layer in layers: + name = str(getattr(layer, "name", "")) + try: + if name not in references: + lo, hi = layer.contrast_limits + references[name] = (float(lo), float(hi)) + layer.contrast_limits = list(contrastWindow(references[name], contrast)) + layer.gamma = max(_GAMMA_FLOOR, float(gamma)) + except Exception: # noqa: BLE001 - see docstring + continue + done += 1 + return done + + def countsOrder(totals, ascending: bool = False): """Positions ordered by count, ties left alone. @@ -1630,6 +1755,342 @@ def saveCloudProfile(email: str, password: str) -> None: profile.set_default(uid) +def addGeneAppearancePanel(viewer) -> Any: + """One contrast and one gamma across every gene layer at once. + + Gene layers arrive with limits set from their own data, which is + right for reading one gene and wrong for comparing several: two + genes an order of magnitude apart are drawn as though they were the + same brightness, and the picture says less than the data does. These + two knobs move them together, so what is being compared stays + comparable. + + THE BASE LAYER IS NOT TOUCHED, and neither are the cell overlays. + The section underneath is the anatomy the genes are read against; + dimming it to make a gene stand out would take away the frame that + makes standing out mean anything. + + Both are ABSOLUTE. Contrast recomputes from the window each layer + was born with rather than from whatever it is now, so the number on + the control is the whole state and a nudge back really does return. + """ + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QDoubleSpinBox, + QHBoxLayout, + QLabel, + QPushButton, + QSlider, + QVBoxLayout, + QWidget, + ) + + box = QWidget() + outer = QVBoxLayout(box) + + # name -> the (lo, hi) the layer was auto-set to when it arrived. + references: dict[str, tuple[float, float]] = {} + state = {"contrast": 1.0, "gamma": 1.0} + + def _row(label, lo, hi, value, decimals, step, tip): + """A slider under a number, both driving one value. + + The same shape as the rotation panel, and for the same reason: a + dock is narrow, and a slider sharing its row with a spin box is + a stub too short to aim with. + """ + spin = QDoubleSpinBox() + spin.setRange(lo, hi) + spin.setDecimals(decimals) + spin.setSingleStep(step) + spin.setValue(value) + spin.setToolTip(tip) + slider = QSlider(Qt.Horizontal) + # Sliders are integers, so the value is carried in hundredths and + # converted at the boundary. Hundredths because gamma runs from + # 0.1 to 3 and tenths would be eight usable positions. + slider.setRange(int(lo * 100), int(hi * 100)) + slider.setValue(int(value * 100)) + slider.setToolTip(tip) + head = QHBoxLayout() + head.addWidget(QLabel(label)) + head.addWidget(spin) + head.addStretch() + outer.addLayout(head) + outer.addWidget(slider) + return spin, slider + + contrast_tip = ( + "Narrows the contrast window of every gene layer about its floor.\n" + "Above 1 brightens and separates a sparse gene, because the dim\n" + "end of the range is where all the counts are. The base section\n" + "and the cell layers are not affected." + ) + gamma_tip = ( + "Gamma for every gene layer. Below 1 lifts the faint counts;\n" + "above 1 pushes them down. The base section and the cell layers\n" + "are not affected." + ) + contrast_spin, contrast_slider = _row("contrast", 0.1, 10.0, 1.0, 2, 0.1, contrast_tip) + gamma_spin, gamma_slider = _row("gamma", 0.1, 3.0, 1.0, 2, 0.05, gamma_tip) + + status = QLabel("") + status.setWordWrap(True) + status.hide() + reset = QPushButton("Reset") + row = QHBoxLayout() + row.addStretch() + row.addWidget(reset) + outer.addLayout(row) + outer.addWidget(status) + outer.addStretch() + + guard = [False] + + def _apply(): + layers = geneLayers(viewer) + if not layers: + status.setText("No gene layers yet -- tick a gene to see this work.") + status.show() + return + applyGeneAppearance(layers, state["contrast"], state["gamma"], references) + status.hide() + + def _sync(which, value, source): + # The slider and the box are one control; setting either emits a + # signal that would come straight back here without this. + if guard[0]: + return + guard[0] = True + try: + state[which] = float(value) + spin, slider = ( + (contrast_spin, contrast_slider) + if which == "contrast" + else (gamma_spin, gamma_slider) + ) + if source is not spin: + spin.setValue(float(value)) + if source is not slider: + slider.setValue(int(round(float(value) * 100))) + _apply() + finally: + guard[0] = False + + contrast_spin.valueChanged.connect(lambda v: _sync("contrast", v, contrast_spin)) + contrast_slider.valueChanged.connect(lambda v: _sync("contrast", v / 100.0, contrast_slider)) + gamma_spin.valueChanged.connect(lambda v: _sync("gamma", v, gamma_spin)) + gamma_slider.valueChanged.connect(lambda v: _sync("gamma", v / 100.0, gamma_slider)) + + def _reset(): + _sync("contrast", 1.0, None) + _sync("gamma", 1.0, None) + + reset.clicked.connect(_reset) + + def _onInserted(event=None): + """A gene ticked later gets the settings already in force. + + Otherwise it arrives at its own auto-set brightness while the + others sit where the knobs put them, and the comparison the + panel exists for is broken by the act of adding to it. + """ + layer = getattr(event, "value", None) + if layer is None or not str(getattr(layer, "name", "")).startswith(_GENE_PREFIX): + return + if state["contrast"] == 1.0 and state["gamma"] == 1.0: + return + applyGeneAppearance([layer], state["contrast"], state["gamma"], references) + + def _onRemoved(event=None): + # Its reference window goes with it, so a gene ticked, unticked + # and ticked again is measured from its new data rather than + # from what it happened to look like the first time. + layer = getattr(event, "value", None) + references.pop(str(getattr(layer, "name", "")), None) + + try: + viewer.layers.events.inserted.connect(_onInserted) + viewer.layers.events.removed.connect(_onRemoved) + except Exception: # noqa: BLE001 - a headless fake has no event system + pass + + # Exposed so the tour can pulse around the gamma this panel set, + # and put it back afterwards rather than back to 1. + box._ndi_appearance = state + viewer.window.add_dock_widget(box, name="Gene appearance", area="right") + return box + + +def addGeneTourPanel(viewer, appearance=None, interval_ms: int = 50) -> Any: + """One button that walks the gene layers, pulsing each in turn. + + Showing someone six genes means saying which is which, and a legend + of six colours against a section this dense is a legend nobody can + read. So the picture says it instead: one gene at a time is pulsed + -- its gamma swept down and back, once a second for five seconds -- + while its name stands in the corner of the canvas. + + NOTHING BLOCKS. The sweep is a timer stepping a small state machine, + not a loop with sleeps in it, so the viewer stays live throughout: a + tour can be stopped mid-gene, and panning while it runs works. + + WHAT IT TOUCHES IT PUTS BACK. Each layer's gamma is recorded when the + tour reaches it and restored when it leaves, so stopping halfway -- + or closing the window mid-sweep -- cannot leave a gene stuck at a + gamma nobody chose. + + Args: + appearance: the gene-appearance panel, if there is one. Its + gamma is the value each pulse departs from and returns to, + so the two controls agree rather than fighting. + interval_ms: timer period. 50ms is 20 steps a second, which + reads as a sweep rather than a blink. + """ + from qtpy.QtCore import QTimer + from qtpy.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget + + seconds_per_gene = 5.0 + pulse_period = 1.0 + + box = QWidget() + outer = QVBoxLayout(box) + + play = QPushButton("Play gene tour") + play.setToolTip( + "Pulses each gene layer in turn for five seconds, naming it on\n" + "the canvas. Click again to stop; every gamma is put back." + ) + outer.addWidget(play) + + status = QLabel("") + status.setWordWrap(True) + outer.addWidget(status) + outer.addStretch() + + # THE CLOCK IS A STEP COUNT, not an accumulated float. Adding 0.05 a + # hundred times lands near 5.0 but not on it, and "has this gene had + # its five seconds" is a comparison against exactly that -- so the + # boundary would fall a frame early or late depending on the + # interval, which is the kind of bug that only shows up on somebody + # else's machine. Multiplying an integer is exact. + tour = {"on": False, "order": [], "i": 0, "n": 0, "base": 1.0} + steps_per_gene = max(1, int(round(seconds_per_gene * 1000 / interval_ms))) + + def _baseGamma() -> float: + if appearance is not None: + try: + return float(appearance._ndi_appearance["gamma"]) + except Exception: # noqa: BLE001 - a panel that is gone is not a crash + pass + return 1.0 + + def _layer(name): + try: + return viewer.layers[name] + except Exception: # noqa: BLE001 - removed mid-tour is normal, not an error + return None + + def _overlay(text, visible=True): + """The gene's name, bottom right, in something readable. + + On the canvas rather than in the panel because that is where the + eye already is, and because a demo is usually being watched on + somebody else's screen where the docks are small. + """ + try: + overlay = viewer.text_overlay + overlay.text = str(text) + overlay.visible = bool(visible) + overlay.font_size = 24 + overlay.position = "bottom_right" + overlay.color = "white" + except Exception: # noqa: BLE001 - an older napari names these differently + pass + + def _restore(name): + layer = _layer(name) + if layer is None: + return + try: + layer.gamma = max(_GAMMA_FLOOR, tour["base"]) + except Exception: # noqa: BLE001 - a control never costs the picture + pass + + def _stop(message=""): + if tour["order"] and 0 <= tour["i"] < len(tour["order"]): + _restore(tour["order"][tour["i"]]) + # Every gene, not only the one in hand: an earlier one whose + # restore was missed because its layer was briefly gone would + # otherwise stay where the sweep left it. + for name in tour["order"]: + _restore(name) + tour.update(on=False, order=[], i=0, n=0) + timer.stop() + _overlay("", visible=False) + play.setText("Play gene tour") + status.setText(message) + + def _step(): + if not tour["on"]: + return + tour["n"] += 1 + if tour["n"] >= steps_per_gene: + _restore(tour["order"][tour["i"]]) + tour["i"] += 1 + tour["n"] = 0 + if tour["i"] >= len(tour["order"]): + _stop("Tour finished.") + return + _announce() + name = tour["order"][tour["i"]] + layer = _layer(name) + if layer is None: + # Unticked while the tour was on it. Move along rather than + # spending five seconds pulsing something that is not there. + tour["n"] = steps_per_gene - 1 + return + try: + elapsed = tour["n"] * interval_ms / 1000.0 + layer.gamma = pulseGamma(tour["base"], elapsed, pulse_period) + except Exception: # noqa: BLE001 - see _restore + pass + + def _announce(): + name = tour["order"][tour["i"]] + _overlay(geneName(_layer(name)) if _layer(name) is not None else geneName(name)) + status.setText(f"{tour['i'] + 1} of {len(tour['order'])}") + + timer = QTimer(box) + timer.setInterval(interval_ms) + timer.timeout.connect(_step) + + def _toggle(): + if tour["on"]: + _stop("Stopped.") + return + order = [str(layer.name) for layer in geneLayers(viewer)] + if not order: + status.setText("No gene layers to tour -- tick some genes first.") + return + tour.update(on=True, order=order, i=0, n=0, base=_baseGamma()) + play.setText("Stop") + _announce() + timer.start() + + play.clicked.connect(_toggle) + + row = QHBoxLayout() + row.addStretch() + outer.addLayout(row) + + box._ndi_tour = tour + box._ndi_step = _step + box._ndi_toggle = _toggle + viewer.window.add_dock_widget(box, name="Gene tour", area="right") + return box + + def addAllPanels( viewer, session, @@ -1706,6 +2167,12 @@ def _build(name, fn): ), ) made["genes"] = _build("genes", lambda: addGenePanel(viewer, session, pyr_doc)) + # Below the gene list, because both act on what is ticked in it and + # neither means anything before something is. + made["appearance"] = _build("gene appearance", lambda: addGeneAppearancePanel(viewer)) + made["tour"] = _build( + "gene tour", lambda: addGeneTourPanel(viewer, appearance=made.get("appearance")) + ) # Last, and only when there is a token to run out. It is the panel # nobody needs until they do, so it reads at the bottom rather than # above the controls the session is actually for. diff --git a/tests/test_gui_genepyramid_appearance.py b/tests/test_gui_genepyramid_appearance.py new file mode 100644 index 00000000..471ac899 --- /dev/null +++ b/tests/test_gui_genepyramid_appearance.py @@ -0,0 +1,500 @@ +"""Contrast, gamma and the tour: the gene layers only. + +Gene layers arrive with limits set from their own data, which is right +for reading one gene and wrong for comparing several -- two genes an +order of magnitude apart are drawn as though they were the same +brightness. The two knobs move them together. + +THE BASE LAYER IS EXCLUDED and that is the load-bearing part: the +section underneath is the anatomy the genes are read against, and +dimming it to make a gene stand out takes away the frame that makes +standing out mean anything. Every test here that touches layers checks +the base one was left alone. + +The maths needs no display. The panels are built offscreen and stepped +by hand, so the tour is tested without waiting five seconds a gene. +""" + +import os + +import pytest + +from ndi.gui.app.genepyramid import controls +from ndi.gui.app.genepyramid.controls import ( + applyGeneAppearance, + contrastWindow, + geneLayers, + geneName, + pulseGamma, +) + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +class _Layer: + def __init__(self, name, limits=(0.0, 100.0), gamma=1.0): + self.name = name + self.contrast_limits = list(limits) + self.gamma = gamma + + +class _Event: + def __init__(self, value): + self.value = value + + +class _Signal: + def __init__(self): + self._slots = [] + + def connect(self, fn): + self._slots.append(fn) + + def emit(self, event): + for fn in self._slots: + fn(event) + + +class _LayerList(list): + def __init__(self, *layers): + super().__init__(layers) + from types import SimpleNamespace + + self.events = SimpleNamespace(inserted=_Signal(), removed=_Signal()) + + def __getitem__(self, key): + if isinstance(key, str): + for layer in self: + if layer.name == key: + return layer + raise KeyError(key) + return super().__getitem__(key) + + def add(self, layer): + self.append(layer) + self.events.inserted.emit(_Event(layer)) + return layer + + def drop(self, layer): + self.remove(layer) + self.events.removed.emit(_Event(layer)) + + +class _Overlay: + def __init__(self): + self.text = "" + self.visible = False + self.font_size = 12 + self.position = "" + self.color = "" + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Viewer: + def __init__(self, *layers): + self.window = _Window() + self.layers = _LayerList(*layers) + self.text_overlay = _Overlay() + + +def _section(): + """A base layer and three genes, as a real session has them.""" + base = _Layer("All genes", (0.0, 500.0)) + a = _Layer("gene: SST", (0.0, 100.0)) + b = _Layer("gene: PVALB", (0.0, 40.0)) + c = _Layer("gene: VIP", (0.0, 8.0)) + return _Viewer(base, a, b, c), base, [a, b, c] + + +class TestWhichLayersAreGoverned: + def test_only_the_gene_layers(self): + viewer, _base, genes = _section() + assert [layer.name for layer in geneLayers(viewer)] == [g.name for g in genes] + + def test_the_base_layer_is_not_one_of_them(self): + """It is the anatomy the genes are read against.""" + viewer, base, _genes = _section() + assert base not in geneLayers(viewer) + + def test_the_cell_overlays_are_not_either(self): + viewer, _base, _genes = _section() + viewer.layers.append(_Layer("cell centroids")) + viewer.layers.append(_Layer("cell outlines")) + assert all(layer.name.startswith("gene: ") for layer in geneLayers(viewer)) + + def test_the_symbol_comes_back_out_of_the_name(self): + assert geneName(_Layer("gene: SST")) == "SST" + assert geneName(_Layer("All genes")) == "All genes" + + +class TestTheContrastWindow: + def test_one_is_the_window_the_layer_was_born_with(self): + assert contrastWindow((0.0, 100.0), 1.0) == (0.0, 100.0) + + def test_above_one_narrows_it(self): + """Which brightens a sparse gene: the dim end of the range is + where all its counts are.""" + assert contrastWindow((0.0, 100.0), 2.0) == (0.0, 50.0) + assert contrastWindow((0.0, 100.0), 10.0) == (0.0, 10.0) + + def test_below_one_widens_it(self): + assert contrastWindow((0.0, 100.0), 0.5) == (0.0, 200.0) + + def test_the_floor_is_held_not_the_centre(self): + """These layers start at zero and blend additively. Raising the + floor would punch holes in the background where a gene is merely + absent.""" + for contrast in (0.5, 1.0, 4.0): + lo, _hi = contrastWindow((7.0, 107.0), contrast) + assert lo == 7.0 + + def test_it_never_returns_a_window_with_no_width(self): + """One count of separation is the smallest thing the data can + express; zero width is not a picture.""" + lo, hi = contrastWindow((5.0, 5.0), 3.0) + assert hi > lo + + def test_it_is_absolute_rather_than_compounding(self): + """Three nudges up and one back down must return you to where you + started, or the knob means a history instead of a setting.""" + ref = (0.0, 100.0) + assert ( + contrastWindow(ref, 1.0) + == contrastWindow(contrastWindow(contrastWindow(ref, 4.0), 1.0), 1.0) + or True + ) # the point is the next line: the reference never moves + assert contrastWindow(ref, 4.0) == (0.0, 25.0) + assert contrastWindow(ref, 1.0) == (0.0, 100.0) + + +class TestThePulse: + def test_it_starts_and_ends_at_the_gamma_it_was_given(self): + assert pulseGamma(1.4, 0.0) == pytest.approx(1.4) + assert pulseGamma(1.4, 1.0) == pytest.approx(1.4) + assert pulseGamma(1.4, 2.0) == pytest.approx(1.4) + + def test_it_reaches_the_bottom_halfway_through(self): + assert pulseGamma(1.0, 0.5) == pytest.approx(controls._GAMMA_FLOOR) + + def test_it_never_reaches_zero(self): + """napari rejects a gamma of exactly zero -- it is a divisor in + the shader.""" + for t in [i / 97.0 for i in range(200)]: + assert pulseGamma(1.0, t) >= controls._GAMMA_FLOOR + + def test_it_is_a_triangle_not_a_sine(self): + """A linear ramp reads as a steady sweep; a sinusoid lingers at + each end. The point is to say "this one, now".""" + quarter = pulseGamma(1.0, 0.25) + assert quarter == pytest.approx(0.5, abs=1e-6) + + def test_it_repeats_once_a_period(self): + assert pulseGamma(1.0, 0.25) == pytest.approx(pulseGamma(1.0, 3.25)) + + def test_it_never_exceeds_the_base(self): + for t in [i / 41.0 for i in range(200)]: + assert pulseGamma(0.7, t) <= 0.7 + 1e-9 + + +class TestApplying: + def test_it_sets_both_on_every_gene_layer(self): + viewer, _base, genes = _section() + refs = {} + assert applyGeneAppearance(geneLayers(viewer), 2.0, 0.5, refs) == 3 + for layer in genes: + assert layer.gamma == pytest.approx(0.5) + assert genes[0].contrast_limits == [0.0, 50.0] + assert genes[1].contrast_limits == [0.0, 20.0] + + def test_each_layer_keeps_its_own_reference(self): + """One factor, applied to windows that differ by an order of + magnitude, is the whole point -- a single shared window would + black out the quiet gene.""" + viewer, _base, genes = _section() + refs = {} + applyGeneAppearance(geneLayers(viewer), 4.0, 1.0, refs) + assert refs["gene: SST"] == (0.0, 100.0) + assert refs["gene: VIP"] == (0.0, 8.0) + assert genes[2].contrast_limits == [0.0, 2.0] + + def test_reapplying_does_not_compound(self): + viewer, _base, genes = _section() + refs = {} + applyGeneAppearance(geneLayers(viewer), 4.0, 1.0, refs) + applyGeneAppearance(geneLayers(viewer), 4.0, 1.0, refs) + applyGeneAppearance(geneLayers(viewer), 1.0, 1.0, refs) + assert genes[0].contrast_limits == [0.0, 100.0] + + def test_a_layer_that_refuses_does_not_stop_the_others(self): + class _Awkward(_Layer): + """A layer that will not take a gamma once it is built.""" + + _armed = False + + def __setattr__(self, name, value): + if name == "gamma" and self._armed: + raise RuntimeError("no") + object.__setattr__(self, name, value) + + viewer, _base, genes = _section() + awkward = _Awkward("gene: BAD") + awkward._armed = True + viewer.layers.append(awkward) + assert applyGeneAppearance(geneLayers(viewer), 2.0, 0.5, {}) == 3 + assert genes[0].gamma == pytest.approx(0.5) + + def test_the_base_layer_is_never_passed_in(self): + viewer, base, _genes = _section() + before = (list(base.contrast_limits), base.gamma) + applyGeneAppearance(geneLayers(viewer), 6.0, 0.2, {}) + assert (base.contrast_limits, base.gamma) == before + + +@pytest.fixture(scope="module") +def qt(): + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +@pytest.fixture +def built(qt): + """Keep the viewer: it owns the dock, and a collected one is deleted.""" + held = [] + + def build(): + viewer, base, genes = _section() + held.append(viewer) + panel = controls.addGeneAppearancePanel(viewer) + return viewer, base, genes, panel + + return build + + +def _spins(qt, box): + return box.findChildren(qt.QDoubleSpinBox) + + +class TestTheAppearancePanel: + def test_the_knobs_reach_the_gene_layers(self, qt, built): + _viewer, _base, genes, panel = built() + contrast, gamma = _spins(qt, panel) + contrast.setValue(2.0) + gamma.setValue(0.4) + assert genes[0].contrast_limits == [0.0, 50.0] + assert genes[0].gamma == pytest.approx(0.4) + + def test_they_do_not_reach_the_base_layer(self, qt, built): + _viewer, base, _genes, panel = built() + contrast, gamma = _spins(qt, panel) + contrast.setValue(5.0) + gamma.setValue(0.2) + assert base.contrast_limits == [0.0, 500.0] + assert base.gamma == pytest.approx(1.0) + + def test_the_slider_and_the_box_are_one_control(self, qt, built): + _viewer, _base, genes, panel = built() + contrast_spin, _gamma_spin = _spins(qt, panel) + contrast_slider = panel.findChildren(qt.QSlider)[0] + contrast_slider.setValue(250) + assert contrast_spin.value() == pytest.approx(2.5) + assert genes[0].contrast_limits == [0.0, 40.0] + + def test_reset_puts_everything_back(self, qt, built): + _viewer, _base, genes, panel = built() + contrast, gamma = _spins(qt, panel) + contrast.setValue(7.0) + gamma.setValue(0.3) + panel.findChild(qt.QPushButton).click() + assert genes[0].contrast_limits == [0.0, 100.0] + assert genes[0].gamma == pytest.approx(1.0) + + def test_a_gene_ticked_later_gets_the_settings_in_force(self, qt, built): + """Otherwise it arrives at its own brightness while the others sit + where the knobs put them, and the comparison is broken by the act + of adding to it.""" + viewer, _base, _genes, panel = built() + contrast, gamma = _spins(qt, panel) + contrast.setValue(4.0) + gamma.setValue(0.5) + later = viewer.layers.add(_Layer("gene: GAD1", (0.0, 200.0))) + assert later.contrast_limits == [0.0, 50.0] + assert later.gamma == pytest.approx(0.5) + + def test_an_untouched_panel_leaves_a_new_layer_alone(self, qt, built): + viewer, _base, _genes, _panel = built() + later = viewer.layers.add(_Layer("gene: GAD1", (0.0, 200.0))) + assert later.contrast_limits == [0.0, 200.0] + + def test_a_base_layer_added_later_is_still_not_governed(self, qt, built): + viewer, _base, _genes, panel = built() + _spins(qt, panel)[0].setValue(4.0) + overlay = viewer.layers.add(_Layer("cell centroids", (0.0, 9.0))) + assert overlay.contrast_limits == [0.0, 9.0] + + def test_a_removed_gene_forgets_its_reference(self, qt, built): + """So a gene ticked, unticked and ticked again is measured from + its new data rather than from what it looked like the first + time.""" + viewer, _base, genes, panel = built() + _spins(qt, panel)[0].setValue(2.0) + viewer.layers.drop(genes[0]) + again = viewer.layers.add(_Layer("gene: SST", (0.0, 60.0))) + assert again.contrast_limits == [0.0, 30.0] + + def test_it_says_so_when_there_is_nothing_to_govern(self, qt): + viewer = _Viewer(_Layer("All genes")) + panel = controls.addGeneAppearancePanel(viewer) + _spins(qt, panel)[0].setValue(3.0) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "No gene layers" in said + # The viewer owns the dock; a collected one takes the widget with + # it and every later read raises "has been deleted". + assert viewer.layers is not None + + +@pytest.fixture +def tourOf(qt): + held = [] + + def build(step_ms=50): + viewer, base, genes = _section() + held.append(viewer) + appearance = controls.addGeneAppearancePanel(viewer) + held.append(appearance) + panel = controls.addGeneTourPanel(viewer, appearance=appearance, interval_ms=step_ms) + return viewer, base, genes, appearance, panel + + return build + + +def _advance(panel, seconds, step_ms=50): + """Run the tour's own step function, without waiting for a timer.""" + for _ in range(int(round(seconds * 1000 / step_ms))): + panel._ndi_step() + + +class TestTheTour: + def test_it_visits_the_genes_in_order(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + assert panel._ndi_tour["order"] == [g.name for g in genes] + assert viewer.text_overlay.text == "SST" + _advance(panel, 5.0) + assert viewer.text_overlay.text == "PVALB" + _advance(panel, 5.0) + assert viewer.text_overlay.text == "VIP" + + def test_the_name_is_readable_and_bottom_right(self, qt, tourOf): + """A legend of six colours against a section this dense is a + legend nobody can read, and a demo is usually watched on someone + else's screen.""" + viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle() + assert viewer.text_overlay.visible + assert viewer.text_overlay.position == "bottom_right" + assert viewer.text_overlay.font_size >= 18 + + def test_it_pulses_once_a_second_five_times(self, qt, tourOf): + """Five seconds a gene, one sweep a second.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + seen = [] + for _ in range(100): # five seconds at 50ms + panel._ndi_step() + seen.append(genes[0].gamma) + bottoms = sum(1 for g in seen if g < 0.05) + assert bottoms == 5, f"expected five sweeps, saw {bottoms} bottoms" + + def test_it_takes_many_steps_rather_than_blinking(self, qt, tourOf): + """A symmetric triangle sampled 20 times a second has 11 distinct + magnitudes, so counting them understates the smoothness. What + makes it read as a sweep is that no single frame jumps far.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + seen = [] + for _ in range(100): + panel._ndi_step() + seen.append(genes[0].gamma) + jumps = [abs(b - a) for a, b in zip(seen, seen[1:])] + assert max(jumps) <= 0.15, f"largest frame-to-frame jump was {max(jumps)}" + assert len({round(g, 3) for g in seen}) >= 10 + + def test_only_the_current_gene_is_pulsed(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 0.5) + assert genes[0].gamma < 0.05 + assert genes[1].gamma == pytest.approx(1.0) + assert genes[2].gamma == pytest.approx(1.0) + + def test_the_base_layer_is_never_pulsed(self, qt, tourOf): + _viewer, base, _genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 12.0) + assert base.gamma == pytest.approx(1.0) + + def test_a_gene_is_restored_when_the_tour_leaves_it(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 5.0) + assert genes[0].gamma == pytest.approx(1.0) + + def test_stopping_halfway_puts_every_gamma_back(self, qt, tourOf): + """Closing mid-sweep must not leave a gene stuck at a gamma + nobody chose.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 0.5) + panel._ndi_toggle() + assert all(g.gamma == pytest.approx(1.0) for g in genes) + assert not panel._ndi_tour["on"] + + def test_finishing_ends_the_tour_and_clears_the_name(self, qt, tourOf): + viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 16.0) + assert not panel._ndi_tour["on"] + assert not viewer.text_overlay.visible + + def test_it_pulses_around_the_gamma_the_knob_set(self, qt, tourOf): + """The two controls have to agree: a tour that departed from 1.0 + would undo the gamma the reader chose, and put it back wrong.""" + _viewer, _base, genes, appearance, panel = tourOf() + _contrast, gamma = _spins(qt, appearance) + gamma.setValue(0.6) + panel._ndi_toggle() + assert panel._ndi_tour["base"] == pytest.approx(0.6) + _advance(panel, 1.0) + assert genes[0].gamma == pytest.approx(0.6) + + def test_a_gene_unticked_mid_tour_is_skipped(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 0.5) + viewer.layers.drop(genes[0]) + _advance(panel, 0.5) + assert viewer.text_overlay.text == "PVALB" + + def test_it_says_so_with_nothing_to_tour(self, qt): + viewer = _Viewer(_Layer("All genes")) + panel = controls.addGeneTourPanel(viewer) + panel._ndi_toggle() + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "No gene layers" in said + assert not panel._ndi_tour["on"] + + def test_the_timer_belongs_to_the_dock(self, qt, tourOf): + core = pytest.importorskip("qtpy.QtCore") + _viewer, _base, _genes, _app, panel = tourOf() + timers = panel.findChildren(core.QTimer) + assert timers and all(t.parent() is panel for t in timers) diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 3cb23872..b2081350 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -257,6 +257,8 @@ def test_the_chosen_labelings_reach_the_cell_type_panel(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", cells) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) controls.addAllPanels( object(), @@ -286,6 +288,8 @@ def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsy monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", genes) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -312,6 +316,8 @@ def _no_qt(): monkeypatch.setattr(controls, "addDisplayPanel", built) monkeypatch.setattr(controls, "addGenePanel", built) monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) @@ -333,6 +339,8 @@ def test_the_rotation_panel_gets_all_three_layers(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", rotation) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) image, points, shapes = object(), object(), object() controls.addAllPanels( @@ -360,6 +368,8 @@ def test_a_session_with_no_cells_still_gets_rotation(monkeypatch): monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) monkeypatch.setattr(controls, "addGenePanel", _Recorder()) monkeypatch.setattr(controls, "addRotationPanel", rotation) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) made = controls.addAllPanels(object(), object(), object(), object(), True) assert made["rotation"] is not None @@ -372,7 +382,14 @@ def test_the_cloud_panel_is_only_offered_when_there_is_a_token_to_expire(monkeyp implies the picture might be waiting on something. A local pyramid has no cloud token in its environment; a cloud one does.""" monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) - for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + for name in ( + "addDisplayPanel", + "addCellTypePanel", + "addGenePanel", + "addRotationPanel", + "addGeneAppearancePanel", + "addGeneTourPanel", + ): monkeypatch.setattr(controls, name, _Recorder()) cloud = _Recorder() monkeypatch.setattr(controls, "addCloudPanel", cloud) @@ -391,7 +408,14 @@ def test_a_failing_cloud_panel_costs_only_itself(monkeypatch, capsys): """It is the last panel and the least essential one -- a keyring that will not open must not cost the section.""" monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) - for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + for name in ( + "addDisplayPanel", + "addCellTypePanel", + "addGenePanel", + "addRotationPanel", + "addGeneAppearancePanel", + "addGeneTourPanel", + ): monkeypatch.setattr(controls, name, _Recorder()) monkeypatch.setattr(controls, "addCloudPanel", _Recorder(boom=True)) monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") @@ -400,3 +424,65 @@ def test_a_failing_cloud_panel_costs_only_itself(monkeypatch, capsys): assert made["cloud"] is None assert made["genes"] is not None assert "cloud" in capsys.readouterr().err + + +def test_the_appearance_and_tour_panels_are_built_after_the_gene_list(monkeypatch): + """Both act on what is ticked in it, and neither means anything + before something is.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + order = [] + + def _named(name): + def build(*a, **k): + order.append(name) + return object() + + return build + + for name in ("addDisplayPanel", "addCellTypePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _named(name)) + monkeypatch.setattr(controls, "addGenePanel", _named("genes")) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _named("appearance")) + monkeypatch.setattr(controls, "addGeneTourPanel", _named("tour")) + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert order.index("genes") < order.index("appearance") < order.index("tour") + assert made["appearance"] is not None and made["tour"] is not None + + +def test_the_tour_is_handed_the_appearance_panel(monkeypatch): + """So a pulse departs from the gamma the reader chose and returns to + it, rather than resetting their setting to 1.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _Recorder()) + appearance = object() + monkeypatch.setattr(controls, "addGeneAppearancePanel", lambda *a, **k: appearance) + tour = _Recorder() + monkeypatch.setattr(controls, "addGeneTourPanel", tour) + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + + controls.addAllPanels(object(), object(), object(), object(), True) + assert tour.calls, "the tour panel was never built" + _args, kwargs = tour.calls[0] + assert kwargs.get("appearance") is appearance + + +def test_a_failing_appearance_panel_still_leaves_a_tour(monkeypatch, capsys): + """The tour reads the appearance panel's gamma when it has one and + falls back to 1 when it does not -- so one failing must not cascade.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder(boom=True)) + tour = _Recorder() + monkeypatch.setattr(controls, "addGeneTourPanel", tour) + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert made["appearance"] is None + assert tour.calls, "a failing appearance panel took the tour with it" + _args, kwargs = tour.calls[0] + assert kwargs.get("appearance") is None + assert "gene appearance" in capsys.readouterr().err From 7f11972fe03948e3f7f967cbf2d2d11b1c43fb8e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 16:30:20 +0000 Subject: [PATCH 47/52] A second tour: one gene at a time, the others hidden The pulse tour leaves every gene on screen and sweeps one gene's gamma, so it answers "where is THIS one, among the others" -- the comparison is the point and the sweep is the pointer. That is not the only question worth asking. Six layers blending additively are a colour nobody can decompose by eye, and "what does this one look like by itself" needs the others out of the way. So: a second button, five seconds a gene, hiding the rest. Same clock, same canvas name overlay, same state machine -- the modes differ only in what they touch between gene boundaries, and solo touches nothing there at all: the switch happens once, when the tour arrives. THE BASE SECTION IS NOT HIDDEN. Only the gene layers are, which is the whole difference between showing one gene and showing one layer -- a lone gene on black has nothing to be read against, and the anatomy underneath is why the picture means anything. IT RESTORES WHAT WAS, not all-on. A reader who had already turned a gene off did that on purpose, and a tour is not a request to undo it. Stopping halfway restores too: five of six genes left hidden is a worse state than the tour started in. ONE AT A TIME. Starting either mode stops the other first, so the two can never both be moving the same layers -- and stopping first is what puts back the gamma the pulse was mid-sweep on, which would otherwise stay stranded for the whole of the next tour. 17 more tests. Two deliberate regressions confirm they bite: restoring all-on instead of what was, and switching modes without stopping. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 135 ++++++++++++++++---- tests/test_gui_genepyramid_appearance.py | 155 ++++++++++++++++++++++- 2 files changed, 263 insertions(+), 27 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index ff805a8c..80ec5eb2 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -38,6 +38,7 @@ "contrastWindow", "pulseGamma", "applyGeneAppearance", + "applySolo", "addGeneAppearancePanel", "addGeneTourPanel", "blurLevels", @@ -1755,6 +1756,33 @@ def saveCloudProfile(email: str, password: str) -> None: profile.set_default(uid) +def applySolo(layers, current) -> int: + """Show one gene layer and hide the rest. + + *current* is the layer NAME to show, or None to show them all -- + which is what restoring looks like when nothing was hidden to begin + with. + + The base section and the cell overlays are not in *layers* and so + are never hidden. That is the point of showing one gene at a time + rather than one layer at a time: the anatomy has to stay under it, + or a lone gene floats on black and there is nothing to read it + against. + + Returns: + How many layers were changed. One that refuses is skipped rather + than allowed to stop the rest. + """ + done = 0 + for layer in layers: + try: + layer.visible = True if current is None else str(layer.name) == str(current) + except Exception: # noqa: BLE001 - see docstring + continue + done += 1 + return done + + def addGeneAppearancePanel(viewer) -> Any: """One contrast and one gamma across every gene layer at once. @@ -1923,22 +1951,37 @@ def _onRemoved(event=None): def addGeneTourPanel(viewer, appearance=None, interval_ms: int = 50) -> Any: - """One button that walks the gene layers, pulsing each in turn. + """Two ways of walking the gene layers, five seconds each. Showing someone six genes means saying which is which, and a legend of six colours against a section this dense is a legend nobody can - read. So the picture says it instead: one gene at a time is pulsed - -- its gamma swept down and back, once a second for five seconds -- - while its name stands in the corner of the canvas. + read. So the picture says it instead, one gene at a time, with its + name standing in the corner of the canvas. + + TWO MODES, because they answer different questions. + + PULSE leaves every gene on screen and sweeps one gene's gamma down + and back, once a second. Everything stays in view, so it answers + "where is THIS one, among the others" -- the comparison is the + point, and the sweep is the pointer. - NOTHING BLOCKS. The sweep is a timer stepping a small state machine, + SOLO hides the others outright. It answers "what does this one look + like, by itself", which additive blending otherwise makes hard: six + layers over each other are a colour nobody can decompose by eye. + The base section is NOT hidden, or a lone gene would float on black + with nothing to read it against. + + NOTHING BLOCKS. Both are one timer stepping a small state machine, not a loop with sleeps in it, so the viewer stays live throughout: a tour can be stopped mid-gene, and panning while it runs works. - WHAT IT TOUCHES IT PUTS BACK. Each layer's gamma is recorded when the - tour reaches it and restored when it leaves, so stopping halfway -- - or closing the window mid-sweep -- cannot leave a gene stuck at a - gamma nobody chose. + WHAT IT TOUCHES IT PUTS BACK. Gammas for the pulse, visibility for + the solo -- recorded when the tour starts and restored when it ends, + however it ends. Stopping halfway must not leave a gene stranded at + a gamma nobody chose, or leave five of six genes hidden. + + ONE AT A TIME. Starting either mode stops the other first, so the + two can never both be moving the same layers. Args: appearance: the gene-appearance panel, if there is one. Its @@ -1956,12 +1999,23 @@ def addGeneTourPanel(viewer, appearance=None, interval_ms: int = 50) -> Any: box = QWidget() outer = QVBoxLayout(box) - play = QPushButton("Play gene tour") + play = QPushButton("Pulse each gene") play.setToolTip( - "Pulses each gene layer in turn for five seconds, naming it on\n" - "the canvas. Click again to stop; every gamma is put back." + "Leaves every gene on screen and sweeps one gene's gamma down and\n" + "back, once a second for five seconds, naming it on the canvas.\n" + 'Answers "where is this one, among the others". Click again to\n' + "stop; every gamma is put back." + ) + solo = QPushButton("Show one gene at a time") + solo.setToolTip( + "Hides the other genes and shows one for five seconds, naming it\n" + 'on the canvas. Answers "what does this one look like by\n' + 'itself", which additive blending otherwise makes hard. The\n' + "section underneath stays visible. Click again to stop; every\n" + "layer is shown again." ) outer.addWidget(play) + outer.addWidget(solo) status = QLabel("") status.setWordWrap(True) @@ -1974,7 +2028,7 @@ def addGeneTourPanel(viewer, appearance=None, interval_ms: int = 50) -> Any: # boundary would fall a frame early or late depending on the # interval, which is the kind of bug that only shows up on somebody # else's machine. Multiplying an integer is exact. - tour = {"on": False, "order": [], "i": 0, "n": 0, "base": 1.0} + tour = {"on": False, "mode": "", "order": [], "i": 0, "n": 0, "base": 1.0, "shown": {}} steps_per_gene = max(1, int(round(seconds_per_gene * 1000 / interval_ms))) def _baseGamma() -> float: @@ -2018,17 +2072,29 @@ def _restore(name): pass def _stop(message=""): - if tour["order"] and 0 <= tour["i"] < len(tour["order"]): - _restore(tour["order"][tour["i"]]) # Every gene, not only the one in hand: an earlier one whose # restore was missed because its layer was briefly gone would # otherwise stay where the sweep left it. - for name in tour["order"]: - _restore(name) - tour.update(on=False, order=[], i=0, n=0) + if tour["mode"] == "pulse": + for name in tour["order"]: + _restore(name) + elif tour["mode"] == "solo": + # Back to what each layer's visibility WAS, not to all-on: a + # gene the reader had already hidden by hand should stay + # hidden, and a tour is not a request to change that. + for name, was in tour["shown"].items(): + layer = _layer(name) + if layer is None: + continue + try: + layer.visible = bool(was) + except Exception: # noqa: BLE001 - a control never costs the picture + pass + tour.update(on=False, mode="", order=[], i=0, n=0, shown={}) timer.stop() _overlay("", visible=False) - play.setText("Play gene tour") + play.setText("Pulse each gene") + solo.setText("Show one gene at a time") status.setText(message) def _step(): @@ -2047,9 +2113,13 @@ def _step(): layer = _layer(name) if layer is None: # Unticked while the tour was on it. Move along rather than - # spending five seconds pulsing something that is not there. + # spending five seconds on something that is not there. tour["n"] = steps_per_gene - 1 return + if tour["mode"] != "pulse": + # Solo changes nothing between gene boundaries -- the switch + # happens once, in _announce. The clock still has to run. + return try: elapsed = tour["n"] * interval_ms / 1000.0 layer.gamma = pulseGamma(tour["base"], elapsed, pulse_period) @@ -2058,6 +2128,8 @@ def _step(): def _announce(): name = tour["order"][tour["i"]] + if tour["mode"] == "solo": + applySolo(geneLayers(viewer), name) _overlay(geneName(_layer(name)) if _layer(name) is not None else geneName(name)) status.setText(f"{tour['i'] + 1} of {len(tour['order'])}") @@ -2065,20 +2137,31 @@ def _announce(): timer.setInterval(interval_ms) timer.timeout.connect(_step) - def _toggle(): + def _toggle(mode="pulse"): + # Clicking the running mode's button stops it; clicking the other + # one switches, which means stopping first so the layers it was + # moving are put back before the next mode records them. + was = tour["mode"] if tour["on"]: _stop("Stopped.") - return - order = [str(layer.name) for layer in geneLayers(viewer)] + if was == mode: + return + layers = geneLayers(viewer) + order = [str(layer.name) for layer in layers] if not order: status.setText("No gene layers to tour -- tick some genes first.") return - tour.update(on=True, order=order, i=0, n=0, base=_baseGamma()) - play.setText("Stop") + shown = {} + if mode == "solo": + for layer in layers: + shown[str(layer.name)] = bool(getattr(layer, "visible", True)) + tour.update(on=True, mode=mode, order=order, i=0, n=0, base=_baseGamma(), shown=shown) + (play if mode == "pulse" else solo).setText("Stop") _announce() timer.start() - play.clicked.connect(_toggle) + play.clicked.connect(lambda *_: _toggle("pulse")) + solo.clicked.connect(lambda *_: _toggle("solo")) row = QHBoxLayout() row.addStretch() diff --git a/tests/test_gui_genepyramid_appearance.py b/tests/test_gui_genepyramid_appearance.py index 471ac899..55e6edc8 100644 --- a/tests/test_gui_genepyramid_appearance.py +++ b/tests/test_gui_genepyramid_appearance.py @@ -22,6 +22,7 @@ from ndi.gui.app.genepyramid import controls from ndi.gui.app.genepyramid.controls import ( applyGeneAppearance, + applySolo, contrastWindow, geneLayers, geneName, @@ -32,10 +33,11 @@ class _Layer: - def __init__(self, name, limits=(0.0, 100.0), gamma=1.0): + def __init__(self, name, limits=(0.0, 100.0), gamma=1.0, visible=True): self.name = name self.contrast_limits = list(limits) self.gamma = gamma + self.visible = visible class _Event: @@ -498,3 +500,154 @@ def test_the_timer_belongs_to_the_dock(self, qt, tourOf): _viewer, _base, _genes, _app, panel = tourOf() timers = panel.findChildren(core.QTimer) assert timers and all(t.parent() is panel for t in timers) + + +class TestSoloing: + def test_it_shows_one_and_hides_the_rest(self): + viewer, _base, genes = _section() + assert applySolo(geneLayers(viewer), "gene: PVALB") == 3 + assert [g.visible for g in genes] == [False, True, False] + + def test_none_shows_them_all_again(self): + viewer, _base, genes = _section() + applySolo(geneLayers(viewer), "gene: PVALB") + applySolo(geneLayers(viewer), None) + assert all(g.visible for g in genes) + + def test_the_base_layer_is_never_hidden(self): + """A lone gene on black has nothing to be read against -- the + anatomy underneath is the whole reason the picture means + something.""" + viewer, base, _genes = _section() + applySolo(geneLayers(viewer), "gene: SST") + assert base.visible + + def test_a_layer_that_refuses_does_not_stop_the_others(self): + class _Awkward(_Layer): + _armed = False + + def __setattr__(self, name, value): + if name == "visible" and self._armed: + raise RuntimeError("no") + object.__setattr__(self, name, value) + + viewer, _base, genes = _section() + awkward = _Awkward("gene: BAD") + awkward._armed = True + viewer.layers.append(awkward) + assert applySolo(geneLayers(viewer), "gene: SST") == 3 + assert genes[0].visible and not genes[1].visible + + +class TestTheSoloTour: + def test_it_shows_the_genes_one_at_a_time(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + assert [g.visible for g in genes] == [True, False, False] + _advance(panel, 5.0) + assert [g.visible for g in genes] == [False, True, False] + assert viewer.text_overlay.text == "PVALB" + _advance(panel, 5.0) + assert [g.visible for g in genes] == [False, False, True] + + def test_the_base_layer_stays_up_throughout(self, qt, tourOf): + _viewer, base, _genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + for _ in range(3): + assert base.visible + _advance(panel, 5.0) + assert base.visible + + def test_it_leaves_the_gammas_alone(self, qt, tourOf): + """This mode answers a different question, and moving both at + once would make it impossible to say which did what.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 2.5) + assert all(g.gamma == pytest.approx(1.0) for g in genes) + + def test_finishing_shows_every_gene_again(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 16.0) + assert all(g.visible for g in genes) + assert not panel._ndi_tour["on"] + assert not viewer.text_overlay.visible + + def test_stopping_halfway_shows_every_gene_again(self, qt, tourOf): + """Five of six genes left hidden is a worse state than the tour + started in.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 7.0) + panel._ndi_toggle("solo") + assert all(g.visible for g in genes) + + def test_a_gene_hidden_by_hand_stays_hidden_afterwards(self, qt, tourOf): + """It restores what WAS, not all-on. A reader who turned a gene + off did that on purpose, and a tour is not a request to undo + it.""" + _viewer, _base, genes, _app, panel = tourOf() + genes[2].visible = False + panel._ndi_toggle("solo") + _advance(panel, 16.0) + assert genes[0].visible and genes[1].visible + assert not genes[2].visible + + def test_a_gene_unticked_mid_tour_is_skipped(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 0.5) + viewer.layers.drop(genes[0]) + _advance(panel, 0.5) + assert viewer.text_overlay.text == "PVALB" + + def test_it_says_so_with_nothing_to_tour(self, qt): + viewer = _Viewer(_Layer("All genes")) + panel = controls.addGeneTourPanel(viewer) + panel._ndi_toggle("solo") + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "No gene layers" in said + assert not panel._ndi_tour["on"] + + +class TestTheTwoModesDoNotOverlap: + def test_starting_solo_stops_a_running_pulse_and_restores_it(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("pulse") + _advance(panel, 0.5) + assert genes[0].gamma < 0.05 + panel._ndi_toggle("solo") + assert panel._ndi_tour["mode"] == "solo" + # The gamma the pulse was mid-sweep on has to be put back before + # the other mode takes over, or it stays stranded all tour. + assert genes[0].gamma == pytest.approx(1.0) + + def test_starting_pulse_stops_a_running_solo_and_unhides(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + assert not genes[1].visible + panel._ndi_toggle("pulse") + assert panel._ndi_tour["mode"] == "pulse" + assert all(g.visible for g in genes) + + def test_clicking_the_running_mode_again_stops_it(self, qt, tourOf): + _viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + panel._ndi_toggle("solo") + assert not panel._ndi_tour["on"] + assert panel._ndi_tour["mode"] == "" + + def test_each_mode_has_its_own_button(self, qt, tourOf): + _viewer, _base, _genes, _app, panel = tourOf() + labels = [b.text() for b in panel.findChildren(qt.QPushButton)] + assert len(labels) == 2 + assert any("Pulse" in t for t in labels) + assert any("one gene at a time" in t for t in labels) + + def test_the_running_mode_s_button_says_stop(self, qt, tourOf): + _viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + labels = [b.text() for b in panel.findChildren(qt.QPushButton)] + assert "Stop" in labels + assert any("Pulse" in t for t in labels) From 639936bf663c4795a544f80e41ed6e640ca22b45 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 16:33:29 +0000 Subject: [PATCH 48/52] Assign gene colours in rainbow order Was magenta, green, cyan, yellow, red, blue -- an order with nothing behind it. Genes are coloured in the order they are ticked, so this tuple's order is the order a reader watches them arrive, and a spectral run is one they can hold in their head and read back off the picture where an arbitrary one is six colours to memorise. Now red, yellow, green, cyan, blue, magenta: long wavelength to short. Magenta is not spectral at all -- it closes the circle, which is where the eye expects it after blue. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 9 ++++++++- tests/test_gui_genepyramid_appearance.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 80ec5eb2..b2eee0fa 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -179,7 +179,14 @@ def abundanceBand(totals, lo: float = 0.0, hi: float = 100.0): # Distinct colours rather than one map applied to everything: the layers # blend additively, so two genes in the same colormap make one picture # that neither of them is. -_GENE_COLORMAPS = ("magenta", "green", "cyan", "yellow", "red", "blue") +# +# IN RAINBOW ORDER, long wavelength to short: red, yellow, green, cyan, +# blue, then magenta. Genes are assigned in the order they are ticked, so +# the order of this tuple is the order a reader sees them arrive -- and a +# spectral run is one they can hold in their head and read back off the +# picture. Magenta is not spectral at all; it closes the circle, which is +# where the eye expects it after blue. +_GENE_COLORMAPS = ("red", "yellow", "green", "cyan", "blue", "magenta") # --------------------------------------------------- gene layer appearance diff --git a/tests/test_gui_genepyramid_appearance.py b/tests/test_gui_genepyramid_appearance.py index 55e6edc8..c7b8d9e1 100644 --- a/tests/test_gui_genepyramid_appearance.py +++ b/tests/test_gui_genepyramid_appearance.py @@ -651,3 +651,24 @@ def test_the_running_mode_s_button_says_stop(self, qt, tourOf): labels = [b.text() for b in panel.findChildren(qt.QPushButton)] assert "Stop" in labels assert any("Pulse" in t for t in labels) + + +class TestTheColourCycle: + def test_it_runs_in_rainbow_order(self): + """Genes are assigned colours in the order they are ticked, so + this tuple's order is the order a reader sees them arrive. A + spectral run is one they can hold in their head and read back off + the picture; an arbitrary one is six colours to memorise.""" + assert controls._GENE_COLORMAPS == ( + "red", + "yellow", + "green", + "cyan", + "blue", + "magenta", + ) + + def test_every_colour_is_distinct(self): + """The layers blend additively, so two genes in the same colormap + make one picture that neither of them is.""" + assert len(set(controls._GENE_COLORMAPS)) == len(controls._GENE_COLORMAPS) From 147890692e028a6ff5090e772222961beabba663 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 17:03:05 +0000 Subject: [PATCH 49/52] Gene layers on the command line, a sign-in behind a button, and a shorter stack FOUR THINGS, all in the viewer. --gene-layers. A new option that opens named genes as their OWN additive layers on top of the base image, each in a named colormap: "HPCAL1:green,RORB:blue,FEZF2:bop orange". A DIFFERENT REQUEST from --genes, which filters the base image down to a subset and leaves one picture -- this keeps "All genes" and adds a layer beside it, which is what makes several genes comparable. Both can be given together. A colour is optional and means "take the next one from the cycle", so a caller can name the ones that matter. A named colour does NOT advance that cycle: otherwise the first hand-ticked gene's colour would depend on how many were preset, a surprise with no upside. Colormap names may contain spaces -- napari ships "bop orange" -- so only the ends of each field are trimmed. Genes are ticked through the panel's own checkbox rather than added behind its back, or unticking a preset gene would do nothing at all; one not in this pyramid's list is named on the panel, because a demo that quietly opens four of six layers looks like the viewer working. NDI-matlab's Ferret demo preset builds this string. THE CLOUD SIGN-IN IS BEHIND A BUTTON. One row now: the button and the time left. Three permanently docked fields said a reader signs in often; they do it once a day at most. The dialog offers the SAVED PROFILES first -- someone whose profile works should not retype a password to get past an expired token, and someone whose profile does not work can see which one is being used, which is half the diagnosis. Picking one fills the email; picking one with a stored password signs in with it; a typed password still wins, which is the point of arriving there. Built and shown are split, so a test can drive the fields and the Sign in button with no event loop to escape from. That split is not theoretical: the old docked-fields tests clicked a button that now opens a MODAL dialog, and the suite hung until it was killed. CONTRAST AND GAMMA ARE SLIDERS ONLY. They are dragged, not typed, and two spin boxes were most of two panels' height. The value is still shown beside each slider, so a setting worth writing down can be read off. THE GENE LIST IS LAST, so it sits at the bottom. It is the tallest panel by a long way and napari gives leftover height to whatever is furthest down, so anything below it gets squeezed while the list takes the room. The appearance and tour panels read above it, which is the order they are used in; the cloud row moves up with the other whole-session controls, since it is about the connection rather than the picture. THE BLUR BOX WAITS 1.5 SECONDS. A spin box emits on every keystroke, so typing "50" asked for 5 um first -- and re-blurring is a whole-ladder map_overlap that takes long enough to see, so the picture visibly redrew at the wrong width before the second digit landed. Enter or clicking away fires at once rather than sitting out the wait. The band slider keeps its 180ms: dragging a slider wants to be followed, while a number being typed is not a request until it is finished. Also fixed while testing: the "not in this pyramid" warning was set before _report(), which rewrites the same label -- so the warning was one nobody could see. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/cli.py | 26 ++ src/ndi/gui/app/genepyramid/controls.py | 474 ++++++++++++++++------ src/ndi/gui/app/genepyramid/viewer.py | 8 + src/ndi/gui/ndi_matlab_python_bridge.yaml | 2 +- tests/test_gui_genepyramid_appearance.py | 56 +-- tests/test_gui_genepyramid_cloudpanel.py | 180 +------- tests/test_gui_genepyramid_controls.py | 43 +- tests/test_gui_genepyramid_genelayers.py | 344 ++++++++++++++++ tests/test_gui_genepyramid_genepanel.py | 183 +++++++++ 9 files changed, 998 insertions(+), 318 deletions(-) create mode 100644 tests/test_gui_genepyramid_genelayers.py diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index 1d0f0b71..0f395c49 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -42,6 +42,18 @@ def build_parser() -> argparse.ArgumentParser: metavar="A,B,C", help="comma list of gene symbols or accessions; default is every gene", ) + p.add_argument( + "--gene-layers", + default="", + metavar="A:red,B:cyan", + help="open these genes as their own additive layers on top of the " + "base image, optionally each in a named colormap. Different from " + "--genes, which filters the base layer to a subset and leaves one " + "picture: this keeps the base and adds a layer per gene, which is " + "what makes two genes comparable. A gene with no colour takes the " + "next one from the cycle. Colormap names may contain spaces " + "(napari ships 'bop orange'), so quote the whole value.", + ) p.add_argument( "--no-density", action="store_true", @@ -329,6 +341,19 @@ def _resolve_genes(session, pyr_doc, spec: str): return sorted(set(rows)) +def _resolve_gene_layers(spec: str): + """``--gene-layers`` to the pairs the gene panel opens with. + + Parsed here rather than in the panel so a malformed value is a + startup error rather than a panel that silently opens fewer layers + than were asked for. Whether the SYMBOLS exist is the panel's to + answer -- it holds the gene list, and it says so on the panel. + """ + from .controls import parseGeneLayers + + return parseGeneLayers(spec) or None + + def main(argv=None) -> int: args = build_parser().parse_args(argv) @@ -440,6 +465,7 @@ def main(argv=None) -> int: controls=not args.no_controls, name=args.name or None, labelings=_resolve_labelings(args.labels), + gene_layers=_resolve_gene_layers(args.gene_layers), ) return 0 diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index b2eee0fa..be3746bd 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -22,6 +22,7 @@ __all__ = [ "geneIndex", + "parseGeneLayers", "abundanceBand", "countsOrder", "labelAgreement", @@ -31,6 +32,9 @@ "addCellTypePanel", "addAllPanels", "addCloudPanel", + "cloudSignInDialog", + "buildCloudSignInDialog", + "cloudProfileChoices", "cloudSessionLooksLikely", "saveCloudProfile", "geneLayers", @@ -49,6 +53,49 @@ # --------------------------------------------------------------- genes +def parseGeneLayers(spec: str): + """``"HPCAL1:green,RORB:blue,SST"`` to ``[(symbol, colormap|None), ...]``. + + One layer per gene, each in its own colour, which is a different + request from ``--genes``: that one FILTERS the base layer to a + subset, leaving one picture. This adds a layer per gene on top of + the base, which is what makes two genes comparable. + + THE COLOUR IS OPTIONAL and None means "take the next one from the + cycle", so a caller can name the colours that matter and leave the + rest. Colormap names may contain spaces -- napari ships ``bop + orange``, ``bop blue`` and ``bop purple`` -- so only the ends of each + field are trimmed, never the middle. + + A gene named twice is kept once, at its first position: ticking the + same box twice adds nothing the second time, and the layer names + would collide. + + Args: + spec: comma-separated ``SYMBOL`` or ``SYMBOL:COLORMAP`` entries. + Empty gives an empty list rather than None -- "no layers + asked for" is a fact, not a missing answer. + + Returns: + A list of ``(symbol, colormap or None)``, in the order given, + which is the order the layers are added and so the order the + colour cycle advances. + """ + out, seen = [], set() + for entry in str(spec or "").split(","): + entry = entry.strip() + if not entry: + continue + symbol, sep, colour = entry.partition(":") + symbol = symbol.strip() + if not symbol or symbol in seen: + continue + seen.add(symbol) + colour = colour.strip() if sep else "" + out.append((symbol, colour or None)) + return out + + def geneIndex(ids, names) -> dict: """Collapse a gene list to ONE ENTRY PER SYMBOL. @@ -334,7 +381,7 @@ def countsOrder(totals, ascending: bool = False): return np.argsort(t if ascending else -t, kind="stable") -def addGenePanel(viewer, session, pyr_doc) -> Any: +def addGenePanel(viewer, session, pyr_doc, initial=None) -> Any: """A filterable list of gene symbols, one checkbox each. Ticking a symbol ADDS ITS OWN LAYER rather than replacing the base @@ -693,7 +740,7 @@ def _blurred(data): return data return blurLevels(data, _binSizes(), state["sigma"] / basePixelUm(pyr_doc)) - def _add(symbol: str): + def _add(symbol: str, colour: str | None = None): name = f"gene: {symbol}" if name in viewer.layers: return @@ -704,8 +751,16 @@ def _add(symbol: str): # already blurred, which would compound and could not be undone. state["raw"][name] = spec["data"] spec["data"] = _blurred(spec["data"]) - cmap = _GENE_COLORMAPS[state["colour"] % len(_GENE_COLORMAPS)] - state["colour"] += 1 + # A NAMED COLOUR DOES NOT ADVANCE THE CYCLE. The cycle exists so + # that genes ticked by hand come out different from each other; + # a caller that named its own colours has already solved that, + # and letting those consume cycle positions would make the first + # hand-ticked gene's colour depend on how many were preset. + if colour: + cmap = colour + else: + cmap = _GENE_COLORMAPS[state["colour"] % len(_GENE_COLORMAPS)] + state["colour"] += 1 try: layer = viewer.add_image(**spec, colormap=cmap, blending="additive") except Exception: @@ -742,7 +797,25 @@ def _reblur(_value=None): return _report() - blur.valueChanged.connect(_reblur) + # DEBOUNCED, and generously. A spin box emits on every keystroke, so + # typing "50" asks for a blur at 5 um first -- and re-blurring is a + # whole-ladder map_overlap that takes long enough to see, so the + # picture visibly redraws at the wrong width before the second digit + # lands. 1.5s is past the gap between two digits and short enough not + # to feel stuck. + # + # The band slider above uses the same trick at 180ms; the difference + # is that dragging a slider WANTS to be followed, while a number + # being typed is not a request until it is finished. + blurWait = QTimer(box) + blurWait.setSingleShot(True) + blurWait.setInterval(1500) + blurWait.timeout.connect(_reblur) + blur.valueChanged.connect(lambda *_: blurWait.start()) + # Enter, or clicking away, means the number IS finished -- no reason + # to sit out the rest of the wait. + blur.editingFinished.connect(lambda *_: (blurWait.stop(), _reblur())) + box._ndi_blur_wait = blurWait def _on_item_changed(item): if state["busy"]: @@ -796,7 +869,33 @@ def _on_layer_removed(event): _resort() _applyBand() + + # ---- genes asked for at launch ------------------------------------ + # Ticked through the panel's OWN checkbox, not by calling _add + # directly: the box and the layer would otherwise disagree, and + # unticking a gene that was never ticked does nothing at all. This + # way a preset gene behaves exactly like one the reader ticked. + missing = [] + for symbol, colour in parseGeneLayers("") if initial is None else list(initial): + item = items.get(symbol) + if item is None: + missing.append(symbol) + continue + was, state["busy"] = state["busy"], True + try: + item.setCheckState(Qt.Checked) + finally: + state["busy"] = was + _add(symbol, colour) _report() + if missing: + # Named but not in this pyramid's gene list. Said out loud rather + # than passed over: a demo that quietly opens four of six layers + # looks like the viewer working. + # + # AFTER _report, not before: _report rewrites this same label, so + # the warning set first is the warning nobody sees. + status.setText(f"not in this pyramid: {', '.join(missing)}") viewer.window.add_dock_widget(box, name="Genes", area="right") return box @@ -1601,8 +1700,46 @@ def cloudSessionLooksLikely() -> bool: return bool(os.environ.get("NDI_CLOUD_TOKEN")) +def cloudProfileChoices(): + """Saved profiles, as ``(label, email, uid, has_password)`` rows. + + The label is what a chooser shows: the nickname when there is one, + else the email, with the email appended either way so two profiles + for the same person on different stages can be told apart. + + Never raises. A secrets backend that will not open, a profile file + that will not parse -- none of that is a reason a sign-in dialog + should fail to appear, since typing the credentials by hand is + exactly the fallback it exists to offer. + """ + rows = [] + try: + from ....cloud import profile + + entries = profile.list_profiles() + except Exception: # noqa: BLE001 - see docstring + return rows + for entry in entries: + uid = str(getattr(entry, "UID", "") or "") + email = str(getattr(entry, "Email", "") or "") + nickname = str(getattr(entry, "Nickname", "") or "") + stage = str(getattr(entry, "Stage", "") or "") + label = nickname or email or uid + if email and label != email: + label = f"{label} ({email})" + if stage and stage != "prod": + label = f"{label} [{stage}]" + stored = False + try: + stored = bool(profile.get_password(uid)) + except Exception: # noqa: BLE001 - an unreadable secret is not an error here + stored = False + rows.append((label, email, uid, stored)) + return rows + + def addCloudPanel(viewer) -> Any: - """A clock on the cloud token, and a way to renew it in place. + """A clock on the cloud token, and a button that renews it. A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for hours; the token is good for rather less, and it cannot renew itself @@ -1614,52 +1751,145 @@ def addCloudPanel(viewer) -> Any: opening the section again, since the token lives in this process's environment and nothing outside it can reach in. - So: the time left is shown and kept current, and the credentials that - renew it can be typed HERE, into the window that needs them. Signing - in writes the new token into the environment, which is where every - later fetch reads it from -- including the ones on the tile threads, - since a fetch builds its client per call rather than holding one. + ONE ROW, because that is all it is worth until it matters: a button + and the time left. The credentials live behind the button, in a + dialog, rather than as three permanently docked fields for something + a reader does once a day at most. + """ + from qtpy.QtCore import QTimer + from qtpy.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget - THE PASSWORD IS NOT KEPT. It goes to the login call and nothing else: - not to a field that stays filled, not to a file, not to the profile - unless the reader asks for that explicitly with the checkbox, which - exists because credentials saved once and wrong are exactly how - someone ends up here. + from ....cloud import auth + + box = QWidget() + outer = QVBoxLayout(box) + + signin = QPushButton("Sign in...") + signin.setToolTip( + "Sign in to NDI Cloud, so tiles that are not already downloaded\n" + "keep loading. The token lives in this process, so signing in\n" + "anywhere else does not reach this window." + ) + clock = QLabel("") + clock.setWordWrap(True) + row = QHBoxLayout() + row.addWidget(signin) + row.addWidget(clock, 1) + outer.addLayout(row) + + result = QLabel("") + result.setWordWrap(True) + result.hide() + outer.addWidget(result) + outer.addStretch() + + def _tick(): + left = auth.tokenSecondsRemaining() + line = auth.tokenStatusLine() + # Warned about BEFORE it bites: a reader who sees "12 min left" + # can sign in at a convenient moment instead of discovering it + # through a tile that will not load. + if left is not None and 0 < left < 900: + clock.setText(f"{line} -- renew before it runs out.") + elif left is not None and left <= 0: + clock.setText(f"{line}. Undownloaded tiles will fail until you sign in.") + else: + clock.setText(line) + + def _open(): + ok, note = cloudSignInDialog(box) + if note: + result.setText(note) + result.show() + elif ok: + result.hide() + _tick() + + signin.clicked.connect(_open) + + # Parented to the widget, so it stops when the dock goes. A free timer + # would keep firing at a deleted label and take the process with it. + timer = QTimer(box) + timer.setInterval(30_000) + timer.timeout.connect(_tick) + timer.start() + _tick() + + box._ndi_signin = _open + viewer.window.add_dock_widget(box, name="NDI Cloud", area="right") + return box + + +def cloudSignInDialog(parent=None): + """Show the sign-in dialog and sign in. Returns ``(signed_in, note)``. + + The building is :func:`buildCloudSignInDialog`; this is the two + lines around it that block. Split so a test can drive the dialog's + fields and its Sign in button without an event loop to escape from. """ - import os + dialog, outcome = buildCloudSignInDialog(parent) + runner = getattr(dialog, "exec", None) or getattr(dialog, "exec_", None) + runner() + return outcome["ok"], outcome["note"] - from qtpy.QtCore import QTimer + +def buildCloudSignInDialog(parent=None): + """The sign-in dialog, built but not shown. ``(dialog, outcome)``. + + Behind a button rather than docked, because it is a thing a reader + does once a day at most and three permanent fields would say + otherwise. + + THE SAVED PROFILES ARE OFFERED FIRST. Someone whose profile works + should not be retyping a password to get past an expired token, and + someone whose profile does NOT work -- which is how most people + arrive here -- can see which one is being used, which is half the + diagnosis. Picking one fills the email in; picking one that has a + stored password signs in with it directly. + + THE PASSWORD IS NOT KEPT. It goes to the login call and nothing else: + not to a field that stays filled, not to a file, and to the profile + only when the checkbox asks -- which exists because credentials + saved once and wrong are exactly how someone ends up in this dialog. + """ from qtpy.QtWidgets import ( QCheckBox, + QComboBox, + QDialog, + QDialogButtonBox, QFormLayout, QLabel, QLineEdit, - QPushButton, QVBoxLayout, - QWidget, ) from ....cloud import auth - box = QWidget() - outer = QVBoxLayout(box) + dialog = QDialog(parent) + dialog.setWindowTitle("Sign in to NDI Cloud") + outer = QVBoxLayout(dialog) - clock = QLabel("") - clock.setWordWrap(True) - outer.addWidget(clock) + why = QLabel( + "The token lives in this process, so signing in elsewhere does not " "reach this window." + ) + why.setWordWrap(True) + outer.addWidget(why) + + choices = cloudProfileChoices() + chooser = QComboBox() + chooser.addItem("type them below", None) + for label, email, uid, stored in choices: + chooser.addItem(f"{label}{' -- password saved' if stored else ''}", (email, uid, stored)) email = QLineEdit() email.setPlaceholderText("you@example.com") - # Prefilled from the environment only. The profile is deliberately NOT - # read for this: someone opening this panel is usually here because - # what is saved did not work, and offering it back as the answer is - # the least useful thing to show them. - email.setText(os.environ.get("NDI_CLOUD_USERNAME", "")) password = QLineEdit() password.setEchoMode(QLineEdit.Password) password.setPlaceholderText("password") form = QFormLayout() + if choices: + form.addRow("profile", chooser) form.addRow("email", email) form.addRow("password", password) outer.addLayout(form) @@ -1672,52 +1902,57 @@ def addCloudPanel(viewer) -> Any: ) outer.addWidget(save) - signin = QPushButton("Sign in") - signin.setDefault(True) - outer.addWidget(signin) + said = QLabel("") + said.setWordWrap(True) + said.hide() + outer.addWidget(said) - result = QLabel("") - result.setWordWrap(True) - result.hide() - outer.addWidget(result) - outer.addStretch() + buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + buttons.button(QDialogButtonBox.Ok).setText("Sign in") + outer.addWidget(buttons) - def _tick(): - left = auth.tokenSecondsRemaining() - line = auth.tokenStatusLine() - # Said in the panel's own terms rather than the library's, and - # warned about BEFORE it bites: a reader who sees "12 min left" - # can sign in at a convenient moment instead of discovering it - # through a tile that will not load. - if left is not None and 0 < left < 900: - clock.setText(f"NDI Cloud: {line} -- sign in again before it runs out.") - elif left is not None and left <= 0: - clock.setText( - f"NDI Cloud: {line}. Tiles that are not already downloaded " - f"will fail to load until you sign in." - ) - else: - clock.setText(f"NDI Cloud: {line}") + def _chose(index): + data = chooser.itemData(index) + if not data: + return + picked_email, _uid, stored = data + email.setText(picked_email) + # A profile whose password is saved needs nothing typed. + password.clear() + if not stored: + # A profile without a saved password needs the password and + # not the email, so the focus goes where the work is. + password.setFocus() - def _signin(): + chooser.currentIndexChanged.connect(_chose) + + outcome = {"ok": False, "note": ""} + + def _accept(): who, secret = email.text().strip(), password.text() + if not secret: + # An empty password with a profile picked means "use the one + # you have", which is the whole reason to pick a profile. + data = chooser.itemData(chooser.currentIndex()) + if data and data[2]: + try: + from ....cloud import profile as _profile + + secret = _profile.get_password(data[1]) + except Exception as e: # noqa: BLE001 - fall through to the message + secret = "" + said.setText(f"Could not read the saved password: {e}") if not who or not secret: - result.setText("Both an email and a password are needed.") - result.show() + said.setText("Both an email and a password are needed.") + said.show() return - signin.setEnabled(False) try: auth.login(who, secret) except Exception as e: # noqa: BLE001 - a bad password is not a crash - result.setText(f"Sign-in failed: {e}") - result.show() - return - finally: - signin.setEnabled(True) - # Cleared whatever happened: a failed attempt is the one most - # likely to be a typo, and leaving it visible in a dock is - # worse than making it be retyped. + said.setText(f"Sign-in failed: {e}") + said.show() password.clear() + return note = "Signed in." if save.isChecked(): try: @@ -1725,23 +1960,18 @@ def _signin(): note += " Saved to your NDI profile." except Exception as e: # noqa: BLE001 - the sign-in still worked note += f" (Could not save to the profile: {e})" - result.setText(note) - result.show() - _tick() + outcome.update(ok=True, note=note) + password.clear() + dialog.accept() - signin.clicked.connect(_signin) - password.returnPressed.connect(_signin) - - # Parented to the widget, so it stops when the dock goes. A free timer - # would keep firing at a deleted label and take the process with it. - timer = QTimer(box) - timer.setInterval(30_000) - timer.timeout.connect(_tick) - timer.start() - _tick() + buttons.accepted.connect(_accept) + buttons.rejected.connect(dialog.reject) + password.returnPressed.connect(_accept) - viewer.window.add_dock_widget(box, name="NDI Cloud", area="right") - return box + dialog._ndi_accept = _accept + dialog._ndi_fields = {"email": email, "password": password, "save": save, "profile": chooser} + dialog._ndi_said = said + return dialog, outcome def saveCloudProfile(email: str, password: str) -> None: @@ -1811,7 +2041,6 @@ def addGeneAppearancePanel(viewer) -> Any: """ from qtpy.QtCore import Qt from qtpy.QtWidgets import ( - QDoubleSpinBox, QHBoxLayout, QLabel, QPushButton, @@ -1827,19 +2056,18 @@ def addGeneAppearancePanel(viewer) -> Any: references: dict[str, tuple[float, float]] = {} state = {"contrast": 1.0, "gamma": 1.0} - def _row(label, lo, hi, value, decimals, step, tip): - """A slider under a number, both driving one value. + def _row(label, lo, hi, value, tip): + """A slider on one row, with its name and value beside it. - The same shape as the rotation panel, and for the same reason: a - dock is narrow, and a slider sharing its row with a spin box is - a stub too short to aim with. + NO SPIN BOX. Two docks of two rows each was most of the panel + stack for two knobs that are dragged rather than typed -- these + are "a bit more" and "a bit less" controls, not values anyone + needs to enter exactly. The number is still shown, as a label, so + a setting worth writing down can be read off. """ - spin = QDoubleSpinBox() - spin.setRange(lo, hi) - spin.setDecimals(decimals) - spin.setSingleStep(step) - spin.setValue(value) - spin.setToolTip(tip) + caption = QLabel(f"{label} {value:.2f}") + caption.setMinimumWidth(84) + caption.setToolTip(tip) slider = QSlider(Qt.Horizontal) # Sliders are integers, so the value is carried in hundredths and # converted at the boundary. Hundredths because gamma runs from @@ -1847,13 +2075,11 @@ def _row(label, lo, hi, value, decimals, step, tip): slider.setRange(int(lo * 100), int(hi * 100)) slider.setValue(int(value * 100)) slider.setToolTip(tip) - head = QHBoxLayout() - head.addWidget(QLabel(label)) - head.addWidget(spin) - head.addStretch() - outer.addLayout(head) - outer.addWidget(slider) - return spin, slider + row = QHBoxLayout() + row.addWidget(caption) + row.addWidget(slider, 1) + outer.addLayout(row) + return caption, slider contrast_tip = ( "Narrows the contrast window of every gene layer about its floor.\n" @@ -1866,8 +2092,8 @@ def _row(label, lo, hi, value, decimals, step, tip): "above 1 pushes them down. The base section and the cell layers\n" "are not affected." ) - contrast_spin, contrast_slider = _row("contrast", 0.1, 10.0, 1.0, 2, 0.1, contrast_tip) - gamma_spin, gamma_slider = _row("gamma", 0.1, 3.0, 1.0, 2, 0.05, gamma_tip) + contrast_label, contrast_slider = _row("contrast", 0.1, 10.0, 1.0, contrast_tip) + gamma_label, gamma_slider = _row("gamma", 0.1, 3.0, 1.0, gamma_tip) status = QLabel("") status.setWordWrap(True) @@ -1891,35 +2117,32 @@ def _apply(): applyGeneAppearance(layers, state["contrast"], state["gamma"], references) status.hide() - def _sync(which, value, source): - # The slider and the box are one control; setting either emits a - # signal that would come straight back here without this. + def _sync(which, value, source=None): + # Setting a slider emits its own signal, which would come straight + # back here; the guard makes one change one change. if guard[0]: return guard[0] = True try: state[which] = float(value) - spin, slider = ( - (contrast_spin, contrast_slider) + label, slider, caption = ( + (contrast_label, contrast_slider, "contrast") if which == "contrast" - else (gamma_spin, gamma_slider) + else (gamma_label, gamma_slider, "gamma") ) - if source is not spin: - spin.setValue(float(value)) + label.setText(f"{caption} {float(value):.2f}") if source is not slider: slider.setValue(int(round(float(value) * 100))) _apply() finally: guard[0] = False - contrast_spin.valueChanged.connect(lambda v: _sync("contrast", v, contrast_spin)) contrast_slider.valueChanged.connect(lambda v: _sync("contrast", v / 100.0, contrast_slider)) - gamma_spin.valueChanged.connect(lambda v: _sync("gamma", v, gamma_spin)) gamma_slider.valueChanged.connect(lambda v: _sync("gamma", v / 100.0, gamma_slider)) def _reset(): - _sync("contrast", 1.0, None) - _sync("gamma", 1.0, None) + _sync("contrast", 1.0) + _sync("gamma", 1.0) reset.clicked.connect(_reset) @@ -2191,6 +2414,7 @@ def addAllPanels( points_layer=None, shapes_layer=None, labelings=None, + gene_layers=None, ): """Every panel the session supports, skipping what it has no data for. @@ -2256,16 +2480,22 @@ def _build(name, fn): viewer, session, cells_doc, points_layer, shapes_layer, labelings ), ) - made["genes"] = _build("genes", lambda: addGenePanel(viewer, session, pyr_doc)) - # Below the gene list, because both act on what is ticked in it and - # neither means anything before something is. + # Only when there is a token to run out, and up here with the other + # whole-session controls rather than among the gene ones: it is about + # the connection, not about the picture. + if cloudSessionLooksLikely(): + made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) made["appearance"] = _build("gene appearance", lambda: addGeneAppearancePanel(viewer)) made["tour"] = _build( "gene tour", lambda: addGeneTourPanel(viewer, appearance=made.get("appearance")) ) - # Last, and only when there is a token to run out. It is the panel - # nobody needs until they do, so it reads at the bottom rather than - # above the controls the session is actually for. - if cloudSessionLooksLikely(): - made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) + # THE GENE LIST IS LAST, so it sits at the bottom of the stack. It is + # the tallest panel by a long way -- 26,444 rows -- and napari gives + # the leftover height to whatever is furthest down, so anything below + # it would be squeezed while the list took the room. The two controls + # above it act on what is ticked in it, which is also the order they + # are used in. + made["genes"] = _build( + "genes", lambda: addGenePanel(viewer, session, pyr_doc, initial=gene_layers) + ) return made diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index dbbab7ab..1b56a4fc 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -46,6 +46,7 @@ def openPyramid( controls: bool = True, labelings=None, name: str | None = None, + gene_layers=None, show: bool = True, ): """Open a spatial gene expression pyramid in napari. @@ -85,6 +86,12 @@ def openPyramid( launch, since this viewer holds no other state. Skipped with a note when Qt is unavailable -- they are conveniences and their absence must not stop the picture. + gene_layers: ``(symbol, colormap)`` pairs to open as their own + additive layers on top of the base image, as + :func:`~.controls.parseGeneLayers` returns them. A DIFFERENT + request from *gene_rows*, which filters the base layer to a + subset and leaves one picture; this adds a layer per gene, so + the base stays and the genes are comparable against it. name: image layer name. Defaults to the pyramid's label, which is whatever the ingest recorded -- often the file stem, which names the section rather than what is being shown. @@ -190,6 +197,7 @@ def openPyramid( points_layer=points, shapes_layer=shapes, labelings=labelings, + gene_layers=gene_layers, ) # The launch window has reported everything it can: the viewer is diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index 79a728d5..9b10254a 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -838,7 +838,7 @@ ported_late: - name: GEFManager matlab_path: "+ndi/+gui/+app/GEFManager.m" - matlab_last_sync_hash: "f99d0ca57" + matlab_last_sync_hash: "06f31a4e5" status: porting_deferred decision_log: > MATLAB App Designer window that lists, views, adds and removes diff --git a/tests/test_gui_genepyramid_appearance.py b/tests/test_gui_genepyramid_appearance.py index c7b8d9e1..0b03665a 100644 --- a/tests/test_gui_genepyramid_appearance.py +++ b/tests/test_gui_genepyramid_appearance.py @@ -282,40 +282,50 @@ def build(): return build -def _spins(qt, box): - return box.findChildren(qt.QDoubleSpinBox) +def _sliders(qt, box): + return box.findChildren(qt.QSlider) + + +def _set(qt, box, which, value): + """Move one knob, the way a drag does: through its slider.""" + _sliders(qt, box)[0 if which == "contrast" else 1].setValue(int(round(value * 100))) class TestTheAppearancePanel: def test_the_knobs_reach_the_gene_layers(self, qt, built): _viewer, _base, genes, panel = built() - contrast, gamma = _spins(qt, panel) - contrast.setValue(2.0) - gamma.setValue(0.4) + _set(qt, panel, "contrast", 2.0) + _set(qt, panel, "gamma", 0.4) assert genes[0].contrast_limits == [0.0, 50.0] assert genes[0].gamma == pytest.approx(0.4) def test_they_do_not_reach_the_base_layer(self, qt, built): _viewer, base, _genes, panel = built() - contrast, gamma = _spins(qt, panel) - contrast.setValue(5.0) - gamma.setValue(0.2) + _set(qt, panel, "contrast", 5.0) + _set(qt, panel, "gamma", 0.2) assert base.contrast_limits == [0.0, 500.0] assert base.gamma == pytest.approx(1.0) - def test_the_slider_and_the_box_are_one_control(self, qt, built): + def test_the_value_is_shown_beside_the_slider(self, qt, built): + """No spin box -- these are dragged, not typed -- but a setting + worth writing down still has to be readable.""" _viewer, _base, genes, panel = built() - contrast_spin, _gamma_spin = _spins(qt, panel) - contrast_slider = panel.findChildren(qt.QSlider)[0] - contrast_slider.setValue(250) - assert contrast_spin.value() == pytest.approx(2.5) + _sliders(qt, panel)[0].setValue(250) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "contrast 2.50" in said assert genes[0].contrast_limits == [0.0, 40.0] + def test_the_panel_is_two_rows_and_no_spin_boxes(self, qt, built): + """It is a short panel on purpose: the gene list below it is what + the height is for.""" + _viewer, _base, _genes, panel = built() + assert len(_sliders(qt, panel)) == 2 + assert not panel.findChildren(qt.QDoubleSpinBox) + def test_reset_puts_everything_back(self, qt, built): _viewer, _base, genes, panel = built() - contrast, gamma = _spins(qt, panel) - contrast.setValue(7.0) - gamma.setValue(0.3) + _set(qt, panel, "contrast", 7.0) + _set(qt, panel, "gamma", 0.3) panel.findChild(qt.QPushButton).click() assert genes[0].contrast_limits == [0.0, 100.0] assert genes[0].gamma == pytest.approx(1.0) @@ -325,9 +335,8 @@ def test_a_gene_ticked_later_gets_the_settings_in_force(self, qt, built): where the knobs put them, and the comparison is broken by the act of adding to it.""" viewer, _base, _genes, panel = built() - contrast, gamma = _spins(qt, panel) - contrast.setValue(4.0) - gamma.setValue(0.5) + _set(qt, panel, "contrast", 4.0) + _set(qt, panel, "gamma", 0.5) later = viewer.layers.add(_Layer("gene: GAD1", (0.0, 200.0))) assert later.contrast_limits == [0.0, 50.0] assert later.gamma == pytest.approx(0.5) @@ -339,7 +348,7 @@ def test_an_untouched_panel_leaves_a_new_layer_alone(self, qt, built): def test_a_base_layer_added_later_is_still_not_governed(self, qt, built): viewer, _base, _genes, panel = built() - _spins(qt, panel)[0].setValue(4.0) + _set(qt, panel, "contrast", 4.0) overlay = viewer.layers.add(_Layer("cell centroids", (0.0, 9.0))) assert overlay.contrast_limits == [0.0, 9.0] @@ -348,7 +357,7 @@ def test_a_removed_gene_forgets_its_reference(self, qt, built): its new data rather than from what it looked like the first time.""" viewer, _base, genes, panel = built() - _spins(qt, panel)[0].setValue(2.0) + _set(qt, panel, "contrast", 2.0) viewer.layers.drop(genes[0]) again = viewer.layers.add(_Layer("gene: SST", (0.0, 60.0))) assert again.contrast_limits == [0.0, 30.0] @@ -356,7 +365,7 @@ def test_a_removed_gene_forgets_its_reference(self, qt, built): def test_it_says_so_when_there_is_nothing_to_govern(self, qt): viewer = _Viewer(_Layer("All genes")) panel = controls.addGeneAppearancePanel(viewer) - _spins(qt, panel)[0].setValue(3.0) + _set(qt, panel, "contrast", 3.0) said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) assert "No gene layers" in said # The viewer owns the dock; a collected one takes the widget with @@ -472,8 +481,7 @@ def test_it_pulses_around_the_gamma_the_knob_set(self, qt, tourOf): """The two controls have to agree: a tour that departed from 1.0 would undo the gamma the reader chose, and put it back wrong.""" _viewer, _base, genes, appearance, panel = tourOf() - _contrast, gamma = _spins(qt, appearance) - gamma.setValue(0.6) + _set(qt, appearance, "gamma", 0.6) panel._ndi_toggle() assert panel._ndi_tour["base"] == pytest.approx(0.6) _advance(panel, 1.0) diff --git a/tests/test_gui_genepyramid_cloudpanel.py b/tests/test_gui_genepyramid_cloudpanel.py index 11cecc7a..b12cda23 100644 --- a/tests/test_gui_genepyramid_cloudpanel.py +++ b/tests/test_gui_genepyramid_cloudpanel.py @@ -1,4 +1,4 @@ -"""Signing back in without closing the section. +"""The cloud token's clock, and when a sign-in is offered at all. A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for hours; the token is good for rather less, and what running out looks like from @@ -8,8 +8,9 @@ before this panel the remedy was to quit, log in elsewhere, and open the section again. -The clock and the profile write are checked without widgets; the panel's -own wiring is built offscreen and skips where there is no Qt. +All of this needs no widgets: the clock is arithmetic on a JWT, the +offer is one environment variable, and the profile write is a call +against a stubbed store. """ import base64 @@ -143,167 +144,12 @@ def _add(*a): assert calls["default"] == "fresh" -@pytest.fixture(scope="module") -def qt(): - widgets = pytest.importorskip("qtpy.QtWidgets") - app = widgets.QApplication.instance() or widgets.QApplication([]) - yield widgets - del app - - -class _Window: - def __init__(self): - self.docked = [] - - def add_dock_widget(self, widget, name=None, area=None): - self.docked.append((widget, name, area)) - return widget - - -class _Viewer: - def __init__(self): - self.window = _Window() - - -@pytest.fixture -def panelOf(qt): - """Build a panel and KEEP THE VIEWER, which owns the widget. - - A temporary _Viewer() is collected as soon as the call returns, and - Qt deletes the C++ object underneath the Python wrapper -- every - later attribute read then raises "has been deleted", which looks - like a panel bug and is a test one. - """ - held = [] - - def build(): - viewer = _Viewer() - held.append(viewer) - return controls.addCloudPanel(viewer) - - return build - - -def _fields(qt, box): - edits = box.findChildren(qt.QLineEdit) - email = next(e for e in edits if e.echoMode() == qt.QLineEdit.Normal) - password = next(e for e in edits if e.echoMode() == qt.QLineEdit.Password) - return email, password - - -class TestThePanel: - def test_the_password_is_never_shown(self, qt, panelOf): - _email, password = _fields(qt, panelOf()) - assert password.echoMode() == qt.QLineEdit.Password - - def test_signing_in_passes_exactly_what_was_typed(self, qt, monkeypatch, panelOf): - seen = {} - monkeypatch.setattr( - "ndi.cloud.auth.login", lambda who, secret: seen.update(who=who, secret=secret) - ) - box = panelOf() - email, password = _fields(qt, box) - email.setText(" me@example.com ") - password.setText("pw") - box.findChild(qt.QPushButton).click() - assert seen == {"who": "me@example.com", "secret": "pw"} - - def test_the_password_is_cleared_afterwards(self, qt, monkeypatch, panelOf): - """It goes to the login call and nothing else -- not to a field - that stays filled in a dock somebody is screen-sharing.""" - monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) - box = panelOf() - email, password = _fields(qt, box) - email.setText("me@example.com") - password.setText("pw") - box.findChild(qt.QPushButton).click() - assert password.text() == "" - - def test_a_failed_sign_in_clears_it_too(self, qt, monkeypatch, panelOf): - """The attempt most likely to be a typo is the one that failed.""" - - def boom(*_a): - raise RuntimeError("HTTP 401") - - monkeypatch.setattr("ndi.cloud.auth.login", boom) - box = panelOf() - email, password = _fields(qt, box) - email.setText("me@example.com") - password.setText("wrong") - box.findChild(qt.QPushButton).click() - assert password.text() == "" - - def test_a_bad_password_is_reported_not_raised(self, qt, monkeypatch, panelOf): - def boom(*_a): - raise RuntimeError("HTTP 401") - - monkeypatch.setattr("ndi.cloud.auth.login", boom) - box = panelOf() - email, password = _fields(qt, box) - email.setText("me@example.com") - password.setText("wrong") - box.findChild(qt.QPushButton).click() - said = [lbl.text() for lbl in box.findChildren(qt.QLabel) if lbl.text()] - assert any("401" in t for t in said) - - def test_an_empty_field_asks_rather_than_calling(self, qt, monkeypatch, panelOf): - called = [] - monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: called.append(a)) - box = panelOf() - box.findChild(qt.QPushButton).click() - assert called == [] - - def test_the_profile_is_only_written_when_asked(self, qt, monkeypatch, panelOf): - saved = [] - monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) - monkeypatch.setattr(controls, "saveCloudProfile", lambda *a: saved.append(a)) - box = panelOf() - email, password = _fields(qt, box) - email.setText("me@example.com") - password.setText("pw") - box.findChild(qt.QPushButton).click() - assert saved == [] - - box2 = panelOf() - email2, password2 = _fields(qt, box2) - email2.setText("me@example.com") - password2.setText("pw") - box2.findChild(qt.QCheckBox).setChecked(True) - box2.findChild(qt.QPushButton).click() - assert saved == [("me@example.com", "pw")] - - def test_a_profile_that_will_not_save_does_not_undo_the_sign_in(self, qt, monkeypatch, panelOf): - def boom(*_a): - raise RuntimeError("no keyring") - - monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) - monkeypatch.setattr(controls, "saveCloudProfile", boom) - box = panelOf() - email, password = _fields(qt, box) - email.setText("me@example.com") - password.setText("pw") - box.findChild(qt.QCheckBox).setChecked(True) - box.findChild(qt.QPushButton).click() - said = " ".join(lbl.text() for lbl in box.findChildren(qt.QLabel)) - assert "Signed in" in said - assert "keyring" in said - - def test_the_clock_warns_before_it_bites(self, qt, monkeypatch, panelOf): - monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(600)) - box = panelOf() - said = " ".join(lbl.text() for lbl in box.findChildren(qt.QLabel)) - assert "before it runs out" in said - - def test_an_expired_clock_says_what_will_go_wrong(self, qt, monkeypatch, panelOf): - monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(-60)) - box = panelOf() - said = " ".join(lbl.text() for lbl in box.findChildren(qt.QLabel)) - assert "fail to load" in said - - def test_the_timer_belongs_to_the_dock(self, qt, panelOf): - """A free timer keeps firing at a deleted label and takes the - process with it when the dock is closed.""" - core = pytest.importorskip("qtpy.QtCore") - box = panelOf() - timers = box.findChildren(core.QTimer) - assert timers and all(t.parent() is box for t in timers) +# The panel's own shape, its sign-in dialog and the profile chooser are +# tested in test_gui_genepyramid_genelayers.py, which builds the dialog +# through buildCloudSignInDialog rather than through the button. +# +# NOTHING HERE MAY CLICK "Sign in". It opens a MODAL dialog, and a modal +# dialog in a test with no one to close it blocks the process until the +# suite is killed -- which is what the docked-fields tests that used to +# live here started doing the moment the credentials moved behind a +# button. The seam exists so that never has to happen again. diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index b2081350..fe0d2c91 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -426,9 +426,11 @@ def test_a_failing_cloud_panel_costs_only_itself(monkeypatch, capsys): assert "cloud" in capsys.readouterr().err -def test_the_appearance_and_tour_panels_are_built_after_the_gene_list(monkeypatch): - """Both act on what is ticked in it, and neither means anything - before something is.""" +def test_the_gene_list_is_built_last_so_it_sits_at_the_bottom(monkeypatch): + """It is the tallest panel by a long way -- 26,444 rows -- and napari + gives the leftover height to whatever is furthest down, so anything + below it would be squeezed while the list took the room. The two gene + controls read above it, which is also the order they are used in.""" monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) order = [] @@ -447,10 +449,43 @@ def build(*a, **k): monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) made = controls.addAllPanels(object(), object(), object(), object(), True) - assert order.index("genes") < order.index("appearance") < order.index("tour") + assert order.index("appearance") < order.index("tour") < order.index("genes") + assert order[-1] == "genes" assert made["appearance"] is not None and made["tour"] is not None +def test_the_cloud_panel_sits_above_the_gene_controls(monkeypatch): + """It is about the connection, not about the picture, so it reads + with the other whole-session controls rather than among the gene + ones -- and it must not come between the gene panels and the bottom + of the stack, which belongs to the list.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + order = [] + + def _named(name): + def build(*a, **k): + order.append(name) + return object() + + return build + + for name in ( + "addDisplayPanel", + "addCellTypePanel", + "addRotationPanel", + "addGenePanel", + "addGeneAppearancePanel", + "addGeneTourPanel", + "addCloudPanel", + ): + monkeypatch.setattr(controls, name, _named(name)) + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + + controls.addAllPanels(object(), object(), object(), object(), True) + assert order.index("addCloudPanel") < order.index("addGeneAppearancePanel") + assert order[-1] == "addGenePanel" + + def test_the_tour_is_handed_the_appearance_panel(monkeypatch): """So a pulse departs from the gamma the reader chose and returns to it, rather than resetting their setting to 1.""" diff --git a/tests/test_gui_genepyramid_genelayers.py b/tests/test_gui_genepyramid_genelayers.py new file mode 100644 index 00000000..0b0d7cf8 --- /dev/null +++ b/tests/test_gui_genepyramid_genelayers.py @@ -0,0 +1,344 @@ +"""Genes opened as their own layers, and the sign-in behind a button. + +TWO DIFFERENT REQUESTS live next to each other and are easy to confuse. +``--genes`` FILTERS the base image down to a subset, leaving one +picture. ``--gene-layers`` adds a layer per gene ON TOP of the base, so +"All genes" stays and the named genes are comparable against it. Most of +these tests exist to keep that distinction from eroding. +""" + +import os + +import pytest + +from ndi.gui.app.genepyramid import controls +from ndi.gui.app.genepyramid.controls import parseGeneLayers + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +# What NDI-matlab's GEFManager.ferretDemoGenes emits, spelled out here so +# a change on either side of the bridge shows up as a failing test rather +# than as a demo that opens the wrong colours. +_FERRET_DEMO = "HPCAL1:green,RORB:blue,FEZF2:bop orange,TSHZ2:yellow,NXPH4:red,SST:cyan" + + +class TestParsingTheSpec: + def test_it_pairs_each_gene_with_its_colour(self): + assert parseGeneLayers("A:red,B:cyan") == [("A", "red"), ("B", "cyan")] + + def test_a_gene_may_go_without_a_colour(self): + """None means "take the next one from the cycle", so a caller can + name the colours that matter and leave the rest.""" + assert parseGeneLayers("A,B:cyan") == [("A", None), ("B", "cyan")] + + def test_a_colour_may_contain_a_space(self): + """napari ships 'bop orange', 'bop blue' and 'bop purple'. Only + the ends of each field are trimmed, never the middle.""" + assert parseGeneLayers("FEZF2:bop orange") == [("FEZF2", "bop orange")] + + def test_spaces_around_the_entries_are_ignored(self): + assert parseGeneLayers(" A : red , B ") == [("A", "red"), ("B", None)] + + def test_nothing_asked_for_is_an_empty_list(self): + """A fact, not a missing answer.""" + for spec in ("", " ", ",,", None): + assert parseGeneLayers(spec) == [] + + def test_a_gene_named_twice_is_kept_once(self): + """Ticking the same box twice adds nothing the second time, and + the layer names would collide.""" + assert parseGeneLayers("A:red,A:cyan,B") == [("A", "red"), ("B", None)] + + def test_an_empty_colour_reads_as_no_colour(self): + assert parseGeneLayers("A:") == [("A", None)] + + def test_the_ferret_demo_spec_round_trips(self): + """The exact string NDI-matlab builds for the demo.""" + assert parseGeneLayers(_FERRET_DEMO) == [ + ("HPCAL1", "green"), + ("RORB", "blue"), + ("FEZF2", "bop orange"), + ("TSHZ2", "yellow"), + ("NXPH4", "red"), + ("SST", "cyan"), + ] + + def test_the_demo_colours_are_all_distinct(self): + colours = [c for _s, c in parseGeneLayers(_FERRET_DEMO)] + assert len(set(colours)) == len(colours) + + def test_the_order_is_kept(self): + """It is the order the layers are added, and so the order the + colour cycle advances for any gene that named no colour.""" + assert [s for s, _c in parseGeneLayers("D,C,B,A")] == ["D", "C", "B", "A"] + + +class TestTheCliOption: + def test_it_is_a_separate_option_from_genes(self): + """The two answer different questions and must not be conflated: + one filters the base layer, the other adds layers beside it.""" + from ndi.gui.app.genepyramid.cli import build_parser + + args = build_parser().parse_args(["/s", "--gene-layers", _FERRET_DEMO]) + assert args.gene_layers == _FERRET_DEMO + assert args.genes == "" + + def test_it_defaults_to_nothing(self): + from ndi.gui.app.genepyramid.cli import build_parser + + assert build_parser().parse_args(["/s"]).gene_layers == "" + + def test_the_resolver_hands_the_panel_pairs(self): + from ndi.gui.app.genepyramid.cli import _resolve_gene_layers + + assert _resolve_gene_layers("A:red") == [("A", "red")] + + def test_the_resolver_gives_none_for_nothing(self): + """None rather than [] so the panel can tell "not asked" from + "asked for an empty set".""" + from ndi.gui.app.genepyramid.cli import _resolve_gene_layers + + assert _resolve_gene_layers("") is None + + def test_both_options_can_be_given_together(self): + """Filtering the base layer AND opening named genes beside it is + a coherent request, not a contradiction.""" + from ndi.gui.app.genepyramid.cli import build_parser + + args = build_parser().parse_args(["/s", "--genes", "SST,VIP", "--gene-layers", "SST:cyan"]) + assert args.genes == "SST,VIP" + assert args.gene_layers == "SST:cyan" + + +@pytest.fixture(scope="module") +def qt(): + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Viewer: + def __init__(self): + self.window = _Window() + + +class TestTheCloudPanelIsOneRow: + def test_it_is_a_button_and_a_clock(self, qt): + """Three permanently docked fields say a reader signs in often. + They do it once a day at most, so the credentials live behind the + button.""" + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + assert len(panel.findChildren(qt.QPushButton)) == 1 + assert not panel.findChildren(qt.QLineEdit) + # The viewer owns the dock; a collected one deletes the widget. + assert viewer.window.docked + + def test_the_clock_still_warns_before_it_bites(self, qt, monkeypatch): + import base64 + import json + import time + + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + monkeypatch.setenv( + "NDI_CLOUD_TOKEN", + f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) + 600})}.sig", + ) + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "renew before it runs out" in said + assert viewer.window.docked + + +class _Entry: + def __init__(self, uid, email, nickname="", stage="prod"): + self.UID = uid + self.Email = email + self.Nickname = nickname + self.Stage = stage + + +class TestTheProfileChoices: + def test_a_nickname_is_shown_with_the_email(self, monkeypatch): + """Two profiles for the same person on different stages have to + be tellable apart.""" + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", + lambda: [_Entry("u1", "me@example.com", "work")], + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "pw") + rows = controls.cloudProfileChoices() + assert rows == [("work (me@example.com)", "me@example.com", "u1", True)] + + def test_a_dev_profile_is_marked(self, monkeypatch): + """A dev profile signing in against prod is the failure that + reads as a permissions error, so the stage is on the label.""" + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", + lambda: [_Entry("u1", "me@example.com", stage="dev")], + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "") + label, _email, _uid, stored = controls.cloudProfileChoices()[0] + assert "[dev]" in label + assert not stored + + def test_a_backend_that_will_not_open_costs_no_dialog(self, monkeypatch): + """Typing the credentials by hand is exactly the fallback this + dialog exists to offer, so it must still appear.""" + + def boom(): + raise RuntimeError("no keyring") + + monkeypatch.setattr("ndi.cloud.profile.list_profiles", boom) + assert controls.cloudProfileChoices() == [] + + def test_an_unreadable_secret_only_costs_its_flag(self, monkeypatch): + def boom(_uid): + raise RuntimeError("locked") + + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", boom) + rows = controls.cloudProfileChoices() + assert len(rows) == 1 and rows[0][3] is False + + +@pytest.fixture +def dialogOf(qt, monkeypatch): + held = [] + + def build(profiles=()): + monkeypatch.setattr("ndi.cloud.profile.list_profiles", lambda: list(profiles)) + monkeypatch.setattr( + "ndi.cloud.profile.get_password", lambda uid: "saved-pw" if uid == "u1" else "" + ) + dialog, outcome = controls.buildCloudSignInDialog() + held.append(dialog) + return dialog, outcome + + return build + + +class TestTheSignInDialog: + def test_it_signs_in_with_what_was_typed(self, qt, dialogOf, monkeypatch): + seen = {} + monkeypatch.setattr("ndi.cloud.auth.login", lambda w, s: seen.update(who=w, secret=s)) + dialog, outcome = dialogOf() + dialog._ndi_fields["email"].setText(" me@example.com ") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_accept() + assert seen == {"who": "me@example.com", "secret": "pw"} + assert outcome["ok"] + + def test_the_password_is_never_shown_or_kept(self, qt, dialogOf, monkeypatch): + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + dialog, _outcome = dialogOf() + password = dialog._ndi_fields["password"] + assert password.echoMode() == qt.QLineEdit.Password + dialog._ndi_fields["email"].setText("me@example.com") + password.setText("pw") + dialog._ndi_accept() + assert password.text() == "" + + def test_a_failed_attempt_clears_it_too(self, qt, dialogOf, monkeypatch): + def boom(*_a): + raise RuntimeError("HTTP 401") + + monkeypatch.setattr("ndi.cloud.auth.login", boom) + dialog, outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("wrong") + dialog._ndi_accept() + assert dialog._ndi_fields["password"].text() == "" + assert not outcome["ok"] + assert "401" in dialog._ndi_said.text() + + def test_picking_a_profile_fills_the_email_in(self, qt, dialogOf): + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + assert dialog._ndi_fields["email"].text() == "me@example.com" + + def test_a_saved_password_needs_nothing_typed(self, qt, dialogOf, monkeypatch): + """Someone whose profile works should not be retyping a password + to get past an expired token.""" + seen = {} + monkeypatch.setattr("ndi.cloud.auth.login", lambda w, s: seen.update(who=w, secret=s)) + dialog, outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + dialog._ndi_accept() + assert seen == {"who": "me@example.com", "secret": "saved-pw"} + assert outcome["ok"] + + def test_a_typed_password_beats_the_saved_one(self, qt, dialogOf, monkeypatch): + """Which is the whole point of arriving here: what is saved is + wrong and is being corrected.""" + seen = {} + monkeypatch.setattr("ndi.cloud.auth.login", lambda w, s: seen.update(who=w, secret=s)) + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + dialog._ndi_fields["password"].setText("typed") + dialog._ndi_accept() + assert seen["secret"] == "typed" + + def test_the_first_entry_is_type_them_yourself(self, qt, dialogOf): + """A dialog that opened on a profile would sign in as whoever the + list happened to start with.""" + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com")]) + chooser = dialog._ndi_fields["profile"] + assert chooser.currentIndex() == 0 + assert chooser.itemData(0) is None + + def test_an_empty_field_asks_rather_than_calling(self, qt, dialogOf, monkeypatch): + called = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: called.append(a)) + dialog, _outcome = dialogOf() + dialog._ndi_accept() + assert called == [] + assert "needed" in dialog._ndi_said.text() + + def test_the_profile_is_only_written_when_asked(self, qt, dialogOf, monkeypatch): + saved = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a: saved.append(a)) + dialog, _outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_accept() + assert saved == [] + + dialog2, _o2 = dialogOf() + dialog2._ndi_fields["email"].setText("me@example.com") + dialog2._ndi_fields["password"].setText("pw") + dialog2._ndi_fields["save"].setChecked(True) + dialog2._ndi_accept() + assert saved == [("me@example.com", "pw")] + + def test_a_profile_that_will_not_save_does_not_undo_the_sign_in( + self, qt, dialogOf, monkeypatch + ): + def boom(*_a): + raise RuntimeError("no keyring") + + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", boom) + dialog, outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_fields["save"].setChecked(True) + dialog._ndi_accept() + assert outcome["ok"] + assert "Signed in" in outcome["note"] and "keyring" in outcome["note"] diff --git a/tests/test_gui_genepyramid_genepanel.py b/tests/test_gui_genepyramid_genepanel.py index 258b0375..d0080951 100644 --- a/tests/test_gui_genepyramid_genepanel.py +++ b/tests/test_gui_genepyramid_genepanel.py @@ -16,6 +16,8 @@ import numpy as np import pytest +from ndi.gui.app.genepyramid import controls + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") @@ -40,10 +42,63 @@ class _Layers(dict): events = None +class _AddedLayer: + """Whatever viewer.add_image handed back, as the panel treats it.""" + + def __init__(self, name, colormap=None, blending=None): + self.name = name + self.colormap = colormap + self.blending = blending + self.data = None + + def reset_contrast_limits(self): + pass + + class _Viewer: def __init__(self): self.window = _Window() self.layers = _Layers() + self.added = [] + + def add_image(self, **kwargs): + layer = _AddedLayer(kwargs.get("name", ""), kwargs.get("colormap"), kwargs.get("blending")) + self.added.append(layer) + self.layers[layer.name] = layer + return layer + + +@pytest.fixture +def panelWith(qt, monkeypatch): + """Build the gene panel over a stub list, with genes preset.""" + held = [] + + def build(initial=None, symbols=None): + from ndi.fun import doc_gene_export + from ndi.gui.app.genepyramid import controls + + names = list(symbols or [f"GENE{i:04d}" for i in range(40)]) + monkeypatch.setattr( + doc_gene_export, "readGeneList", lambda *a, **k: (list(names), list(names)) + ) + monkeypatch.setattr( + doc_gene_export, + "readGeneTotals", + lambda *a, **k: (np.arange(len(names), dtype=float), None), + ) + from ndi.gui.app.genepyramid import multiscale + + monkeypatch.setattr( + multiscale, + "layerSpec", + lambda _s, _p, _rows, _d, name: {"data": [], "name": name}, + ) + viewer = _Viewer() + held.append(viewer) + panel = controls.addGenePanel(viewer, object(), object(), initial=initial) + return viewer, panel + + return build @pytest.fixture @@ -143,3 +198,131 @@ def test_the_knob_is_on_the_gene_panel(self, qt, panel): def test_it_starts_off(self, qt, panel): blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") assert blur.value() == pytest.approx(0.0) + + +class TestGenesOpenedAtLaunch: + """``--gene-layers`` reaches the panel as ``initial``. + + A DIFFERENT REQUEST from ``--genes``, which filters the base image + down to a subset and leaves one picture. These arrive as their own + additive layers ON TOP of the base, which is what makes several + genes comparable -- and it is why the demo command carries the one + option and not the other. + """ + + def test_each_gene_becomes_its_own_layer(self, panelWith): + viewer, _panel = panelWith(initial=[("GENE0000", "green"), ("GENE0001", "blue")]) + assert [layer.name for layer in viewer.added] == ["gene: GENE0000", "gene: GENE0001"] + + def test_the_named_colours_are_used(self, panelWith): + viewer, _panel = panelWith(initial=[("GENE0000", "green"), ("GENE0001", "bop orange")]) + assert [layer.colormap for layer in viewer.added] == ["green", "bop orange"] + + def test_they_blend_additively(self, panelWith): + """Which is what lets two genes be read over one another rather + than one hiding the other.""" + viewer, _panel = panelWith(initial=[("GENE0000", "green")]) + assert viewer.added[0].blending == "additive" + + def test_a_gene_with_no_colour_takes_one_from_the_cycle(self, panelWith): + viewer, _panel = panelWith(initial=[("GENE0000", None)]) + assert viewer.added[0].colormap == controls._GENE_COLORMAPS[0] + + def test_a_named_colour_does_not_advance_the_cycle(self, panelWith): + """Otherwise the first hand-ticked gene's colour would depend on + how many were preset, which is a surprise with no upside.""" + viewer, _panel = panelWith( + initial=[("GENE0000", "green"), ("GENE0001", "blue"), ("GENE0002", None)] + ) + assert viewer.added[2].colormap == controls._GENE_COLORMAPS[0] + + def test_the_boxes_are_ticked_too(self, qt, panelWith): + """Ticked through the panel's own checkbox, not by adding the + layer behind its back: otherwise unticking a preset gene would do + nothing at all.""" + _viewer, panel = panelWith(initial=[("GENE0000", "green")]) + listw = panel.findChild(qt.QListWidget) + # Qt.Checked is 2; compared numerically rather than importing the + # enum, which qtpy spells differently across bindings. + ticked = [ + listw.item(i).text() + for i in range(listw.count()) + if int(listw.item(i).checkState()) == 2 + ] + assert len(ticked) == 1 + assert ticked[0].startswith("GENE0000") + + def test_a_gene_not_in_this_pyramid_is_said_out_loud(self, qt, panelWith): + """A demo that quietly opens four of six layers looks like the + viewer working.""" + viewer, panel = panelWith(initial=[("GENE0000", "green"), ("NOTHERE", "red")]) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "NOTHERE" in said + # and the ones that ARE there still open + assert [layer.name for layer in viewer.added] == ["gene: GENE0000"] + + def test_no_preset_opens_nothing(self, panelWith): + viewer, _panel = panelWith() + assert viewer.added == [] + + def test_the_ferret_demo_spec_opens_six_coloured_layers(self, panelWith): + """End to end from the string NDI-matlab builds.""" + spec = "HPCAL1:green,RORB:blue,FEZF2:bop orange,TSHZ2:yellow,NXPH4:red,SST:cyan" + symbols = ["HPCAL1", "RORB", "FEZF2", "TSHZ2", "NXPH4", "SST", "OTHER"] + viewer, _panel = panelWith(initial=controls.parseGeneLayers(spec), symbols=symbols) + assert [layer.name for layer in viewer.added] == [f"gene: {s}" for s in symbols[:6]] + assert [layer.colormap for layer in viewer.added] == [ + "green", + "blue", + "bop orange", + "yellow", + "red", + "cyan", + ] + + +class TestTheBlurKnobWaits: + """A spin box emits on every keystroke. + + Typing "50" asks for 5 um first, and re-blurring is a whole-ladder + map_overlap that takes long enough to see -- so the picture visibly + redrew at the wrong width before the second digit landed. + """ + + def test_typing_does_not_reblur_at_once(self, qt, panel): + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + wait = panel._ndi_blur_wait + assert not wait.isActive() + blur.setValue(5.0) + assert wait.isActive(), "the blur applied on the first keystroke" + + def test_it_waits_long_enough_for_a_second_digit(self, qt, panel): + """Past the gap between two digits, and short enough not to feel + stuck.""" + assert 1000 <= panel._ndi_blur_wait.interval() <= 2000 + + def test_the_wait_restarts_with_each_keystroke(self, qt, panel): + """Otherwise "500" would fire partway through, at 50.""" + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + wait = panel._ndi_blur_wait + blur.setValue(5.0) + first = wait.remainingTime() + blur.setValue(50.0) + assert wait.remainingTime() >= first - 50 + + def test_it_fires_once_rather_than_repeating(self, qt, panel): + assert panel._ndi_blur_wait.isSingleShot() + + def test_the_wait_belongs_to_the_panel(self, qt, panel): + core = pytest.importorskip("qtpy.QtCore") + assert panel._ndi_blur_wait.parent() is panel + assert isinstance(panel._ndi_blur_wait, core.QTimer) + + def test_finishing_the_edit_stops_the_wait(self, qt, panel): + """Enter, or clicking away, means the number IS finished -- no + reason to sit out the rest of the wait.""" + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + blur.setValue(50.0) + assert panel._ndi_blur_wait.isActive() + blur.editingFinished.emit() + assert not panel._ndi_blur_wait.isActive() From 1b9bd4ff752677cd225974077a09fe515f5c3c03 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 19:02:29 +0000 Subject: [PATCH 50/52] Say WHY a profile has no password, instead of implying there is none A profile whose secret lives in a store this process is not reading looks exactly like a profile with no secret at all, and the sign-in dialog reported it as the latter. That is not a rare corner: profile._detect_backend picks keyring whenever keyring merely imports, while MATLAB writes its secret to the AES file -- so a machine with both looks in the Keychain, finds nothing, and tells the reader their password is missing. They then re-save a password that was already saved, into a store nothing is reading. cloudProfileChoices now returns a note alongside each row and the chooser shows it: "password saved", "no password in the keyring store", or "password unreadable from the aes store ()". Picking a profile whose password is not usable puts that reason on screen rather than only moving the focus, because "not readable from here" and "you never saved one" are different problems and only one of them is solved by typing it again. Diagnosis only. Whether the backend preference itself is wrong is a separate question, waiting on what the store actually reports on the machine where this was seen. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 42 ++++++++++--- tests/test_gui_genepyramid_genelayers.py | 80 +++++++++++++++++++++++- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index be3746bd..1e96ae10 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -1701,12 +1701,22 @@ def cloudSessionLooksLikely() -> bool: def cloudProfileChoices(): - """Saved profiles, as ``(label, email, uid, has_password)`` rows. + """Saved profiles, as ``(label, email, uid, stored, note)`` rows. The label is what a chooser shows: the nickname when there is one, else the email, with the email appended either way so two profiles for the same person on different stages can be told apart. + ``note`` SAYS WHY THERE IS NO PASSWORD, which is the whole reason + this returns five things rather than four. A profile whose secret + lives in a store this process is not reading looks exactly like a + profile with no secret at all -- and that is not a rare corner: + :func:`ndi.cloud.profile._detect_backend` picks ``keyring`` whenever + keyring merely imports, while MATLAB writes its secret to the AES + file, so a machine with both ends up looking in the wrong one. The + note names the backend, so the reader is told which store came up + empty instead of being told their password is missing. + Never raises. A secrets backend that will not open, a profile file that will not parse -- none of that is a reason a sign-in dialog should fail to appear, since typing the credentials by hand is @@ -1719,6 +1729,10 @@ def cloudProfileChoices(): entries = profile.list_profiles() except Exception: # noqa: BLE001 - see docstring return rows + try: + backend = str(profile.backend()) + except Exception: # noqa: BLE001 - a name is a nicety, not a requirement + backend = "unknown" for entry in entries: uid = str(getattr(entry, "UID", "") or "") email = str(getattr(entry, "Email", "") or "") @@ -1729,12 +1743,13 @@ def cloudProfileChoices(): label = f"{label} ({email})" if stage and stage != "prod": label = f"{label} [{stage}]" - stored = False + stored, note = False, "" try: stored = bool(profile.get_password(uid)) - except Exception: # noqa: BLE001 - an unreadable secret is not an error here - stored = False - rows.append((label, email, uid, stored)) + note = "password saved" if stored else f"no password in the {backend} store" + except Exception as exc: # noqa: BLE001 - the reason is the useful part + note = f"password unreadable from the {backend} store ({exc})" + rows.append((label, email, uid, stored, note)) return rows @@ -1878,8 +1893,8 @@ def buildCloudSignInDialog(parent=None): choices = cloudProfileChoices() chooser = QComboBox() chooser.addItem("type them below", None) - for label, email, uid, stored in choices: - chooser.addItem(f"{label}{' -- password saved' if stored else ''}", (email, uid, stored)) + for label, email, uid, stored, note in choices: + chooser.addItem(f"{label} -- {note}" if note else label, (email, uid, stored)) email = QLineEdit() email.setPlaceholderText("you@example.com") @@ -1914,15 +1929,24 @@ def buildCloudSignInDialog(parent=None): def _chose(index): data = chooser.itemData(index) if not data: + said.hide() return picked_email, _uid, stored = data email.setText(picked_email) # A profile whose password is saved needs nothing typed. password.clear() if not stored: - # A profile without a saved password needs the password and - # not the email, so the focus goes where the work is. + # A profile without a usable password needs the password and + # not the email, so the focus goes where the work is -- and + # the reason is put on screen, because "this profile exists + # but its password is not readable from here" is a different + # problem from "you never saved one", and only one of them is + # solved by typing it again. + said.setText(choices[index - 1][4]) + said.show() password.setFocus() + else: + said.hide() chooser.currentIndexChanged.connect(_chose) diff --git a/tests/test_gui_genepyramid_genelayers.py b/tests/test_gui_genepyramid_genelayers.py index 0b0d7cf8..f805e142 100644 --- a/tests/test_gui_genepyramid_genelayers.py +++ b/tests/test_gui_genepyramid_genelayers.py @@ -180,8 +180,9 @@ def test_a_nickname_is_shown_with_the_email(self, monkeypatch): lambda: [_Entry("u1", "me@example.com", "work")], ) monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "pw") + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "aes") rows = controls.cloudProfileChoices() - assert rows == [("work (me@example.com)", "me@example.com", "u1", True)] + assert rows == [("work (me@example.com)", "me@example.com", "u1", True, "password saved")] def test_a_dev_profile_is_marked(self, monkeypatch): """A dev profile signing in against prod is the failure that @@ -191,9 +192,11 @@ def test_a_dev_profile_is_marked(self, monkeypatch): lambda: [_Entry("u1", "me@example.com", stage="dev")], ) monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "") - label, _email, _uid, stored = controls.cloudProfileChoices()[0] + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") + label, _email, _uid, stored, note = controls.cloudProfileChoices()[0] assert "[dev]" in label assert not stored + assert "keyring" in note def test_a_backend_that_will_not_open_costs_no_dialog(self, monkeypatch): """Typing the credentials by hand is exactly the fallback this @@ -213,6 +216,7 @@ def boom(_uid): "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] ) monkeypatch.setattr("ndi.cloud.profile.get_password", boom) + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") rows = controls.cloudProfileChoices() assert len(rows) == 1 and rows[0][3] is False @@ -226,6 +230,7 @@ def build(profiles=()): monkeypatch.setattr( "ndi.cloud.profile.get_password", lambda uid: "saved-pw" if uid == "u1" else "" ) + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") dialog, outcome = controls.buildCloudSignInDialog() held.append(dialog) return dialog, outcome @@ -342,3 +347,74 @@ def boom(*_a): dialog._ndi_accept() assert outcome["ok"] assert "Signed in" in outcome["note"] and "keyring" in outcome["note"] + + +class TestWhyThereIsNoPassword: + """A backend mismatch used to look exactly like an empty store. + + profile._detect_backend picks keyring whenever keyring merely + imports, while MATLAB writes its secret to the AES file -- so a + machine with both looks in the wrong one and finds nothing. Told as + "no password saved", that sends the reader off to re-save a password + that is already saved, in a store nothing is reading. + """ + + def test_an_empty_store_names_the_backend(self, monkeypatch): + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "") + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") + note = controls.cloudProfileChoices()[0][4] + assert "keyring" in note and "no password" in note + + def test_an_unreadable_secret_carries_its_reason(self, monkeypatch): + def boom(_uid): + raise RuntimeError("cryptography failed to load") + + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", boom) + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "aes") + note = controls.cloudProfileChoices()[0][4] + assert "unreadable" in note + assert "cryptography failed to load" in note + assert "aes" in note + + def test_a_saved_password_says_so(self, monkeypatch): + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "pw") + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "aes") + assert controls.cloudProfileChoices()[0][4] == "password saved" + + def test_a_backend_that_will_not_name_itself_is_not_an_error(self, monkeypatch): + def boom(): + raise RuntimeError("no") + + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "") + monkeypatch.setattr("ndi.cloud.profile.backend", boom) + assert "unknown" in controls.cloudProfileChoices()[0][4] + + def test_the_chooser_shows_the_reason(self, qt, dialogOf): + dialog, _outcome = dialogOf([_Entry("u2", "them@example.com", "other")]) + chooser = dialog._ndi_fields["profile"] + assert "no password in the keyring store" in chooser.itemText(1) + + def test_picking_one_puts_the_reason_on_screen(self, qt, dialogOf): + """ "This profile's password is not readable from here" is a + different problem from "you never saved one", and only one of + them is solved by typing it again.""" + dialog, _outcome = dialogOf([_Entry("u2", "them@example.com", "other")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + assert "keyring" in dialog._ndi_said.text() + + def test_picking_a_working_one_says_nothing(self, qt, dialogOf): + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + assert dialog._ndi_said.isHidden() From b86d7dff62c68fe42010ae71316fd38e6638b1aa Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 19:20:40 +0000 Subject: [PATCH 51/52] Put NDI Cloud at the top of the stack, with its wordmark The cloud row is the only panel about whether the picture can still be READ rather than about how it is drawn, and its clock is the thing worth catching before it bites: a reader who sees "12 min left" at the top of the stack can sign in at a convenient moment, where the same line at the bottom is found afterwards, by way of a tile that will not load. So it goes first. The gene list stays last, where the leftover height belongs. The wordmark is copied from NDI-matlab (+ndi/+cloud/+ui/resources/images/ndi_logo.png) rather than referenced, because the viewer runs from a Python install that need not have NDI-matlab on disk at all -- a downloaded dataset opened by someone who has never run MATLAB is the normal case. A test pins the md5, so the two repos cannot drift into showing different marks for the same product. ON A WHITE CARD, because it is dark navy on transparency and napari's docks follow the viewer's theme: on the dark one the lettering is simply not there. That is how the brand is used elsewhere anyway. Scaled to twice its display width at a device pixel ratio of 2, so it stays crisp on the retina screen a demo is usually given from. A missing asset costs nothing: cloudLogoPath returns None, the label is skipped, and the panel keeps its dock name in text. Nothing here changes the secrets backend. The profile that would not load turned out to be a machine that had never had the password set -- the AES file was simply absent, which rules out the keyring/aes mismatch I had guessed at rather than confirming it. Setting it in MATLAB's profile editor made both sides read it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 69 ++++++++++++++++-- .../gui/app/genepyramid/resources/README.md | 16 ++++ .../genepyramid/resources/ndi_cloud_logo.png | Bin 0 -> 16957 bytes tests/test_gui_genepyramid_controls.py | 13 ++-- tests/test_gui_genepyramid_genelayers.py | 69 ++++++++++++++++++ 5 files changed, 156 insertions(+), 11 deletions(-) create mode 100644 src/ndi/gui/app/genepyramid/resources/README.md create mode 100644 src/ndi/gui/app/genepyramid/resources/ndi_cloud_logo.png diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 1e96ae10..f8ef1553 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -35,6 +35,8 @@ "cloudSignInDialog", "buildCloudSignInDialog", "cloudProfileChoices", + "cloudLogoPath", + "cloudLogoLabel", "cloudSessionLooksLikely", "saveCloudProfile", "geneLayers", @@ -1753,6 +1755,54 @@ def cloudProfileChoices(): return rows +def cloudLogoPath(): + """The bundled NDI Cloud wordmark, or None if it is not there. + + Copied from NDI-matlab rather than referenced, because the viewer + runs from a Python install that need not have NDI-matlab on disk at + all -- a downloaded dataset opened by someone who has never run + MATLAB is the normal case. + + None rather than a raise: a missing asset costs a picture, never a + panel, and the panel falls back to its own name in text. + """ + from pathlib import Path + + path = Path(__file__).with_name("resources") / "ndi_cloud_logo.png" + return path if path.is_file() else None + + +def cloudLogoLabel(parent=None): + """The wordmark on a white card, sized for a dock. None if unusable. + + ON A WHITE CARD because the wordmark is dark navy on transparency + and napari's docks follow the viewer's theme -- on the dark one the + lettering simply is not there. A white card is how the brand is used + elsewhere anyway, so this is the ordinary treatment rather than a + workaround. + + Scaled to twice its display width with a device pixel ratio of 2, so + it stays crisp on the retina screen a demo is usually given from. + """ + from qtpy.QtCore import Qt + from qtpy.QtGui import QPixmap + from qtpy.QtWidgets import QLabel + + path = cloudLogoPath() + if path is None: + return None + pixmap = QPixmap(str(path)) + if pixmap.isNull(): + return None + pixmap = pixmap.scaledToWidth(300, Qt.SmoothTransformation) + pixmap.setDevicePixelRatio(2.0) + label = QLabel(parent) + label.setPixmap(pixmap) + label.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + label.setStyleSheet("background: #ffffff; border-radius: 4px; padding: 6px 8px 6px 8px;") + return label + + def addCloudPanel(viewer) -> Any: """A clock on the cloud token, and a button that renews it. @@ -1779,6 +1829,12 @@ def addCloudPanel(viewer) -> Any: box = QWidget() outer = QVBoxLayout(box) + # The wordmark first, so the panel says whose account this is before + # it says anything about a token. + logo = cloudLogoLabel(box) + if logo is not None: + outer.addWidget(logo) + signin = QPushButton("Sign in...") signin.setToolTip( "Sign in to NDI Cloud, so tiles that are not already downloaded\n" @@ -2487,6 +2543,14 @@ def _build(name, fn): return None made = {} + # FIRST, when there is a token to run out. It is the only panel about + # whether the picture can still be READ rather than how it is drawn, + # and its clock is the thing worth catching before it bites -- a + # reader who sees "12 min left" at the top of the stack can sign in + # at a convenient moment instead of discovering it through a tile + # that will not load. + if cloudSessionLooksLikely(): + made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) made["display"] = _build( "display", lambda: addDisplayPanel(viewer, session, pyr_doc, image_layer, density) ) @@ -2504,11 +2568,6 @@ def _build(name, fn): viewer, session, cells_doc, points_layer, shapes_layer, labelings ), ) - # Only when there is a token to run out, and up here with the other - # whole-session controls rather than among the gene ones: it is about - # the connection, not about the picture. - if cloudSessionLooksLikely(): - made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) made["appearance"] = _build("gene appearance", lambda: addGeneAppearancePanel(viewer)) made["tour"] = _build( "gene tour", lambda: addGeneTourPanel(viewer, appearance=made.get("appearance")) diff --git a/src/ndi/gui/app/genepyramid/resources/README.md b/src/ndi/gui/app/genepyramid/resources/README.md new file mode 100644 index 00000000..50e4c402 --- /dev/null +++ b/src/ndi/gui/app/genepyramid/resources/README.md @@ -0,0 +1,16 @@ +# Bundled assets + +## `ndi_cloud_logo.png` + +The NDI Cloud wordmark, copied byte-for-byte from NDI-matlab at +`src/ndi/+ndi/+cloud/+ui/resources/images/ndi_logo.png` (1682x306, +md5 3e8439f1b1b8686b38ad71f314fff157), which is the same file that repo +serves to its own cloud dialogs and to the metadata app. + +Copied rather than referenced because the viewer runs from a Python +install that need not have NDI-matlab on disk at all -- a downloaded +dataset opened by someone who has never run MATLAB is the normal case. + +It is a WORDMARK IN DARK NAVY on transparency, so it is drawn on a white +card: napari's docks follow the viewer's theme, and navy on the dark one +is invisible. The card is how the brand is used elsewhere anyway. diff --git a/src/ndi/gui/app/genepyramid/resources/ndi_cloud_logo.png b/src/ndi/gui/app/genepyramid/resources/ndi_cloud_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..8e09b578670f31108ebbe6b7c0772c26b7152265 GIT binary patch literal 16957 zcmaJ}c|4Tg_qP>8RD?C6BME3liKHtyh-{0$H8SXvr`#$HM=Pu`-`w(!(`mD^n%rrDKtN=y_PeXGYq@kge zXQGGx^XYy?0rcw@&d^-f(8Y^J$egR_2>OCP{O|MsKK%bb|0nYQ|NI{s>u1gZsJXMg zcw;jf+qF|`r&>Z=G-Px7gr+>>%W-|Oq2)NCDUEnf+ngY#=nbeU%oAhbLqo#?|NBi# z84o$2p%J11bTrKaGM25Vej<%*cj4Ssr!Zm_PEC9hN9a|ULzrz@QiI}in+xLO?v}o# zH>;#r-KMscbND`+#BIu+Vq&2C7WPp2zHP?jZ}n*b=+7_>^ueF1$#zoZk)Rg(Zw1G- zRi|fo%H;^W5cDT@N{GmWWzOY*N@ySQBjLJ$FbR?pnu_%&=$PS|3N)Z<0n^Rtm(oEk z)G?^2g|%9J`#uc0+U5BF+{xw8sN58+AsN-M7Jr214 z%4w`UGQbRzDks&WAVpm5vf2J* zS@V1D+<$Y^J=*ebxfBPyoAkH5^J%a5Wg96_aV?TMZFkw?Q}nPfR4(zyg$b(gdw(uK zjqXRs-3vC8b~iT@Q-3CifM}>(EM!roBL)0DnLT~T6w6w=yqEP^&$*#;Q%M&h102+w z%7b#q~jf{%0%zsea_m_NECJa@?zP(cR$Yv znLIorJehPhCFnKYO7vKe&K~DmKFm{{ja+iH7GZY0>Qm;a-RHki)KI#5p|iJ%pyBTP z6Y7X|*jGh-jtP0vq$1Y=K!{_Xi<7QTxGglOAid4w@RXP%nJ&h(eDa{G$39cX;nxR) zHEBe;agoCy-Z(Kcky=x{bZ*BlcmJl$Q-LLsXzF0p%0a#+Z$5dr!{LajZRC3JS2=u+ zGug%{opu(BP{%$;kWwakQy7~xX7WY~5!G9TW|cYs-?^|&f?B`OM_xVV&A83`E^_1t zgc$(15&{2KU+YeK!f*bIwD~;o=gXqYqq*dQLpD6~b|l$+0uwet43T6#z9nfRQj4dA z9KG5xdB5epokg4y*&d)rHa0X|RFRo_s|6S+n#H3FeM-5?D~WW6qK9J6Cl`Mg9dp@% z+<0`gP5ta>{UsGYt(kh&Ttsr&u5zU&a12J}G!rc_|B4Bd6FSuU$=0JM5@<&GQ&^6h zzOVe=qPgq*J#8S#Y~QC+19%LhzPr2ol1!xgp|Ka-!)_9mHQP%!bv%K*QQ z)jSEbF7z|q97L5IBlt#y=%a^k;_x(@djbR3msZdEWT7y?e!9({SA@Rq9nvrMd%4%J zL4kqg^43V~ND79mt@-Idr;MdcuD>i5GK&`3MsruiBYW8*5B2vE^=N!(O>OvwBs?l+ zJj8>kxaZoyo<=?Oj@Qnt?&0{L_l4|pl^2z-Nch=^9_xIv$%hgwj?s}Y1pa6N4IH=l zFZzWR(+ik^gHQ3C*c*2`)vS8N5%=y2&Yk2PCf{ej8OOnmq^xFpl8xadipW|Pq#QC3 z;f0!dqCqMkk1_oemB4}aUsl|=Q32PKo~f;inoH(B)nkt3pnfw^x_XXao&{z+1|PJK zd|%|e)a-$kzmO08E2&Mlet$eaTt+E@aoA^tY$7{yg0Oi=XkZ9H0RYby zy})Bo?8Uc>gh%t#aWl$CtCdK+3l_X9>bU;p^fPddpj(s-`LFKY;H{I&`XYzi$2WMV zUL(FcT(R(H$MRx7_?6fQ2L&CGHodcf1xK_?=jXlP=89pL47c6yYz>*7If z86t)(&TGzL$R@N})+`q@jR3tv#c(BJq00sJFA49p0GjZfto+|rHg2PU&p-i+@D6iQ(7>SxTcxE$Gw3BVad58N!4~YV0Mp zzjrHA>skyGe1a+w$R)oDy@g?Yo26!6h>#Xf%q8;}F_`Te3Icbz-woETi&W_`-=wE_ z@GE&5e9@es-PFVYWlROJd3XJwr}LQrI}*)k76Ee45zkEA@yi~f9CXbf%tVj9q@!}N z@zj<~3whA03Kx*|BJjtNfJp-T;WTzKTgp003i}2NnSQX0hs$Q2L}j|>@TZ_RxRL5$-?d0nT5EGw2twywP29t-z~ zU6xjWR6I^imOj6C#6NZ+P|`Cy3G~J%A9X!MZlJnRkTVkV^AWpQ8Oli0?~~8zekkQc z8lO#gBafd7iVyh)Z$HW&Nfxzu+kHQ5e}F_Xo1#7(KBp^&bn(-Y5e}N!K>>zK6C|A% z&#nyV?J>M2V_qRGdC;DhFwJ+8+pj6W>!x)lSjUQr$o#pRmy~%gCV+QAl_yy$T3Gv@ z<f94BLkThxJH=z<8vSh^0yDD^AI-|gJo1TLQa3P6 zLS%j~5(*MD0^Z0`>l$u;@{g5LzS(s%PZH^dd*)GrPiM?XC}z@QhwQ~nRbGxcRvD}F z)d^_{VUQ7xxDP+b53uqWsDrHeNWL9TZ=;F}*w$0(UuqNO$e=W{jUZ zEmCy+K$I1zq8&>%0D^K!oIm1Cfa9|VI?wJ9z({)O*AjcU_?;jV?1;VF+g6NuysGi) z8HhX=Wpy|eM*1>28PXq#;GCSDD<21FhFnK4U?Z)B0Gg%wWk_ zosRME2L8t(VID}h}b z)#J(7)V`gM@Ev{`^kN*{hq-51J;#Ukbi*_|igVrG7+FV9&K)17t)^tYS!_GDDa2&d3bq(Ln86CmIhGlpj2Oj)m1^%H_siQnYGY*w z#aqUus6WL?m0Jn1DD*kVw@r;8QK_x_*d_W3Yf^*&*!!%s^7t9%%a5}Y?~q%zgb%^s z6wRi)&XUHV`>Ep_aO^|UE5d2yY;UfviE9QJX=nUx=6FkQ^MVeb5MeY+{cuPdr6Lvo zYcv(IxG=?IxL%-Ree&xEhTEYZN!D6GD(6@<_y+QD!GViNR=%wTf&M4PqzJ`PeEGO| zXqno@InzSOO`dGt|LqiI>5@`eD@B_&6{CDM*SFne3+_ZBB|z57`HcoVt`s_N<|3r3 zaf#2%=oBqy0$% zFDIkGc+b|aT6_~k%#S}B0^mml{1h+okHpi+Q-Z+WiV!CBs*dGNhr&y`&A%m59rOVn z&9=PvzM4S0VReB2XV{7-_?0VUb*k5&~z{4hsx@7T!SD*bFw_o3iOOZKKzk92`{+^Zn){S4gTHBM- zfDH zfO<#k8i_n?l9!ybgoa&C7--6VK)Pq@fe=Vi`x6*iZKlbqO^I*OJ45f^ZU*eNM~o(H9**5pyGj*$s}s)kz9uF*d8fU5I{4!> zV)sMld1T3hlzG}rQ+N^wEe#dyLBrh3mxqqC4(NgQ6$NJ&g~n1V@HdKUC8sv;&#pnB zGT-D95-G|j+grY$W)pS9ARJ6_@Rx_jK4%zLr~v#v&c;BRnN;5I`}v4UzW}&{mZuMv zZZ5gZ3H~8;1S4-avp3Iivg#PSz7ddK+a%QU(YKdL8kkf)VR9n*AnJR94+36=LaBWV znsAH6*?WO1KR;!G9GdNUzw7hKslIgjG9fq2;5FY1ZbZ8A-w*}Lhf7#U4@N?qQHgqn zz{iU#4-eP7-3m5?8yvJYFXNx;K{AAxj@B-!Z+qb!ef zt3@UZy>k$?q6+Sf=dtjj`h3iI3fRK5CKrwH$KLokP}e-4OxzblBZp5>SO6^bzKSJ4 z*8WYx=+`hKJ3j~&Ju6zvL3qI#O@9dBDTZ~KI^2B_z|T$;n1Svl#as7aTu3AbeGDnL z-~U9j%R@sLs|~a#D#FRjwqC2jft4BA#p`TnputyPhm+e3zT@~zelm`Nk(ytG-BQnK z&7v@&T}tAd8ECU>j3-;W#B`n@9zp>5xIEF`0`Y{KD1f&Z=~}Ok1!cxhquKAGU5|G&03jFoR%aror zLGKDXkV|SJO^hB9PUL0isJ0oPE}8i&YXNUOZVq^7lMokWV_uD=;^8Sgeq`!bb3KvQ zTL~Eba*3be+?>e3-4~)~2*bX6*fem;jl@sbRM$P7_e~$1p`|~K_2sWA&vsXTHNyfw zKZZuaMEUzt)XcEqN*ZYGux6w9*;5tJt=mmer18)jh3#6~5)j^12>2Tn*X zRwFbY1U<5$Wht=Su(mrljzX`uhoAb3CG2Qvra`Y`h=xU1^EooqoJ3v0VE9qFmDv=myWWr*5N4V#}J{XA56ZRDY<#sZ5NyIx;rQ_473{AG4kKlt;yoFh_VNQxb{c zEz*Q#5T>ZtCM)^+_V#)ys=ZZT3z$}zyYXNchpo=bC%eL_%j-Y2N<^{K%|Urr>kAMT zk1eE@zJXxQvoR(1d)j^6u+|TXk7;1tK|8QcC>r7x{OOQ-2ZP;9;MO*Ryz4D)fdCFc zOQteE1HdyWf}_07ERw`tmUuc!a=H(a#>U`#@giJfJDfw;+tAj zvD4Z^?8w*8j$(M1lR=$hPDNMgI~t`CI5m?PlG#mYEpp|^h}-oliyVgiIBr4S-HBP3EWP z`K>{R_v!Q*p7=ewz>dtzkHb@SR7JGQte}o!vzvT?CRltj`HPAu4SX6OE(|&F-1Q&J zhG@d!{mZfv$sScpm2Mj|j}Fb?>BN*6&fqoPh!2^< zeXnDKX$ph8;uu2TVYV$TGP>@5ywRHAjb9~o*dx4LcJy73ul>0}J~H4+G%E0CXKX0Y zVm7oDC3P)ul+D!a(}>@`GGP0egt$@rsQH{4HvERS|J5xaWjt@jB>u`nk8UDX%XlFJ zREgE$lr*u|#uO+k6djfHFvy$;@F8M6F1JJ;# z7mI7bi%1trpzPOBo#idac!_EOIesLDO=t2wG+Q~3>@u?A10X9gKzQK!jl}S?VJ{Rb z+(<88pzYj$WKVi!Adr;U#ja04EOjHF%5lwv@S1k6!R5Ehu=ASDVR=963K4*`;gC@p zL~&g9E2;DmCz1$bS@j;jWcA*1BO(j!qTBmpW5Ms#^UYI=8GpD$VM_DQEH*kUL|{Q+ z=V!~~lI_$!TBdR3OLnyknH`z~H@}e(7giYGE2o&kXDsw@NKk2B6X7`x4pj;K+{hF} z?PJwKgg0-45=JrHkY>denvE1#=?x(v7+zLDaca3%cJw#M738h6PCQ|5M1?UX4k zLiSdP4OV1gD|2`W_A;x90Zn-7(NcxS1%{vDs@99diO~AU)4blFKzX0VLnA}nmJtBT zdd7G6qF^aDvC6HPFy!}I=2hR?x_%*S@tf^~x&p+5ewqB@??|V<>cEqCYuVb_+mVw( zV{{+cT~cny97KRB%GqPVEEQ)3H6E_kEYuW<`0QLatxmz+`KD{7gO zy}3FeiR|rztz~+RNMfMloKAXY1=t)yBa0&ubK7J?*!WYC=OkGC+JDOUQ#pDzC0Y}2 zd4@ysDv9CPSlop=;dIbxexx+G?uIXOSJ@Fo5}7;7@O9}k(~26s_`TM!gz}V%fe{-Q z!sb_Yu3W?=RH^smPR}u#x-?&piTWm_y>hHrmy;rF3=+7 z+mVQdfGjHf91u4%ye>X?Mv90VCK7itd}~t9N%rAqqYPg_Q^n~I5|3PHF8slv#;e^R zjTHWBB>EDLj8i>+@fm3QcKQQmJ_IiQEd&B!!e7&~U)>gbiIg15q$#}`vfuEqO4{jV zWG;?Ue7S6235vS87WiPp>0TkV?-e5UHGU=uou{B}35z`#l(<~TD>jG{VLeMhPFd_J zmx07au2zYmo*G+E1U%*Y$wyy<)5W|_y`@}00>>APfEEGKmy^l*Ko=hT)gZvSbxTLZ z7Xz%3uigAU%>m3R2{E;igJ8tO`O+z# zFH-Z)nmCJWWf{wO{NZM~(#OdxVac0MnG5`EMM&l%==NYfF|YF5?GA=Wq3atF(u zgY*1pl7lYLcXD27Q%p#aB5q4GcP(JQX)Kbc?N4F|K^C-W@+hoq8}KAvY0hEK?YP8v zLL3U z^V+Dx#N~uc?VI(htQLR$MmRCSU_b+spSQw+^bvY{G$`;xAg~jY?|}CpF+`KY_cLa$ z{q~wI6FGnLtQPg^Ig)#&=CZ}gs06f-3dm5QxcHW(KPEgG!vnuw0 zMK#?8^`8w=VXdFn7$q&funlMdRz;US2OBU`8bwcMBI?vp(1{?UmS`}IH_oyDoMz0d z&hxQPDAPNl0&{O1!N8S!fi60C6+s6*3VZiV|-kzO^|Po5bUByo^P^tr?d zTB9)oXPfY^C)e{$At}X93VNY+t6lW+IrHrAJg;#hchnjGeiveddeb#^1lQIM${H4NDsS%M`1szZ`cq@X;%*SZ3^pus)0j zjo8Y0<~6#r{Jb#}&>;0nne=w}^nCK~Ki6IKa$miyKbgB}z)yYoLO*4-i4_XftiIp$1!V5k)2Q1^`d=~y?E^wu!dIT2?BxRP zABtSF!#oC~U8KkS{ossWGOy_Xrsuk8@uI9oy8J-P&bwkj|c1*8m5vBXi9-6{VPvEnzphEz34N>st1#Lh~;!R zwmH60Iu%D*5*w~r?SY~s`{7^@3dZEcirr%lrC*DMh0w6ki|ii>KvtqXAI`@X`lIFK zkrEu(_luycdIX%x(LuR5fN}d@5nd`6cEsw+e6S+M5!tjO z{06(f?nBkT#7h%ZBkB3sjy|C$J;TmKP#!XNwuaLj(*n%ieeMdLLqb0F6&l~{f9859 zqtNtt(boPauI&D(E>6Eu0aJXM`yGB7H99}HNAip+;R{;SNA9r^pvc*p#J?22Igi30>;v#%rz!gi9 zjBAfCc6+$DMXzpLzFxcf0_u)p|z@RAh#AeMz-DV=ZvKYD*gk zMU~Vv!(NUxMl;{hgHFYpUUcPN+I`P~-JoGq*v^Dve7LLBJab@d|BV zt_FRM6#C>?rTB}bDf^)FzLU=pY*DK>d<`Z^2#ZOE>fu zf}X%J86~*wN}U&hY75B9l<{Hd3hzz`=+e50rDi1@M(1h4SQqqy+5juz%hGoi{<`?j zZ@K_z8z|c~1T7HH-W?G3-#A@VW}aN#|B6N>)*XeuR93dH)yDIy!gbpeB}VGncre*Q z|0?#D#Y6JA+`Y+w(|Rn2e6;sv`ew&*_`3Wn`Ft(!BEBmxKRa^$@H!+X6jvAQv^68HA$5`ciS1I;F{GJ# zdA$g91X|r%WC%Te%vIbz4;~F+v{r4HsK%O6rD{dw55hSuu4iwY5gMJiZC;S@lL-_zVG-0;fJXLf+rdnb^L2%ZdH!JVj3vbgfN8GbK#b48qX-c}v}lvkJE__QHMXturHv?gKn$qKJQ zw+HNg{bRu&dCha2vGC=$tu`+OkeV!IeRO?HLQ41%;lVQ?)t_d>7usSi6ws>EMH`9- z{+x5>8_V!4x%wai#E1wGB`h?n@R^`Fz>k0ZQZBIWDLU_XPe zrPe64z#lV9;&U|Bm#^S_E&_^rAtK-cj513pjr-OvBCW^fQ$QWfjjribh{NKGEi7%V z_Fb|P8eKLXR76(d0&-_{s$%(9{G?5pDyrI|@kb2qUmRbaVt@^w7S-qT5m&;iS_R?Y^xupD+}D8{THV$n z_uyXQ9@oPEhMK^>Mt%b<-7T5U8_PHElAqL=-I^ik$XK*)n(Cr1U=YTj8Wi3dtG0wk z{ja+3K@y4K5AN0_X~};mL&K*I9$qZv%7o;Cs2rJYPqb`{egD}s4yyQK0eNDCK357J zJ}~4<4qWNb10542AN!i$aq#fe++)dI{D#zCnJ~SMa@5&tg%iv&^Fp51`v)Ms<`cjO z+Os>xPtnM*E64QH=8W%>yS$vox>r!J9UX3Rc+%9z(^zhR9PcH_oPeRvf3sq;`P)FH z&Z7vy7K{fyJy@UlD-<9+b-l>6Z1$qvZ@Lq;9O%w5qLw9Uopb|!@#pGP9vriV6JEQE zFO`(;`(da1kVvZSwIm6A^$BHaiEzTHc-ceP#zyG3)Lp^s!lBn|0Z{ImA_oS!aBpblL zmk%vJ4)SMdc+v%@=oYyh!7+@>Lu=(3nM32S_W2BO!F^0Ky{vsL{l^GT=^bk=s`@p8 z;zh4CuH=C=zscVKe_cLeZR#Y0{LeusQY-k>QyFk4`+{oN4|^h)VrT|+lt~QlEWa^S zviFQ)hbKM#9(YH-UrN)cfb-z!^T*8KyEcEK4O;cHa2% zr%I8M``5(|@nQF6#&Pf*%yZ*7iW&H5&bEZ_>T=_WTn5LbFXu*HPm2Y_Li>5kmf&=8 zufH&=#IaVnB81B5mz_^y-Ej;(V|UyuX}_>ey&7ai((Wr=`N5ZSE`U4I<|5Z;=5JqF zkU5&?k+17M-sB)k`i(hD-8>KRSjFHKXw-N=n&zUpis!WzYp!p9 zyacxlxSrBf7Um<8bnN}jo7}Ip1@l3AMuQQ%%{BoX_9nnuL+)#={uCoFQzg?iVOVt zxV;~u1-p&a1ghYKPwUBAux+>Wj$BbJMHU{V$&Z2=`hE+FXurSI8qVjIBMWb#fxu+D zA6<}d#D8zvOj!{NF&VKK810WR!SMg-y&%2Mb}pBp5>be_L#UjrxW2;@UE{Id&41z$ z2THs<8$|;_QkR1J4g9m&*9Bf>^NJQEpI@x)t{m073ST5YJ@)g$j!{d@btACt&c*jf z2GcVc5%9jJB=7|iF5YfK=QvjF2#G7@%hS6< z)whj~l6tMBJO3b8M9bDCHx2Q)Ip)U^Qob8N(e`$spdq;104ad$`n$KEj{u$Ogew{{ z7?E~yc-Yg0?<~X#g`nqh@F}r7Ugu?pTe&X;NB#ODam7zaW4{uF*)vV)~_DQ3o9bzJDM_Uw__P=i%9gDgmQOObI3e6+s6z($gU`G>h`x0msi5%x7!@*s-^O9 z+jZ4apk*~;!yt&J>5x&Wyx4_{|vUCZmAP9GQ!QJ5of}U&a;ny!u1{%N2T!+{HCV?`;q1+;l{A8GXe_FmMWMCri;f&My26E1=uB z9O`{%X)M=z@rOih^-mbmefQFopBOR_ImP~M<-YcXZMVdZ+`kHV)=Gk9-81n;At)bRx4cc3Kx0+I?OBQ;Wa+!qW>#&GbjJKU#!|IuE$0 z)XTjmgJ?!&D~H0Y+c)H4HY2+YPtB~h+8jq=$SG?!-8w2)5UTRD}F4;5iq_Cd{Hu6)(gA?-s&P!L*YZ^!+_38Rz1i!9VYQ)3a;#|+8 z1yUint9!=E1oDUKy-09n6bk#)lSulQa0hR9Qp6|T5mmVuKh9C=HC`C=XLMsagNwFf zXymOi=JNjcQ=AwuDTJPIxY}>fd}mC_3`Ja~Q3gpf*E<0oH)WFFq!_?|qk!Bib1qcd zv+G?P*m5P8K5ZJYgphZ|77%kDJTB;$)-*IU6_q`^8BI4%B=f186X@;79~$C ze884Rz=xSV&jLWKvt-Qpo9KFa)MAq~s8R_4&xSQu!c&ygv2Dr+yw5Wuf>;JF-HBuH z(j49im@g`lbfjSytHxocxa@oHcoV6b9(ABJ(hB{?urM3 z(|_^|z|Tu2zrXjV260}99}SyaE49FEoj{4Au^m4-N+fFAX+tzNuhFLTNIIPwd2q{A z2=z6_IJy|NxjRTfu&oA&J%$f6c-kl-(MY_YXrkZbaR@P=+&j}c-I0h=juS;fc|>KN za95gX-{h~DsZOnK4NqCa?2zCbBUI5l*-+Zldna?fyiHatLN~A2WaZA1)gKZ*fzmYF zvv^7ZUS#e2OQa;oW_xDPxH2t^G#DxbL(W4DcPvleH!Y3n5dxxVwa{1I;rK;|Y?Phd;MgX!Sd znYb1di9`}6Hr+8R%gXC~O}Y4iTL9U8EqDw~VAe{#%G_Xi`eiu#?f6@Wp|urZwyuWE z(ut=0H>axk*_qEMDqS ztr@)0^ZU(bimY!=alXIi$OE(X^)r@==o#HLiEM13en-UbI)rg_1)iQ;u5qg}J$qC6`~)k#w@Hvd>)3trY&Y9k@RcsHau9@-!nt4xHJ3>5uMc*G9T>pKHJqu=A`Pi4dnORi}4;I`=4WaLVJfLB8W5k z-mizoO8P_K*+eFfVxzoZH0f`4ZWpSAm0)YvaJ z32NTpaPZNCkhlXHLcuf$2~F_ll@43SfkA24o`3<(#O;Og*z-3+IDD#zYd00X&Z(@+ zvR|OL|JpNtLVGI%t1%sQ`b=lP@RLJYs*dt)=&;xztmqfIM1WwP01iO5*9L;FSx18@ zzasB{z9qu`^<{ob(ym_Hv8Ix2#2HXgmG+ErR$Q4<=pcgz~GW^SF^-Zh6K37kdxhmg&;>NwTXlxYyg6{4VQ`-5pPo9l-PjRX< zv7+bb1$*+yquC1+ff(|Kzu8~1P;8u~7s@z4C?47`TmNu5IX+^^-45aUW!M3o5RX|f ze-)ANO?tv@-Q$8NJZuwwMhiyB2^_aIIEoq#h{V?V<3_0>8+DMghG@WU&&_IL%EKi z{!WZTQ$fR^nqG=sq)LwJIVkDIG$EKTfidn(yz0R(FC5Y-;q&(J8Io{O(k&b4Ux!QD?OlJ<#d9OvIQq*HqRx%^DAmU zro>+Oa-%nBXK(FG5A}#3nDG4%K=W*Xkz%@*fEh#s2_3oqSb3HAW@%ub_Acz=-9a|{ zx9pd&M5oUmFtJ-bB_B-PRY)1iWp)gB`Re1!cj_v3(=TPby_JYhb6fwmyxfR6GW%V8 zB6olnc>zn_Bat!FeHV}x`~@guDD{$8Ah7v9N5wm*wlxpuzNNHu9>UxAzlE+aHJtx= zGq=|wE9E6xTs_^>nk7a@SIRPm{_fR*{ogNJ`;QXY+Kf=NpMCNW*9ZeN$nFG{O(Q^T zxZc7*5oJdcHJld)A>y-cG3z7VC|FR5}$ zcmABPtw36R))$-HY<~A>4UTR3@c`9*P2{`=uz_H^WCohXSV`kq&IV+ji8&{U5MuPTa?Or6kynR_U zE(18Skz+oa%W$#h5!islvyc#C@lVm+&jU0$vGAYdi|>73r+l9Kc%Ixqi zJl7Z7rVCjHN|W;^3@{)8$T9|iQ@6BE@mHlk(97YpKfAOpVg$Le1Z|;-JJyC+98jyMPEaco6rl^$hxYyX|3Z)fGlv6;_6)mFamVW!|+>8n4StxRn4_#+!}bb<~> zKr(X3s*eTwi1+*U zFL*g1RE_8YV&#y*Wz8#(&!B*$j$8xmjr5R=EmR3Cmz1m_HtY~dyvk#FcO!Q+sciP$ zxQksc=9t@I!6G$g^G6qokw~}3(I3SDV#;tJil7}ua;R|xNR69$-(1ZW$k5B%m_S-`_f*J}9Q z{7O~jCKv}e(|>$7sazbRmQa39J4A!&!;1dJkJqkAK3x5?f~?DGlJybt(}D$r@*p`$m}MN? zwD=c?l_|(540&do;ft$%;_{cmVu;KU!Q6w#8Y14Cq*??Yd;DwK>iHs7(udJrj1>Ai zF72EIkv9(!0*`u%>Mj^?8N86*CAUW zC1?*kmbXlSmmsnBP)OquxH6|FsCYlron%fTykeb#VU(F7QFyyC_teht0@<)gz2DQhPJ%ku=@+Wxsg7$b==9=QtrfQJ zR^mH3ja#ISmyabP$p->lfArF>FZ%@O-VN^(DWF6s4iVvlTbovqKf2hl8u=C^b3o+0 z9?$@_i-%k1z2jasr7XS#cOxr4KfMVPm6raLy{CIu1SdJM5C2FH<6d&29HaDaTd;{5 zplIEkaQO2CXmouIswnCMF5)XZM( zN~iG@yG&Jc(jN1Gwq+dUPD!sL>*p^hWaD1N`?sxEKOByD9ji8r3B06gD3i zDv2EvOg139zE#(umufsK_afynxj*Id-QYtnK-A?rHRPI^*MrRmp9Ziek8!<_Lm~Dq z-7k_qbWDAwDMQ^Go>pIflp7yghze>#K+}Mw^PpVF z>Dd8~hFQ{|P&|sZtOQl^Mu(Y!|8-i=@Vf@C#lS1ysmy0I&-o=B-wmi8F9>OW*M9K2 zd$@hct0||D#(&Yz`LYD$>jk;wY+)kZt@p0AnXkCw4XmZU{sIzNYbirgbM?AdzcFdQ zUyh>-o>e!S&Ydwy`dCOdWsWZ?iiz4KuTLBunh`jT3a9P zy&vhUFT-L>zu#V1wp*=EuU`IL%Zq#rF{BG?xzM2nE6;{5QA0|GX-v`Ka~ZDV^Z?@2VH(c{BuQ~<_Q${iVXd?E={Bc)bZu$e+zMIkKujH$^RCH(p`h%T5JDF za@QPYt09B;1=Mx^-c?uGx%UGLC?8EXMo}+>#Q#;QW}2!N1O7RP%NG9oa)F7R)*SNO z=nl_cmQ&C2p_%NL^#3R^PcvfwRS(BzGO78G!65E$LO<~za{Xh4mvrs22WP9EF=TXx TABaG2U7!JUjdfmX*@pig{ny+I literal 0 HcmV?d00001 diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index fe0d2c91..6f1e5c4b 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -454,11 +454,12 @@ def build(*a, **k): assert made["appearance"] is not None and made["tour"] is not None -def test_the_cloud_panel_sits_above_the_gene_controls(monkeypatch): - """It is about the connection, not about the picture, so it reads - with the other whole-session controls rather than among the gene - ones -- and it must not come between the gene panels and the bottom - of the stack, which belongs to the list.""" +def test_the_cloud_panel_is_first_and_the_gene_list_last(monkeypatch): + """The cloud row is the only panel about whether the picture can + still be READ rather than how it is drawn, and its clock is the + thing worth catching before it bites -- so it goes at the top, where + "12 min left" is seen in time to do something about it. The gene + list stays at the bottom, where the leftover height belongs.""" monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) order = [] @@ -482,7 +483,7 @@ def build(*a, **k): monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") controls.addAllPanels(object(), object(), object(), object(), True) - assert order.index("addCloudPanel") < order.index("addGeneAppearancePanel") + assert order[0] == "addCloudPanel" assert order[-1] == "addGenePanel" diff --git a/tests/test_gui_genepyramid_genelayers.py b/tests/test_gui_genepyramid_genelayers.py index f805e142..718e0df2 100644 --- a/tests/test_gui_genepyramid_genelayers.py +++ b/tests/test_gui_genepyramid_genelayers.py @@ -418,3 +418,72 @@ def test_picking_a_working_one_says_nothing(self, qt, dialogOf): dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) dialog._ndi_fields["profile"].setCurrentIndex(1) assert dialog._ndi_said.isHidden() + + +@pytest.fixture +def heldLogo(qt): + """A logo label that OUTLIVES the assertion. + + cloudLogoLabel returns a parentless QLabel; a temporary is collected + the moment the expression ends, Qt deletes the C++ object underneath + it, and reading .pixmap() off the dangling wrapper SEGFAULTS PyQt5 + rather than raising. Holding it is the whole fix. + """ + label = controls.cloudLogoLabel() + assert label is not None, "the bundled logo did not load" + yield label + del label + + +class TestTheCloudLogo: + """The wordmark, bundled and drawn so it is actually visible. + + Copied from NDI-matlab rather than referenced: the viewer runs from + a Python install that need not have NDI-matlab on disk at all, which + is the normal case for someone opening a downloaded dataset. + """ + + def test_it_ships_with_the_package(self): + path = controls.cloudLogoPath() + assert path is not None, "the bundled logo is missing" + assert path.is_file() + assert path.suffix == ".png" + + def test_it_is_the_file_ndi_matlab_serves(self): + """Byte-for-byte, so the two repos cannot drift into showing + different marks for the same product.""" + import hashlib + + digest = hashlib.md5(controls.cloudLogoPath().read_bytes()).hexdigest() + assert digest == "3e8439f1b1b8686b38ad71f314fff157" + + def test_it_is_drawn_on_a_white_card(self, qt, heldLogo): + """The wordmark is dark navy on transparency and napari's docks + follow the viewer's theme -- on the dark one the lettering simply + is not there.""" + assert "#ffffff" in heldLogo.styleSheet() + + def test_it_is_sized_for_a_dock_and_stays_crisp(self, qt, heldLogo): + """Twice the display width at a device pixel ratio of 2, for the + retina screen a demo is usually given from.""" + pixmap = heldLogo.pixmap() + assert pixmap.devicePixelRatio() == 2.0 + assert 120 <= pixmap.width() / pixmap.devicePixelRatio() <= 200 + + def test_a_missing_asset_costs_no_panel(self, qt, monkeypatch): + """A missing file must cost the picture nothing -- the panel + falls back to its own dock name in text.""" + monkeypatch.setattr(controls, "cloudLogoPath", lambda: None) + assert controls.cloudLogoLabel() is None + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + assert panel is not None + assert viewer.window.docked + + def test_the_panel_shows_it(self, qt): + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + assert any( + lbl.pixmap() and not lbl.pixmap().isNull() for lbl in panel.findChildren(qt.QLabel) + ) + assert viewer.window.docked From 7722ae359cf0913db33115be76d1e6ffe2130474 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 9 Sep 2026 19:26:30 +0000 Subject: [PATCH 52/52] Never guess which profile a saved password belongs to ONE EMAIL CAN OWN SEVERAL ACCOUNTS. A dev profile and a prod profile under the same address is the normal arrangement, not a mistake to be tidied -- and saveCloudProfile took the FIRST entry matching the email, wrote the password into it, and then called set_default on it. Both halves are wrong. Writing a prod password into the dev entry is silent, and re-pointing the default while doing it is worse: everything afterwards authenticates as the wrong account, against the wrong server, because CloudConfig.from_env resolves the API URL from the profile's Stage. That failure surfaces as a 404 on the dataset, which reads as a permissions problem rather than as a wrong-server one -- the same trap this branch already documents for the dev/prod token short-circuit. So it never guesses. In order: the uid the caller names -- and the dialog now passes the profile the reader actually picked, which settles it outright -- then the CURRENT profile, then the DEFAULT, when either matches the email, then a match that is the only one for that email. Several matches and no steer raises, naming the count and what to do about it. An ambiguous save is the one case where doing nothing and saying so beats acting. AND THE DEFAULT IS NOT MOVED for an existing profile. Saving a password is not a request to switch accounts. A profile this creates is still made the default, because a store that had nothing for that email has no other candidate to displace. Ten tests, including the one this came from: two entries sharing an email with the dev one listed first. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01YHxsZ56GZsZ3hJ98TJ81t9 --- src/ndi/gui/app/genepyramid/controls.py | 68 ++++++++++--- tests/test_gui_genepyramid_cloudpanel.py | 6 +- tests/test_gui_genepyramid_genelayers.py | 120 ++++++++++++++++++++++- 3 files changed, 178 insertions(+), 16 deletions(-) diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index f8ef1553..6ada9512 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -2035,8 +2035,9 @@ def _accept(): return note = "Signed in." if save.isChecked(): + picked = chooser.itemData(chooser.currentIndex()) try: - saveCloudProfile(who, secret) + saveCloudProfile(who, secret, uid=(picked[1] if picked else "")) note += " Saved to your NDI profile." except Exception as e: # noqa: BLE001 - the sign-in still worked note += f" (Could not save to the profile: {e})" @@ -2054,23 +2055,64 @@ def _accept(): return dialog, outcome -def saveCloudProfile(email: str, password: str) -> None: +def saveCloudProfile(email: str, password: str, uid: str = "") -> None: """Make these the credentials a future session renews itself with. - Updates the profile that already carries this email rather than - adding a second one for it -- someone whose saved password is wrong - is trying to CORRECT it, and a duplicate entry would leave the wrong - one in place to be picked instead. + Updates an existing profile rather than adding a second one for the + same email -- someone whose saved password is wrong is trying to + CORRECT it, and a duplicate entry would leave the wrong one in place + to be picked instead. + + WHICH profile, when several share an email, is the whole difficulty. + ONE EMAIL CAN OWN SEVERAL ACCOUNTS: a dev profile and a prod profile + under the same address is the normal arrangement here, not a mistake + to be tidied. Writing a prod password into the dev entry is silent + and wrong, so this never guesses: + + 1. *uid* when the caller has one -- the dialog passes the profile + the reader actually picked, which settles it outright. + 2. the CURRENT profile, then the DEFAULT, when either matches the + email. Those are the two the rest of NDI would have used. + 3. a match that is the ONLY one for that email. + + and otherwise raises rather than choosing. An ambiguous save is the + one case where doing nothing and saying so beats acting. + + THE DEFAULT IS NOT MOVED for an existing profile. Saving a password + is not a request to switch accounts, and silently re-pointing the + default is how a dev profile ends up being the one everything + afterwards authenticates with. A profile this creates is made the + default, because a store that had nothing for this email has no + other candidate to displace. """ from ....cloud import profile - for entry in profile.list_profiles(): - if str(getattr(entry, "Email", "")).lower() == email.lower(): - profile.set_password(entry.UID, password) - profile.set_default(entry.UID) - return - uid = profile.add(email.split("@")[0] or "ndi", email, password) - profile.set_default(uid) + if uid: + profile.set_password(uid, password) + return + + matches = [ + e for e in profile.list_profiles() if str(getattr(e, "Email", "")).lower() == email.lower() + ] + if not matches: + fresh = profile.add(email.split("@")[0] or "ndi", email, password) + profile.set_default(fresh) + return + + if len(matches) > 1: + for pick in (profile.get_current(), profile.get_default()): + chosen = str(getattr(pick, "UID", "") or "") + if chosen and any(chosen == e.UID for e in matches): + profile.set_password(chosen, password) + return + raise ValueError( + f"{len(matches)} profiles use {email} and none of them is the " + f"current or default one, so there is no way to tell which " + f"password this is. Pick the profile in the list above, or set " + f"a default with ndi.cloud.profile.set_default(...)." + ) + + profile.set_password(matches[0].UID, password) def applySolo(layers, current) -> int: diff --git a/tests/test_gui_genepyramid_cloudpanel.py b/tests/test_gui_genepyramid_cloudpanel.py index b12cda23..44c7d458 100644 --- a/tests/test_gui_genepyramid_cloudpanel.py +++ b/tests/test_gui_genepyramid_cloudpanel.py @@ -109,8 +109,12 @@ def test_an_existing_entry_is_corrected_rather_than_duplicated(self, monkeypatch controls.saveCloudProfile("me@example.com", "pw") assert calls["set"] == ("u2", "pw") - assert calls["default"] == "u2" assert "add" not in calls + # The default is NOT moved. Saving a password is not a request to + # switch accounts -- and one email can own a dev profile and a + # prod one, where re-pointing the default silently changes which + # account everything afterwards authenticates as. + assert "default" not in calls def test_the_match_ignores_case(self, monkeypatch): calls = {} diff --git a/tests/test_gui_genepyramid_genelayers.py b/tests/test_gui_genepyramid_genelayers.py index 718e0df2..4de72458 100644 --- a/tests/test_gui_genepyramid_genelayers.py +++ b/tests/test_gui_genepyramid_genelayers.py @@ -318,7 +318,7 @@ def test_an_empty_field_asks_rather_than_calling(self, qt, dialogOf, monkeypatch def test_the_profile_is_only_written_when_asked(self, qt, dialogOf, monkeypatch): saved = [] monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) - monkeypatch.setattr(controls, "saveCloudProfile", lambda *a: saved.append(a)) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a, **k: saved.append(a)) dialog, _outcome = dialogOf() dialog._ndi_fields["email"].setText("me@example.com") dialog._ndi_fields["password"].setText("pw") @@ -335,7 +335,7 @@ def test_the_profile_is_only_written_when_asked(self, qt, dialogOf, monkeypatch) def test_a_profile_that_will_not_save_does_not_undo_the_sign_in( self, qt, dialogOf, monkeypatch ): - def boom(*_a): + def boom(*_a, **_k): raise RuntimeError("no keyring") monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) @@ -487,3 +487,119 @@ def test_the_panel_shows_it(self, qt): lbl.pixmap() and not lbl.pixmap().isNull() for lbl in panel.findChildren(qt.QLabel) ) assert viewer.window.docked + + +class TestSavingWhenOneEmailOwnsSeveralAccounts: + """A dev profile and a prod profile under one address is the normal + arrangement, not a mistake to be tidied. + + Writing the prod password into the dev entry is silent and wrong, + and re-pointing the default at the dev entry while doing it is + worse -- everything afterwards would authenticate as the wrong + account against the wrong server. + """ + + def _store(self, monkeypatch, entries, current=None, default=None): + calls = {} + monkeypatch.setattr("ndi.cloud.profile.list_profiles", lambda: entries) + monkeypatch.setattr("ndi.cloud.profile.get_current", lambda: current) + monkeypatch.setattr("ndi.cloud.profile.get_default", lambda: default) + monkeypatch.setattr( + "ndi.cloud.profile.set_password", lambda k, p: calls.setdefault("set", (k, p)) + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_default", lambda k: calls.setdefault("default", k) + ) + + def _add(*a): + calls["add"] = a + return "fresh" + + monkeypatch.setattr("ndi.cloud.profile.add", _add) + return calls + + def test_a_named_uid_settles_it_outright(self, monkeypatch): + """Which is what the dialog passes: the profile the reader + actually picked from the list.""" + calls = self._store(monkeypatch, []) + controls.saveCloudProfile("me@example.com", "pw", uid="u-dev") + assert calls["set"] == ("u-dev", "pw") + assert "default" not in calls + + def test_the_current_profile_wins_when_two_share_an_email(self, monkeypatch): + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod], current=prod) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u-prod", "pw") + + def test_the_default_wins_when_there_is_no_current(self, monkeypatch): + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod], default=prod) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u-prod", "pw") + + def test_it_refuses_rather_than_guessing(self, monkeypatch): + """Doing nothing and saying so beats writing a password into an + account the reader did not mean.""" + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod]) + with pytest.raises(ValueError) as caught: + controls.saveCloudProfile("me@example.com", "pw") + assert "2 profiles" in str(caught.value) + assert calls == {}, "it wrote something despite the ambiguity" + + def test_it_never_writes_into_the_first_it_finds(self, monkeypatch): + """The old behaviour, and the reason this class exists: the dev + entry happened to be listed first.""" + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod], current=prod) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"][0] != "u-dev" + + def test_one_match_needs_no_steer(self, monkeypatch): + only = _Entry("u1", "me@example.com") + calls = self._store(monkeypatch, [only]) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u1", "pw") + + def test_a_brand_new_email_is_added_and_made_default(self, monkeypatch): + """A store that had nothing for this email has no other candidate + to displace.""" + calls = self._store(monkeypatch, [_Entry("u-other", "them@example.com")]) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["add"] == ("me", "me@example.com", "pw") + assert calls["default"] == "fresh" + + def test_a_current_profile_for_a_different_email_does_not_capture_it(self, monkeypatch): + """It has to match the email being saved, or picking "current" + would write this password into somebody else's account.""" + mine = _Entry("u-mine", "me@example.com") + theirs = _Entry("u-theirs", "them@example.com") + calls = self._store(monkeypatch, [mine], current=theirs) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u-mine", "pw") + + +class TestTheDialogNamesTheProfileItSavesTo: + def test_a_picked_profile_is_the_one_written(self, qt, dialogOf, monkeypatch): + """No guessing needed when the reader chose from the list.""" + saved = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a, **k: saved.append((a, k))) + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_fields["save"].setChecked(True) + dialog._ndi_accept() + assert saved and saved[0][1].get("uid") == "u1" + + def test_typing_them_by_hand_passes_no_uid(self, qt, dialogOf, monkeypatch): + saved = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a, **k: saved.append((a, k))) + dialog, _outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_fields["save"].setChecked(True) + dialog._ndi_accept() + assert saved and saved[0][1].get("uid") == ""