diff --git a/pyproject.toml b/pyproject.toml index fb381fb3..22e7b807 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,16 @@ dependencies = [ "pydantic>=2.0", "openMINDS>=0.2.0", "scipy>=1.9.0", + # ndi.cloud.profile documents the AES-128/CBC file backend as its + # DEFAULT, and that backend needs cryptography. Without it the + # detector falls through to the in-memory backend, which persists + # nothing and reads nothing -- so a clean install could never load a + # saved cloud password, and said so only through a logger warning. + # Not an extra: ndi.cloud is part of the package rather than an + # opt-in surface, and a default that nothing installs is not a + # default. keyring is preferred when present but stays optional, + # since it is the OS-keychain path rather than the portable one. + "cryptography>=41.0", ] [project.optional-dependencies] diff --git a/src/ndi/cloud/__init__.py b/src/ndi/cloud/__init__.py index c36ef984..6862cde1 100644 --- a/src/ndi/cloud/__init__.py +++ b/src/ndi/cloud/__init__.py @@ -29,6 +29,7 @@ from .auth import ( authenticate, changePassword, + credentialReport, isTokenExpired, login, logout, @@ -56,6 +57,7 @@ "CloudSyncError", "CloudUploadError", "authenticate", + "credentialReport", "isTokenExpired", "login", "logout", diff --git a/src/ndi/cloud/api/files.py b/src/ndi/cloud/api/files.py index 7a030723..2a5c6fae 100644 --- a/src/ndi/cloud/api/files.py +++ b/src/ndi/cloud/api/files.py @@ -180,6 +180,8 @@ def getFile( url: NonEmptyStr, target_path: str | Path, timeout: int = 120, + *, + progress=None, ) -> bool: """Download a file from a presigned URL. @@ -187,6 +189,17 @@ def getFile( without a backup. Callers that must not clobber an existing file have to check for it themselves. + ``progress``, if given, is called as ``progress(done, total)`` after each + chunk, in bytes; *total* is the ``Content-Length`` the server sent, or + None when it sent none. It is keyword-only so it cannot be mistaken for + ``timeout``, and it is called from whichever thread is downloading -- + a caller that renders it is responsible for its own locking. + + WHY HERE. This is the one place every on-demand fetch passes through: + the cell table, the contour file, the gene list and every pyramid tile + all arrive by fetch_cloud_file -> getFile. Reporting anywhere else + would cover some of the wait and not the rest. + MATLAB equivalent: +cloud/+api/+files/getFile.m """ import logging @@ -201,9 +214,25 @@ def getFile( resp = requests.get(url, timeout=timeout, stream=True) if resp.status_code == 200: + total = None + if progress is not None: + # Absent on a chunked response, and a caller must cope with that + # rather than the bar guessing a denominator. + try: + total = int(resp.headers.get("Content-Length", "") or 0) or None + except (TypeError, ValueError): + total = None + done = 0 with open(target_path, "wb") as fh: for chunk in resp.iter_content(chunk_size=8192): fh.write(chunk) + if progress is not None: + done += len(chunk) + # A reporting callback must never cost the download. + try: + progress(done, total) + except Exception: # noqa: BLE001 + progress = None return True # Log the failure with S3 error details when available diff --git a/src/ndi/cloud/auth.py b/src/ndi/cloud/auth.py index 676343d9..c5fc8564 100644 --- a/src/ndi/cloud/auth.py +++ b/src/ndi/cloud/auth.py @@ -22,6 +22,163 @@ logger = logging.getLogger(__name__) + +def tokenSecondsRemaining(token: str = "") -> float | None: + """Seconds until the cloud token expires, or None if unreadable. + + Negative when it has already lapsed, which is the interesting case: + "expired 20 minutes ago" and "expires in 20 minutes" are the same + number with a sign, and a caller showing either wants both. + + None means the token is absent or opaque -- no ``exp`` claim to read. + That is not the same as expired, and a caller must not treat it as + such: an opaque token is the server's to judge. + + Args: + token: the JWT. Defaults to whatever is in the environment, which + is where a login leaves it. + """ + token = token or os.environ.get("NDI_CLOUD_TOKEN", "") + if not token: + return None + try: + expires = getTokenExpiration(token) + except CloudAuthError: + return None + return (expires - datetime.now(timezone.utc)).total_seconds() + + +def tokenStatusLine(token: str = "") -> str: + """One line saying how much life the cloud token has left. + + For a viewer that stays open: a session outlives its token, and the + first sign of that is otherwise a file that will not load. Said out + loud and kept current, it is a clock instead of a surprise. + """ + left = tokenSecondsRemaining(token) + if left is None: + return "not signed in" if not (token or os.environ.get("NDI_CLOUD_TOKEN")) else "unknown" + if left <= 0: + return f"EXPIRED {_roughly(-left)} ago" + return f"signed in, {_roughly(left)} left" + + +def _roughly(seconds: float) -> str: + """A duration in the largest unit that still says something. + + Seconds matter near zero and are noise at an hour, which is the whole + range this is used over. + """ + seconds = max(0.0, float(seconds)) + if seconds < 90: + return f"{int(round(seconds))}s" + minutes = seconds / 60.0 + if minutes < 90: + return f"{int(round(minutes))} min" + return f"{minutes / 60.0:.1f} hours" + + +def credentialReport() -> str: + """What this process can and cannot authenticate with, as text. + + An authentication failure reaches most people through something else + -- a tile that will not load, an API call that raises -- and from + there it is hard to tell WHERE the credentials were supposed to come + from. This walks the same three steps :func:`authenticate` walks and + reports each one, so the answer is a line rather than a bisection. + + It answers, in particular, the question that a re-login does not: the + token lives in ``os.environ``, so it belongs to ONE PROCESS. Logging + in from a shell, or from MATLAB, leaves a separately launched viewer + exactly as unauthenticated as it was. Run this IN THE ENVIRONMENT THAT + FAILED, not in the one you logged in from. + + Nothing here prints a password or a token; the token is reported as + present or absent and by its expiry, never by its value. + + Returns: + A short multi-line report, safe to paste. + """ + config = CloudConfig.from_env() + lines = [ + f"api : {config.api_url}", + f"token in env : {'yes' if os.environ.get('NDI_CLOUD_TOKEN') else 'no'}", + ] + if config.token: + try: + expires = getTokenExpiration(config.token) + state = "EXPIRED" if isTokenExpired(config.token) else "valid" + lines.append(f"token : {state}, exp {expires:%Y-%m-%d %H:%M UTC}") + except CloudAuthError as exc: + lines.append(f"token : unreadable ({exc})") + lines.append(f"organization : {config.org_id or '(none)'}") + lines.append(f"username in env: {'yes' if os.environ.get('NDI_CLOUD_USERNAME') else 'no'}") + email, password = _profile_credentials() + if email and password: + lines.append(f"profile : {email}") + else: + lines.append("profile : none usable") + + # The verdict last, because it is what the reader came for, and it is + # the only line that can disagree with the ones above -- a credential + # that is present but wrong fails here and nowhere else. + try: + _token, org = authenticate(config) + except CloudAuthError as exc: + lines.append(f"authenticate : FAILED -- {exc}") + else: + lines.append(f"authenticate : ok (organization {org})") + return "\n".join(lines) + + +def _noCredentialsMessage(config: CloudConfig) -> str: + """Say WHICH step of :func:`authenticate` came up empty. + + Three quite different situations end at the same raise, and the reader + of the message is usually looking at a failed file fetch rather than at + a login: a token that expired while the process ran, a token that was + never set, or credentials that exist but did not resolve. Naming the + one that happened is the difference between "log back in and relaunch" + and "you have not set this up". + + The remedy is the same sentence in every case, because it is the one + that also prevents the next occurrence: with a username and password + reachable, :func:`authenticate` renews an expired token by itself and + step 1 stops mattering. + """ + fix = ( + "Set NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD, or save a default cloud " + "profile (ndi.cloud.profile.add(...) then set_default(...)), so the " + "token can be renewed without you. NDI_CLOUD_TOKEN alone works only " + "until it expires." + ) + if not config.token: + return f"Not logged in to NDI Cloud: no token, no username/password, no profile. {fix}" + + if isTokenExpired(config.token): + when = "" + try: + when = f" (it expired at {getTokenExpiration(config.token):%Y-%m-%d %H:%M UTC})" + except CloudAuthError: + when = " (its expiry could not be read, which counts as expired)" + # The token is held in os.environ, so a login in another shell does + # NOT reach a process that is already running -- which is exactly + # the shape of this failure when a viewer has been open a while. + return ( + f"The NDI Cloud token has expired{when}, and there are no " + f"credentials to renew it with. A login in another shell will " + f"not reach this process: log in and start it again. " + f"{fix}" + ) + + # A token that is neither absent nor expired: step 1 wanted an org id + # too, and did not get one. + return ( + "The NDI Cloud token is present and unexpired but carries no " + f"organization id, so it cannot be used. {fix}" + ) + + # --------------------------------------------------------------------------- # JWT helpers (no cryptographic verification — matches MATLAB behaviour) # --------------------------------------------------------------------------- @@ -293,12 +450,40 @@ def logout(config: CloudConfig | None = None) -> None: config.org_id = "" +def _profile_credentials() -> tuple[str, str]: + """Return ``(email, password)`` from the saved cloud profile. + + The session's current profile wins over the persisted default, matching + :func:`ndi.cloud.profile.get_current` / :func:`~ndi.cloud.profile.get_default`. + + Returns three empty strings when there is no usable profile. Every + failure mode here is benign -- no profile file, no default set, a + secrets backend that cannot decrypt -- and none of them should turn a + missing credential into a traceback from an unrelated call site, so they + all fall through to the caller's own "no credentials" error. + """ + try: + from . import profile as _profile + + entry = _profile.get_current() or _profile.get_default() + if entry is None: + return "", "" + password = _profile.get_password(entry.UID) + if not entry.Email or not password: + return "", "" + return entry.Email, password + except Exception as exc: # noqa: BLE001 - see docstring + logger.debug("No usable cloud profile: %s", exc) + return "", "" + + def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: """Return an active token and organization ID, attempting login if needed. Priority (matching MATLAB ``authenticate.m``): 1. Existing valid token in config/env (local JWT exp pre-check). 2. Username + password from env → login. + 3. The saved cloud profile (current, else default) → login. Args: config: Optional config. @@ -326,10 +511,35 @@ def authenticate(config: CloudConfig | None = None) -> tuple[str, str]: updated = login(email, password, config) return updated.token, updated.org_id - raise CloudAuthError( - "No valid token and no credentials available. " - "Set NDI_CLOUD_TOKEN or NDI_CLOUD_USERNAME/NDI_CLOUD_PASSWORD." - ) + # 3. Fall back to the saved cloud profile. + # + # MATLAB's authenticate.m has this step -- authenticatedWithSecret, + # which reads the MATLAB Vault -- ahead of the environment one. Python + # had no equivalent at all, so a user who had set up a profile (the + # documented way to keep a cloud password) still got "no credentials + # available" from anything that authenticated implicitly: every + # @_auto_client API call, and, most visibly, the ndic:// file handler + # that fetches pyramid tiles for a downloaded dataset. + # + # It goes last rather than first because an explicitly exported + # NDI_CLOUD_USERNAME is a deliberate override and should keep winning + # over whatever is on disk. + # The profile's Stage picks prod vs dev, but that is resolved in + # CloudConfig.from_env rather than here, because it has to apply to a + # config that short-circuits at step 1 on an already-exported token. + email, password = _profile_credentials() + if email and password: + updated = login(email, password, config) + return updated.token, updated.org_id + + # WHICH OF THE THREE FAILED IS THE WHOLE DIAGNOSIS, and the old message + # ran them together. "No valid token and no credentials available" is + # true whether you never logged in or logged in two hours ago, and + # those want opposite responses -- and it lands where it is least + # legible: on a tile fetch, in a napari toast, one per tile, an hour + # into a session that was working. An EXPIRED token in particular is + # invisible from that sentence, so the tile reads as missing data. + raise CloudAuthError(_noCredentialsMessage(config)) # --------------------------------------------------------------------------- diff --git a/src/ndi/cloud/config.py b/src/ndi/cloud/config.py index a5f4d6a8..9fed1b83 100644 --- a/src/ndi/cloud/config.py +++ b/src/ndi/cloud/config.py @@ -26,6 +26,26 @@ } +def _profile_stage() -> str: + """The Stage ('prod'/'dev') of the cloud profile that would be used. + + NDI-matlab's profile.switchProfile does ``setenv('CLOUD_API_ENVIRONMENT', + prof.Stage)`` and api/url.m reads it back, so Stage naming the API + environment is the established contract, not an invention here. + + Empty string when there is no usable profile. Every failure mode is + benign -- no profile file, no default set, a secrets backend that will + not load -- and none should stop a config from being built. + """ + try: + from . import profile as _profile + + entry = _profile.get_current() or _profile.get_default() + return entry.Stage if entry is not None else "" + except Exception: # noqa: BLE001 - see docstring + return "" + + @dataclass class CloudConfig: """NDI Cloud connection configuration. @@ -51,10 +71,18 @@ class CloudConfig: @classmethod def from_env(cls) -> CloudConfig: """Create a CloudConfig from environment variables.""" - # Determine API URL + # Determine API URL. Precedence: an explicit URL, then an explicit + # environment, then the saved profile's Stage, then prod. + # + # The profile step has to be HERE rather than at the point of login. + # login() exports NDI_CLOUD_TOKEN, so every later from_env() carries a + # valid token and authenticate() returns at its first step without + # ever reaching the credential path -- which meant a dev profile + # logged in against dev and then issued every subsequent request + # against prod, where the dataset 404s. api_url = os.environ.get("NDI_CLOUD_URL", "") if not api_url: - env = os.environ.get("CLOUD_API_ENVIRONMENT", "prod") + env = os.environ.get("CLOUD_API_ENVIRONMENT", "") or _profile_stage() or "prod" api_url = _API_URLS.get(env, _API_URLS["prod"]) upload_no_zip_raw = os.environ.get("NDI_CLOUD_UPLOAD_NO_ZIP", "") diff --git a/src/ndi/cloud/filehandler.py b/src/ndi/cloud/filehandler.py index 0c63ae52..b936d79b 100644 --- a/src/ndi/cloud/filehandler.py +++ b/src/ndi/cloud/filehandler.py @@ -19,6 +19,7 @@ from __future__ import annotations +import contextlib import logging from pathlib import Path from typing import TYPE_CHECKING @@ -123,6 +124,37 @@ def updateFileInfoForRemoteFiles(doc_props: dict, cloud_dataset_id: str) -> None files["file_info"] = fi_list[0] +#: Installed by :func:`watchFetches`. Module-level rather than a parameter +#: because DID fixes the file handler's signature -- it counts positional +#: parameters to decide between the two- and three-argument forms -- so +#: there is nowhere to thread one through from a caller. +_fetch_observer = None + + +@contextlib.contextmanager +def watchFetches(observer): + """Report every on-demand cloud fetch to *observer* for the duration. + + *observer* is called as ``observer(event, uri, done, total)``, where + *event* is ``"start"``, ``"chunk"`` or ``"done"``, sizes are bytes, and + *total* is None when the server sent no Content-Length. + + Restores the previous observer on exit, including on an exception, so + nested use and a failed launch both leave the hook as they found it. + + Fetches happen on whichever thread asked for the file -- the launch + thread for the cell table and the gene list, a _TileFetcher worker for + a tile -- so an observer that renders anything owns its own locking. + """ + global _fetch_observer + previous = _fetch_observer + _fetch_observer = observer + try: + yield + finally: + _fetch_observer = previous + + def fetch_cloud_file( ndic_uri: str, target_path: str | Path, @@ -197,7 +229,20 @@ def fetch_cloud_file( tmp_path = target.with_suffix(target.suffix + ".tmp") logger.debug("Fetching cloud file %s -> %s", ndic_uri, target) - success = getFile(download_url, tmp_path, timeout=300) + observer = _fetch_observer + if observer is None: + success = getFile(download_url, tmp_path, timeout=300) + else: + observer("start", ndic_uri, 0, None) + try: + success = getFile( + download_url, + tmp_path, + timeout=300, + progress=lambda done, total: observer("chunk", ndic_uri, done, total), + ) + finally: + observer("done", ndic_uri, 0, None) if success: tmp_path.rename(target) diff --git a/src/ndi/cloud/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/ndi_matlab_python_bridge.yaml index 15925159..6ee04a27 100644 --- a/src/ndi/cloud/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/ndi_matlab_python_bridge.yaml @@ -1292,7 +1292,7 @@ functions: # --- profile.py (singleton manager for cloud login profiles) --- - name: profile matlab_path: "+ndi/+cloud/profile.m" - matlab_last_sync_hash: "921a02244" + matlab_last_sync_hash: "e6a580b34" python_path: "ndi/cloud/profile.py" functions: - matlab: "ndi.cloud.profile.list()" diff --git a/src/ndi/cloud/profile.py b/src/ndi/cloud/profile.py index 7ef6aa50..bcbab321 100644 --- a/src/ndi/cloud/profile.py +++ b/src/ndi/cloud/profile.py @@ -193,8 +193,117 @@ def _write_secrets_file(filename: Path, payload: dict) -> None: _write_owner_only(filename, json.dumps(payload, indent=2)) +def _as_profile_list(raw) -> list: + """MATLAB writes ONE profile as an object, not an array of one. + + ``jsonencode`` of a 1x1 struct produces ``{...}`` and of a 1xN struct + array produces ``[{...}, ...]``, so NDI-matlab's profiles file has a + different shape for its `Profiles` field depending on how many + profiles the user has. MATLAB's own reader has normalizeProfiles for + exactly this; this is its counterpart. + + Without it, a single-profile file -- which is what every new user + has, having just made their first -- was iterated as a list, yielding + the dict's KEYS, every one of which failed the isinstance(dict) check + and was skipped. The result was zero profiles and no error: the file + was found, parsed, and silently read as empty. + """ + if isinstance(raw, dict): + return [raw] + if isinstance(raw, list): + return raw + return [] + + +# MATLAB's reserved words, for the last rule of makeValidName. NDI's own +# secret keys ("NDI Cloud ") can never collide with one, but a port that +# implements four of five rules is a trap for whoever ports the fifth. +_MATLAB_KEYWORDS = frozenset( + { + "break", + "case", + "catch", + "classdef", + "continue", + "else", + "elseif", + "end", + "for", + "function", + "global", + "if", + "otherwise", + "parfor", + "persistent", + "return", + "spmd", + "switch", + "try", + "while", + } +) + +# MATLAB's namelengthmax. +_MATLAB_NAMELENGTHMAX = 63 + + +def _make_valid_name(name: str) -> str: + """Port of MATLAB ``makeValidName(name, 'ReplacementStyle', 'underscore')``. + + NDI-matlab's profile.m runs every secret key through this before using + it as a struct field, because that is what reading the secrets file + back through ``jsondecode`` requires -- a JSON key has to become a + valid MATLAB identifier. Python has no such constraint, but it has to + produce the SAME name or it cannot read the file MATLAB wrote. + + The rule that matters, and the one a reasonable guess gets wrong: + whitespace is **deleted** and the following letter capitalised, not + replaced. So ``"NDI Cloud 41269..."`` becomes ``NDICloud41269...``, + not ``NDI_Cloud_41269...``. + """ + if not name: + return "x" + + # 1. Whitespace is deleted; the next alphabetic character is capitalised. + chars: list[str] = [] + capitalize_next = False + for ch in name: + if ch.isspace(): + capitalize_next = True + continue + if capitalize_next and ch.isalpha(): + ch = ch.upper() + capitalize_next = False + chars.append(ch) + out = "".join(chars) + + # 2. Anything left that is not an ASCII alphanumeric or an underscore + # becomes one underscore each (ReplacementStyle 'underscore'). + out = "".join(c if (c.isascii() and c.isalnum()) or c == "_" else "_" for c in out) + + # 3. A name must begin with a letter. + if not out or not (out[0].isascii() and out[0].isalpha()): + out = "x" + out + + # 4. A name must not BE a keyword. + if out in _MATLAB_KEYWORDS: + out += "_" + + # 5. namelengthmax. + return out[:_MATLAB_NAMELENGTHMAX] + + def _safe_field(name: str) -> str: - """Map a secret key to a JSON-safe field name.""" + """Map a secret key to the field name NDI-matlab uses for it.""" + return _make_valid_name(name) + + +def _legacy_safe_field(name: str) -> str: + """The field name THIS module used to write, before the MATLAB port. + + Reads fall back to it so a password saved by an older NDI-python is + still found. Nothing writes it any more. + """ return name.replace(" ", "_").replace(":", "_") @@ -280,7 +389,7 @@ def _load_from_disk(self) -> None: except (ValueError, OSError) as exc: logger.warning("Could not load cloud profiles from %s: %s", self.filename, exc) return - raw = data.get("Profiles") or [] + raw = _as_profile_list(data.get("Profiles")) self.profiles = [] for item in raw: if not isinstance(item, dict): @@ -360,6 +469,9 @@ def _set_secret(self, key: str, value: str) -> None: elif self.backend == "aes": store = _read_secrets_file(self.secrets_filename) store[_safe_field(key)] = _aes_encrypt(value) + # Drop any entry under the name this module used to write, so a + # stale copy of a changed password cannot be read back later. + store.pop(_legacy_safe_field(key), None) _write_secrets_file(self.secrets_filename, store) else: # memory self._memory_store[key] = value @@ -375,6 +487,10 @@ def _get_secret(self, key: str) -> str: if self.backend == "aes": store = _read_secrets_file(self.secrets_filename) entry = store.get(_safe_field(key)) + if entry is None: + # A password written by an older NDI-python is under the old + # name. Still readable; the next write moves it across. + entry = store.get(_legacy_safe_field(key)) if entry is None: raise KeyError(f'No secret stored for "{key}".') return _aes_decrypt(entry) @@ -393,6 +509,7 @@ def _remove_secret(self, key: str) -> None: elif self.backend == "aes": store = _read_secrets_file(self.secrets_filename) store.pop(_safe_field(key), None) + store.pop(_legacy_safe_field(key), None) _write_secrets_file(self.secrets_filename, store) else: self._memory_store.pop(key, None) @@ -597,9 +714,23 @@ def reload() -> None: def reset() -> None: - """Clear the in-memory singleton state. Does NOT touch disk.""" + """Clear the in-memory singleton state. Does NOT touch disk. + + THIS INCLUDES A FORCED BACKEND. :func:`use_backend` is a test hook and + the override it sets is in-memory singleton state like any other, so + reset returns it to :func:`_detect_backend`. + + Leaving it out let the memory backend outlive the tests that selected + it: the singleton lives for the whole process, so a test run left + every later :func:`set_password` writing to a dict that is discarded + at exit -- with no error, and with :func:`get_password` returning the + value it had just stored, so it looked as though the password had + been saved. NDI-matlab had the same hole and the same symptom; see + ndi.cloud.profile.reset there. + """ obj = _get_singleton() obj.profiles = [] obj.current_uid = "" obj.default_uid = "" obj._memory_store = {} + obj.backend = _detect_backend() diff --git a/src/ndi/fun/doc_gene.py b/src/ndi/fun/doc_gene.py index 2835ed80..223221df 100644 --- a/src/ndi/fun/doc_gene.py +++ b/src/ndi/fun/doc_gene.py @@ -399,7 +399,9 @@ def makePyramid( gene_list_doc, subjectID: str, binSizes=(1, 2, 4, 8, 16, 32), - grid: int = 9, + grid: int | None = None, + tileBudgetBytes: int = 50 * 2**20, + gridRange=(3, 64), basePixelSize=(0.5, 0.5), pixelSizeUnits: str = "micrometer", label: str = "", @@ -407,6 +409,7 @@ def makePyramid( chipSerial: str = "", pipelineVersion: str = "", origin=None, + progressFcn=None, ): """Bin flat spatial records into a tiled pyramid and store its documents. @@ -432,9 +435,28 @@ def makePyramid( the step's AREA factor while leaving the mean unchanged, so uniform small ratios zoom most smoothly. grid: tile grid, ``grid`` by ``grid`` at EVERY level. Constant - because nonzeros barely fall with bin size, so every level - wants comparable tiling and a viewport maps to tile indices - once, independent of zoom. + across levels because nonzeros barely fall with bin size, so + every level wants comparable tiling and a viewport maps to + tile indices once, independent of zoom. ``None``, the default, + SIZES IT FROM THE DATA: a grid is a fixed fraction of the + extent, so the same 9x9 that gives a mouse section 20 MB tiles + gave a real ferret hemisphere 257 MB ones. + tileBudgetBytes: what the automatic grid aims for in the BIGGEST + tile at the finest level, not the average one. Tissue does not + spread evenly over its bounding box, so the densest tile of a + real section runs a few times the median (2.5x, measured) and + budgeting the median misses by that factor. Bigger tiles mean + fewer files to store and upload; smaller tiles mean less to + fetch before a viewer can draw. Ignored when ``grid`` is set. + gridRange: bounds on the automatic grid. The upper bound is a file + COUNT limit rather than a geometric one -- ``grid**2`` files + per level, every one of them written, stored and uploaded. + progressFcn: called as ``progressFcn(fraction, text)`` with + ``fraction`` in [0, 1] across this call. This is the slow half + of an ingest and reporting only at its ends makes minutes of + it indistinguishable from a hang, so the grid choice, every + level and every band within a level report. ``None`` is + silent. basePixelSize: (x, y) physical size of one base pixel. origin: ``(min_x, min_y)`` of the tiled region, or None to take it from the data. Prefer the acquisition's own bounding box when @@ -482,6 +504,26 @@ def makePyramid( extent_y = int(y.max()) - min_y + 1 bins = sorted(int(b) for b in binSizes) + if grid is None: + _tick(progressFcn, 0.0, "Choosing the tile grid...") + grid, est_bytes = _choose_grid( + x, y, min_x, min_y, extent_x, extent_y, bins[0], tileBudgetBytes, gridRange + ) + # Said out loud, because the file COUNT is a cost this choice + # cannot see: grid**2 files per level, all of which have to be + # written, stored and uploaded. A caller who cares more about that + # than about fetch latency raises tileBudgetBytes or fixes grid. + _tick( + progressFcn, + 0.0, + f"Tile grid {grid}x{grid}: {grid * grid:,} files per level, " + f"biggest tile about {_human_bytes(est_bytes)}.", + ) + else: + grid = int(grid) + if grid < 1: + raise ValueError("grid must be a positive integer, or None to size it from the data") + pyr_doc = ( _blank( "spatialGeneExpressionPyramid", @@ -510,6 +552,7 @@ def makePyramid( pyr_doc = pyr_doc.set_dependency_value("geneList_id", gene_list_doc.id) pyr_doc = pyr_doc.set_dependency_value("subject_id", subjectID) + _tick(progressFcn, 0.0, "Summing per-gene totals...") totals_path = _write_gene_totals(gene_index, count, n_genes) try: pyr_doc = _store_doc(session, pyr_doc, ["gene_totals.tsv"], [totals_path]) @@ -517,27 +560,36 @@ def makePyramid( if os.path.exists(totals_path): os.unlink(totals_path) - tile_docs = [ - _make_level( - session, - x, - y, - gene_index, - count, - min_x, - min_y, - extent_x, - extent_y, - b, - grid, - n_genes, - pyr_doc, - basePixelSize, - pixelSizeUnits, - subjectID, + # Equal shares per level. Every level sorts the SAME record array -- + # coarsening merges pixels but leaves the genes distinct, so the record + # count barely falls with bin size and bin32 costs about what bin1 + # costs. Weighting the shares by output size would report a lie. + tile_docs = [] + for k, b in enumerate(bins): + lo = 0.05 + 0.95 * k / len(bins) + hi = 0.05 + 0.95 * (k + 1) / len(bins) + tile_docs.append( + _make_level( + session, + x, + y, + gene_index, + count, + min_x, + min_y, + extent_x, + extent_y, + b, + grid, + n_genes, + pyr_doc, + basePixelSize, + pixelSizeUnits, + subjectID, + _level_progress(progressFcn, lo, hi, k + 1, len(bins), b), + ) ) - for b in bins - ] + _tick(progressFcn, 1.0, "Pyramid complete.") return pyr_doc, tile_docs @@ -558,70 +610,51 @@ def _make_level( base_pixel_size, pixel_size_units, subject_id, + progress=None, ): - """Build and store one resolution level.""" + """Build and store one resolution level. + + ONE BAND OF TILE ROWS AT A TIME, and this is what decides how much + memory a section needs. A tile spans ``th`` level rows and the full + width of its row, so records in different tile rows never share a tile + and can be sorted, collapsed and written entirely apart. Doing the + whole level at once means the sort's working set scales with the + section; doing it a band at a time means it scales with the section + DIVIDED BY THE GRID -- and because the grid is chosen from the data, a + section big enough to need the chunking is exactly the one that gets a + fine grid to chunk with. + + The cost is ``grid`` passes over the coordinates to select the bands, + cheap next to ``grid`` sorts, and a build that is somewhat slower and + fits. Nothing about what is written changes: a band holds whole tiles. + """ lw = -(-extent_x // b) # ceil lh = -(-extent_y // b) tw = -(-lw // grid) th = -(-lh // grid) - n_tiles = grid * grid - - px = (x - min_x) // b - py = (y - min_y) // b - tcol = px // tw - trow = py // th - xl = px - tcol * tw - yl = py - trow * th - tid = trow * grid + tcol - - # ONE sort, on a TILE-MAJOR key, so tile boundaries fall out of the - # sorted order with a searchsorted instead of needing a second sort. - # - # int64 from the first term. This product reaches ~1e13 on a real - # section; under NumPy's weak promotion an int32 array times a Python - # int stays int32, which wraps SILENTLY at 2.15e9. Distinct - # (pixel, gene) pairs then collide and the dedup below merges them -- - # about 1.7M spurious merges on one measured section, matching the - # n**2/2**33 collision estimate. Assert rather than trust: no fixture - # small enough to run quickly can reach the overflow. - key = ((tid.astype(np.int64) * th + yl) * tw + xl) * n_genes + gi - if key.dtype != np.int64: - raise TypeError(f"sort key must be int64, got {key.dtype}") - span = np.int64(th) * tw * n_genes - if n_tiles * span >= 2**63: - raise OverflowError(f"sort key would reach {n_tiles * span:,}, past int64") - - order = np.argsort(key, kind="stable") - key = key[order] - xl = xl[order] - yl = yl[order] - g = gi[order] - # int32 is enough: a group sums at most b*b base records. - cc = c[order].astype(np.int64) - - # Collapse duplicate (pixel, gene) pairs created by binning. - new = np.empty(len(key), bool) - new[0] = True - np.not_equal(key[1:], key[:-1], out=new[1:]) - starts = np.flatnonzero(new) - cc = np.add.reduceat(cc, starts) - xl, yl, g, key = xl[starts], yl[starts], g[starts], key[starts] - cc = np.minimum(cc, 65535) # data_type_count is uint16 - - tid_sorted = key // span - bounds = np.searchsorted(tid_sorted, np.arange(n_tiles + 1)) names, paths = [], [] tmpdir = tempfile.mkdtemp() try: - for t in range(n_tiles): - lo, hi = int(bounds[t]), int(bounds[t + 1]) - if lo == hi: - continue # tiles with no data are not written - p = os.path.join(tmpdir, f"tile.bin_{t}") - writeTileFile(p, xl[lo:hi], yl[lo:hi], g[lo:hi], cc[lo:hi]) - names.append(f"tile.bin_{t}") - paths.append(p) + for trow in range(grid): + _tick(progress, 0.05 + 0.85 * trow / grid, f"tile row {trow + 1} of {grid}...") + # The band's bounds in SOURCE units. Computing each record's + # level row to compare against would allocate the full-length + # array this loop exists not to allocate; the bounds map back + # into source units exactly, so the comparison happens on the + # coordinates as they are. + y_lo = min_y + trow * th * b + y_hi = y_lo + th * b + sel = (y >= y_lo) & (y < y_hi) + if not sel.any(): + continue + for name, data in _make_band( + x[sel], y[sel], gi[sel], c[sel], min_x, min_y, b, grid, tw, th, n_genes + ): + p = os.path.join(tmpdir, name) + writeTileFile(p, *data) + names.append(name) + paths.append(p) doc = ( _blank( @@ -650,12 +683,14 @@ def _make_level( "data_type_coordinate": "uint16", "tile_compression": "none", "tile_format_version": 1, + "tile_index_origin": TILE_INDEX_ORIGIN, }, ) + session.newdocument() ) doc = doc.set_dependency_value("spatialGeneExpressionPyramid_id", pyr_doc.id) doc = doc.set_dependency_value("subject_id", subject_id) + _tick(progress, 0.92, f"storing {len(names)} tiles...") return _store_doc(session, doc, names, paths) finally: for p in paths: @@ -665,6 +700,179 @@ def _make_level( os.rmdir(tmpdir) +def _make_band(x, y, gi, c, min_x, min_y, b, grid, tw, th, n_genes): + """Collapse one band of tile rows, yielding ``(name, arrays)`` per tile. + + THE KEY SORTS TILE-MAJOR: tile index, then the pixel within the tile, + then the gene. Two things follow. + + Tile boundaries fall out of the sorted order, so no second sort and no + pass per grid cell is needed to find them -- which matters once the + grid is sized from the data and can reach 64x64, i.e. 4096 cells. + + And the key already encodes the tile, the tile-local pixel AND the + gene, so none of them has to survive the sort: they are DECODED back + out of it afterwards. Two full-length arrays go through the sort where + six used to. The decode is exact for as long as the int64 guard below + holds, which is the condition the key already needed. + + int64 from the first term. This product reaches ~1e13 on a real + section; under NumPy's weak promotion an int32 array times a Python int + stays int32, which wraps SILENTLY at 2.15e9. Distinct (pixel, gene) + pairs then collide and the dedup merges them -- about 1.7M spurious + merges on one measured section, matching the n**2/2**33 collision + estimate. Assert rather than trust: no fixture small enough to run + quickly can reach the overflow. + """ + py = (y - min_y) // b + trow = py // th + yl = py - trow * th + del py + px = (x - min_x) // b + tcol = px // tw + xl = px - tcol * tw + del px + tid = trow * grid + tcol + del trow, tcol + + key = ((tid.astype(np.int64) * th + yl) * tw + xl) * n_genes + gi + if key.dtype != np.int64: + raise TypeError(f"sort key must be int64, got {key.dtype}") + span = np.int64(th) * tw * n_genes + if grid * grid * span >= 2**63: + raise OverflowError(f"sort key would reach {grid * grid * span:,}, past int64") + del tid, xl, yl + + order = np.argsort(key, kind="stable") + key = key[order] + # int64 because a group sums at most b*b base records. + cc = c[order].astype(np.int64) + del order + + # Collapse duplicate (pixel, gene) pairs created by binning. + new = np.empty(len(key), bool) + new[0] = True + np.not_equal(key[1:], key[:-1], out=new[1:]) + starts = np.flatnonzero(new) + del new + cc = np.minimum(np.add.reduceat(cc, starts), 65535) # data_type_count is uint16 + key = key[starts] + del starts + + # Decode the survivors back out of the key. xl and yl come out already + # tile-local, which is the form writeTileFile wants. + g, key = key % n_genes, key // n_genes + xl, key = key % tw, key // tw + yl, tid = key % th, key // th + del key + + # One contiguous slice per occupied tile. Tiles with no data are not + # written at all, so the stored series has holes in it and a + # document's file list, not a walk, is what reports which exist. + t_start = np.flatnonzero(np.r_[True, tid[1:] != tid[:-1]]) + t_end = np.r_[t_start[1:], len(tid)] + for lo, hi in zip(t_start, t_end): + name = f"tile.bin_{int(tid[lo]) + TILE_INDEX_ORIGIN}" + yield name, (xl[lo:hi], yl[lo:hi], g[lo:hi], cc[lo:hi]) + + +def _tick(fcn, frac, txt): + """Silent when no handle was given, so every phase can report progress + without each call site testing for a display first.""" + if fcn is None: + return + fcn(float(frac), txt) + + +def _level_progress(fcn, lo, hi, k, n_levels, b): + """Re-scale a caller's [0, 1] handle onto the ``[lo, hi]`` slice of it, + so a level reports its own progress in its own terms without knowing + where in the whole build it sits. ``None`` stays ``None``.""" + if fcn is None: + return None + + def sub(frac, txt): + fcn(lo + (hi - lo) * frac, f"Level {k} of {n_levels} (bin {b}): {txt}") + + return sub + + +def _human_bytes(n): + """Progress text, so a rounded human unit rather than a byte count.""" + units = ("B", "KB", "MB", "GB", "TB") + k = 0 + n = float(n) + while n >= 1024 and k < len(units) - 1: + n /= 1024 + k += 1 + return f"{n:.1f} {units[k]}" + + +def _choose_grid(x, y, min_x, min_y, extent_x, extent_y, b1, budget, g_range): + """Size the tile grid from where the data actually is. + + The thing being budgeted is the LARGEST tile at the FINEST level: the + one a viewer waits on and the one that has to be uploaded. Two facts + make the obvious estimate -- total bytes over ``grid**2`` -- wrong by a + factor of a few: + + 1. Tissue does not fill its bounding box, and where it is present it + is not uniform. On a real ferret hemisphere the densest tile ran + 2.5x the median, so budgeting the mean tile overshoots by 2.5x. + 2. Coarser levels are cheaper per record but not by much -- binning + merges pixels while the genes in them stay distinct -- so bin1 + binds and the other levels come along. + + So this histograms the records over the extent and budgets the + heaviest cell, rather than assuming they are spread evenly. + + Returns: + ``(grid, estimated_biggest_tile_bytes)``. + """ + H = 256 # histogram cells per side + + # Subsample. Choosing a grid does not need every record, and a full + # pass would cost two more full-length arrays at exactly the moment + # this is trying to save them. Records in a .gef are ordered by GENE, + # so a fixed stride is spatially unbiased. + n = len(x) + step = max(1, -(-n // 20_000_000)) + cx = np.minimum(((x[::step] - min_x) * H) // extent_x, H - 1).astype(np.int64) + cy = np.minimum(((y[::step] - min_y) * H) // extent_y, H - 1).astype(np.int64) + h = np.bincount(cy * H + cx, minlength=H * H).reshape(H, H) + frac = h / h.sum() # share of all records, per histogram cell + del cx, cy, h + + # Bytes a record costs in tile format version 1: 4 for the gene index, + # 2 for the count, plus the per-pixel row header amortised over the + # genes detected in that pixel. Measured at ~10.6 on a full section; + # 11 is used so the estimate errs towards a smaller tile. + bytes_per_record = 11 + + g_lo = max(int(g_range[0]), 1) + g_hi = max(int(g_range[1]), g_lo) + # A tile may not exceed 65536 level-pixels on a side: tile-local + # coordinates are uint16. This is a floor on the search, so the + # automatic grid never proposes a geometry the writer would refuse. + side = max(-(-extent_x // b1), -(-extent_y // b1)) + g_lo = max(g_lo, -(-side // 65536)) + g_hi = max(g_hi, g_lo) + + est = float("inf") + for g in range(g_lo, g_hi + 1): + # Both partitions are uniform over the same extent, so a histogram + # cell belongs to the tile containing its centre. The map is the + # same along both axes and separable, so aggregating H x H down to + # g x g is one indicator matrix applied on each side. + m = np.minimum(((np.arange(H) * 2 + 1) * g) // (2 * H), g - 1) + a = np.zeros((H, g)) + a[np.arange(H), m] = 1.0 + est = float((a.T @ frac @ a).max()) * n * bytes_per_record + if est <= budget: + return g, est + return g_hi, est + + def _write_gene_totals(gene_index, count, n_genes): """Per-gene totals for THIS dataset. @@ -687,6 +895,58 @@ def _write_gene_totals(gene_index, count, n_genes): return path +# The file suffix is ONE-BASED. A DID file series names its first member +# NAME_1, not NAME_0 -- did.document/addFileSeries takes "ONE-BASED member +# numbers" and refuses anything else ("Member indices must be positive +# integers (one-based)"). The tile INDEX stays zero-based, because it is a +# grid position and index_order defines it as row*tile_columns + column, so +# the suffix is that index plus this origin. +# +# Getting this wrong is not cosmetic: NDI-matlab's uploader walks a legacy +# NAME# series as NAME1, NAME2, ... and stops at the first name that is not +# there, because that gap was how it learned where the series ended. A +# zero-based pyramid therefore hands it a series starting at a name it never +# probes. +TILE_INDEX_ORIGIN = 1 + + +def tileFileName(tiles_props, index): + """Name of the file holding tile *index* of a tiles document. + + *index* is the zero-based grid index (``row * tile_columns + column`` + for row-major); the returned name carries the document's own origin. + + Pyramids written before the one-based convention was honoured name + their first tile ``tile.bin_0`` and record no ``tile_index_origin``, + which reads as 0 and keeps them readable. The origin has to be recorded + rather than detected: a sparse pyramid whose first tile happens to be + empty has no ``_0`` under either convention, so a guess from the stored + names would shift every tile by one on the documents it got wrong -- + a picture that is quietly wrong rather than one that is missing. + """ + try: + origin = int(tiles_props.get("tile_index_origin", 0) or 0) + except (TypeError, ValueError): + origin = 0 + return f"tile.bin_{int(index) + origin}" + + +def tileIndexFromName(tiles_props, name): + """The zero-based grid index of the tile stored under *name*. + + The inverse of :func:`tileFileName`. Readers that walk a document's + stored names and work backwards to (row, column) need this: with a + one-based origin the suffix is one MORE than the grid index, and + reading the suffix as the index puts every tile one cell along its + row -- and, for the last tile of a row, into the next row. + """ + try: + origin = int(tiles_props.get("tile_index_origin", 0) or 0) + except (TypeError, ValueError): + origin = 0 + return int(str(name).rsplit("_", 1)[1]) - origin + + def _find_level(session, pyr_doc, bin_size): """The tiles document for one bin size, and its property struct.""" q = ndi_query("").isa("spatialGeneExpressionTiles") & ndi_query("").depends_on( @@ -746,7 +1006,7 @@ def readViewport(session, pyr_doc, bin_size, rect=None, gene_rows=None, density= for r in range(y0 // th, min(max(y1 - 1, y0) // th, rows - 1) + 1): for c in range(x0 // tw, min(max(x1 - 1, x0) // tw, cols - 1) + 1): - name = f"tile.bin_{r * cols + c}" + name = tileFileName(lv, r * cols + c) if name not in stored: info["tiles_empty"] += 1 continue @@ -803,8 +1063,7 @@ def exportRegion(session, pyr_doc, bin_size, rect=None): rows_idx, cols_idx, vals, coords = [], [], [], [] n_px = 0 for name in sorted(tile_doc.current_file_list()): - t_id = int(name.rsplit("_", 1)[1]) - tr, tc = divmod(t_id, cols) + tr, tc = divmod(tileIndexFromName(lv, name), cols) tx0, ty0 = tc * tw, tr * th if tx0 >= x1 or tx0 + tw <= x0 or ty0 >= y1 or ty0 + th <= y0: continue @@ -1359,16 +1618,24 @@ def readContourFile( vx = np.frombuffer(raw, vt, count=total, offset=pos) vy = np.frombuffer(raw, vt, count=total, offset=pos + total * vt.itemsize) - polys = [ - np.stack([vx[offsets[i] : offsets[i + 1]], vy[offsets[i] : offsets[i + 1]]], 1) - for i in range(n) - ] + # One (total, 2) array, then SPLIT INTO VIEWS of it. A stack per cell + # cost 2.0s on the opossum section's 493,126 cells; this costs 0.4s, + # and the whole-file array is what lets readContours place every + # vertex in one vectorised addition instead of half a million small + # ones. The views share that array, so nothing is copied. + xy = np.empty((total, 2), vt) + xy[:, 0] = vx + xy[:, 1] = vy + polys = np.split(xy, offsets[1:-1]) if n else [] info = { "nCells": n, "nVerticesTotal": total, "vertexType": vertexType, "offsetType": offsetType, "nVerticesPerCell": nVerticesPerCell, + # The flat form, for callers that would otherwise rebuild it. + "vertices": xy, + "offsets": offsets, } return polys, info @@ -1630,3 +1897,180 @@ def _cells_doc_count(cells_doc): except Exception: pass return None + + +def readContours(session, cells_doc, cells=None): + """Boundary polygons for a cells document, in SOURCE coordinates. + + The document-level companion to :func:`readContourFile`, which parses + the bytes. This adds the two things a caller needs and the file does + not carry: it finds ``contours.bin`` in the document, and it applies + ``contour_reference``. + + THAT SECOND PART IS THE POINT. Vertices are usually stored RELATIVE to + their cell's centroid -- that is what makes them fit in int16, and + :func:`writeContourFile` refuses absolute coordinates that would wrap + -- so a caller that draws :func:`readContourFile`'s output directly + puts every outline in a small cluster near the origin. That is a + picture of nothing, drawn without an error. Here the centroid is added + back, so what comes out is always in the same frame as the centroids + from :func:`readCells` and as the pyramid itself. + + Args: + session: an ndi.session or ndi.dataset holding the document. + cells_doc: a spatialGeneExpressionCells document. + cells: the columns :func:`readCells` returns for this same + document, when the caller already has them. Only the + centroids are used, and only for a centroid-referenced file. + Passing them avoids a second parse of cells.tsv, which is the + larger of the two files and the slower to parse. + + Returns: + ``(polys, info)``. *polys* is one ``(N, 2)`` array of ``[x, y]`` + per cell in cell_index order; a cell with no usable boundary gets + a ``(0, 2)`` array rather than being dropped, so row i here is + row i of cells.tsv. *info* is :func:`readContourFile`'s, plus + ``contourReference`` and ``nEmpty``. + + Raises: + ValueError: if the document was written without contours, if the + reference is not one this understands, or if there are not as + many centroids as cells to place them on. + """ + c = cells_doc.document_properties["spatialGeneExpressionCells"] + if not bool(c.get("contours_present")): + raise ValueError( + f"spatialGeneExpressionCells {cells_doc.id} has contours_present " + f"0: this cell table was written without boundaries. Re-ingest " + f"the cellbin with contours to get them." + ) + + reference = c.get("contour_reference") or "centroid" + if reference not in ("centroid", "absolute"): + raise ValueError(f"contour_reference is {reference!r}; expected 'centroid' or 'absolute'.") + + fh = session.database_openbinarydoc(cells_doc, "contours.bin") + try: + # The dtypes are document FIELDS, not constants, so they are passed + # through rather than assumed -- the same reason readContourFile + # takes them as arguments. + polys, info = readContourFile( + fh, + nVerticesPerCell=int(c.get("n_vertices_per_cell") or 0), + vertexType=c.get("data_type_vertex") or _CONTOUR_VERTEX_TYPE, + offsetType=c.get("data_type_offset") or _CONTOUR_OFFSET_TYPE, + ) + finally: + session.database_closebinarydoc(fh) + + verts = np.asarray(info["vertices"], dtype=float) + offsets = info["offsets"] + n = info["nCells"] + + if reference == "centroid": + # Re-reading cells.tsv here cost a second parse of a 26 MB file -- + # 3.4s on the opossum section, for centroids the caller had + # already read. Passed in, it is free. + cols = cells if cells is not None else readCells(session, cells_doc)[0] + cx = np.asarray(cols["x"], dtype=float) + cy = np.asarray(cols["y"], dtype=float) + if len(cx) != n: + raise ValueError( + f"contours.bin holds {n} cells but cells.tsv " + f"holds {len(cx)} rows. Centroid-relative vertices cannot be " + f"placed without a centroid each." + ) + # One addition over every vertex, rather than one per cell. Each + # vertex takes its own cell's centroid through np.repeat, which + # is what makes a per-cell offset expressible as a whole-array + # operation: 1.5s in place of 5.3s. + counts = np.diff(offsets) + verts[:, 0] += np.repeat(cx, counts) + verts[:, 1] += np.repeat(cy, counts) + + polys = np.split(verts, offsets[1:-1]) if n else [] + + info = dict(info) + info["contourReference"] = reference + info["nEmpty"] = sum(1 for p in polys if len(p) == 0) + return polys, info + + +def readCellTypeLabels(session, labels_doc): + """Read ``labels.tsv`` -- one class name per cell. + + The counterpart to :func:`makeCellTypeLabels`, which had no reader. + + ROW ORDER IS THE CONTRACT with cells.tsv, and cell_index is written + explicitly rather than inferred, so this returns labels placed BY + cell_index rather than in file order. A file that arrived reordered + would otherwise give every cell its neighbour's type -- valid-looking + and wrong, which is the same failure the writer refuses to create. + + An unlabelled cell is the empty string, not a missing row: a labeling + that covers part of the table is normal, and dropping those rows would + shift the rest. + + Args: + session: an ndi.session or ndi.dataset holding the document. + labels_doc: a cellTypeLabels document. + + Returns: + ``(labels, info)``. *labels* is a list of str, one per cell, in + cell_index order. *info* carries labelName, isUnsupervised, + categories (sorted, blanks excluded), nUnlabeled and nCells. + """ + c = labels_doc.document_properties["cellTypeLabels"] + + fh = session.database_openbinarydoc(labels_doc, "labels.tsv") + try: + text = fh.read().decode("utf-8") + finally: + session.database_closebinarydoc(fh) + + rows = [ln for ln in text.splitlines() if ln.strip()] + if not rows: + raise ValueError(f"labels.tsv of {labels_doc.id} is empty.") + header = rows[0].split("\t") + try: + i_idx = header.index("cell_index") + i_lab = header.index("label") + except ValueError as e: + raise ValueError( + f"labels.tsv of {labels_doc.id} has header {header!r}; expected " + f"cell_index and label." + ) from e + + n = int(c.get("n_cells") or (len(rows) - 1)) + labels = [""] * n + for ln in rows[1:]: + parts = ln.split("\t") + idx = int(parts[i_idx]) + if 0 <= idx < n: + labels[idx] = parts[i_lab] if i_lab < len(parts) else "" + + info = { + "labelName": c.get("label_name", ""), + "isUnsupervised": bool(c.get("is_unsupervised")), + "taxonomyLevel": c.get("taxonomy_level", ""), + "assignmentMethod": c.get("assignment_method", ""), + "nCells": n, + "nUnlabeled": sum(1 for v in labels if not v.strip()), + "categories": sorted({v for v in labels if v.strip()}), + } + return labels, info + + +def findCellTypeLabels(session, cells_doc): + """Every cellTypeLabels document belonging to a cells document. + + Returned rather than merged: a cellbin routinely carries a transferred + atlas call AND one or more unsupervised clusterings, and they are not + interchangeable, so a caller picks. + """ + from ..query import ndi_query + + return session.database_search( + ndi_query("").isa("cellTypeLabels") + & ndi_query("").depends_on("cells_document_id", cells_doc.id) + ) diff --git a/src/ndi/fun/doc_gene_export.py b/src/ndi/fun/doc_gene_export.py index 28adf928..e2d4d550 100644 --- a/src/ndi/fun/doc_gene_export.py +++ b/src/ndi/fun/doc_gene_export.py @@ -28,9 +28,9 @@ import numpy as np -from .doc_gene import _find_level, exportRegion, readTileFile +from .doc_gene import _find_level, exportRegion, readTileFile, tileIndexFromName -__all__ = ["exportGef", "exportHdf5", "readGeneList"] +__all__ = ["exportGef", "exportHdf5", "readGeneList", "readGeneTotals"] def readGeneList(session, pyr_doc): @@ -77,6 +77,60 @@ def readGeneList(session, pyr_doc): return ids, names +def readGeneTotals(session, pyr_doc): + """Reads per gene row for THIS pyramid, from ``gene_totals.tsv``. + + Totals belong to the pyramid rather than to the geneList because they + are a property of the counts: several datasets may share one gene + list, and they do not share its totals. + + OLDER PYRAMIDS DO NOT HAVE THIS FILE. It is returned as None rather + than as zeros, because a gene with no reads and a gene whose reads + were never recorded are different things and a caller that cannot + tell them apart would draw an abundance distribution out of nothing. + + Args: + session: an ndi.session or ndi.dataset. + pyr_doc: a spatialGeneExpressionPyramid document. + + Returns: + ``(totals, nRecords)``, both int64 arrays with one entry per gene + row in zero-based order, or ``(None, None)`` when the pyramid + carries no totals. *nRecords* is how many stored records the gene + appears in, which is not the same as its reads. + """ + try: + stored = set(pyr_doc.current_file_list()) + except Exception: + stored = set() + if "gene_totals.tsv" not in stored: + return None, None + + fh = session.database_openbinarydoc(pyr_doc, "gene_totals.tsv") + if fh is None: + return None, None + try: + text = fh.read().decode("utf-8") + finally: + session.database_closebinarydoc(fh) + + rows = [ln for ln in text.splitlines() if ln.strip()] + totals, records = [], [] + for line in rows[1:]: + f = line.split("\t") + # gene_index is written explicitly and is zero-based; asserting it + # is what stops a reordered file from putting one gene's totals on + # another gene, which every later number would inherit silently. + if int(f[0]) != len(totals): + raise ValueError( + f"gene_totals.tsv row {len(totals)} declares gene_index " + f"{f[0]}; the file is out of order or not zero-based" + ) + totals.append(int(f[1])) + records.append(int(f[2]) if len(f) > 2 else 0) + return np.asarray(totals, np.int64), np.asarray(records, np.int64) + + def _gather(session, pyr_doc, bin_size): """Every stored record of one level, in SOURCE coordinates. @@ -92,8 +146,7 @@ def _gather(session, pyr_doc, bin_size): xs, ys, gs, cs = [], [], [], [] for name in sorted(tile_doc.current_file_list(), key=lambda s: int(s.rsplit("_", 1)[1])): - t_id = int(name.rsplit("_", 1)[1]) - tr, tc = divmod(t_id, cols) + tr, tc = divmod(tileIndexFromName(lv, name), cols) fh = session.database_openbinarydoc(tile_doc, name) try: t = readTileFile(fh) diff --git a/src/ndi/fun/ndi_matlab_python_bridge.yaml b/src/ndi/fun/ndi_matlab_python_bridge.yaml index 837bc9d7..9f352e72 100644 --- a/src/ndi/fun/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/ndi_matlab_python_bridge.yaml @@ -2690,18 +2690,45 @@ functions: - name: makePyramid matlab_path: "+ndi/+fun/+doc/+gene/makePyramid.m" - matlab_last_sync_hash: "6ae508708" + matlab_last_sync_hash: "38aa4e8fb" python_path: "ndi/fun/doc_gene.py" decision_log: > Exact match. Options keep their MATLAB spelling (binSizes, grid, - basePixelSize, subjectID). subjectID is REQUIRED on both sides: the - pyramid document declares subject_id mustbenotempty, because a - section is measured from an animal and NDI records that on the - measurement rather than leaving it to be inferred. + tileBudgetBytes, gridRange, basePixelSize, subjectID). subjectID is + REQUIRED on both sides: the pyramid document declares subject_id + mustbenotempty, because a section is measured from an animal and NDI + records that on the measurement rather than leaving it to be + inferred. + + NDI-matlab 38aa4e8fb sized the tile grid from the data and built a level + one band of tile rows at a time; both are ported here. grid now + defaults to [] / None on both sides, because a grid is a fixed + FRACTION of the extent and the 9x9 that gives a mouse section 20 MB + tiles gave a real ferret hemisphere 257 MB ones. tileBudgetBytes + (50 MB) budgets the BIGGEST tile at the finest level rather than the + average one: tissue does not fill its bounding box and the densest + tile of a real section ran 2.5x the median. gridRange (3..64) bounds + it, the upper bound being a file COUNT limit -- grid**2 files per + level, all of them written, stored and uploaded. + + The two languages reached the same place from different sides. + Python already sorted on a TILE-MAJOR key, so tile boundaries fell + out of the sorted order; MATLAB scanned every record against every + grid cell, which a data-sized grid makes untenable, so MATLAB + adopted the Python key. Both now also DECODE the tile, the pixel and + the gene back out of that key instead of carrying them alongside it, + and both build a level one band of tile rows at a time. Neither + changes a byte of what is written -- a tile is a contiguous + rectangle, so global row-major order inside it IS local row-major + order -- which the shared conformance tile still holds them to. + + progressFcn(fraction, text) is on both, silent when not given. The + pyramid is the slow half of an ingest (951 s of a 971 s ferret + build) and reported only at its ends. - name: readViewport matlab_path: "+ndi/+fun/+doc/+gene/readViewport.m" - matlab_last_sync_hash: "b9972abf1" + matlab_last_sync_hash: "10687267a" python_path: "ndi/fun/doc_gene.py" input_arguments: - name: bin_size @@ -2754,9 +2781,57 @@ functions: one level cannot collide), same existence check, same fetch-again-on-miss. + - name: tileFileName + matlab_path: "+ndi/+fun/+doc/+gene/tileFileName.m" + matlab_last_sync_hash: "10687267a" + python_path: "ndi/fun/doc_gene.py" + input_arguments: + - name: tiles_props + type_matlab: "struct" + type_python: "dict" + - name: index + type_python: "int" + output_arguments: + - name: name + type_python: "str" + decision_log: > + Exact match. The tile INDEX is zero-based on both sides -- it is a + grid position, and index_order defines it as row*tile_columns + + column, so it is internal data per the indexing_policy. The FILE + SUFFIX is one-based, because a DID file series names its first + member NAME_1 and addFileSeries rejects anything else. The + difference between the two is the document's tile_index_origin, + which this function adds. + + - name: tileIndexFromName + matlab_path: "+ndi/+fun/+doc/+gene/tileIndexFromName.m" + matlab_last_sync_hash: "10687267a" + python_path: "ndi/fun/doc_gene.py" + decision_log: > + Exact match; the inverse of tileFileName, for readers that walk a + document's stored names and work back to (row, column). + + - name: tileIndexOrigin + matlab_path: "+ndi/+fun/+doc/+gene/tileIndexOrigin.m" + matlab_last_sync_hash: "10687267a" + status: matlab_only + decision_log: > + Python needs nothing here, by design. The function reads an + optional numeric document field with a default, which in MATLAB + takes isfield, isnumeric and isscalar guards and so earns a name; + in Python it is one dict.get call from the standard library, + written inline in tileFileName and tileIndexFromName. That puts it + on the matlab_only side of the line rather than + ported_differently: the replacement is the language, not something + in this repo. + + The BEHAVIOUR is mirrored and tested on both sides -- an absent, + empty, non-numeric or non-scalar tile_index_origin reads as 0, + which is the legacy zero-based tile naming. + - name: exportRegion matlab_path: "+ndi/+fun/+doc/+gene/exportRegion.m" - matlab_last_sync_hash: "b9972abf1" + matlab_last_sync_hash: "10687267a" python_path: "ndi/fun/doc_gene.py" decision_log: > Exact match. Returns raw counts, never density. @@ -2919,7 +2994,7 @@ functions: - name: makeCells matlab_path: "+ndi/+fun/+doc/+gene/makeCells.m" - matlab_last_sync_hash: "6ae508708" + matlab_last_sync_hash: "b589db9bb" python_path: "ndi/fun/doc_gene.py" decision_log: > Writes cells.tsv, optionally contours.bin, and stores the document. @@ -2971,11 +3046,11 @@ functions: # ======================================================================= # +ndi/+fun/+doc/+gene -- spatial-transcriptomics ingest additions # - # A group of MATLAB functions that lift the .gef / .h5ad ingest path out - # of ndi.gui.app.GeneIngest so a script, a batch importer or a Python - # caller can drive it without opening the GUI (VH-Lab/NDI-matlab - # PR #933). The four recorded here have no Python counterpart yet: - # ndi.gui.app.GeneIngest itself is not ported either (see the entry in + # A group of five MATLAB functions that lift the .gef / .h5ad ingest + # path out of ndi.gui.app.GEFManager so a script, a batch importer or a + # Python caller can drive it without opening the GUI (VH-Lab/NDI-matlab + # PR #933). None of the five has a Python counterpart yet: + # ndi.gui.app.GEFManager itself is not ported either (see the entry in # src/ndi/gui/app/ndi_matlab_python_bridge.yaml), and the ingest path # only becomes reachable from Python once the GUI class or a # non-interactive equivalent lands. The tile / cell / pyramid readers @@ -2994,7 +3069,7 @@ functions: - name: fromCellBin matlab_path: "+ndi/+fun/+doc/+gene/fromCellBin.m" - matlab_last_sync_hash: "6ae5087" + matlab_last_sync_hash: "b589db9bb" status: porting_deferred decision_log: > MATLAB reads a SAW cellbin .h5ad with @@ -3005,9 +3080,13 @@ functions: about row order. Ports as a group with the other three entries under this heading; no current Python caller. + NDI-matlab b589db9bb changed one line of the help text -- + recordSource now says it creates a fileReference rather than a + generic_file, following makeSourceFile. Comment only; no + behavioural change, and nothing to port here. - name: fromGEF matlab_path: "+ndi/+fun/+doc/+gene/fromGEF.m" - matlab_last_sync_hash: "a178e46" + matlab_last_sync_hash: "38aa4e8fb" status: porting_deferred decision_log: > MATLAB reads a Stereo-seq .gef with ndr.format.stereoseq.readGEF @@ -3016,16 +3095,38 @@ functions: path in one call. Ports with the other three entries under this heading. + NDI-matlab b589db9bb changed three lines of the help text: two + naming ndi.gui.app.GeneIngest, which was renamed GEFManager, and + one saying recordSource creates a fileReference rather than a + generic_file, following makeSourceFile. Comments only; no + behavioural change, and nothing to port here. + + NDI-matlab 38aa4e8fb stopped promoting the reader's int32/uint16 records + to double before calling makePyramid -- 32 bytes a record held for + the whole build where 14 does, 14 GB on a 7.7e8-record section -- + and hands makePyramid a progressFcn scaled onto the last 30% of its + own. Both belong to the deferred port, not to doc_gene.py. - name: makeSourceFile matlab_path: "+ndi/+fun/+doc/+gene/makeSourceFile.m" - matlab_last_sync_hash: "6ae5087" + matlab_last_sync_hash: "b589db9bb" status: porting_deferred decision_log: > - MATLAB creates the generic_file document that the spatial gene - expression documents' source_file_id dependency points at, so a - pyramid can name the .gef it was built from. Describes by default - (name + checksum + dates), attaches only when attachFile=true. - Ports with the other three entries under this heading. + MATLAB creates the document that the spatial gene expression + documents' source_file_id dependency points at, so a pyramid can + name the .gef it was built from. + + WHICH document changed, in NDI-matlab b589db9bb. Describing a file the + database does not hold is now a fileReference -- a new class with no + files section at all -- and only attachFile=true produces a + generic_file with the bytes attached. The old shape was a + generic_file with nothing attached, which is invalid, since + generic_file requires generic_file.ext; nothing enforced it until + DID-matlab #182 closed a fail-open, at which point adding such a + session to a dataset failed at the add rather than at the write + that made it. + + Ports with the other three entries under this heading. A Python + port must create a fileReference, not a generic_file. - name: readNotes matlab_path: "+ndi/+fun/+doc/+gene/readNotes.m" diff --git a/src/ndi/gui/app/genepyramid/cli.py b/src/ndi/gui/app/genepyramid/cli.py index b6d1d615..0f395c49 100644 --- a/src/ndi/gui/app/genepyramid/cli.py +++ b/src/ndi/gui/app/genepyramid/cli.py @@ -42,6 +42,18 @@ def build_parser() -> argparse.ArgumentParser: metavar="A,B,C", help="comma list of gene symbols or accessions; default is every gene", ) + p.add_argument( + "--gene-layers", + default="", + metavar="A:red,B:cyan", + help="open these genes as their own additive layers on top of the " + "base image, optionally each in a named colormap. Different from " + "--genes, which filters the base layer to a subset and leaves one " + "picture: this keeps the base and adds a layer per gene, which is " + "what makes two genes comparable. A gene with no colour takes the " + "next one from the cycle. Colormap names may contain spaces " + "(napari ships 'bop orange'), so quote the whole value.", + ) p.add_argument( "--no-density", action="store_true", @@ -60,6 +72,35 @@ def build_parser() -> argparse.ArgumentParser: "Otherwise a document id, or a directory holding cells.tsv as " "extract_cells.py writes it.", ) + p.add_argument( + "--outlines", + action="store_true", + help="draw cell boundary polygons, not just centroids. Needs " + "--cells and a cells document whose contours_present is 1.", + ) + p.add_argument( + "--name", + default="", + metavar="TEXT", + help="name for the image layer. Default is the pyramid's label, " + "which the ingest usually took from the file, so it names the " + "SECTION rather than what is being shown.", + ) + p.add_argument( + "--labels", + default="", + metavar="A,B", + help="which cell type labelings to show, by name; 'none' shows no " + "labeling panel. Default shows every one the cells document has, " + "each with its own switch, and names any pair that says the same " + "thing about the section -- a subclass call and the clustering it " + "was transferred onto -- so you can switch one off.", + ) + p.add_argument( + "--no-controls", + action="store_true", + help="do not dock the gene / density panel", + ) p.add_argument("--list", action="store_true", help="list the pyramids and exit") p.add_argument( "--report", @@ -69,12 +110,66 @@ def build_parser() -> argparse.ArgumentParser: return p -def _open_session(path): +def _asDataset(path): + """Open PATH as a dataset. A named function, not an inline import, so + a test can stand in for it: what _open_session does AROUND the two + readings -- which it prefers, and that a failure of one does not lose + the other -- is not about how either is constructed.""" + from ndi.dataset import ndi_dataset_dir + + return ndi_dataset_dir(path) + + +def _asSession(path): + """Open PATH as a session. Named for the same reason as _asDataset, + and with more cause: ``ndi.session.dir`` resolves to the CLASS rather + than the module (the package attribute shadows the submodule), so + patching it where it is imported does not work.""" from ndi.session.dir import ndi_session_dir return ndi_session_dir(path) +def _open_session(path): + """Open PATH, whether it holds a session or a downloaded dataset. + + THE TWO LOOK IDENTICAL ON DISK. A dataset keeps its database at + /.ndi exactly as a session does, so ndi_session_dir opens a + downloaded dataset without complaint -- and then finds almost + nothing in it. A dataset's documents may live in LINKED SESSIONS, + and only ndi_dataset.database_search follows those links; the + session reader looks in the dataset's own database and stops there. + + The symptom is a downloaded dataset reporting "no + spatialGeneExpressionPyramid" while the same directory opened in + MATLAB as an ndi.dataset.dir lists the pyramid and its levels. The + path is right, the data is there, and the reader is looking one + level too shallow. + + So the dataset reading is tried first and kept when it finds + pyramids. A plain session opened as a dataset simply has no linked + sessions and answers the same as before, but that is not relied on: + if the dataset reading raises, or finds nothing where the session + reading finds something, the session reading wins. + """ + opened = None + for describe in (_asDataset, _asSession): + try: + candidate = describe(path) + except Exception: # noqa: BLE001 - the other reading may still work + continue + try: + if _pyramids(candidate): + return candidate + except Exception: # noqa: BLE001 - likewise + continue + if opened is None: + opened = candidate + if opened is None: + raise ValueError(f"{path} could not be opened as either an NDI session or a dataset") + return opened + + def _pyramids(session): from ndi.query import ndi_query @@ -139,9 +234,83 @@ def _resolve_cells(session, pyr_doc, spec: str): ) cols, info = readCells(session, docs[0]) info["source"] = docs[0].id + # The document itself, not just its id: the cell-type panel needs it to + # find the cellTypeLabels that depend on it, and re-querying by id in + # the viewer would be the same search done twice. + info["document"] = docs[0] return cols, info +def _resolve_labelings(spec: str): + """--labels into what openPyramid wants: names, none of them, or None. + + ``none`` is an EMPTY LIST rather than None, because the two mean + different things here: no names asked for, against no choice made. + """ + spec = (spec or "").strip() + if not spec: + return None + if spec.lower() == "none": + return [] + return [s.strip() for s in spec.split(",") if s.strip()] + + +def _resolve_outlines(session, pyr_doc, spec: str, cells_info, cells=None): + """Boundary polygons for the cells document --cells resolved. + + Returns None after printing why, so main can exit 1. A directory of + extracted cells is refused rather than half-supported: contours.bin + is read out of the DOCUMENT, and pretending otherwise would fail + later with a message about a missing file rather than about the + thing the caller actually asked for. + """ + import os + + from ndi.fun.doc_gene import readContours + from ndi.query import ndi_query + + if spec != "auto" and os.path.isdir(spec): + print( + "--outlines reads contours.bin from the cells DOCUMENT, and " + f"{spec} is a directory. Drop the path from --cells to use the " + "pyramid's own cells document.", + file=sys.stderr, + ) + return None + + if cells_info is not None and not cells_info.get("contoursPresent"): + print( + "this cells document has contours_present 0: it was written " + "without boundaries, so there is nothing to draw. Re-ingest the " + "cellbin with contours.", + file=sys.stderr, + ) + return None + + docs = session.database_search( + ndi_query("").isa("spatialGeneExpressionCells") + & ndi_query("").depends_on("spatialGeneExpressionPyramid_id", pyr_doc.id) + ) + if spec != "auto": + docs = [d for d in docs if d.id == spec] + if len(docs) != 1: + print(f"expected one cells document for --outlines, found {len(docs)}", file=sys.stderr) + return None + + try: + polys, info = readContours(session, docs[0], cells=cells) + except Exception as e: + print(f"could not read contours: {e}", file=sys.stderr) + return None + + drawn = sum(1 for p in polys if len(p)) + print( + f"[outlines] {drawn} of {info['nCells']} cells have a boundary " + f"({info['contourReference']}-referenced, {info['nVerticesTotal']} vertices)" + ) + return polys + + def _resolve_genes(session, pyr_doc, spec: str): """Gene symbols or accessions to ZERO-BASED rows. @@ -172,11 +341,28 @@ def _resolve_genes(session, pyr_doc, spec: str): return sorted(set(rows)) +def _resolve_gene_layers(spec: str): + """``--gene-layers`` to the pairs the gene panel opens with. + + Parsed here rather than in the panel so a malformed value is a + startup error rather than a panel that silently opens fewer layers + than were asked for. Whether the SYMBOLS exist is the panel's to + answer -- it holds the gene list, and it says so on the panel. + """ + from .controls import parseGeneLayers + + return parseGeneLayers(spec) or None + + def main(argv=None) -> int: args = build_parser().parse_args(argv) - session = _open_session(args.session) - docs = _pyramids(session) + from .progress import stage + + with stage(f"opening the session at {args.session}"): + session = _open_session(args.session) + with stage("finding the pyramids"): + docs = _pyramids(session) if not docs: print(f"no spatialGeneExpressionPyramid in {args.session}", file=sys.stderr) @@ -201,8 +387,21 @@ def main(argv=None) -> int: print(_describe(d), file=sys.stderr) return 1 + if args.outlines and not args.cells: + print( + "--outlines needs --cells: the boundaries live on the cells " + "document, and centroid-relative vertices need the centroids " + "to be placed at all.", + file=sys.stderr, + ) + return 1 + gene_rows = _resolve_genes(session, pyr, args.genes) - cells, cells_info = _resolve_cells(session, pyr, args.cells) + import contextlib + + reading_cells = stage("reading the cell table") if args.cells else contextlib.nullcontext() + with reading_cells: + cells, cells_info = _resolve_cells(session, pyr, args.cells) density = not args.no_density if args.report: @@ -225,7 +424,12 @@ def main(argv=None) -> int: if cells_info.get("segmentationMethod"): print(f" segmented by {cells_info['segmentationMethod']}") if cells_info.get("contoursPresent"): - print(" contours present but not read; centroids only") + print( + f" contours present ({cells_info.get('contourReference')}" + f"-referenced); --outlines draws them" + ) + else: + print(" no contours in this document; centroids only") print(f" {'bin':>5} {'height':>8} {'width':>8} {'tiles':>12}") for lv in levels: print( @@ -239,7 +443,30 @@ def main(argv=None) -> int: overlay = None if cells is not None: overlay = {"x": cells["x"], "y": cells["y"]} - openPyramid(session, pyr, gene_rows=gene_rows, density=density, cells=overlay) + + polys = None + if args.outlines: + with stage("reading the cell boundaries"): + # The centroids are handed over rather than re-read: the + # boundaries are stored relative to them, and cells.tsv is + # the slower of the two files to parse. + polys = _resolve_outlines(session, pyr, args.cells, cells_info, cells) + if polys is None: + return 1 + + openPyramid( + session, + pyr, + gene_rows=gene_rows, + density=density, + cells=overlay, + outlines=polys, + cells_doc=(cells_info or {}).get("document"), + controls=not args.no_controls, + name=args.name or None, + labelings=_resolve_labelings(args.labels), + gene_layers=_resolve_gene_layers(args.gene_layers), + ) return 0 diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py new file mode 100644 index 00000000..6ada9512 --- /dev/null +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -0,0 +1,2626 @@ +"""Dock panels: choose density or counts, filter cell types, pick genes. + +Built with qtpy directly rather than magicgui, to match the widgets the +FICTURE viewer in the bscholl project already uses -- a filterable list +with a checkbox per entry, and radio buttons for the two display modes. +Those are the shapes people here already know, and a different idiom for +the same job is a cost with no return. + +PANEL ORDER, top to bottom on the right: the display mode, then the cell +types, then the genes. The first two are short and are read once; the +gene list is long and is scrolled, so putting it last keeps the others +on screen. + +Everything Qt is imported inside the functions. This module is imported +by :mod:`viewer`, which a headless caller may import to compose layers by +hand, and a top-level ``qtpy`` import would drag Qt into that. +""" + +from __future__ import annotations + +from typing import Any + +__all__ = [ + "geneIndex", + "parseGeneLayers", + "abundanceBand", + "countsOrder", + "labelAgreement", + "selectLabelings", + "addGenePanel", + "addDisplayPanel", + "addCellTypePanel", + "addAllPanels", + "addCloudPanel", + "cloudSignInDialog", + "buildCloudSignInDialog", + "cloudProfileChoices", + "cloudLogoPath", + "cloudLogoLabel", + "cloudSessionLooksLikely", + "saveCloudProfile", + "geneLayers", + "geneName", + "contrastWindow", + "pulseGamma", + "applyGeneAppearance", + "applySolo", + "addGeneAppearancePanel", + "addGeneTourPanel", + "blurLevels", + "basePixelUm", +] + + +# --------------------------------------------------------------- genes + + +def parseGeneLayers(spec: str): + """``"HPCAL1:green,RORB:blue,SST"`` to ``[(symbol, colormap|None), ...]``. + + One layer per gene, each in its own colour, which is a different + request from ``--genes``: that one FILTERS the base layer to a + subset, leaving one picture. This adds a layer per gene on top of + the base, which is what makes two genes comparable. + + THE COLOUR IS OPTIONAL and None means "take the next one from the + cycle", so a caller can name the colours that matter and leave the + rest. Colormap names may contain spaces -- napari ships ``bop + orange``, ``bop blue`` and ``bop purple`` -- so only the ends of each + field are trimmed, never the middle. + + A gene named twice is kept once, at its first position: ticking the + same box twice adds nothing the second time, and the layer names + would collide. + + Args: + spec: comma-separated ``SYMBOL`` or ``SYMBOL:COLORMAP`` entries. + Empty gives an empty list rather than None -- "no layers + asked for" is a fact, not a missing answer. + + Returns: + A list of ``(symbol, colormap or None)``, in the order given, + which is the order the layers are added and so the order the + colour cycle advances. + """ + out, seen = [], set() + for entry in str(spec or "").split(","): + entry = entry.strip() + if not entry: + continue + symbol, sep, colour = entry.partition(":") + symbol = symbol.strip() + if not symbol or symbol in seen: + continue + seen.add(symbol) + colour = colour.strip() if sep else "" + out.append((symbol, colour or None)) + return out + + +def geneIndex(ids, names) -> dict: + """Collapse a gene list to ONE ENTRY PER SYMBOL. + + Real annotations name the same gene on several rows -- this opossum + list repeats 324 symbols in its first 2,000 -- because a symbol can + carry several accessions: transcript variants, a duplicated model, a + scaffold copy. The rows are separate columns of the pyramid, so a + picker that offered one item per ROW listed the same gene many times + over and left the user to guess which was the gene. + + Every row a symbol names is kept, and callers sum them: taking the + first would show part of the gene's signal and look like the whole. + + Args: + ids: accessions, one per gene row, in zero-based row order. + names: symbols, same length. A blank falls back to the accession. + + Returns: + A dict, sorted case-insensitively by symbol, mapping symbol to + ``{"rows": [...], "accessions": [...]}``. + """ + by: dict[str, dict] = {} + for row, (acc, nm) in enumerate(zip(ids, names)): + acc = str(acc or "").strip() + symbol = str(nm or "").strip() or acc or f"row {row}" + entry = by.setdefault(symbol, {"rows": [], "accessions": []}) + entry["rows"].append(row) + if acc and acc not in entry["accessions"]: + entry["accessions"].append(acc) + return dict(sorted(by.items(), key=lambda kv: kv[0].lower())) + + +def abundanceBand(totals, lo: float = 0.0, hi: float = 100.0): + """Which entries fall in a percentile band OF THE READS. + + NOT a percentile of the list, and the difference is the whole point. + Rank least to most abundant and accumulate the totals; an entry's + position is where its own reads sit in that cumulative sweep. So + ``lo=90`` keeps the handful of genes that between them carry the top + 10% OF ALL READS. Ranking by list position instead would keep 10% of + the genes, which in a real section is a completely different and much + larger set: expression is heavy-tailed, and a few hundred genes out + of thirty thousand carry most of the signal. + + An entry occupies an INTERVAL of that sweep -- the mass before it to + the mass after -- and sits at the interval's midpoint. Using either + endpoint would let one dominant gene straddle a cut and be kept or + dropped on the strength of its own bulk. + + TIES SHARE ONE MIDPOINT, computed over the whole tied group. + Otherwise ties break on sort order, which is arbitrary, and two genes + with identical totals land on opposite sides of the same threshold. + It matters most at the bottom, where thousands of genes tie near zero. + + A consequence that looks like a bug the first time: an entry whose + own reads are a large share of the total cannot be removed by a small + top cut. One gene carrying 30% of all reads has the interval + [70, 100] and the midpoint 85, so ``hi=99`` keeps it -- correctly, + since dropping it would remove 30% of the reads, not 1%. *info* + reports that gene's share and the cut that would drop it, so a panel + can say so rather than appear inert. + + Args: + totals: reads per entry, array-like. + lo, hi: band edges in percent. + + Returns: + ``(keep, info)``. *keep* is a boolean array over *totals*. + """ + import numpy as np + + tot = np.asarray(totals, dtype=np.float64).ravel() + n = len(tot) + grand = float(tot.sum()) + if n == 0 or grand <= 0: + return np.ones(n, bool), {"available": False, "nTotal": n} + + loudest = int(np.argmax(tot)) + top_share = float(tot[loudest] / grand * 100.0) + info = { + "available": True, + "nTotal": n, + "lo": float(lo), + "hi": float(hi), + "topRow": loudest, + "topShare": top_share, + # Where the loudest entry sits, so a panel can name the cut that + # would actually drop it. Its interval is [100 - share, 100]. + "topMid": 100.0 - top_share / 2.0, + } + if lo <= 0.0 and hi >= 100.0: + return ( + np.ones(n, bool), + {**info, "nKept": n, "pctReads": 100.0, "droppedTop": []}, + ) + + order = np.argsort(tot, kind="stable") + srt = tot[order] + cum = np.cumsum(srt) + before = cum - srt + start = np.flatnonzero(np.r_[True, srt[1:] != srt[:-1]]) + stop = np.r_[start[1:], n] - 1 + group_mid = (before[start] + cum[stop]) / 2.0 / grand * 100.0 + mid = np.repeat(group_mid, np.r_[start[1:], n] - start) + + in_band = (mid >= lo) & (mid <= hi) + keep = np.zeros(n, bool) + keep[order[in_band]] = True + + # Which loud entries the top cut removed. At hi=99 that is a handful + # of names, and naming them is the useful feedback -- "you just + # dropped MBP" rather than "1,204 genes hidden". + dropped = [] + if hi < 100.0: + for i in order[::-1][:40]: + if not keep[i]: + dropped.append(int(i)) + if len(dropped) == 8: + break + return keep, { + **info, + "nKept": int(keep.sum()), + "pctReads": float(tot[keep].sum() / grand * 100.0) if keep.any() else 0.0, + "droppedTop": dropped, + } + + +# Distinct colours rather than one map applied to everything: the layers +# blend additively, so two genes in the same colormap make one picture +# that neither of them is. +# +# IN RAINBOW ORDER, long wavelength to short: red, yellow, green, cyan, +# blue, then magenta. Genes are assigned in the order they are ticked, so +# the order of this tuple is the order a reader sees them arrive -- and a +# spectral run is one they can hold in their head and read back off the +# picture. Magenta is not spectral at all; it closes the circle, which is +# where the eye expects it after blue. +_GENE_COLORMAPS = ("red", "yellow", "green", "cyan", "blue", "magenta") + + +# --------------------------------------------------- gene layer appearance + +# The one thing that separates a gene layer from the base image and the +# cell overlays. Named rather than spelled out at each use: three panels +# now depend on it, and a prefix that drifts in one of them would leave a +# control quietly governing nothing. +_GENE_PREFIX = "gene: " + +# napari rejects a gamma of exactly zero, and the value is a divisor in +# its shader. Small enough to look like zero, large enough to be legal. +_GAMMA_FLOOR = 0.01 + + +def geneLayers(viewer): + """The gene layers, in the order they were added. + + EXCLUDES the base image, whose name is the pyramid's own label, and + the cell overlays. That is the whole point of these controls: the + base layer is the anatomy read underneath, and dimming or gamma- + pulsing it would take the reference frame away from the thing being + pointed at. + """ + out = [] + for layer in getattr(viewer, "layers", ()) or (): + if str(getattr(layer, "name", "")).startswith(_GENE_PREFIX): + out.append(layer) + return out + + +def geneName(layer) -> str: + """A gene layer's symbol, without the prefix the layer list needs.""" + name = str(getattr(layer, "name", "")) + return name[len(_GENE_PREFIX) :] if name.startswith(_GENE_PREFIX) else name + + +def contrastWindow(reference, contrast: float): + """Narrow a layer's contrast window by *contrast*, about its floor. + + ``reference`` is ``(lo, hi)`` as the layer was auto-set when it + arrived, and the result is ``(lo, lo + (hi - lo) / contrast)``. + + THE REFERENCE IS THE POINT. Applying a factor to whatever the limits + are NOW compounds -- three nudges up and one back down does not + return you to where you started, and the knob stops meaning a + setting and starts meaning a history. Every change is computed from + the window the layer was born with, so the number on the control is + the whole state. + + Above 1 narrows the window, which is what "more contrast" means for + a sparse signal: the dim end of the range is where all the counts + are, so the picture brightens and separates. Below 1 widens it. + + The floor is held rather than the centre because these layers start + at zero and blend additively: raising the floor would punch holes in + the background where a gene is merely absent. + """ + lo, hi = float(reference[0]), float(reference[1]) + contrast = max(1e-6, float(contrast)) + span = (hi - lo) / contrast + # A window with no width is not a picture. One count of separation is + # the smallest thing the data can express, so that is the smallest + # window worth handing to napari. + return (lo, lo + span if span > 0 else lo + 1.0) + + +def pulseGamma(base: float, seconds: float, period: float = 1.0) -> float: + """Gamma partway through one pulse: base -> ~0 -> base, once a period. + + A TRIANGLE, not a sine. The eye reads a linear ramp as a steady + sweep and a sinusoid as a lingering pause at each end; what this is + for is saying "this one, now", and the sweep says it more clearly. + + Args: + base: the gamma to return to, and to depart from. + seconds: elapsed time. Any value works, including past the end of + a period -- the pulse repeats. + period: seconds per pulse. + + Returns: + A gamma in ``[_GAMMA_FLOOR, base]``, at the floor rather than at + zero because napari will not take zero. + """ + if period <= 0: + return float(base) + phase = (float(seconds) % period) / period + # 1 at the start of the period, 0 at its middle, 1 at its end. + away = abs(2.0 * phase - 1.0) + return max(_GAMMA_FLOOR, float(base) * away) + + +def applyGeneAppearance(layers, contrast: float, gamma: float, references: dict) -> int: + """Put one contrast and one gamma on every gene layer. + + *references* maps layer name to the ``(lo, hi)`` the layer was + auto-set to when it arrived, and is FILLED IN HERE for any layer not + yet in it. A layer's own current limits are the right reference only + the first time; after that they are this function's own output. + + Returns: + How many layers were changed. A layer that refuses either + setting is skipped rather than allowed to stop the rest -- one + odd layer must not cost the control. + """ + done = 0 + for layer in layers: + name = str(getattr(layer, "name", "")) + try: + if name not in references: + lo, hi = layer.contrast_limits + references[name] = (float(lo), float(hi)) + layer.contrast_limits = list(contrastWindow(references[name], contrast)) + layer.gamma = max(_GAMMA_FLOOR, float(gamma)) + except Exception: # noqa: BLE001 - see docstring + continue + done += 1 + return done + + +def countsOrder(totals, ascending: bool = False): + """Positions ordered by count, ties left alone. + + STABLE, and that is the whole content of this function. The list + arrives alphabetical, so a stable sort leaves genes with equal counts + in alphabetical order instead of an arbitrary one -- and at the + bottom of a real section thousands of them tie, which is precisely + where an arbitrary order is most annoying to read. + + Sorting ASCENDING is a separate sort rather than the descending one + reversed: reversing would reverse the ties too, so the alphabetical + order inside each tied group would come back backwards in one + direction and forwards in the other. + + Args: + totals: counts per entry, array-like. + ascending: quietest first rather than loudest first. + + Returns: + An int array of positions into *totals*. + """ + import numpy as np + + t = np.asarray(totals) + return np.argsort(t if ascending else -t, kind="stable") + + +def addGenePanel(viewer, session, pyr_doc, initial=None) -> Any: + """A filterable list of gene symbols, one checkbox each. + + Ticking a symbol ADDS ITS OWN LAYER rather than replacing the base + image, which is what makes two genes comparable: they land as + separate additive layers in different colours and the base stays put + underneath. Unticking removes that layer again, and removing the + layer in napari's own list unticks the box, so the two never disagree. + + Variants are already combined by :func:`geneIndex`; every row a + symbol names is summed into its layer, and its counts are the sum + over those rows too. + + TWO FILTERS, applied together. The text box answers "where is this + gene"; the abundance band answers "which genes carry the signal". + They compose, because narrowing to the loud genes and then searching + within them is the normal way to use both. Sorting is separate from + both and survives them: alphabetical to find a gene you can name, by + counts to see which ones there are -- which is what the band leaves + you wanting, since a few hundred loud genes in alphabetical order + still hide their own ranking. + """ + import numpy as np + from qtpy.QtCore import Qt, QTimer + from qtpy.QtWidgets import ( + QDoubleSpinBox, + QHBoxLayout, + QLabel, + QLineEdit, + QListWidget, + QListWidgetItem, + QPushButton, + QVBoxLayout, + QWidget, + ) + + from ....fun.doc_gene_export import readGeneList, readGeneTotals + from .multiscale import layerSpec + + ids, names = readGeneList(session, pyr_doc) + index = geneIndex(ids, names) + n_variants = sum(1 for e in index.values() if len(e["rows"]) > 1) + + # Counts per SYMBOL, summed over its variant rows -- the same rows the + # layer sums, so the number in the list is the number the picture is + # drawn from. + totals, _records = readGeneTotals(session, pyr_doc) + if totals is not None and len(totals) != len(ids): + # Indexing a short totals array by gene row would either raise or, + # worse, put one gene's reads on another. Neither is worth risking + # for a column. + print( + f"[genepyramid] gene_totals.tsv has {len(totals)} rows but the " + f"gene list has {len(ids)}; counts and the abundance band are " + f"omitted rather than guessed." + ) + totals = None + if totals is None: + symbol_totals = None + else: + symbol_totals = np.array([int(totals[e["rows"]].sum()) for e in index.values()], np.int64) + + box = QWidget() + outer = QVBoxLayout(box) + base_head = f"{len(index):,} genes" + if n_variants: + base_head += f" ({n_variants:,} with variants, summed)" + header = QLabel(base_head) + + def _headText(info=None) -> str: + # The band's own count lives here rather than in a line of its + # own: it is the same fact the header already states, narrowed. + if not info or not info.get("available") or info["nKept"] == info["nTotal"]: + return base_head + return ( + f"{info['nKept']:,} of {info['nTotal']:,} genes · " + f"{info['pctReads']:.1f}% of reads" + ) + + outer.addWidget(header) + + search = QLineEdit() + search.setPlaceholderText("filter by symbol or accession") + outer.addWidget(search) + + # Column headers, not a menu: two buttons that read like a table's + # header row and behave like one -- click to sort by that column, + # click again to reverse it. Four orders, which is what looking at a + # ranked list actually needs; a menu would have listed all four and + # made the reader pick the wording rather than the direction. + sort_row = QHBoxLayout() + name_btn = QPushButton("Gene") + count_btn = QPushButton("Counts") + for b in (name_btn, count_btn): + b.setFlat(True) + sort_row.addWidget(b) + count_btn.setEnabled(symbol_totals is not None) + sort_row.addStretch() + outer.addLayout(sort_row) + + listw = QListWidget() + # THE LIST IS WHAT THE PANEL IS FOR. Left to its own size hint it came + # up four rows tall, wedged between the band above and the blur below, + # and finding a gene among 26,444 meant scrolling a porthole. A dock + # panel is only as tall as it is asked to be, so it is asked. + listw.setMinimumHeight(260) + items: dict[str, Any] = {} + symbols = list(index) + entries = list(index.values()) + state = {"busy": False, "colour": 0, "keep": None, "sigma": 0.0, "raw": {}} + + def _makeItem(i: int): + symbol = symbols[i] + rows, accs = entries[i]["rows"], entries[i]["accessions"] + text = symbol + if symbol_totals is not None: + text += f" \u2014 {int(symbol_totals[i]):,}" + if len(rows) > 1: + text += f" ({len(rows)} variants)" + item = QListWidgetItem(text) + item.setFlags(item.flags() | Qt.ItemIsUserCheckable) + item.setCheckState(Qt.Unchecked) + item.setData(Qt.UserRole, symbol) + # Accessions are searchable too: a symbol is what people usually + # type, but not everything here has one. + item.setData(Qt.UserRole + 1, " ".join([symbol] + accs).lower()) + # The symbol's ORIGINAL position, carried on the item so that the + # abundance mask -- computed once, in that order -- is still + # readable after the list has been re-sorted into another one. + item.setData(Qt.UserRole + 2, i) + tip = [ + symbol, + f"{len(rows)} pyramid column(s): {', '.join(accs) or '(no accession)'}", + ] + if totals is not None: + # By ROW, not zipped against the accessions: duplicates are + # dropped from that list, so position there is not position + # here and pairing them would attach the wrong number. + tip.append("counts per column: " + ", ".join(f"{int(totals[r]):,}" for r in rows)) + item.setToolTip("\n".join(tip)) + return item + + def _populate(order): + """Fill the list in ORDER, a sequence of original positions. + + Rebuilt rather than reordered in place. Moving thirty thousand + items one at a time is quadratic, and sorting them through a + Python comparison is no cheaper than making them again -- while + this is the same single batched pass the panel already pays once. + + THE TICKS COME BACK FROM THE LAYERS, not from remembered state: + the drawn layers are what a tick means, so reading them is what + stops the boxes and the picture from disagreeing after a sort. + """ + was, state["busy"] = state["busy"], True + listw.setUpdatesEnabled(False) + try: + listw.clear() + items.clear() + for i in order: + item = _makeItem(int(i)) + symbol = symbols[int(i)] + if f"gene: {symbol}" in viewer.layers: + item.setCheckState(Qt.Checked) + listw.addItem(item) + items[symbol] = item + finally: + listw.setUpdatesEnabled(True) + state["busy"] = was + _refilter() + + # (column, ascending). Names start A-Z; counts, when asked for, start + # loudest-first, because that is the question a counts column is + # being clicked to answer. + sort_state = {"by": "name", "asc": True} + + def _sortBy(column: str): + if sort_state["by"] == column: + sort_state["asc"] = not sort_state["asc"] + else: + sort_state["by"] = column + sort_state["asc"] = column == "name" + _resort() + + def _resort(): + by, asc = sort_state["by"], sort_state["asc"] + if by == "counts" and symbol_totals is not None: + order = countsOrder(symbol_totals, ascending=asc) + else: + # geneIndex already sorted by name, and symbols are unique, so + # Z-A is that order backwards -- no tie to preserve. + order = range(len(symbols)) if asc else range(len(symbols) - 1, -1, -1) + arrow = " \u25b2" if asc else " \u25bc" + name_btn.setText("Gene" + (arrow if by == "name" else "")) + count_btn.setText("Counts" + (arrow if by == "counts" else "")) + _populate(order) + + name_btn.clicked.connect(lambda *_: _sortBy("name")) + count_btn.clicked.connect(lambda *_: _sortBy("counts")) + + # ---- the abundance band ------------------------------------------ + # THE SLIDER GETS ITS OWN LINE. Sharing one with the label left it a + # stub in a narrow dock -- and this slider carries its own value + # labels at each end, so what was left after "Abundance band" was the + # numbers and no track between them: unreachable rather than merely + # cramped. Same lesson as the rotation slider. + # + # What the band is and what it just did stay hover text rather than a + # paragraph: the explanation is read once and the panel is read every + # time, and the list underneath is what the space is for. + if symbol_totals is None: + note = QLabel("no gene_totals.tsv: no counts, no band") + note.setToolTip( + "This pyramid was written without gene_totals.tsv, so per-gene " + "abundance is unknown. The counts column and the abundance " + "band are both omitted rather than invented." + ) + outer.addWidget(note) + slider = lo_box = hi_box = None + else: + outer.addWidget(QLabel("Abundance band")) + band_row = QHBoxLayout() + slider = lo_box = hi_box = None + try: + # napari depends on superqt, so the two-handle widget is + # normally there. Still optional: the spin boxes below are + # fully functional, and a missing optional import should not + # cost the control. + from superqt import QLabeledDoubleRangeSlider + + slider = QLabeledDoubleRangeSlider(Qt.Horizontal) + slider.setRange(0.0, 100.0) + slider.setValue((0.0, 100.0)) + slider.setDecimals(1) + band_row.addWidget(slider, stretch=1) + except Exception as e: # pragma: no cover - depends on the install + print(f"[genepyramid] no superqt range slider ({e}); using spin boxes") + lo_box = QDoubleSpinBox(minimum=0.0, maximum=100.0, decimals=1, singleStep=0.5) + hi_box = QDoubleSpinBox(minimum=0.0, maximum=100.0, decimals=1, singleStep=0.5) + hi_box.setValue(100.0) + band_row.addWidget(lo_box) + band_row.addWidget(hi_box) + outer.addLayout(band_row) + + outer.addWidget(listw, stretch=1) + + # ---- blur --------------------------------------------------------- + # Gene expression is a handful of counts in scattered single pixels. + # Drawn honestly it is dust; blurred it reads as a pattern. On the + # GENE layers only -- the base layer is already dense and the cell + # layers are not what anyone is squinting at. + blur_row = QHBoxLayout() + blur_row.addWidget(QLabel("blur")) + blur = QDoubleSpinBox() + blur.setRange(0.0, 200.0) + blur.setDecimals(1) + blur.setSingleStep(1.0) + blur.setSuffix(" um") + blur.setToolTip( + "Spreads each gene's counts over its neighbourhood, so a sparse\n" + "gene reads as a pattern instead of dust. The width is a DISTANCE,\n" + "so it stays the same as you zoom between levels. 0 is off.\n" + "Spreading lowers the peak a long way -- the contrast limits are\n" + "reset each time so the layer stays visible." + ) + blur_row.addWidget(blur, 1) + outer.addLayout(blur_row) + + status = QLabel("") + status.setWordWrap(True) + clear = QPushButton("Remove all gene layers") + row = QHBoxLayout() + row.addStretch() + row.addWidget(clear) + outer.addLayout(row) + outer.addWidget(status) + + # ---- filtering ---------------------------------------------------- + + def _band(): + if slider is not None: + v = slider.value() + return float(v[0]), float(v[1]) + if lo_box is not None: + lo, hi = lo_box.value(), hi_box.value() + # Keep the handles ordered rather than rejecting the edit: a + # user dragging the low box past the high one means to move + # the band, and a silent no-op reads as a broken widget. + return (hi, lo) if lo > hi else (lo, hi) + return 0.0, 100.0 + + def _applyBand(): + if symbol_totals is None: + state["keep"] = None + _refilter() + return + lo, hi = _band() + keep, info = abundanceBand(symbol_totals, lo, hi) + state["keep"] = keep + for w in (slider, lo_box, hi_box): + if w is not None: + w.setToolTip(_bandText(info, symbols)) + header.setText(_headText(info)) + _refilter() + + def _refilter(): + needle = search.text().strip().lower() + keep = state["keep"] + listw.setUpdatesEnabled(False) + try: + for item in items.values(): + i = item.data(Qt.UserRole + 2) + ok = True if keep is None else bool(keep[i]) + if ok and needle: + ok = needle in item.data(Qt.UserRole + 1) + item.setHidden(not ok) + finally: + listw.setUpdatesEnabled(True) + + # A slider emits continuously while dragged and the band is arithmetic + # over thirty thousand entries plus a pass over the list, so it is + # applied once the handle settles rather than on every tick. + debounce = QTimer(box) + debounce.setSingleShot(True) + debounce.setInterval(180) + debounce.timeout.connect(_applyBand) + if slider is not None: + slider.valueChanged.connect(lambda *_: debounce.start()) + elif lo_box is not None: + lo_box.valueChanged.connect(lambda *_: debounce.start()) + hi_box.valueChanged.connect(lambda *_: debounce.start()) + + search.textChanged.connect(lambda *_: _refilter()) + + # ---- layers ------------------------------------------------------- + + def _shown() -> int: + return sum(1 for it in items.values() if it.checkState() == Qt.Checked) + + def _report(extra: str = ""): + n = _shown() + status.setText(f"{n} gene layer(s) shown" + (f" -- {extra}" if extra else "")) + + def _binSizes(): + # Cached: levelTable reads the level documents, and the blur knob + # would otherwise re-read them on every tick of the spin box. + if state.get("bins") is None: + from ....fun.doc_gene import levelTable + + levels, _f = levelTable(session, pyr_doc) + state["bins"] = [lv["binSize"] for lv in levels] + return state["bins"] + + def _blurred(data): + """The ladder as it should be drawn at the current width.""" + if not state["sigma"]: + return data + return blurLevels(data, _binSizes(), state["sigma"] / basePixelUm(pyr_doc)) + + def _add(symbol: str, colour: str | None = None): + name = f"gene: {symbol}" + if name in viewer.layers: + return + rows = index[symbol]["rows"] + spec = layerSpec(session, pyr_doc, rows, True, name) + # The UNBLURRED ladder is kept, because the knob is absolute: each + # change re-blurs from the original rather than blurring what was + # already blurred, which would compound and could not be undone. + state["raw"][name] = spec["data"] + spec["data"] = _blurred(spec["data"]) + # A NAMED COLOUR DOES NOT ADVANCE THE CYCLE. The cycle exists so + # that genes ticked by hand come out different from each other; + # a caller that named its own colours has already solved that, + # and letting those consume cycle positions would make the first + # hand-ticked gene's colour depend on how many were preset. + if colour: + cmap = colour + else: + cmap = _GENE_COLORMAPS[state["colour"] % len(_GENE_COLORMAPS)] + state["colour"] += 1 + try: + layer = viewer.add_image(**spec, colormap=cmap, blending="additive") + except Exception: + # An unknown colormap must not cost the layer. + layer = viewer.add_image(**spec, blending="additive") + try: + layer.reset_contrast_limits() + except Exception: + pass + extra = f"{symbol}: {len(rows)} variants summed" if len(rows) > 1 else "" + _report(extra) + + def _remove(symbol: str): + name = f"gene: {symbol}" + if name in viewer.layers: + del viewer.layers[name] + state["raw"].pop(name, None) + _report() + + def _reblur(_value=None): + state["sigma"] = float(blur.value()) + for name, raw in list(state["raw"].items()): + if name not in viewer.layers: + state["raw"].pop(name, None) + continue + layer = viewer.layers[name] + try: + layer.data = _blurred(raw) + # Spreading a sparse signal lowers its peak a long way, so + # limits set for the unblurred layer would show black. + layer.reset_contrast_limits() + except Exception as e: # noqa: BLE001 - a knob never costs a layer + status.setText(f"blur failed: {e}") + return + _report() + + # DEBOUNCED, and generously. A spin box emits on every keystroke, so + # typing "50" asks for a blur at 5 um first -- and re-blurring is a + # whole-ladder map_overlap that takes long enough to see, so the + # picture visibly redraws at the wrong width before the second digit + # lands. 1.5s is past the gap between two digits and short enough not + # to feel stuck. + # + # The band slider above uses the same trick at 180ms; the difference + # is that dragging a slider WANTS to be followed, while a number + # being typed is not a request until it is finished. + blurWait = QTimer(box) + blurWait.setSingleShot(True) + blurWait.setInterval(1500) + blurWait.timeout.connect(_reblur) + blur.valueChanged.connect(lambda *_: blurWait.start()) + # Enter, or clicking away, means the number IS finished -- no reason + # to sit out the rest of the wait. + blur.editingFinished.connect(lambda *_: (blurWait.stop(), _reblur())) + box._ndi_blur_wait = blurWait + + def _on_item_changed(item): + if state["busy"]: + return + state["busy"] = True + try: + symbol = item.data(Qt.UserRole) + if item.checkState() == Qt.Checked: + _add(symbol) + else: + _remove(symbol) + except Exception as e: + status.setText(f"failed: {e}") + finally: + state["busy"] = False + + listw.itemChanged.connect(_on_item_changed) + + def _on_clear(): + state["busy"] = True + try: + for symbol, it in items.items(): + if it.checkState() == Qt.Checked: + it.setCheckState(Qt.Unchecked) + _remove(symbol) + finally: + state["busy"] = False + _report() + + clear.clicked.connect(_on_clear) + + def _on_layer_removed(event): + """A layer closed in napari's own list unticks its box.""" + name = getattr(getattr(event, "value", None), "name", "") + if not str(name).startswith("gene: "): + return + item = items.get(str(name)[len("gene: ") :]) + if item is None or item.checkState() != Qt.Checked: + return + was, state["busy"] = state["busy"], True + try: + item.setCheckState(Qt.Unchecked) + finally: + state["busy"] = was + _report() + + try: + viewer.layers.events.removed.connect(_on_layer_removed) + except Exception: # pragma: no cover - depends on the napari build + pass + + _resort() + _applyBand() + + # ---- genes asked for at launch ------------------------------------ + # Ticked through the panel's OWN checkbox, not by calling _add + # directly: the box and the layer would otherwise disagree, and + # unticking a gene that was never ticked does nothing at all. This + # way a preset gene behaves exactly like one the reader ticked. + missing = [] + for symbol, colour in parseGeneLayers("") if initial is None else list(initial): + item = items.get(symbol) + if item is None: + missing.append(symbol) + continue + was, state["busy"] = state["busy"], True + try: + item.setCheckState(Qt.Checked) + finally: + state["busy"] = was + _add(symbol, colour) + _report() + if missing: + # Named but not in this pyramid's gene list. Said out loud rather + # than passed over: a demo that quietly opens four of six layers + # looks like the viewer working. + # + # AFTER _report, not before: _report rewrites this same label, so + # the warning set first is the warning nobody sees. + status.setText(f"not in this pyramid: {', '.join(missing)}") + viewer.window.add_dock_widget(box, name="Genes", area="right") + return box + + +def _bandText(info, symbols) -> str: + """What the band actually selected, in words. + + A percentile band is not self-explanatory: 5-95% sounds like it keeps + most genes and in fact keeps a small minority, since the bottom 5% of + READS is spread over many thousands of barely detected ones. Showing + both numbers, and naming what the top cut removed, is what makes the + control readable. + """ + if not info.get("available"): + return "no abundance information for this pyramid." + parts = [ + f"{info['nKept']:,} of {info['nTotal']:,} genes · " + f"{info['pctReads']:.1f}% of all reads" + ] + dropped = info.get("droppedTop") or [] + if dropped: + parts.append("dropped from the top: " + ", ".join(symbols[i] for i in dropped)) + # The confusing case: a top cut that removes nothing because one gene + # is too large to fit inside it. Say why, and say what cut would work, + # rather than letting the control look broken. + elif info["hi"] < 100.0 and info.get("topShare"): + parts.append( + f"nothing dropped at the top: {symbols[info['topRow']]} alone is " + f"{info['topShare']:.1f}% of all reads, so no gene fits in the top " + f"{100 - info['hi']:.1f}%. Set the high edge below " + f"{info['topMid']:.1f} to drop it." + ) + if info["nKept"] == 0: + parts.append("empty band -- nothing listed.") + return "\n".join(parts) + + +# ------------------------------------------------------------- display + + +def addDisplayPanel(viewer, session, pyr_doc, image_layer, density: bool = True) -> Any: + """Radio buttons for density against raw counts. + + Two exclusive states, so radio buttons rather than a checkbox: the + label then says what each choice IS, instead of naming one and + leaving the other implied. Why it matters is hover text on the two + buttons rather than a paragraph in the panel: it is read once. + + Switching only swaps the layer's data. A level is summed over the + selected genes into a 2D array, so the shape does not change and the + camera, the zoom and the level the viewer is on all survive. + """ + from qtpy.QtWidgets import QButtonGroup, QLabel, QRadioButton, QVBoxLayout, QWidget + + from .multiscale import levelArrays + + box = QWidget() + outer = QVBoxLayout(box) + + r_density = QRadioButton("density (counts per base pixel)") + r_counts = QRadioButton("raw counts (summed per bin)") + r_density.setChecked(density) + r_counts.setChecked(not density) + + group = QButtonGroup(box) + group.addButton(r_density) + group.addButton(r_counts) + + # Why the choice exists, as hover text rather than a paragraph: it is + # read once and then never again, and a standing explanation of a + # two-button control is mostly in the way. + why = ( + "Binning sums, so raw counts make a coarse level binSize^2 brighter\n" + "than a fine one and the picture jumps at each level change.\n" + "Density divides that out, so one contrast range serves every level." + ) + for w in (r_density, r_counts): + w.setToolTip(why) + + # Kept, but HIDDEN until something goes wrong. It used to sit under the + # buttons restating which one was just clicked, which the buttons + # already show. Deleting it outright would make a failed switch silent, + # and a control that can fail without saying so is worse than a + # redundant line -- so it appears only when there is something to say. + status = QLabel("") + status.setWordWrap(True) + status.hide() + + def _switch(want_density: bool): + try: + image_layer.data = levelArrays( + session, pyr_doc, _currentRows(image_layer), want_density + ) + image_layer.reset_contrast_limits() + except Exception as e: + status.setText(f"failed: {e}") + status.show() + return + status.hide() + + r_density.toggled.connect(lambda on: _switch(True) if on else None) + r_counts.toggled.connect(lambda on: _switch(False) if on else None) + + outer.addWidget(r_density) + outer.addWidget(r_counts) + outer.addWidget(status) + outer.addStretch() + + viewer.window.add_dock_widget(box, name="Display", area="right") + return box + + +def _currentRows(image_layer): + """The gene rows the base layer was built from, if it recorded them. + + The base layer is whatever openPyramid drew; nothing here changes its + gene selection, so None (all genes) is the honest default rather than + a guess that would quietly widen the picture. + """ + return getattr(image_layer, "_ndi_gene_rows", None) + + +# ---------------------------------------------------------- cell types + + +# Two labelings this close say the same thing about the section. The +# number is reported every time rather than applied silently, so a pair +# that only nearly agrees is visible as a pair that nearly agrees. +REDUNDANT_AT = 0.99 + + +def labelAgreement(a, b) -> float: + """How completely knowing *a* tells you *b*, as a fraction of cells. + + Exact equality of partitions was too strict for the case it was + written for. A subclass call transferred cell by cell onto a + clustering agrees with it almost everywhere and disagrees on a + handful of boundary cells, which is not a second opinion -- it is the + same opinion with noise -- but it is not the same partition either, + so an equality test showed both and the reader was back where they + started. + + For each group of *a*, the majority *b* value is the one *a* would + predict; this returns the fraction of cells that value is right for. + 1.0 means *a* determines *b* exactly, which is what a per-cluster + renaming gives. + + The measure is DIRECTIONAL. A fine clustering determines a coarse + call without the reverse holding, so callers test both ways. + """ + a = list(a) + b = list(b) + if len(a) != len(b): + raise ValueError( + f"labelings of {len(a)} and {len(b)} cells cannot be compared; " + f"they are not labelings of the same cells" + ) + if not a: + return 1.0 + groups: dict[Any, dict] = {} + for va, vb in zip(a, b): + counts = groups.setdefault(va, {}) + counts[vb] = counts.get(vb, 0) + 1 + return sum(max(c.values()) for c in groups.values()) / len(a) + + +def selectLabelings(found, wanted=None): + """Which labelings to draw, and which are the same one twice. + + Pure, so the decision is testable without a display -- it is the part + that can be wrong. + + *found* is a list of ``(labels, info)`` as + :func:`~ndi.fun.doc_gene.readCellTypeLabels` returns them. + + With *wanted* -- a list of label names -- the caller has chosen, and + nothing is collapsed or reordered away from that choice. Names that + are not there come back in *missing* rather than being ignored: a + typo that silently showed everything would look like the flag not + working. + + Without it, everything found is kept and the redundancy is REPORTED + rather than acted on. Dropping one automatically was wrong twice + over: below the threshold it did nothing and the reader still saw + two, and above it the labeling vanished with no way to bring it + back. Naming the pair and letting the reader switch one off is the + same information with the decision left where it belongs. + + Supervised calls are ordered first, so a redundant pair is measured + against the labeling that carries biological names rather than + against a cluster index. + + Returns: + ``(kept, redundant, missing)``. *redundant* holds + ``(name, other_name, forward, reverse)``, the two agreements + measured in both directions so the note can quote a number. + """ + found = sorted(found, key=lambda t: (bool(t[1]["isUnsupervised"]), t[1]["labelName"] or "")) + + if wanted is not None: + want = [str(w).strip() for w in wanted if str(w).strip()] + by_name = {(t[1]["labelName"] or ""): t for t in found} + kept = [by_name[w] for w in want if w in by_name] + return kept, [], [w for w in want if w not in by_name] + + kept, redundant = [], [] + for labels, info in found: + name = info["labelName"] or "(unnamed)" + for kept_labels, kept_info in kept: + forward = labelAgreement(labels, kept_labels) + reverse = labelAgreement(kept_labels, labels) + if max(forward, reverse) >= REDUNDANT_AT: + redundant.append((name, kept_info["labelName"] or "(unnamed)", forward, reverse)) + break + kept.append((labels, info)) + return kept, redundant, [] + + +def addCellTypePanel( + viewer, session, cells_doc, points_layer, shapes_layer=None, labelings=None +) -> Any: + """One checkbox per class, per labeling, over the cell layers. + + A cellbin routinely carries several labelings -- a transferred atlas + call and one or more clusterings -- so each becomes its own group + rather than being merged. They are not interchangeable: a clustering + carries no biological identity. + + ONE ROW EACH, with the classes behind a toggle. A cellbin carries a + couple of labelings with dozens of classes between them, and listing + every class of every one fills the dock before anyone has asked to + see them. + + The row's own checkbox switches the whole labeling off, which drops + its contribution to the filter rather than hiding the cells it + named: an unwanted second opinion should stop having an opinion. + Everything starts on, and a labeling that says the same thing as + another is NAMED rather than removed, so the reader is told which + one to untick instead of finding one of them gone. + + Hiding works by ALPHA, not by removing points. The row order of the + points layer is the row order of cells.tsv and of every labels.tsv + beside it, and deleting rows would break that correspondence for the + next labeling the user toggles. + """ + from qtpy.QtWidgets import ( + QCheckBox, + QHBoxLayout, + QLabel, + QPushButton, + QScrollArea, + QVBoxLayout, + QWidget, + ) + + from ....fun.doc_gene import findCellTypeLabels, readCellTypeLabels + + docs = findCellTypeLabels(session, cells_doc) + if not docs: + return None + + import numpy as np + + found = [] + for d in docs: + try: + labels, info = readCellTypeLabels(session, d) + except Exception as e: + print(f"[genepyramid] skipping a cellTypeLabels document: {e}") + continue + found.append((labels, info)) + if not found: + return None + + kept, redundant, missing = selectLabelings(found, labelings) + + box = QWidget() + outer = QVBoxLayout(box) + + def _note(msg): + print(f"[genepyramid] {msg}") + w = QLabel(msg) + w.setWordWrap(True) + outer.addWidget(w) + + # Named, not acted on: it says WHICH of the two to switch off. + agrees = {} + for name, other, forward, reverse in redundant: + agrees[name] = (other, max(forward, reverse)) + _note( + f"{name} agrees with {other} on {100 * max(forward, reverse):.1f}% " + f"of cells -- untick one of them." + ) + if missing: + have = ", ".join((i["labelName"] or "(unnamed)") for _lb, i in found) + _note(f"no labeling named {', '.join(missing)}; this cells document has: {have}") + if not kept: + _note("no labelings to show.") + + n_points = len(points_layer.data) + # visible[i] is False when ANY ticked-off class of a labeling that is + # still switched ON covers cell i, so two labelings filter jointly + # rather than the last one clicked winning. + hidden_by = {} + switched_off = set() + + def _refresh(): + hide = np.zeros(n_points, dtype=bool) + for (label_name, _category), mask in hidden_by.items(): + if mask is None or label_name in switched_off: + continue + hide |= mask + for layer in (points_layer, shapes_layer): + if layer is None: + continue + try: + colors = np.asarray(layer.face_color).copy() + if colors.shape[0] == len(hide): + colors[:, 3] = np.where(hide, 0.0, 1.0) + layer.face_color = colors + edge = np.asarray(layer.edge_color).copy() + if edge.shape[0] == len(hide): + edge[:, 3] = np.where(hide, 0.0, 1.0) + layer.edge_color = edge + layer.refresh() + except Exception: + pass + + for labels, info in kept: + name = info["labelName"] or "(unnamed)" + kind = "clustering" if info["isUnsupervised"] else "cell type call" + + # ONE ROW PER LABELING, and the classes are behind it. A cellbin + # carries a couple of labelings with dozens of classes between + # them, and listing every class of every one of them fills the + # dock with checkboxes nobody asked to see yet. + head = QHBoxLayout() + master = QCheckBox(f"{name} ({len(info['categories'])} {kind})") + master.setChecked(True) + tip = [f"{name} -- {kind}, {len(info['categories'])} classes"] + if info["nUnlabeled"]: + tip.append(f"{info['nUnlabeled']:,} cells are unlabelled by this") + if name in agrees: + other, share = agrees[name] + tip.append(f"agrees with {other} on {100 * share:.1f}% of cells") + master.setToolTip("\n".join(tip)) + expand = QPushButton("classes \u25b8") + expand.setFlat(True) + head.addWidget(master, stretch=1) + head.addWidget(expand) + outer.addLayout(head) + + arr = np.asarray(labels, dtype=object) + inner = QWidget() + col = QVBoxLayout(inner) + for category in info["categories"]: + mask = arr == category + cb = QCheckBox(f"{category} ({int(mask.sum()):,})") + cb.setChecked(True) + key = (name, category) + + def _toggled(on, key=key, mask=mask): + hidden_by[key] = None if on else mask[:n_points] + _refresh() + + cb.toggled.connect(_toggled) + col.addWidget(cb) + scroll = QScrollArea() + scroll.setWidgetResizable(True) + scroll.setWidget(inner) + scroll.setMaximumHeight(180) + scroll.setVisible(False) + outer.addWidget(scroll) + + def _onExpand(*_, scroll=scroll, expand=expand): + shown = not scroll.isVisible() + scroll.setVisible(shown) + expand.setText("classes \u25be" if shown else "classes \u25b8") + + def _onMaster(on, name=name, scroll=scroll, expand=expand): + # Switching a labeling OFF drops its whole contribution to the + # filter rather than hiding its cells: an unwanted second + # opinion should stop having an opinion, not start hiding + # everything it named. + if on: + switched_off.discard(name) + else: + switched_off.add(name) + scroll.setVisible(False) + expand.setText("classes \u25b8") + expand.setEnabled(on) + _refresh() + + expand.clicked.connect(_onExpand) + master.toggled.connect(_onMaster) + + outer.addStretch() + + viewer.window.add_dock_widget(box, name="Cell types", area="right") + return box + + +# ----------------------------------------------------------------- all + + +def rotationAffine(angle_deg, center): + """A 3x3 homogeneous affine rotating ``(row, col)`` about ``center``. + + Pure, and separate from the panel, because the thing most likely to be + wrong here is the SIGN and the PIVOT, and neither needs a display to + check. + + napari layer coordinates are ``(row, col)`` with row increasing + DOWNWARD -- the pyramid records ``origin_corner: upper-left`` and the + viewer honours it. The two sign flips that come from that cancel, so + the matrix is the ordinary one and a positive angle turns the section + ANTICLOCKWISE on screen. Checked rather than reasoned about: a point + to the right of the pivot lands above it at +90 degrees. + + Args: + angle_deg: rotation in degrees, positive anticlockwise on screen. + center: ``(row, col)`` pivot in WORLD coordinates -- the same space + ``viewer.camera.center`` reports, which is what makes "rotate + about what I am looking at" expressible. + + Returns: + ``numpy.ndarray`` of shape (3, 3), suitable for ``layer.affine``. + """ + import numpy as np + + t = np.deg2rad(float(angle_deg)) + c, s = np.cos(t), np.sin(t) + r = np.array([[c, -s], [s, c]], float) + # A = T(center) . R . T(-center), so the offset is center - R @ center: + # the pivot is the one point the rotation leaves alone. + cy, cx = float(center[0]), float(center[1]) + offset = np.array([cy, cx]) - r @ np.array([cy, cx]) + a = np.eye(3) + a[:2, :2] = r + a[:2, 2] = offset + return a + + +def rotationCenter(viewer): + """The current view centre as ``(row, col)`` in world coordinates. + + ``viewer.camera.center`` is reported as ``(z, y, x)`` even for 2D data, + where z is 0, so the last two entries are the ones that mean anything + here. Taking the LAST two rather than indexing 1 and 2 keeps this + right if a 2-tuple is ever handed back instead. + """ + c = tuple(float(v) for v in viewer.camera.center) + if len(c) < 2: + raise ValueError(f"camera centre has no row/column to read: {c!r}") + return (c[-2], c[-1]) + + +def applyRotation(layers, angle_deg, center): + """Put ONE shared affine on every layer, so they turn together. + + napari applies ``affine`` after each layer's own scale and translate, + so the same matrix means the same world-space rotation on all of them + even though the image ladder, the centroids and the outlines arrive in + different units. That is the whole reason this is one matrix rather + than a rotation per layer: the section, the cells and their outlines + are one picture, and a control that could slide them apart would be a + control that can produce a wrong picture. + + ``None`` entries are skipped -- outlines and centroids are optional and + a session without them is the normal case, not an error. + + Returns: + The affine that was applied. + """ + a = rotationAffine(angle_deg, center) + for layer in layers: + if layer is None: + continue + layer.affine = a + return a + + +def addRotationPanel(viewer, layers) -> Any: + """A slider and a box for turning the whole picture. + + Sections are not mounted square, and a reader lining one up against an + atlas plate or a companion section wants an arbitrary angle rather than + the 90-degree steps napari offers on the canvas. + + THE PIVOT IS TAKEN WHEN THE INTERACTION STARTS, not on every tick. + Rotating the layers does not move the camera, so re-reading the view + centre mid-drag would usually give the same answer -- but "usually" is + the problem: pan during a drag and the picture would jump as the pivot + moved under it. Captured once per interaction, a drag is a drag. + + The angle is ABSOLUTE, not accumulated: every change rebuilds the + affine from zero, so the picture cannot drift out of square through + repeated small adjustments the way an incremental compose would. + """ + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QDoubleSpinBox, + QHBoxLayout, + QLabel, + QPushButton, + QSlider, + QVBoxLayout, + QWidget, + ) + + box = QWidget() + outer = QVBoxLayout(box) + + slider = QSlider(Qt.Horizontal) + slider.setMinimum(-180) + slider.setMaximum(180) + slider.setValue(0) + spin = QDoubleSpinBox() + spin.setRange(-180.0, 180.0) + spin.setDecimals(1) + # The step is what the box's own - and + buttons move by, and five + # degrees is a visible turn -- half a degree took ten clicks to show + # anything. Anyone wanting finer than five types the number, which the + # one decimal place still accepts. + spin.setSingleStep(5.0) + spin.setSuffix(" deg") + reset = QPushButton("Reset") + + why = ( + "Turns the image, the centroids and the outlines together, about\n" + "the centre of the current view. Positive is anticlockwise.\n" + "The pivot is taken when you start moving the control, so panning\n" + "between adjustments re-centres it." + ) + for w in (slider, spin): + w.setToolTip(why) + + # Hidden until it has something to report, for the same reason as the + # display panel's: the spin box already shows the angle. + status = QLabel("") + status.setWordWrap(True) + status.hide() + + # One-element lists rather than nonlocal: these are read and written + # from several nested callbacks and a mutable holder keeps them in one + # place instead of scattering `nonlocal` declarations. + pivot = [None] + guard = [False] + # The affine currently in force, or None while the picture is square. + # Kept rather than recomputed, because a layer added later has to get + # THE SAME matrix -- rebuilding it from the angle would use whatever + # the view centre is by then, and two layers rotated by equal angles + # about different pivots do not line up. + current = [None] + + def _layers(): + """Every layer the rotation governs, as of right now. + + NOT the list this panel was built with. That list is the layers + that existed at launch, and a gene ticked afterwards is not in it + -- it arrived square while the section around it was turned, which + is a picture that is wrong rather than merely unrotated. The + viewer's own list is the answer to "what is on screen", so it is + what gets asked, with the passed-in layers kept as a seed for + anything not registered with the viewer. + """ + seen, out = set(), [] + for group in (layers or (), getattr(viewer, "layers", ()) or ()): + for layer in group: + if layer is None or id(layer) in seen: + continue + seen.add(id(layer)) + out.append(layer) + return out + + def _pivot(): + if pivot[0] is None: + pivot[0] = rotationCenter(viewer) + return pivot[0] + + def _apply(angle): + try: + current[0] = applyRotation(_layers(), angle, _pivot()) + except Exception as e: # noqa: BLE001 - a control never costs the picture + status.setText(f"failed: {e}") + status.show() + return + status.hide() + + def _onInserted(event=None): + """Turn a newly added layer to match the ones already turned. + + Ticking a gene while the section is rotated adds its layer square, + and the two then disagree about where the tissue is -- the gene + lands beside the anatomy rather than on it. Since the matrix is + shared and absolute, catching the layer as it arrives is enough: + it gets exactly what everything else has. + """ + a = current[0] + if a is None: + return + layer = getattr(event, "value", None) + # Fall back to sweeping the viewer: napari's event carries the new + # layer, but a caller that inserts by other means may not, and a + # missed layer is the failure this exists to prevent. + targets = [layer] if layer is not None else _layers() + for target in targets: + try: + target.affine = a + except Exception: # noqa: BLE001 - a stray entry never costs the rest + continue + + # Wrapped: the fakes a headless test builds have no event system, and + # neither does a viewer whose layer list is a plain container. + try: + viewer.layers.events.inserted.connect(_onInserted) + except Exception: # noqa: BLE001 - see above + pass + + def _sync(angle, source): + # The slider and the box show the same number, so each has to move + # the other -- and setting a widget's value emits its own signal, + # which would come straight back here. The guard makes the pair one + # control rather than two that argue. + if guard[0]: + return + guard[0] = True + try: + if source is not slider: + slider.setValue(int(round(angle))) + if source is not spin: + spin.setValue(float(angle)) + _apply(float(angle)) + finally: + guard[0] = False + + def _release(): + # Interaction over: the next one re-reads the view centre. + pivot[0] = None + + slider.sliderPressed.connect(lambda: pivot[0] or _pivot()) + slider.sliderReleased.connect(_release) + slider.valueChanged.connect(lambda v: _sync(v, slider)) + spin.editingFinished.connect(lambda: (_sync(spin.value(), spin), _release())) + + def _reset(): + pivot[0] = None + _sync(0.0, None) + pivot[0] = None + + reset.clicked.connect(_reset) + + # TWO ROWS, NOT ONE. Sharing a row with the box and the button left + # the slider a stub too short to aim with: a dock panel is narrow, and + # the number and the Reset button take a fixed width out of it whatever + # is left over. On its own row the slider gets the panel's full width, + # which is what makes it draggable. + row = QHBoxLayout() + row.addWidget(spin) + row.addWidget(reset) + row.addStretch() + outer.addLayout(row) + outer.addWidget(slider) + outer.addWidget(status) + outer.addStretch() + + viewer.window.add_dock_widget(box, name="Rotation", area="right") + return box + + +def blurLevels(levels, binSizes, sigmaBasePixels: float, truncate: float = 3.0): + """Gaussian-blur a multiscale ladder by the same DISTANCE at every level. + + Gene expression is a handful of counts in scattered single pixels. + Drawn honestly it is dust, and a reader looking for where a gene is + expressed cannot see a pattern in it. Blurring spreads each pixel over + its neighbourhood so the pattern reads, at the cost of no longer being + able to point at one bin. + + THE WIDTH IS IN BASE PIXELS, NOT LEVEL PIXELS, and that is the whole + difficulty. Each level is binned differently, so a fixed number of + level-pixels would be a different physical distance on each one -- the + blur would visibly change width as the viewer switched levels while + zooming, which is precisely what a multiscale ladder exists to avoid. + Dividing by the level's bin size makes one setting mean one distance + everywhere. + + BLURRED PER BLOCK, WITH A HALO. Levels are dask arrays whose blocks + are tiles, and a plain per-block filter would see each tile's edge as + the end of the world -- leaving a seam at every tile boundary, in a + grid, which reads as an artefact of the data rather than of the + drawing. map_overlap gives each block ``truncate * sigma`` of its + neighbours to work from, which is exactly the reach of the kernel, so + the result matches blurring the level whole. + + The kernel is unity: widening spreads each pixel's counts rather than + adding to them, so width and brightness stay separate knobs. Contrast limits will want resetting afterwards + all the same -- spreading a sparse signal lowers its peak a long way. + + Args: + levels: dask arrays, finest first, as ``layerSpec`` returns. + binSizes: each level's bin size in base pixels, same order. + sigmaBasePixels: blur width, in BASE pixels. 0 or less returns the + ladder untouched, which is what "no blur" should cost. + truncate: kernel cutoff in sigmas; also the halo, so the two + cannot disagree. + + Returns: + A new list of dask arrays. The input is not modified. + """ + import numpy as np + from scipy.ndimage import gaussian_filter + + if sigmaBasePixels is None or sigmaBasePixels <= 0: + return list(levels) + + out = [] + for arr, b in zip(levels, binSizes): + sigma = float(sigmaBasePixels) / float(b) + if sigma <= 0: + out.append(arr) + continue + # +1 so integer truncation can never make the halo shorter than + # the kernel's reach, which is the one way a seam gets back in. + depth = int(np.ceil(truncate * sigma)) + 1 + out.append( + arr.map_overlap( + gaussian_filter, + depth=depth, + boundary=0, + sigma=sigma, + truncate=truncate, + mode="constant", + dtype=arr.dtype, + ) + ) + return out + + +def basePixelUm(pyr_doc, fallback: float = 0.5) -> float: + """One base pixel's physical width, so a blur can be set in microns. + + The blur knob is in micrometres because that is the unit a reader + thinks in -- a spread of ten microns means something about tissue, + ten base pixels means something about the chip. :func:`blurLevels` + works in base pixels, and this is the conversion between them. + + Only the X size is used. A pyramid whose pixels are not square would + want an anisotropic sigma, and neither the ladder nor the knob is + built for that; Stereo-seq's DNB grid is square, so the case has not + arisen. If it does, this is where it would be noticed. + + Args: + pyr_doc: a spatialGeneExpressionPyramid document. + fallback: used when the document does not say, or says something + that is not a positive number. 0.5 um is the Stereo-seq DNB + pitch, which is what every pyramid seen so far records. A + wrong-but-plausible scale makes the knob mean the wrong + distance; a zero would make it divide by zero. + + Returns: + Micrometres per base pixel, always positive. + """ + try: + p = pyr_doc.document_properties["spatialGeneExpressionPyramid"] + um = float(p["base_pixel_size_x"]) + except Exception: # noqa: BLE001 - a missing field costs the default, not the knob + return float(fallback) + return um if um > 0 else float(fallback) + + +def _importQtWidgets() -> None: + """Import qtpy.QtWidgets, or raise ImportError. + + A named function rather than an inline import inside addAllPanels so a + test can stand in for it. What addAllPanels does AROUND the panels -- + the order they are built in, and that one throwing does not stop the + rest or take the window down -- is not about Qt, and a CI job that + installs no Qt binding should still hold that behaviour to account. + Without this seam those tests passed anywhere a binding happened to be + installed and silently asserted nothing anywhere else. + """ + import qtpy.QtWidgets # noqa: F401 + + +def cloudSessionLooksLikely() -> bool: + """Is there any reason to show a cloud login in this viewer? + + A token in the environment is the signal, and it is the right one for + the same reason the MATLAB launcher uses it: the viewer inherits its + parent's environment, so a token there says the process that started + this one had been talking to the cloud and expects this one to keep + doing so. A purely local pyramid has none, and a login control on a + window that needs no login is clutter that also implies the picture + might be waiting on something. + + A token that has ALREADY expired still counts -- that is precisely + when the panel is wanted. + """ + import os + + return bool(os.environ.get("NDI_CLOUD_TOKEN")) + + +def cloudProfileChoices(): + """Saved profiles, as ``(label, email, uid, stored, note)`` rows. + + The label is what a chooser shows: the nickname when there is one, + else the email, with the email appended either way so two profiles + for the same person on different stages can be told apart. + + ``note`` SAYS WHY THERE IS NO PASSWORD, which is the whole reason + this returns five things rather than four. A profile whose secret + lives in a store this process is not reading looks exactly like a + profile with no secret at all -- and that is not a rare corner: + :func:`ndi.cloud.profile._detect_backend` picks ``keyring`` whenever + keyring merely imports, while MATLAB writes its secret to the AES + file, so a machine with both ends up looking in the wrong one. The + note names the backend, so the reader is told which store came up + empty instead of being told their password is missing. + + Never raises. A secrets backend that will not open, a profile file + that will not parse -- none of that is a reason a sign-in dialog + should fail to appear, since typing the credentials by hand is + exactly the fallback it exists to offer. + """ + rows = [] + try: + from ....cloud import profile + + entries = profile.list_profiles() + except Exception: # noqa: BLE001 - see docstring + return rows + try: + backend = str(profile.backend()) + except Exception: # noqa: BLE001 - a name is a nicety, not a requirement + backend = "unknown" + for entry in entries: + uid = str(getattr(entry, "UID", "") or "") + email = str(getattr(entry, "Email", "") or "") + nickname = str(getattr(entry, "Nickname", "") or "") + stage = str(getattr(entry, "Stage", "") or "") + label = nickname or email or uid + if email and label != email: + label = f"{label} ({email})" + if stage and stage != "prod": + label = f"{label} [{stage}]" + stored, note = False, "" + try: + stored = bool(profile.get_password(uid)) + note = "password saved" if stored else f"no password in the {backend} store" + except Exception as exc: # noqa: BLE001 - the reason is the useful part + note = f"password unreadable from the {backend} store ({exc})" + rows.append((label, email, uid, stored, note)) + return rows + + +def cloudLogoPath(): + """The bundled NDI Cloud wordmark, or None if it is not there. + + Copied from NDI-matlab rather than referenced, because the viewer + runs from a Python install that need not have NDI-matlab on disk at + all -- a downloaded dataset opened by someone who has never run + MATLAB is the normal case. + + None rather than a raise: a missing asset costs a picture, never a + panel, and the panel falls back to its own name in text. + """ + from pathlib import Path + + path = Path(__file__).with_name("resources") / "ndi_cloud_logo.png" + return path if path.is_file() else None + + +def cloudLogoLabel(parent=None): + """The wordmark on a white card, sized for a dock. None if unusable. + + ON A WHITE CARD because the wordmark is dark navy on transparency + and napari's docks follow the viewer's theme -- on the dark one the + lettering simply is not there. A white card is how the brand is used + elsewhere anyway, so this is the ordinary treatment rather than a + workaround. + + Scaled to twice its display width with a device pixel ratio of 2, so + it stays crisp on the retina screen a demo is usually given from. + """ + from qtpy.QtCore import Qt + from qtpy.QtGui import QPixmap + from qtpy.QtWidgets import QLabel + + path = cloudLogoPath() + if path is None: + return None + pixmap = QPixmap(str(path)) + if pixmap.isNull(): + return None + pixmap = pixmap.scaledToWidth(300, Qt.SmoothTransformation) + pixmap.setDevicePixelRatio(2.0) + label = QLabel(parent) + label.setPixmap(pixmap) + label.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + label.setStyleSheet("background: #ffffff; border-radius: 4px; padding: 6px 8px 6px 8px;") + return label + + +def addCloudPanel(viewer) -> Any: + """A clock on the cloud token, and a button that renews it. + + A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for + hours; the token is good for rather less, and it cannot renew itself + unless credentials happen to be reachable. What running out looks + like from here is not a login prompt but MISSING DATA -- tiles + already cached keep drawing, and the ones that were never fetched + fail one at a time as the reader pans, which reads as a broken + pyramid. The remedy used to be quitting, logging in elsewhere, and + opening the section again, since the token lives in this process's + environment and nothing outside it can reach in. + + ONE ROW, because that is all it is worth until it matters: a button + and the time left. The credentials live behind the button, in a + dialog, rather than as three permanently docked fields for something + a reader does once a day at most. + """ + from qtpy.QtCore import QTimer + from qtpy.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget + + from ....cloud import auth + + box = QWidget() + outer = QVBoxLayout(box) + + # The wordmark first, so the panel says whose account this is before + # it says anything about a token. + logo = cloudLogoLabel(box) + if logo is not None: + outer.addWidget(logo) + + signin = QPushButton("Sign in...") + signin.setToolTip( + "Sign in to NDI Cloud, so tiles that are not already downloaded\n" + "keep loading. The token lives in this process, so signing in\n" + "anywhere else does not reach this window." + ) + clock = QLabel("") + clock.setWordWrap(True) + row = QHBoxLayout() + row.addWidget(signin) + row.addWidget(clock, 1) + outer.addLayout(row) + + result = QLabel("") + result.setWordWrap(True) + result.hide() + outer.addWidget(result) + outer.addStretch() + + def _tick(): + left = auth.tokenSecondsRemaining() + line = auth.tokenStatusLine() + # Warned about BEFORE it bites: a reader who sees "12 min left" + # can sign in at a convenient moment instead of discovering it + # through a tile that will not load. + if left is not None and 0 < left < 900: + clock.setText(f"{line} -- renew before it runs out.") + elif left is not None and left <= 0: + clock.setText(f"{line}. Undownloaded tiles will fail until you sign in.") + else: + clock.setText(line) + + def _open(): + ok, note = cloudSignInDialog(box) + if note: + result.setText(note) + result.show() + elif ok: + result.hide() + _tick() + + signin.clicked.connect(_open) + + # Parented to the widget, so it stops when the dock goes. A free timer + # would keep firing at a deleted label and take the process with it. + timer = QTimer(box) + timer.setInterval(30_000) + timer.timeout.connect(_tick) + timer.start() + _tick() + + box._ndi_signin = _open + viewer.window.add_dock_widget(box, name="NDI Cloud", area="right") + return box + + +def cloudSignInDialog(parent=None): + """Show the sign-in dialog and sign in. Returns ``(signed_in, note)``. + + The building is :func:`buildCloudSignInDialog`; this is the two + lines around it that block. Split so a test can drive the dialog's + fields and its Sign in button without an event loop to escape from. + """ + dialog, outcome = buildCloudSignInDialog(parent) + runner = getattr(dialog, "exec", None) or getattr(dialog, "exec_", None) + runner() + return outcome["ok"], outcome["note"] + + +def buildCloudSignInDialog(parent=None): + """The sign-in dialog, built but not shown. ``(dialog, outcome)``. + + Behind a button rather than docked, because it is a thing a reader + does once a day at most and three permanent fields would say + otherwise. + + THE SAVED PROFILES ARE OFFERED FIRST. Someone whose profile works + should not be retyping a password to get past an expired token, and + someone whose profile does NOT work -- which is how most people + arrive here -- can see which one is being used, which is half the + diagnosis. Picking one fills the email in; picking one that has a + stored password signs in with it directly. + + THE PASSWORD IS NOT KEPT. It goes to the login call and nothing else: + not to a field that stays filled, not to a file, and to the profile + only when the checkbox asks -- which exists because credentials + saved once and wrong are exactly how someone ends up in this dialog. + """ + from qtpy.QtWidgets import ( + QCheckBox, + QComboBox, + QDialog, + QDialogButtonBox, + QFormLayout, + QLabel, + QLineEdit, + QVBoxLayout, + ) + + from ....cloud import auth + + dialog = QDialog(parent) + dialog.setWindowTitle("Sign in to NDI Cloud") + outer = QVBoxLayout(dialog) + + why = QLabel( + "The token lives in this process, so signing in elsewhere does not " "reach this window." + ) + why.setWordWrap(True) + outer.addWidget(why) + + choices = cloudProfileChoices() + chooser = QComboBox() + chooser.addItem("type them below", None) + for label, email, uid, stored, note in choices: + chooser.addItem(f"{label} -- {note}" if note else label, (email, uid, stored)) + + email = QLineEdit() + email.setPlaceholderText("you@example.com") + password = QLineEdit() + password.setEchoMode(QLineEdit.Password) + password.setPlaceholderText("password") + + form = QFormLayout() + if choices: + form.addRow("profile", chooser) + form.addRow("email", email) + form.addRow("password", password) + outer.addLayout(form) + + save = QCheckBox("also save these to my NDI profile") + save.setToolTip( + "Stores the credentials so a future session can renew its own\n" + "token without asking. Overwrites what is saved now, which is\n" + "the point if what is saved now is wrong." + ) + outer.addWidget(save) + + said = QLabel("") + said.setWordWrap(True) + said.hide() + outer.addWidget(said) + + buttons = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel) + buttons.button(QDialogButtonBox.Ok).setText("Sign in") + outer.addWidget(buttons) + + def _chose(index): + data = chooser.itemData(index) + if not data: + said.hide() + return + picked_email, _uid, stored = data + email.setText(picked_email) + # A profile whose password is saved needs nothing typed. + password.clear() + if not stored: + # A profile without a usable password needs the password and + # not the email, so the focus goes where the work is -- and + # the reason is put on screen, because "this profile exists + # but its password is not readable from here" is a different + # problem from "you never saved one", and only one of them is + # solved by typing it again. + said.setText(choices[index - 1][4]) + said.show() + password.setFocus() + else: + said.hide() + + chooser.currentIndexChanged.connect(_chose) + + outcome = {"ok": False, "note": ""} + + def _accept(): + who, secret = email.text().strip(), password.text() + if not secret: + # An empty password with a profile picked means "use the one + # you have", which is the whole reason to pick a profile. + data = chooser.itemData(chooser.currentIndex()) + if data and data[2]: + try: + from ....cloud import profile as _profile + + secret = _profile.get_password(data[1]) + except Exception as e: # noqa: BLE001 - fall through to the message + secret = "" + said.setText(f"Could not read the saved password: {e}") + if not who or not secret: + said.setText("Both an email and a password are needed.") + said.show() + return + try: + auth.login(who, secret) + except Exception as e: # noqa: BLE001 - a bad password is not a crash + said.setText(f"Sign-in failed: {e}") + said.show() + password.clear() + return + note = "Signed in." + if save.isChecked(): + picked = chooser.itemData(chooser.currentIndex()) + try: + saveCloudProfile(who, secret, uid=(picked[1] if picked else "")) + note += " Saved to your NDI profile." + except Exception as e: # noqa: BLE001 - the sign-in still worked + note += f" (Could not save to the profile: {e})" + outcome.update(ok=True, note=note) + password.clear() + dialog.accept() + + buttons.accepted.connect(_accept) + buttons.rejected.connect(dialog.reject) + password.returnPressed.connect(_accept) + + dialog._ndi_accept = _accept + dialog._ndi_fields = {"email": email, "password": password, "save": save, "profile": chooser} + dialog._ndi_said = said + return dialog, outcome + + +def saveCloudProfile(email: str, password: str, uid: str = "") -> None: + """Make these the credentials a future session renews itself with. + + Updates an existing profile rather than adding a second one for the + same email -- someone whose saved password is wrong is trying to + CORRECT it, and a duplicate entry would leave the wrong one in place + to be picked instead. + + WHICH profile, when several share an email, is the whole difficulty. + ONE EMAIL CAN OWN SEVERAL ACCOUNTS: a dev profile and a prod profile + under the same address is the normal arrangement here, not a mistake + to be tidied. Writing a prod password into the dev entry is silent + and wrong, so this never guesses: + + 1. *uid* when the caller has one -- the dialog passes the profile + the reader actually picked, which settles it outright. + 2. the CURRENT profile, then the DEFAULT, when either matches the + email. Those are the two the rest of NDI would have used. + 3. a match that is the ONLY one for that email. + + and otherwise raises rather than choosing. An ambiguous save is the + one case where doing nothing and saying so beats acting. + + THE DEFAULT IS NOT MOVED for an existing profile. Saving a password + is not a request to switch accounts, and silently re-pointing the + default is how a dev profile ends up being the one everything + afterwards authenticates with. A profile this creates is made the + default, because a store that had nothing for this email has no + other candidate to displace. + """ + from ....cloud import profile + + if uid: + profile.set_password(uid, password) + return + + matches = [ + e for e in profile.list_profiles() if str(getattr(e, "Email", "")).lower() == email.lower() + ] + if not matches: + fresh = profile.add(email.split("@")[0] or "ndi", email, password) + profile.set_default(fresh) + return + + if len(matches) > 1: + for pick in (profile.get_current(), profile.get_default()): + chosen = str(getattr(pick, "UID", "") or "") + if chosen and any(chosen == e.UID for e in matches): + profile.set_password(chosen, password) + return + raise ValueError( + f"{len(matches)} profiles use {email} and none of them is the " + f"current or default one, so there is no way to tell which " + f"password this is. Pick the profile in the list above, or set " + f"a default with ndi.cloud.profile.set_default(...)." + ) + + profile.set_password(matches[0].UID, password) + + +def applySolo(layers, current) -> int: + """Show one gene layer and hide the rest. + + *current* is the layer NAME to show, or None to show them all -- + which is what restoring looks like when nothing was hidden to begin + with. + + The base section and the cell overlays are not in *layers* and so + are never hidden. That is the point of showing one gene at a time + rather than one layer at a time: the anatomy has to stay under it, + or a lone gene floats on black and there is nothing to read it + against. + + Returns: + How many layers were changed. One that refuses is skipped rather + than allowed to stop the rest. + """ + done = 0 + for layer in layers: + try: + layer.visible = True if current is None else str(layer.name) == str(current) + except Exception: # noqa: BLE001 - see docstring + continue + done += 1 + return done + + +def addGeneAppearancePanel(viewer) -> Any: + """One contrast and one gamma across every gene layer at once. + + Gene layers arrive with limits set from their own data, which is + right for reading one gene and wrong for comparing several: two + genes an order of magnitude apart are drawn as though they were the + same brightness, and the picture says less than the data does. These + two knobs move them together, so what is being compared stays + comparable. + + THE BASE LAYER IS NOT TOUCHED, and neither are the cell overlays. + The section underneath is the anatomy the genes are read against; + dimming it to make a gene stand out would take away the frame that + makes standing out mean anything. + + Both are ABSOLUTE. Contrast recomputes from the window each layer + was born with rather than from whatever it is now, so the number on + the control is the whole state and a nudge back really does return. + """ + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QHBoxLayout, + QLabel, + QPushButton, + QSlider, + QVBoxLayout, + QWidget, + ) + + box = QWidget() + outer = QVBoxLayout(box) + + # name -> the (lo, hi) the layer was auto-set to when it arrived. + references: dict[str, tuple[float, float]] = {} + state = {"contrast": 1.0, "gamma": 1.0} + + def _row(label, lo, hi, value, tip): + """A slider on one row, with its name and value beside it. + + NO SPIN BOX. Two docks of two rows each was most of the panel + stack for two knobs that are dragged rather than typed -- these + are "a bit more" and "a bit less" controls, not values anyone + needs to enter exactly. The number is still shown, as a label, so + a setting worth writing down can be read off. + """ + caption = QLabel(f"{label} {value:.2f}") + caption.setMinimumWidth(84) + caption.setToolTip(tip) + slider = QSlider(Qt.Horizontal) + # Sliders are integers, so the value is carried in hundredths and + # converted at the boundary. Hundredths because gamma runs from + # 0.1 to 3 and tenths would be eight usable positions. + slider.setRange(int(lo * 100), int(hi * 100)) + slider.setValue(int(value * 100)) + slider.setToolTip(tip) + row = QHBoxLayout() + row.addWidget(caption) + row.addWidget(slider, 1) + outer.addLayout(row) + return caption, slider + + contrast_tip = ( + "Narrows the contrast window of every gene layer about its floor.\n" + "Above 1 brightens and separates a sparse gene, because the dim\n" + "end of the range is where all the counts are. The base section\n" + "and the cell layers are not affected." + ) + gamma_tip = ( + "Gamma for every gene layer. Below 1 lifts the faint counts;\n" + "above 1 pushes them down. The base section and the cell layers\n" + "are not affected." + ) + contrast_label, contrast_slider = _row("contrast", 0.1, 10.0, 1.0, contrast_tip) + gamma_label, gamma_slider = _row("gamma", 0.1, 3.0, 1.0, gamma_tip) + + status = QLabel("") + status.setWordWrap(True) + status.hide() + reset = QPushButton("Reset") + row = QHBoxLayout() + row.addStretch() + row.addWidget(reset) + outer.addLayout(row) + outer.addWidget(status) + outer.addStretch() + + guard = [False] + + def _apply(): + layers = geneLayers(viewer) + if not layers: + status.setText("No gene layers yet -- tick a gene to see this work.") + status.show() + return + applyGeneAppearance(layers, state["contrast"], state["gamma"], references) + status.hide() + + def _sync(which, value, source=None): + # Setting a slider emits its own signal, which would come straight + # back here; the guard makes one change one change. + if guard[0]: + return + guard[0] = True + try: + state[which] = float(value) + label, slider, caption = ( + (contrast_label, contrast_slider, "contrast") + if which == "contrast" + else (gamma_label, gamma_slider, "gamma") + ) + label.setText(f"{caption} {float(value):.2f}") + if source is not slider: + slider.setValue(int(round(float(value) * 100))) + _apply() + finally: + guard[0] = False + + contrast_slider.valueChanged.connect(lambda v: _sync("contrast", v / 100.0, contrast_slider)) + gamma_slider.valueChanged.connect(lambda v: _sync("gamma", v / 100.0, gamma_slider)) + + def _reset(): + _sync("contrast", 1.0) + _sync("gamma", 1.0) + + reset.clicked.connect(_reset) + + def _onInserted(event=None): + """A gene ticked later gets the settings already in force. + + Otherwise it arrives at its own auto-set brightness while the + others sit where the knobs put them, and the comparison the + panel exists for is broken by the act of adding to it. + """ + layer = getattr(event, "value", None) + if layer is None or not str(getattr(layer, "name", "")).startswith(_GENE_PREFIX): + return + if state["contrast"] == 1.0 and state["gamma"] == 1.0: + return + applyGeneAppearance([layer], state["contrast"], state["gamma"], references) + + def _onRemoved(event=None): + # Its reference window goes with it, so a gene ticked, unticked + # and ticked again is measured from its new data rather than + # from what it happened to look like the first time. + layer = getattr(event, "value", None) + references.pop(str(getattr(layer, "name", "")), None) + + try: + viewer.layers.events.inserted.connect(_onInserted) + viewer.layers.events.removed.connect(_onRemoved) + except Exception: # noqa: BLE001 - a headless fake has no event system + pass + + # Exposed so the tour can pulse around the gamma this panel set, + # and put it back afterwards rather than back to 1. + box._ndi_appearance = state + viewer.window.add_dock_widget(box, name="Gene appearance", area="right") + return box + + +def addGeneTourPanel(viewer, appearance=None, interval_ms: int = 50) -> Any: + """Two ways of walking the gene layers, five seconds each. + + Showing someone six genes means saying which is which, and a legend + of six colours against a section this dense is a legend nobody can + read. So the picture says it instead, one gene at a time, with its + name standing in the corner of the canvas. + + TWO MODES, because they answer different questions. + + PULSE leaves every gene on screen and sweeps one gene's gamma down + and back, once a second. Everything stays in view, so it answers + "where is THIS one, among the others" -- the comparison is the + point, and the sweep is the pointer. + + SOLO hides the others outright. It answers "what does this one look + like, by itself", which additive blending otherwise makes hard: six + layers over each other are a colour nobody can decompose by eye. + The base section is NOT hidden, or a lone gene would float on black + with nothing to read it against. + + NOTHING BLOCKS. Both are one timer stepping a small state machine, + not a loop with sleeps in it, so the viewer stays live throughout: a + tour can be stopped mid-gene, and panning while it runs works. + + WHAT IT TOUCHES IT PUTS BACK. Gammas for the pulse, visibility for + the solo -- recorded when the tour starts and restored when it ends, + however it ends. Stopping halfway must not leave a gene stranded at + a gamma nobody chose, or leave five of six genes hidden. + + ONE AT A TIME. Starting either mode stops the other first, so the + two can never both be moving the same layers. + + Args: + appearance: the gene-appearance panel, if there is one. Its + gamma is the value each pulse departs from and returns to, + so the two controls agree rather than fighting. + interval_ms: timer period. 50ms is 20 steps a second, which + reads as a sweep rather than a blink. + """ + from qtpy.QtCore import QTimer + from qtpy.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget + + seconds_per_gene = 5.0 + pulse_period = 1.0 + + box = QWidget() + outer = QVBoxLayout(box) + + play = QPushButton("Pulse each gene") + play.setToolTip( + "Leaves every gene on screen and sweeps one gene's gamma down and\n" + "back, once a second for five seconds, naming it on the canvas.\n" + 'Answers "where is this one, among the others". Click again to\n' + "stop; every gamma is put back." + ) + solo = QPushButton("Show one gene at a time") + solo.setToolTip( + "Hides the other genes and shows one for five seconds, naming it\n" + 'on the canvas. Answers "what does this one look like by\n' + 'itself", which additive blending otherwise makes hard. The\n' + "section underneath stays visible. Click again to stop; every\n" + "layer is shown again." + ) + outer.addWidget(play) + outer.addWidget(solo) + + status = QLabel("") + status.setWordWrap(True) + outer.addWidget(status) + outer.addStretch() + + # THE CLOCK IS A STEP COUNT, not an accumulated float. Adding 0.05 a + # hundred times lands near 5.0 but not on it, and "has this gene had + # its five seconds" is a comparison against exactly that -- so the + # boundary would fall a frame early or late depending on the + # interval, which is the kind of bug that only shows up on somebody + # else's machine. Multiplying an integer is exact. + tour = {"on": False, "mode": "", "order": [], "i": 0, "n": 0, "base": 1.0, "shown": {}} + steps_per_gene = max(1, int(round(seconds_per_gene * 1000 / interval_ms))) + + def _baseGamma() -> float: + if appearance is not None: + try: + return float(appearance._ndi_appearance["gamma"]) + except Exception: # noqa: BLE001 - a panel that is gone is not a crash + pass + return 1.0 + + def _layer(name): + try: + return viewer.layers[name] + except Exception: # noqa: BLE001 - removed mid-tour is normal, not an error + return None + + def _overlay(text, visible=True): + """The gene's name, bottom right, in something readable. + + On the canvas rather than in the panel because that is where the + eye already is, and because a demo is usually being watched on + somebody else's screen where the docks are small. + """ + try: + overlay = viewer.text_overlay + overlay.text = str(text) + overlay.visible = bool(visible) + overlay.font_size = 24 + overlay.position = "bottom_right" + overlay.color = "white" + except Exception: # noqa: BLE001 - an older napari names these differently + pass + + def _restore(name): + layer = _layer(name) + if layer is None: + return + try: + layer.gamma = max(_GAMMA_FLOOR, tour["base"]) + except Exception: # noqa: BLE001 - a control never costs the picture + pass + + def _stop(message=""): + # Every gene, not only the one in hand: an earlier one whose + # restore was missed because its layer was briefly gone would + # otherwise stay where the sweep left it. + if tour["mode"] == "pulse": + for name in tour["order"]: + _restore(name) + elif tour["mode"] == "solo": + # Back to what each layer's visibility WAS, not to all-on: a + # gene the reader had already hidden by hand should stay + # hidden, and a tour is not a request to change that. + for name, was in tour["shown"].items(): + layer = _layer(name) + if layer is None: + continue + try: + layer.visible = bool(was) + except Exception: # noqa: BLE001 - a control never costs the picture + pass + tour.update(on=False, mode="", order=[], i=0, n=0, shown={}) + timer.stop() + _overlay("", visible=False) + play.setText("Pulse each gene") + solo.setText("Show one gene at a time") + status.setText(message) + + def _step(): + if not tour["on"]: + return + tour["n"] += 1 + if tour["n"] >= steps_per_gene: + _restore(tour["order"][tour["i"]]) + tour["i"] += 1 + tour["n"] = 0 + if tour["i"] >= len(tour["order"]): + _stop("Tour finished.") + return + _announce() + name = tour["order"][tour["i"]] + layer = _layer(name) + if layer is None: + # Unticked while the tour was on it. Move along rather than + # spending five seconds on something that is not there. + tour["n"] = steps_per_gene - 1 + return + if tour["mode"] != "pulse": + # Solo changes nothing between gene boundaries -- the switch + # happens once, in _announce. The clock still has to run. + return + try: + elapsed = tour["n"] * interval_ms / 1000.0 + layer.gamma = pulseGamma(tour["base"], elapsed, pulse_period) + except Exception: # noqa: BLE001 - see _restore + pass + + def _announce(): + name = tour["order"][tour["i"]] + if tour["mode"] == "solo": + applySolo(geneLayers(viewer), name) + _overlay(geneName(_layer(name)) if _layer(name) is not None else geneName(name)) + status.setText(f"{tour['i'] + 1} of {len(tour['order'])}") + + timer = QTimer(box) + timer.setInterval(interval_ms) + timer.timeout.connect(_step) + + def _toggle(mode="pulse"): + # Clicking the running mode's button stops it; clicking the other + # one switches, which means stopping first so the layers it was + # moving are put back before the next mode records them. + was = tour["mode"] + if tour["on"]: + _stop("Stopped.") + if was == mode: + return + layers = geneLayers(viewer) + order = [str(layer.name) for layer in layers] + if not order: + status.setText("No gene layers to tour -- tick some genes first.") + return + shown = {} + if mode == "solo": + for layer in layers: + shown[str(layer.name)] = bool(getattr(layer, "visible", True)) + tour.update(on=True, mode=mode, order=order, i=0, n=0, base=_baseGamma(), shown=shown) + (play if mode == "pulse" else solo).setText("Stop") + _announce() + timer.start() + + play.clicked.connect(lambda *_: _toggle("pulse")) + solo.clicked.connect(lambda *_: _toggle("solo")) + + row = QHBoxLayout() + row.addStretch() + outer.addLayout(row) + + box._ndi_tour = tour + box._ndi_step = _step + box._ndi_toggle = _toggle + viewer.window.add_dock_widget(box, name="Gene tour", area="right") + return box + + +def addAllPanels( + viewer, + session, + pyr_doc, + image_layer, + density, + cells_doc=None, + points_layer=None, + shapes_layer=None, + labelings=None, + gene_layers=None, +): + """Every panel the session supports, skipping what it has no data for. + + Added in the order they should read down the right-hand side: the + display mode, the cell types, then the genes. napari stacks docks in + the order they arrive, so this order is the layout. + + A PANEL MUST NOT COST THE PICTURE. By the time this runs the viewer is + on screen and the image layer is drawn; a panel that throws would + unwind out of openPyramid before napari.run(), so the window appears + and the process exits -- a viewer that "opens briefly and closes", + with the reason lost unless someone was watching stderr. + + That is not hypothetical. Each panel reads files out of the database + to build itself: genes.tsv and gene_totals.tsv for the genes, one + labels.tsv per labeling for the cell types. Those reads can fail for + reasons that have nothing to do with the image -- a document whose + binary is not local, a cloud-backed session, a file an older ingest + never wrote -- and none of them is a reason to refuse to draw the + section. So each panel is built independently and a failure costs + that panel, named, with its traceback. + """ + import sys + import traceback + + try: + _importQtWidgets() + except ImportError as e: + print( + f"[genepyramid] control panels unavailable ({e}). The image is " + f"unaffected; --genes and --no-density still work at launch.", + file=sys.stderr, + ) + return {} + + def _build(name, fn): + try: + return fn() + except Exception as e: # noqa: BLE001 - a panel is never worth the window + print( + f"[genepyramid] the {name} panel could not be built " + f"({type(e).__name__}: {e}). The image is unaffected.", + file=sys.stderr, + ) + traceback.print_exc() + return None + + made = {} + # FIRST, when there is a token to run out. It is the only panel about + # whether the picture can still be READ rather than how it is drawn, + # and its clock is the thing worth catching before it bites -- a + # reader who sees "12 min left" at the top of the stack can sign in + # at a convenient moment instead of discovering it through a tile + # that will not load. + if cloudSessionLooksLikely(): + made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) + made["display"] = _build( + "display", lambda: addDisplayPanel(viewer, session, pyr_doc, image_layer, density) + ) + # Rotation sits with the other whole-picture controls, above the ones + # that choose WHAT is drawn: it changes how the section is presented, + # not which genes or cells are in it. + made["rotation"] = _build( + "rotation", + lambda: addRotationPanel(viewer, [image_layer, shapes_layer, points_layer]), + ) + if cells_doc is not None and points_layer is not None: + made["cellTypes"] = _build( + "cell types", + lambda: addCellTypePanel( + viewer, session, cells_doc, points_layer, shapes_layer, labelings + ), + ) + made["appearance"] = _build("gene appearance", lambda: addGeneAppearancePanel(viewer)) + made["tour"] = _build( + "gene tour", lambda: addGeneTourPanel(viewer, appearance=made.get("appearance")) + ) + # THE GENE LIST IS LAST, so it sits at the bottom of the stack. It is + # the tallest panel by a long way -- 26,444 rows -- and napari gives + # the leftover height to whatever is furthest down, so anything below + # it would be squeezed while the list took the room. The two controls + # above it act on what is ticked in it, which is also the order they + # are used in. + made["genes"] = _build( + "genes", lambda: addGenePanel(viewer, session, pyr_doc, initial=gene_layers) + ) + return made diff --git a/src/ndi/gui/app/genepyramid/multiscale.py b/src/ndi/gui/app/genepyramid/multiscale.py index 32fc44b4..88d141f1 100644 --- a/src/ndi/gui/app/genepyramid/multiscale.py +++ b/src/ndi/gui/app/genepyramid/multiscale.py @@ -279,7 +279,7 @@ def levelArrays(session, pyr_doc, gene_rows=None, density: bool = True) -> list[ levels, _frame = levelTable(session, pyr_doc) fetcher = _TileFetcher(session) - from ....fun.doc_gene import _find_level + from ....fun.doc_gene import _find_level, tileFileName out = [] for lv in levels: @@ -288,7 +288,7 @@ def levelArrays(session, pyr_doc, gene_rows=None, density: bool = True) -> list[ rows, cols = lv["tileRows"], lv["tileColumns"] lh, lw = lv["levelHeight"], lv["levelWidth"] - tile_doc, _props = _find_level(session, pyr_doc, b) + tile_doc, lv_props = _find_level(session, pyr_doc, b) stored = set(tile_doc.current_file_list()) scale = b if density else 1 @@ -312,8 +312,10 @@ def block( _th=th, _tw=tw, _scale=scale, + _name=tileFileName, + _lv=lv_props, ): - name = f"tile.bin_{r * _cols + c}" + name = _name(_lv, r * _cols + c) if name not in _stored: return np.zeros((_th, _tw), np.float32) # missing tile costs nothing try: diff --git a/src/ndi/gui/app/genepyramid/progress.py b/src/ndi/gui/app/genepyramid/progress.py new file mode 100644 index 00000000..492389b5 --- /dev/null +++ b/src/ndi/gui/app/genepyramid/progress.py @@ -0,0 +1,333 @@ +"""Say what the launch is doing, and how long each part took. + +Opening a large section takes tens of seconds before anything is drawn, +and every one of those seconds is spent somewhere specific: reading a +26 MB cell table, placing twelve million contour vertices, handing half +a million polygons to napari. Without this the whole wait is one opaque +block and the only available theory is "NDI is slow". + +Timings go to STDERR so they interleave with the other launch notes and +stay out of anything a caller pipes. On by default: a program that makes +you wait owes you an account of the wait, and these lines cost one print +each. +""" + +from __future__ import annotations + +import contextlib +import os +import sys +import threading +import time + +__all__ = ["stage", "note", "cloudFetches", "humanBytes", "closeLaunchWindow"] + +#: Set NDI_GENEPYRAMID_QUIET=1 to silence the launch narration. +_QUIET_ENV = "NDI_GENEPYRAMID_QUIET" + +#: "gui", "text" or "off". Unset picks gui when a Qt binding loads and a +#: window can be created, text when stderr is a terminal, off otherwise. +_MODE_ENV = "NDI_GENEPYRAMID_PROGRESS" + + +def _quiet() -> bool: + return bool(os.environ.get(_QUIET_ENV, "").strip()) + + +def note(text: str) -> None: + """One launch line, no timing.""" + if not _quiet(): + print(f"[genepyramid] {text}", file=sys.stderr, flush=True) + + +@contextlib.contextmanager +def stage(label: str): + """Time a step of the launch and report it when it finishes. + + The label is printed BEFORE the work, so a step that hangs is named + on screen rather than leaving the user with the previous step's line + and no idea which one is stuck. + """ + if _quiet(): + yield + return + # The stderr narration stays on in every mode. It is the record of + # where the time went, and it costs one print per step whether or not + # anyone is watching a window. + print(f"[genepyramid] {label} ...", file=sys.stderr, flush=True) + if _mode() == "gui": + _guiStage(label) + t0 = time.perf_counter() + try: + # Every stage gets the byte reporting. On a directory-backed + # session nothing is fetched and this costs one hook install; on a + # cloud-backed one it is the difference between a label that sits + # there for a minute and something that moves. + with cloudFetches(label): + yield + finally: + dt = time.perf_counter() - t0 + print(f"[genepyramid] {label}: {dt:.1f}s", file=sys.stderr, flush=True) + + +def humanBytes(n: int) -> str: + """A byte count at a glance. 26214400 is not a number anyone reads.""" + step = 1024.0 + value = float(n) + for unit in ("B", "KB", "MB", "GB"): + if value < step or unit == "GB": + return f"{value:.0f} {unit}" if unit == "B" else f"{value:.1f} {unit}" + value /= step + return f"{value:.1f} GB" + + +@contextlib.contextmanager +def cloudFetches(label: str = "fetching"): + """Draw a progress line for every cloud file fetched inside this block. + + A session opened from a local directory fetches nothing and this costs + one hook install; there is no output and no reason to ask the caller + which kind of session they have. + + A CLOUD session is the case this exists for. The cell table, the + contour file and the gene list are each a whole download before the + window can appear, and stage() alone prints a label and then nothing + moves for however long that takes -- which is indistinguishable from a + hang. This redraws one line in place with bytes and, when the server + sent a Content-Length, a percentage. + + ONE LINE, AND ONLY FOR THE FILE BEING WAITED ON. Tiles are fetched + from _TileFetcher threads once the viewer is up, several possibly at + once; interleaving a line each would scroll the launch notes away. So + the renderer tracks a count of files and shows the newest, and every + write is under a lock because the callers are threads. + """ + mode = _mode() + if mode == "off": + yield + return + + try: + from ....cloud.filehandler import watchFetches + except Exception: # noqa: BLE001 - reporting is never worth the launch + yield + return + + state = {"files": 0, "done": 0, "total": None, "shown": False} + lock = threading.Lock() + launch_thread = threading.get_ident() + + def render(): + n, done, total = state["files"], state["done"], state["total"] + if total: + pct = min(100.0, 100.0 * done / total) + body = f"{humanBytes(done)} of {humanBytes(total)} ({pct:.0f}%)" + else: + body = humanBytes(done) + suffix = f", file {n}" if n > 1 else "" + sys.stderr.write(f"\r[genepyramid] {label}: {body}{suffix} ") + sys.stderr.flush() + state["shown"] = True + + def observer(event, _uri, done, total): + with lock: + if event == "start": + state["files"] += 1 + state["done"] = 0 + state["total"] = None + elif event == "chunk": + state["done"] = done + state["total"] = total + if mode == "gui": + # Only from the launch thread. Touching a widget from + # a _TileFetcher worker is a crash, not a wrong number. + if threading.get_ident() == launch_thread: + _guiBytes(done, total) + else: + render() + + try: + with watchFetches(observer): + yield + finally: + with lock: + if state["shown"]: + # Leave the cursor on a clean line, so the next stage note + # does not land on top of half a progress bar. + sys.stderr.write("\r" + " " * 78 + "\r") + sys.stderr.flush() + + +# ------------------------------------------------------------------ gui + +#: The launch window, once built. One per process: the launch is one +#: sequence on one thread, and a second window would be a second claim +#: about what is happening. +_window = None +_window_failed = False + + +def _mode() -> str: + """Which indicator to use: ``"gui"``, ``"text"`` or ``"off"``.""" + asked = os.environ.get(_MODE_ENV, "").strip().lower() + if asked in ("gui", "text", "off"): + return asked + if _quiet(): + return "off" + # GUI first: the viewer is normally started from NDI-matlab's View + # button, where there is no terminal at all and a stderr bar is + # written to nobody. Falling back to text keeps a terminal launch + # informative. + if _launchWindow() is not None: + return "gui" + return "text" if sys.stderr.isatty() else "off" + + +def _displayLikely() -> bool: + """Whether building a QApplication is safe to attempt. + + THIS GUARD IS NOT OPTIONAL. Qt does not raise when it cannot reach a + display: it prints "could not connect to display" and calls abort(). + That kills the process, so try/except around QApplication() catches + nothing -- the launch would die instead of falling back to text, which + is a worse failure than having no progress window at all. + + macOS and Windows always have a window server, and they are where the + viewer is started from NDI-matlab's View button. On X11/Wayland the + environment says: DISPLAY or WAYLAND_DISPLAY, or an explicit + QT_QPA_PLATFORM (offscreen included, which is how this is tested). + """ + if sys.platform in ("darwin", "win32"): + return True + return bool( + os.environ.get("DISPLAY") + or os.environ.get("WAYLAND_DISPLAY") + or os.environ.get("QT_QPA_PLATFORM") + ) + + +def _launchWindow(): + """The launch window, built on first use, or None if Qt will not. + + Built BEFORE napari is imported, because the wait this reports on + starts before there is a viewer to host it -- opening the session, + the cell table and the contour file are all spent before napari is + even imported. That means creating the QApplication ourselves, which + napari then reuses. + + The high-DPI attributes are set here for that reason: napari sets + them before creating its own app, and on a Retina display an app + created without them renders at the wrong scale. Setting them is + harmless where they are already the default (Qt6 deprecates both). + """ + global _window, _window_failed + if _window is not None or _window_failed: + return _window + try: + from qtpy.QtCore import Qt + from qtpy.QtWidgets import ( + QApplication, + QLabel, + QProgressBar, + QVBoxLayout, + QWidget, + ) + + app = QApplication.instance() + if app is None and not _displayLikely(): + # No app to borrow and no display to make one against. See + # _displayLikely: attempting it anyway aborts the process. + _window_failed = True + return None + if app is None: + for attr in ("AA_EnableHighDpiScaling", "AA_UseHighDpiPixmaps"): + with contextlib.suppress(AttributeError): + QApplication.setAttribute(getattr(Qt, attr), True) + app = QApplication([]) + + w = QWidget() + w.setWindowTitle("Opening pyramid") + layout = QVBoxLayout(w) + title = QLabel("Opening pyramid") + font = title.font() + font.setBold(True) + title.setFont(font) + step = QLabel("starting ...") + bar = QProgressBar() + bar.setRange(0, 0) # busy until a byte count gives it a range + layout.addWidget(title) + layout.addWidget(step) + layout.addWidget(bar) + w.resize(420, 110) + w.show() + app.processEvents() + + _window = {"app": app, "widget": w, "step": step, "bar": bar} + except Exception: # noqa: BLE001 - no display, no binding, no window + _window_failed = True + _window = None + return _window + + +def _guiStage(label: str) -> None: + """Show the step that is about to run, and MAKE SURE IT IS PAINTED. + + setText queues a repaint; processEvents services what is queued NOW. + Between them they can still leave the old text on screen, because the + step that follows blocks the thread for as long as it takes and no + event loop runs meanwhile -- so any paint the window needs after that + point (it gets exposed, the compositor asks again, the OS marks the + app unresponsive) is never serviced. + + The effect is a window frozen on whichever label was current when the + FIRST slow step began, which made a launch that was working look + stuck on "importing napari" for minutes while the label underneath + had long since moved on. + + repaint() paints synchronously, before returning, so the new label is + on the glass before the blocking work starts. It does not animate -- + nothing can, without an event loop -- but each step is legible while + it runs instead of one step being legible for all of them. + """ + win = _launchWindow() + if win is None: + return + with contextlib.suppress(Exception): + win["step"].setText(label) + win["bar"].setRange(0, 0) # unknown length again + win["app"].processEvents() + win["widget"].repaint() + + +def _guiBytes(done: int, total) -> None: + win = _launchWindow() + if win is None: + return + with contextlib.suppress(Exception): + if total: + win["bar"].setRange(0, 100) + win["bar"].setValue(int(100 * done / total)) + win["bar"].setFormat(f"{humanBytes(done)} of {humanBytes(total)} (%p%)") + else: + win["bar"].setRange(0, 0) + win["bar"].setFormat(humanBytes(done)) + win["app"].processEvents() + # Synchronous, for the same reason as _guiStage: a byte count that + # only lands when the transfer finishes has reported nothing. + win["widget"].repaint() + + +def closeLaunchWindow() -> None: + """Take the launch window down. Safe to call when there never was one. + + Called once the viewer is built and about to be handed to the user: + a progress window that outlives the thing it was reporting on is + worse than none, because it makes a finished launch look stuck. + """ + global _window + win, _window = _window, None + if win is None: + return + with contextlib.suppress(Exception): + win["widget"].close() + win["app"].processEvents() diff --git a/src/ndi/gui/app/genepyramid/resources/README.md b/src/ndi/gui/app/genepyramid/resources/README.md new file mode 100644 index 00000000..50e4c402 --- /dev/null +++ b/src/ndi/gui/app/genepyramid/resources/README.md @@ -0,0 +1,16 @@ +# Bundled assets + +## `ndi_cloud_logo.png` + +The NDI Cloud wordmark, copied byte-for-byte from NDI-matlab at +`src/ndi/+ndi/+cloud/+ui/resources/images/ndi_logo.png` (1682x306, +md5 3e8439f1b1b8686b38ad71f314fff157), which is the same file that repo +serves to its own cloud dialogs and to the metadata app. + +Copied rather than referenced because the viewer runs from a Python +install that need not have NDI-matlab on disk at all -- a downloaded +dataset opened by someone who has never run MATLAB is the normal case. + +It is a WORDMARK IN DARK NAVY on transparency, so it is drawn on a white +card: napari's docks follow the viewer's theme, and navy on the dark one +is invisible. The card is how the brand is used elsewhere anyway. diff --git a/src/ndi/gui/app/genepyramid/resources/ndi_cloud_logo.png b/src/ndi/gui/app/genepyramid/resources/ndi_cloud_logo.png new file mode 100644 index 00000000..8e09b578 Binary files /dev/null and b/src/ndi/gui/app/genepyramid/resources/ndi_cloud_logo.png differ diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index 3c8247e2..1b56a4fc 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -9,7 +9,9 @@ from typing import Any -from .multiscale import layerSpec, sourceToWorld +import numpy as np + +from .multiscale import layerSpec, sourceToWorld, worldTransform __all__ = ["require_napari", "openPyramid"] @@ -39,6 +41,12 @@ def openPyramid( gene_rows=None, density: bool = True, cells: dict[str, Any] | None = None, + outlines=None, + cells_doc=None, + controls: bool = True, + labelings=None, + name: str | None = None, + gene_layers=None, show: bool = True, ): """Open a spatial gene expression pyramid in napari. @@ -56,6 +64,37 @@ def openPyramid( straight in, because the image layer carries the origin in its translate and centroids that skip that transform land somewhere plausible and wrong. + outlines: optional list of ``(N, 2)`` ``[x, y]`` vertex arrays in + SOURCE coordinates, as :func:`~ndi.fun.doc_gene.readContours` + returns them, drawn as a Shapes layer. Routed through + :func:`~.multiscale.sourceToWorld` for the same reason + centroids are. Empty polygons are dropped: napari treats a + zero-vertex shape as malformed rather than as nothing. + cells_doc: the spatialGeneExpressionCells document the overlay + came from. Only needed for the cell-type panel, which reads + the cellTypeLabels documents that depend on it -- the cells + themselves arrive as plain arrays and carry no way back to + their own document. + labelings: label names to show in the cell-type panel, or None to + let it decide. Two labelings that say the same thing about the + section -- a subclass call transferred onto the clustering it + was transferred onto -- are otherwise drawn twice, so by + default the panel keeps one and says so. Naming them here + overrides that entirely. + controls: dock the gene, display and cell-type panels. Without + them the gene selection and the density choice are fixed at + launch, since this viewer holds no other state. Skipped with + a note when Qt is unavailable -- they are conveniences and + their absence must not stop the picture. + gene_layers: ``(symbol, colormap)`` pairs to open as their own + additive layers on top of the base image, as + :func:`~.controls.parseGeneLayers` returns them. A DIFFERENT + request from *gene_rows*, which filters the base layer to a + subset and leaves one picture; this adds a layer per gene, so + the base stays and the genes are comparable against it. + name: image layer name. Defaults to the pyramid's label, which is + whatever the ingest recorded -- often the file stem, which + names the section rather than what is being shown. show: call ``napari.run()``. False returns the viewer without blocking, which is what a test or a caller composing several layers wants. @@ -63,20 +102,109 @@ def openPyramid( Returns: The napari Viewer. """ - napari = require_napari() + from .progress import closeLaunchWindow, note, stage + + with stage("starting napari"): + napari = require_napari() viewer = napari.Viewer() - viewer.add_image(**layerSpec(session, pyr_doc, gene_rows, density)) + # NAPARI'S OWN THEME IS LEFT ALONE, which means the canvas stays dark. + # Setting it to "light" did put the docks on a light ground, but it + # took the CANVAS with it -- and a dark canvas is what imaging data is + # read against. What was actually wanted was the PANELS on the NDI + # Cloud palette, and those are styled individually in + # ndi.gui.app.genepyramid.controls.applyCloudStyle, which does not + # touch the canvas. Two different surfaces, and only one of them + # should be white. + # The image ladder is LAZY: layerSpec resolves tile paths and the + # level table, and reads no tile bytes. Nothing here is the wait. + with stage("preparing the image levels (read on demand)"): + spec = layerSpec(session, pyr_doc, gene_rows, density, name) + # SPLIT FROM THE LADDER, because only one of these two is lazy and the + # label used to claim both were. Building the ladder reads nothing. + # add_image then makes napari's thumbnail, and napari builds that by + # materialising the WHOLE COARSEST LEVEL -- Image._post_init calls + # refresh, which reaches _level_materializer and computes the dask + # array for the thumbnail level. On a real section that is gigabytes + # of tiles, and on a downloaded dataset it is gigabytes of fetching. + # + # It is the longest step of a cold launch and it used to sit behind a + # label that said "(lazy)", which is why it read as a hang. Naming it + # does not make it fast; it makes it legible. + with stage("drawing the overview (reads the whole lowest-resolution level)"): + image = viewer.add_image(**spec) + + shapes = None + if outlines is not None: + keep = [p for p in outlines if len(p)] + if keep: + # np.column_stack per polygon rather than list(zip(row, col)): + # the tuple form built twelve million Python tuples for the + # opossum section and cost 13.8s, against 1.6s for this. + with stage(f"placing {len(keep):,} outlines"): + (sy, sx), _t = worldTransform(session, pyr_doc) + paths = [np.column_stack([p[:, 1] * sy, p[:, 0] * sx]) for p in keep] + # shape_type polygon closes the ring itself, which matches the + # format: writeContourFile does not repeat the first vertex. + # + # THIS IS THE SLOW ONE and nothing here can make it fast: + # napari triangulates every polygon as it takes them. + note( + f"drawing {len(keep):,} cell outlines. Each one has to be cut " + f"into triangles before it can be drawn, which is the long " + f"part of the launch. Leave out --outlines to skip it." + ) + with stage("drawing the cell outlines"): + shapes = viewer.add_shapes( + paths, + shape_type="polygon", + name="cell outlines", + face_color="transparent", + edge_color="cyan", + edge_width=1, + ) + points = None if cells is not None: - row, col = sourceToWorld(session, pyr_doc, cells["x"], cells["y"]) - viewer.add_points( - list(zip(row, col)), - name="cell centroids", - size=cells.get("size", 8), - face_color=cells.get("face_color", "red"), - border_width=0, - ) + with stage(f"placing {len(cells['x']):,} centroids"): + row, col = sourceToWorld(session, pyr_doc, cells["x"], cells["y"]) + coords = np.column_stack([row, col]) + with stage("drawing the cell centroids"): + points = viewer.add_points( + coords, + name="cell centroids", + size=cells.get("size", 8), + face_color=cells.get("face_color", "red"), + border_width=0, + ) + + if controls: + from .controls import addAllPanels + + # Remembered on the layer so the display panel can rebuild the + # ladder with the SAME gene selection when it switches between + # density and counts; recomputing with all genes would silently + # widen what is being shown. + image._ndi_gene_rows = gene_rows + with stage("building the control panels"): + addAllPanels( + viewer, + session, + pyr_doc, + image, + density, + cells_doc=cells_doc, + points_layer=points, + shapes_layer=shapes, + labelings=labelings, + gene_layers=gene_layers, + ) + + # The launch window has reported everything it can: the viewer is + # built and the panels are on it. Leaving it up past this point would + # make a finished launch look stuck, and it would sit over the picture + # it exists to get you to. + closeLaunchWindow() if show: napari.run() diff --git a/src/ndi/gui/ndi_matlab_python_bridge.yaml b/src/ndi/gui/ndi_matlab_python_bridge.yaml index 50809c7a..9b10254a 100644 --- a/src/ndi/gui/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/ndi_matlab_python_bridge.yaml @@ -836,19 +836,42 @@ ported_late: stage markers rather than field presence). The deferral is unchanged: JRCLUST itself is still not reachable from Python. - - name: GeneIngest - matlab_path: "+ndi/+gui/+app/GeneIngest.m" - matlab_last_sync_hash: "eae346a" + - name: GEFManager + matlab_path: "+ndi/+gui/+app/GEFManager.m" + matlab_last_sync_hash: "06f31a4e5" status: porting_deferred decision_log: > - MATLAB App Designer window that lists, adds and removes spatial - transcriptomics pyramids in a session, and drives the .gef / - cellbin .h5ad ingest path with a confirm step between file choice - and the write. Added in VH-Lab/NDI-matlab PR #933 alongside the - +fun/+doc/+gene entry points (fromGEF, fromCellBin, makeSourceFile, - readNotes, tilePath) that this app calls, all recorded - not_yet_ported in src/ndi/fun/. Python has the pyramid readers and - writers (ndi.fun.doc_gene) but no session-scoped ingest UI over - them; the app is the natural place for one and lands with a - non-interactive or navigator-integrated equivalent. Interactive - viewing is separately napari's, through NDI-python. + MATLAB App Designer window that lists, views, adds and removes + spatial transcriptomics pyramids in a session, and drives the .gef + / cellbin .h5ad ingest path with a confirm step between file choice + and the write. Added in VH-Lab/NDI-matlab PR #933 as GeneIngest and + renamed there in PR #977, which also gave it a View button. Named + alongside the +fun/+doc/+gene entry points (fromGEF, fromCellBin, + makeSourceFile, readNotes, tilePath) that this app calls, all + recorded not_yet_ported in src/ndi/fun/. Python has the pyramid + readers and writers (ndi.fun.doc_gene) but no session-scoped ingest + UI over them; the app is the natural place for one and lands with a + non-interactive or navigator-integrated equivalent. + ONE PART OF IT IS ALREADY PORTED, in the other direction. View + draws nothing in MATLAB: it builds a command line for the napari + viewer in THIS package -- ndi.gui.app.genepyramid.cli, the + napariViewGEF console script -- and runs it. So what stays unported + is the ingest and the listing; the viewing the MATLAB app offers is + this side's, reached across a process boundary rather than + reimplemented. + + NDI-matlab 92ddd79c1 is styling only: a paper() helper puts the + uitable, uitextarea, uilistbox and uieditfield on the cloud + palette's white body, since the figure and its layouts were already + on c.offWhite while the controls kept MATLAB's own grey. The Python + side made the SAME correction independently and in its own idiom -- + the napari viewer sets the light theme and styles its dock panels + from ndi.gui.cloud_colors, the identical palette. Nothing to port: + these are two different windows that agreed to look alike, not one + window and its port. + + NDI-matlab cf0983b5c made the View dialog non-modal -- onView no + longer ends in uiwait -- and gave the app an onClose that takes the + dialog with it. Nothing to port: the Python side draws no dialog at + all. View builds a command line and runs napariViewGEF, which is + already its own process and has never blocked anything. diff --git a/src/ndi/ndi_common/database_documents/data/fileReference.json b/src/ndi/ndi_common/database_documents/data/fileReference.json new file mode 100644 index 00000000..845ea9a8 --- /dev/null +++ b/src/ndi/ndi_common/database_documents/data/fileReference.json @@ -0,0 +1,25 @@ +{ + "document_class": { + "definition": "$NDIDOCUMENTPATH\/data\/fileReference.json", + "validation": "$NDISCHEMAPATH\/data\/fileReference_schema.json", + "class_name": "fileReference", + "property_list_name": "fileReference", + "class_version": 1, + "superclasses": [ + { "definition": "$NDIDOCUMENTPATH\/base.json"} + ] + }, + "depends_on": [ + { "name": "document_id", "value": "" } + ], + "fileReference": { + "filename": "", + "originalPath": "", + "formatOntology": "", + "dateCreated": 0, + "dateUpdated": 0, + "fileSize": 0, + "checksum": "", + "checksumAlgorithm": "MD5" + } +} diff --git a/src/ndi/ndi_common/database_documents/data/fileReference.md b/src/ndi/ndi_common/database_documents/data/fileReference.md new file mode 100644 index 00000000..10746a64 --- /dev/null +++ b/src/ndi/ndi_common/database_documents/data/fileReference.md @@ -0,0 +1,66 @@ +# fileReference + +The `fileReference` document class records a file that exists **outside** +the database: its name, its size, its dates and its checksum. It holds no +file of its own. + +## Why it is not generic_file + +`generic_file` is for a file the database **holds**. Its class declares +`generic_file.ext` with `mustbenotempty: 1`, and the cloud download path +(`ndi.cloud.download.downloadGenericFiles`) exists to fetch those bytes +back out. A `generic_file` document without its file is not a lighter +version of one — it is an invalid one. + +That distinction was learned the hard way. `ndi.fun.doc.gene.makeSourceFile` +used to describe a source `.gef` by creating a `generic_file` document and +deliberately not attaching the file, on the reasoning that ingesting a copy +of a 9.4 GB container beside the pyramid derived from it doubles the +storage to hold bytes the pyramid already summarises. The reasoning was +right; the class was wrong. The documents were schema-invalid from the +moment they were made, and stayed invisible only because DID's file +validation used to fail open — it fell through to "valid" when a required +file was absent (fixed in did-matlab #182). When that was tightened, every +affected session became uncopyable into a dataset, with an error naming a +file_list that was in fact perfectly correct. + +So the two cases now have two classes, and the difference is in the name: + +| | holds the bytes | records the identity | +| --- | --- | --- | +| `generic_file` | yes, `generic_file.ext` required | — | +| `fileReference` | no files at all | filename, size, dates, checksum | + +## What identifies the file + +**The checksum, not the path.** A filename tells you what somebody called +a file; a checksum tells you whether the file you have now is the file the +document was made from. Two SAW runs of the same chip produce files with +the same name and different contents, which is exactly the confusion this +is meant to survive. `originalPath` is recorded as a hint about where the +file was, not a promise about where it is — paths do not survive moving +between machines. + +`checksumAlgorithm` is stored alongside the checksum so a later reader can +verify it without guessing. An empty `checksum` means *not computed*; it +never means the file was empty. + +`fileSize` is a cheap second field: two files of different sizes are +certainly not the same file, and the size is readable without opening it. + +## Pointing at one + +`fileReference` carries the same optional `document_id` dependency +`generic_file` does, so it can name a document it is about. In the other +direction, documents point at it through their own dependencies — +`spatialGeneExpressionPyramid`, `spatialGeneExpressionTiles` and +`spatialGeneExpressionCells` all declare an optional `source_file_id`, +which may name either class. Those dependencies are untyped, so a session +built before this class existed, pointing `source_file_id` at a +`generic_file`, keeps working. + +## See also + +`ndi.fun.doc.gene.makeSourceFile`, which creates a `fileReference` by +default and a `generic_file` when asked to attach the file; +`ndi.cloud.download.downloadGenericFiles`, which is for the other class. diff --git a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json index c0a7ce78..b925c9c8 100644 --- a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json +++ b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.json @@ -49,6 +49,7 @@ "data_type_offset": "uint32", "data_type_coordinate": "uint16", "tile_compression": "none", - "tile_format_version": 1 + "tile_format_version": 1, + "tile_index_origin": 1 } } diff --git a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md index 9bdab14c..7b063ecd 100644 --- a/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md +++ b/src/ndi/ndi_common/database_documents/data/spatialGeneExpressionTiles.md @@ -116,11 +116,26 @@ The gene dictionary is **not** a file of this document. It lives in the `geneList` document that the parent pyramid depends on, so that a dissociated RNA-seq dataset built on the same annotation can share it. -`#` is a non-negative integer computed from the tile's (row, column) by +`#` is the tile's INDEX plus the document's `tile_index_origin`. The index +is a non-negative grid position computed from the tile's (row, column) by the parent's `index_order`; for `row-major` it is -`row * tile_columns + column`. Tiles containing no data are not written. -Use `ndi.document/current_file_list` to discover which tiles exist — -there is no separate index file. +`row * tile_columns + column`. + +`tile_index_origin` is **1**, because a DID file series is one-based: its +first member is `NAME_1`, and `did.document/addFileSeries` refuses +anything else ("Member indices must be positive integers (one-based)"). +So the tile at (0,0) is stored as `tile.bin_1`. Pyramids written before +this convention was honoured record no `tile_index_origin`, which reads as +0; they name that tile `tile.bin_0` and stay readable. Read the origin +from the document rather than inferring it from the names — a sparse +pyramid whose first tile is empty has no `_0` under either convention, so +a guess would shift every tile by one on the documents it got wrong. + +Tiles containing no data are not written, so the series is SPARSE: an +absent `#` is normal and means that tile held nothing. Use +`ndi.document/current_file_list` to discover which tiles exist — there is +no separate index file, and walking the numbers until one is missing +finds the first hole rather than the end. ### tile.bin_N layout (tile_format_version 1) diff --git a/src/ndi/ndi_common/schema_documents/data/fileReference_schema.json b/src/ndi/ndi_common/schema_documents/data/fileReference_schema.json new file mode 100644 index 00000000..b894efdf --- /dev/null +++ b/src/ndi/ndi_common/schema_documents/data/fileReference_schema.json @@ -0,0 +1,73 @@ +{ + "classname": "fileReference", + "superclasses": [ "base" ], + "depends_on": [ + { "name": "document_id", "mustbenotempty": 0} + ], + "fileReference": [ + { + "name": "filename", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The file's name with extension, without a directory (e.g. 'Opossum.tissue.gef')." + }, + { + "name": "originalPath", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The full path the file was read from, if it is worth recording. It is a hint about where the file was, not a promise about where it is: paths do not survive being moved between machines, which is why the checksum rather than this field identifies the file." + }, + { + "name": "formatOntology", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The ontology node describing the file format in the form of ontology:nodeID (e.g 'EMPTY:0000002'). Empty when not known; a wrong term is worse than none because it is machine-readable and will be believed." + }, + { + "name": "dateCreated", + "type": "double", + "default_value": [0], + "parameters": [0,10000000,1], + "queryable": 1, + "documentation": "The time the file was created, as a datenum. The file's own date, not the date the document was made." + }, + { + "name": "dateUpdated", + "type": "double", + "default_value": [0], + "parameters": [0,10000000,1], + "queryable": 1, + "documentation": "The time the file was last updated, as a datenum." + }, + { + "name": "fileSize", + "type": "double", + "default_value": [0], + "parameters": [0,1e18,1], + "queryable": 1, + "documentation": "The file's size in bytes, or 0 if it was not recorded. A cheap second identifying field: two files that differ in size are certainly not the same file, and it is readable without opening the file." + }, + { + "name": "checksum", + "type": "string", + "default_value": "", + "parameters": "", + "queryable": 1, + "documentation": "The checksum of the file's contents, in the algorithm named by checksumAlgorithm. Empty when it was not computed; empty means unknown and never means the file had no content." + }, + { + "name": "checksumAlgorithm", + "type": "string", + "default_value": "MD5", + "parameters": "", + "queryable": 1, + "documentation": "The algorithm the checksum was computed with, so a future reader can verify it without guessing. 'MD5' is what ndi.fun.file.MD5 produces." + } + ] +} diff --git a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json index e3412b17..4152c944 100644 --- a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json +++ b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionPyramid_schema.json @@ -181,7 +181,7 @@ "default_value": "row-major", "parameters": "", "queryable": 1, - "documentation": "How (row,column) maps to the suffix N in tile.bin_N. 'row-major': N = row*tile_columns + column. 'column-major': N = column*tile_rows + row." + "documentation": "How (row,column) maps to the tile INDEX N. 'row-major': N = row*tile_columns + column. 'column-major': N = column*tile_rows + row. The FILE holding tile N is named tile.bin_, where the origin is a field of the tiles document; a DID file series is one-based, so it is 1 for documents written since that was honoured and absent (0) for older ones." }, { "name": "origin_corner", diff --git a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json index 6d9257bf..58543e24 100644 --- a/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json +++ b/src/ndi/ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json @@ -216,6 +216,19 @@ ], "queryable": 1, "documentation": "The version of the tile binary layout. See spatialGeneExpressionTiles.md." + }, + { + "name": "tile_index_origin", + "type": "integer", + "default_value": 1, + "parameters": [ + 0, + 1, + 0, + 0 + ], + "queryable": 1, + "documentation": "The suffix of the file holding tile N is N plus this. A DID file series is ONE-BASED -- its first member is NAME_1 -- so this is 1 for every document written since that was honoured. Pyramids written before it name their first tile tile.bin_0 and carry no value here, which reads as 0 and keeps them readable. The tile INDEX itself stays zero-based in either case; it is a grid position, defined by index_order as row*tile_columns + column." } ] } diff --git a/src/ndi/ndi_matlab_python_bridge.yaml b/src/ndi/ndi_matlab_python_bridge.yaml index 960e7a81..1a01385b 100644 --- a/src/ndi/ndi_matlab_python_bridge.yaml +++ b/src/ndi/ndi_matlab_python_bridge.yaml @@ -1155,7 +1155,7 @@ classes: - name: preferences type: class matlab_path: "+ndi/preferences.m" - matlab_last_sync_hash: "9a9150e7c" + matlab_last_sync_hash: "362060795" python_path: "ndi/preferences.py" python_class: "ndi_preferences" inherits: "" diff --git a/src/ndi/preferences.py b/src/ndi/preferences.py index af831d76..5ece3191 100644 --- a/src/ndi/preferences.py +++ b/src/ndi/preferences.py @@ -226,6 +226,19 @@ def _register_defaults(self) -> None: "subclass ndi.gui.app.SessionApp are added to the built-in " "ndi.gui.app and ndi.app packages.", ) + self._add_item( + "GUI", + "GEFManager", + "ViewerLauncher", + "/usr/local/bin/napariViewGEF", + "str", + "Program the GEF Manager's View button runs to open a pyramid in " + "napari. Normally a small shell wrapper rather than the " + "napariViewGEF console script itself: MATLAB exports library " + "paths of its own, and a Python process started from MATLAB loads " + "MATLAB's copies of libraries it must not use, so the wrapper " + "scrubs the environment before running the real entry point.", + ) def _add_item( self, diff --git a/src/ndi/util/ndi_matlab_python_bridge.yaml b/src/ndi/util/ndi_matlab_python_bridge.yaml index ed405c9e..3efcdeb3 100644 --- a/src/ndi/util/ndi_matlab_python_bridge.yaml +++ b/src/ndi/util/ndi_matlab_python_bridge.yaml @@ -725,3 +725,29 @@ not_applicable: commit. It still holds exactly find_instance_name.m and find_techniques_names.m, the two files this entry accounts for. Nothing to port. + +# ============================================================================= +# Deferred MATLAB functions +# ============================================================================= +porting_deferred: + + - name: hdf5Contents + matlab_path: "+ndi/+util/hdf5Contents.m" + matlab_last_sync_hash: "742ac0d81" + status: porting_deferred + decision_log: > + Prints a table of contents for an HDF5 container -- groups, + datasets, attributes -- and a verdict naming which layout it is: + square-bin GEF, CellBin GEF, AnnData, or none of those. It exists + because a Stereo-seq run leaves several files with the same stem and + different middles (Y00709C1E2.tissue.gef beside + Y00709C1E2.adjusted.cellbin.gef), all of them HDF5, and the + extension does not say which one ndi.fun.doc.gene.fromGEF can read. + + Not ported yet because it is a MATLAB-console convenience: the + question it answers is asked interactively, before committing to an + ingest, and NDI-matlab is where that ingest is driven from + (ndi.gui.app.GEFManager). Nothing in the Python package calls it. + h5py makes the port straightforward whenever a Python-side ingest + GUI wants the same answer; it is the printing and the verdict + wording that would need mirroring, not the walk. diff --git a/tests/test_cloud_auth_profile_fallback.py b/tests/test_cloud_auth_profile_fallback.py new file mode 100644 index 00000000..34e9c8a3 --- /dev/null +++ b/tests/test_cloud_auth_profile_fallback.py @@ -0,0 +1,375 @@ +"""Cloud credentials and the API they are sent to both come from the profile. + +Two gaps, one after the other. + +FIRST: ndi.cloud.profile is the documented place to keep a cloud password, +but nothing in the authentication path read it. Every implicit login -- each +@_auto_client API call, and the ndic:// file handler that fetches pyramid +tiles for a downloaded dataset -- went token -> environment -> raise, and +said "no credentials available" with a working default profile on disk. +MATLAB's authenticate.m has had the equivalent step (authenticatedWithSecret) +all along. + +SECOND, once that worked: the profile's Stage picks prod vs dev, and applying +it only on the login path is not enough. login() exports NDI_CLOUD_TOKEN, so +every config built afterwards short-circuits at authenticate()'s first step +and never reaches the credential path -- a dev profile logged in against dev +and then issued every subsequent request against prod. NDI-matlab settles +this the same way, in profile.switchProfile: + + setenv('CLOUD_API_ENVIRONMENT', prof.Stage) + +so the stage is resolved where the config is built, not where the login is. +""" + +import base64 +import json +import os +import time +import unittest +from unittest import mock + +from ndi.cloud import auth +from ndi.cloud.config import CloudConfig +from ndi.cloud.exceptions import CloudAuthError +from ndi.cloud.profile import ProfileEntry + +_ENV_KEYS = ( + "NDI_CLOUD_TOKEN", + "NDI_CLOUD_ORGANIZATION_ID", + "NDI_CLOUD_USERNAME", + "NDI_CLOUD_PASSWORD", + "NDI_CLOUD_URL", + "CLOUD_API_ENVIRONMENT", +) + +_PROD = "https://api.ndi-cloud.com/v1" +_DEV = "https://dev-api.ndi-cloud.com/v1" + + +class _NoEnvMixin(unittest.TestCase): + def setUp(self): + patcher = mock.patch.dict(os.environ, {}, clear=False) + patcher.start() + self.addCleanup(patcher.stop) + for key in _ENV_KEYS: + os.environ.pop(key, None) + + +def _unexpired_token() -> str: + """A JWT whose exp claim is far enough out that isTokenExpired says no.""" + + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + return f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) + 3600})}.sig" + + +def _login_spy(token="tok", org="org-1"): + """Stand in for auth.login, recording what it was handed.""" + calls = [] + + def fake(email=None, password=None, config=None): + calls.append((email, password, config.api_url if config else None)) + config = config or CloudConfig() + config.token = token + config.org_id = org + return config + + return fake, calls + + +class TestProfileCredentials(_NoEnvMixin): + def test_a_current_profile_is_preferred_over_the_default(self): + current = ProfileEntry(UID="u1", Email="current@example.com", Stage="prod") + default = ProfileEntry(UID="u2", Email="default@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=current), + mock.patch("ndi.cloud.profile.get_default", return_value=default), + mock.patch("ndi.cloud.profile.get_password", return_value="pw"), + ): + self.assertEqual(auth._profile_credentials(), ("current@example.com", "pw")) + + def test_the_default_profile_is_used_when_no_current_one_is_set(self): + default = ProfileEntry(UID="u2", Email="default@example.com", Stage="dev") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=default), + mock.patch("ndi.cloud.profile.get_password", return_value="pw"), + ): + self.assertEqual(auth._profile_credentials(), ("default@example.com", "pw")) + + def test_no_profile_at_all_reports_emptiness_rather_than_raising(self): + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + ): + self.assertEqual(auth._profile_credentials(), ("", "")) + + def test_a_secrets_backend_that_throws_is_not_allowed_to_escape(self): + entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=entry), + mock.patch( + "ndi.cloud.profile.get_password", side_effect=RuntimeError("locked keyring") + ), + ): + self.assertEqual(auth._profile_credentials(), ("", "")) + + def test_a_profile_with_no_stored_password_counts_as_no_credentials(self): + entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=entry), + mock.patch("ndi.cloud.profile.get_password", return_value=""), + ): + self.assertEqual(auth._profile_credentials(), ("", "")) + + +class TestAuthenticateReachesTheProfile(_NoEnvMixin): + def test_the_profile_logs_in_when_the_environment_is_empty(self): + fake, calls = _login_spy() + with ( + mock.patch.object(auth, "login", fake), + mock.patch.object(auth, "_profile_credentials", return_value=("a@example.com", "pw")), + ): + self.assertEqual(auth.authenticate(), ("tok", "org-1")) + self.assertEqual(calls[0][:2], ("a@example.com", "pw")) + + def test_an_exported_username_still_wins_over_the_profile(self): + os.environ["NDI_CLOUD_USERNAME"] = "env@example.com" + os.environ["NDI_CLOUD_PASSWORD"] = "envpw" + fake, calls = _login_spy() + with ( + mock.patch.object(auth, "login", fake), + mock.patch.object(auth, "_profile_credentials", return_value=("a@example.com", "pw")), + ): + auth.authenticate() + self.assertEqual(calls[0][:2], ("env@example.com", "envpw")) + + def test_a_dev_profiles_login_is_issued_against_dev(self): + fake, calls = _login_spy() + with ( + mock.patch("ndi.cloud.config._profile_stage", return_value="dev"), + mock.patch.object(auth, "login", fake), + mock.patch.object(auth, "_profile_credentials", return_value=("a@example.com", "pw")), + ): + auth.authenticate() + self.assertEqual(calls[0][2], _DEV) + + def test_with_neither_environment_nor_profile_the_error_names_both(self): + with mock.patch.object(auth, "_profile_credentials", return_value=("", "")): + with self.assertRaises(CloudAuthError) as ctx: + auth.authenticate() + message = str(ctx.exception) + self.assertIn("NDI_CLOUD_USERNAME", message) + self.assertIn("profile", message) + + +class TestTheStageSurvivesTheTokenShortCircuit(_NoEnvMixin): + """The bug a dev user actually hits. + + Each on-demand pyramid tile builds its own client, so this struck from + the second request onwards, after a login that looked fine, as: + + Not found (HTTP 404): {"error":"Dataset ... does not exist or user + does not have access"} + + which reads as a permissions problem rather than a wrong-server one. + """ + + def test_a_config_built_after_login_still_points_at_dev(self): + os.environ["NDI_CLOUD_TOKEN"] = "already-have-one" + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, _DEV) + + def test_authenticate_short_circuits_without_losing_the_dev_url(self): + os.environ["NDI_CLOUD_TOKEN"] = _unexpired_token() + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + config = CloudConfig.from_env() + _, org = auth.authenticate(config) + self.assertEqual(org, "org-1") + self.assertEqual(config.api_url, _DEV) + + +class TestApiUrlPrecedence(_NoEnvMixin): + def test_an_explicit_url_beats_the_profile_stage(self): + os.environ["NDI_CLOUD_URL"] = "https://example.invalid/v1" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, "https://example.invalid/v1") + + def test_an_explicit_environment_beats_the_profile_stage(self): + os.environ["CLOUD_API_ENVIRONMENT"] = "prod" + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + def test_the_profile_stage_beats_the_prod_default(self): + with mock.patch("ndi.cloud.config._profile_stage", return_value="dev"): + self.assertEqual(CloudConfig.from_env().api_url, _DEV) + + def test_with_no_profile_at_all_it_is_prod(self): + with mock.patch("ndi.cloud.config._profile_stage", return_value=""): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + def test_an_unrecognised_stage_falls_back_to_prod_rather_than_a_bad_url(self): + with mock.patch("ndi.cloud.config._profile_stage", return_value="staging"): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + def test_a_profile_store_that_will_not_load_does_not_stop_a_config(self): + with mock.patch("ndi.cloud.profile.get_current", side_effect=RuntimeError("locked")): + self.assertEqual(CloudConfig.from_env().api_url, _PROD) + + +class TestProfileStage(_NoEnvMixin): + def test_the_current_profile_is_preferred_over_the_default(self): + from ndi.cloud.config import _profile_stage + + current = ProfileEntry(UID="u1", Email="a@example.com", Stage="dev") + default = ProfileEntry(UID="u2", Email="b@example.com", Stage="prod") + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=current), + mock.patch("ndi.cloud.profile.get_default", return_value=default), + ): + self.assertEqual(_profile_stage(), "dev") + + def test_no_profile_yields_no_stage_rather_than_a_guess(self): + from ndi.cloud.config import _profile_stage + + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + ): + self.assertEqual(_profile_stage(), "") + + +def _expired_token(ago_seconds: int = 3600) -> str: + """A JWT whose exp claim is in the past.""" + + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + return f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) - ago_seconds})}.sig" + + +class TestWhatTheFailureSays(_NoEnvMixin): + """Three quite different situations end at the same raise. + + The reader of this message is usually looking at a FAILED FILE FETCH, + not at a login: a napari toast on a tile, one per tile, an hour into a + session that was working. "No valid token and no credentials + available" is equally true of never having logged in and of a token + that lapsed while the viewer was open, and those want opposite + responses -- so the message has to say which happened. + """ + + def _raised(self, config): + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=None), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + ): + with self.assertRaises(CloudAuthError) as caught: + auth.authenticate(config) + return str(caught.exception) + + def test_an_expired_token_is_named_as_expired(self): + """The one that reads as missing data if it is not said out loud: + the tile is there, the token lapsed.""" + message = self._raised(CloudConfig(token=_expired_token(), org_id="org-1")) + self.assertIn("expired", message.lower()) + self.assertNotIn("Not logged in", message) + + def test_an_expired_token_says_when(self): + message = self._raised(CloudConfig(token=_expired_token(), org_id="org-1")) + self.assertIn("UTC", message) + + def test_an_expired_token_says_another_shell_will_not_do(self): + """The token lives in os.environ, so logging in elsewhere does not + reach a process that is already running. Without that sentence the + obvious remedy looks like it did not work.""" + message = self._raised(CloudConfig(token=_expired_token(), org_id="org-1")) + self.assertIn("another shell", message) + self.assertIn("start it again", message) + + def test_no_token_at_all_says_so_instead(self): + message = self._raised(CloudConfig(token="", org_id="")) + self.assertIn("Not logged in", message) + self.assertNotIn("expired", message.lower().split("only until")[0]) + + def test_a_token_with_no_organization_is_its_own_case(self): + """Step 1 wants both. A token that passes the expiry check and is + still refused is otherwise indistinguishable from a bad password.""" + message = self._raised(CloudConfig(token=_unexpired_token(), org_id="")) + self.assertIn("organization id", message) + + def test_every_case_says_how_to_stop_it_happening_again(self): + """Credentials it can reach are the fix for all three: with them, + authenticate renews the token itself and step 1 stops mattering.""" + for config in ( + CloudConfig(token="", org_id=""), + CloudConfig(token=_expired_token(), org_id="org-1"), + CloudConfig(token=_unexpired_token(), org_id=""), + ): + message = self._raised(config) + self.assertIn("NDI_CLOUD_USERNAME", message) + self.assertIn("profile", message) + + +class TestTheCredentialReport(_NoEnvMixin): + """One line instead of a bisection. + + An authentication failure reaches most people through something else + -- a tile that will not load -- and from there it is hard to say where + the credentials were meant to come from. The report walks the same + three steps authenticate walks and names each. + """ + + def _report(self, profile_entry=None, password=""): + with ( + mock.patch("ndi.cloud.profile.get_current", return_value=profile_entry), + mock.patch("ndi.cloud.profile.get_default", return_value=None), + mock.patch("ndi.cloud.profile.get_password", return_value=password), + ): + return auth.credentialReport() + + def test_it_says_every_step_came_up_empty(self): + report = self._report() + self.assertIn("token in env : no", report) + self.assertIn("username in env: no", report) + self.assertIn("profile : none usable", report) + self.assertIn("FAILED", report) + + def test_an_expired_token_is_reported_with_its_expiry(self): + os.environ["NDI_CLOUD_TOKEN"] = _expired_token() + report = self._report() + self.assertIn("token in env : yes", report) + self.assertIn("EXPIRED", report) + self.assertIn("UTC", report) + + def test_a_live_token_reads_as_valid(self): + os.environ["NDI_CLOUD_TOKEN"] = _unexpired_token() + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + report = self._report() + self.assertIn("valid", report) + self.assertIn("authenticate : ok", report) + + def test_a_usable_profile_is_named(self): + entry = ProfileEntry(UID="u1", Email="a@example.com", Stage="prod") + with mock.patch.object(auth, "login", _login_spy()[0]): + report = self._report(profile_entry=entry, password="pw") + self.assertIn("profile : a@example.com", report) + + def test_it_never_prints_the_secret(self): + """It is meant to be pasted into a chat or an issue.""" + token = _unexpired_token() + os.environ["NDI_CLOUD_TOKEN"] = token + os.environ["NDI_CLOUD_ORGANIZATION_ID"] = "org-1" + os.environ["NDI_CLOUD_USERNAME"] = "a@example.com" + os.environ["NDI_CLOUD_PASSWORD"] = "hunter2" + report = self._report() + self.assertNotIn(token, report) + self.assertNotIn("hunter2", report) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cloud_profile_matlab_shape.py b/tests/test_cloud_profile_matlab_shape.py new file mode 100644 index 00000000..6aae151d --- /dev/null +++ b/tests/test_cloud_profile_matlab_shape.py @@ -0,0 +1,93 @@ +"""Reading a profiles file that NDI-matlab wrote. + +The two languages deliberately share ~/.ndi/NDI_Cloud_Profiles.json, so +what MATLAB's jsonencode produces is what this reader has to accept. It +produces a different SHAPE depending on how many profiles there are: a +1x1 struct becomes a JSON object, a 1xN struct array becomes a JSON +array. MATLAB's own reader has normalizeProfiles for that; these tests +pin the Python counterpart. + +The single-profile case is the one that matters most, because it is +what every new user has after making their first. +""" + +from __future__ import annotations + +import json + +from ndi.cloud import profile +from ndi.cloud.profile import _as_profile_list, _ProfileSingleton + +ONE = { + "UID": "abc123", + "Nickname": "steve", + "Email": "steve@walthamdatascience.com", + "Stage": "prod", + "PasswordSecret": "NDI Cloud abc123", +} + + +def _write(tmp_path, profiles_value, default_uid="abc123"): + f = tmp_path / "NDI_Cloud_Profiles.json" + f.write_text(json.dumps({"Profiles": profiles_value, "DefaultUID": default_uid})) + return f + + +def _load(tmp_path, monkeypatch): + monkeypatch.setenv("NDI_PREFDIR", str(tmp_path)) + obj = _ProfileSingleton(backend="memory") + obj._load_from_disk() + obj._adopt_default_as_current() + return obj + + +def test_a_single_profile_written_as_an_object_is_read(tmp_path, monkeypatch): + """MATLAB writes one profile as {...}, not [{...}]. Iterating that as + a list yields its keys, and every key was skipped as 'not a dict' -- + so the file was found, parsed, and silently read as empty.""" + _write(tmp_path, ONE) + obj = _load(tmp_path, monkeypatch) + assert [p.Nickname for p in obj.profiles] == ["steve"] + assert obj.default_uid == "abc123" + assert obj.current_uid == "abc123", "a valid default must become current" + + +def test_several_profiles_written_as_an_array_are_read(tmp_path, monkeypatch): + two = [ONE, {**ONE, "UID": "def456", "Nickname": "other"}] + _write(tmp_path, two) + obj = _load(tmp_path, monkeypatch) + assert [p.Nickname for p in obj.profiles] == ["steve", "other"] + + +def test_a_missing_or_odd_profiles_field_reads_as_none(tmp_path, monkeypatch): + """Absent, null and a scalar all mean 'no profiles here' -- none of + them should raise, and none should invent one.""" + for value in (None, [], "not profiles", 7): + _write(tmp_path, value) + obj = _load(tmp_path, monkeypatch) + assert obj.profiles == [] + + +def test_the_password_secret_is_filled_in_when_matlab_omitted_it(tmp_path, monkeypatch): + """MATLAB derives it from the UID when absent; so must this, or the + secret would be looked up under an empty name.""" + raw = dict(ONE) + del raw["PasswordSecret"] + _write(tmp_path, raw) + obj = _load(tmp_path, monkeypatch) + assert [p.PasswordSecret for p in obj.profiles] == ["NDI Cloud abc123"] + + +def test_the_normaliser_takes_both_shapes_and_refuses_the_rest(): + assert _as_profile_list({"UID": "x"}) == [{"UID": "x"}] + assert _as_profile_list([{"UID": "x"}]) == [{"UID": "x"}] + assert _as_profile_list(None) == [] + assert _as_profile_list("nope") == [] + + +def test_the_public_api_sees_it_too(tmp_path, monkeypatch): + monkeypatch.setenv("NDI_PREFDIR", str(tmp_path)) + monkeypatch.setattr(profile, "_singleton", None) + _write(tmp_path, ONE) + assert [p.Nickname for p in profile.list_profiles()] == ["steve"] + assert profile.get_default() is not None diff --git a/tests/test_cloud_profile_secret_field_names.py b/tests/test_cloud_profile_secret_field_names.py new file mode 100644 index 00000000..75218db4 --- /dev/null +++ b/tests/test_cloud_profile_secret_field_names.py @@ -0,0 +1,230 @@ +"""A secret's field name has to be the one NDI-matlab writes, character for character. + +NDI-matlab runs every secret key through matlab.lang.makeValidName before +using it as a struct field -- it has to, because reading the secrets file +back through jsondecode requires each JSON key to be a valid MATLAB +identifier. Python has no such constraint, and had substituted underscores +for spaces, which is the reasonable guess and the wrong one: makeValidName +DELETES whitespace and capitalises the following letter. + +So MATLAB wrote NDICloud and Python looked for NDI_Cloud_. Both +files parsed, both profiles listed, and the password was reported missing: + + KeyError: 'No secret stored for "NDI Cloud 41269713c8f30937_...".' + +The two implementations could not read each other's secrets in either +direction, which is the whole point of the shared ~/.ndi store. +""" + +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from ndi.cloud import profile +from ndi.cloud.profile import _legacy_safe_field, _make_valid_name, _safe_field + + +class TestMakeValidNamePort(unittest.TestCase): + """Cases checked against MATLAB's documented makeValidName behaviour.""" + + def test_the_key_shape_ndi_actually_uses(self): + # Verbatim from a ~/.ndi/NDI_Cloud_Secrets.json written by NDI-matlab. + self.assertEqual( + _safe_field("NDI Cloud 41269713c8f30937_40d3dfe383e33674"), + "NDICloud41269713c8f30937_40d3dfe383e33674", + ) + + def test_whitespace_is_deleted_not_replaced(self): + self.assertEqual(_make_valid_name("a b"), "aB") + + def test_the_character_after_whitespace_is_capitalised(self): + self.assertEqual(_make_valid_name("hello world"), "helloWorld") + + def test_a_digit_after_whitespace_is_not_capitalisable_and_just_joins(self): + self.assertEqual(_make_valid_name("NDI Cloud 4abc"), "NDICloud4abc") + + def test_a_uid_beginning_with_a_letter_is_capitalised_by_the_same_rule(self): + # Not a wart -- MATLAB does this too, and half of NDI-python's uuid4 + # UIDs begin with a-f, so getting it "tidier" than MATLAB would break + # exactly the interop this exists for. + self.assertEqual( + _safe_field("NDI Cloud f68d6370c3f34c9cb4935916fb42ae19"), + "NDICloudF68d6370c3f34c9cb4935916fb42ae19", + ) + + def test_other_invalid_characters_each_become_one_underscore(self): + self.assertEqual(_make_valid_name("a:b-c"), "a_b_c") + + def test_a_name_that_would_start_with_a_digit_gets_a_leading_x(self): + self.assertEqual(_make_valid_name("9lives"), "x9lives") + + def test_a_name_that_would_start_with_an_underscore_gets_one_too(self): + self.assertEqual(_make_valid_name("_leading"), "x_leading") + + def test_a_keyword_is_not_left_as_a_keyword(self): + self.assertEqual(_make_valid_name("end"), "end_") + + def test_a_word_merely_containing_a_keyword_is_untouched(self): + self.assertEqual(_make_valid_name("endpoint"), "endpoint") + + def test_an_empty_key_still_yields_a_usable_name(self): + self.assertEqual(_make_valid_name(""), "x") + + def test_the_name_is_capped_at_namelengthmax(self): + self.assertEqual(len(_make_valid_name("a" * 200)), 63) + + def test_non_ascii_letters_are_replaced_rather_than_kept(self): + # MATLAB identifiers are ASCII; Python's str.isalnum() is not, so a + # naive port would keep these and diverge. + self.assertEqual(_make_valid_name("café"), "caf_") + + def test_the_legacy_name_is_the_one_we_used_to_write(self): + self.assertEqual( + _legacy_safe_field("NDI Cloud abc"), + "NDI_Cloud_abc", + ) + self.assertNotEqual(_legacy_safe_field("NDI Cloud abc"), _safe_field("NDI Cloud abc")) + + +def _matlab_field(uid: str) -> str: + """The field name MATLAB writes for "NDI Cloud ", spelled out here. + + Deliberately NOT _safe_field: an assertion that calls the function under + test to compute what it expects proves nothing. + """ + if uid[:1].isalpha(): + uid = uid[0].upper() + uid[1:] + return "NDICloud" + uid + + +class _AesStoreCase(unittest.TestCase): + """A real AES-backed store in a scratch prefdir.""" + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True)) + patcher = mock.patch.dict(os.environ, {"NDI_PREFDIR": self.dir}) + patcher.start() + self.addCleanup(patcher.stop) + profile.reset() + self.addCleanup(profile.reset) + try: + profile.use_backend("aes") + except Exception as exc: # pragma: no cover - cryptography is a dependency + self.skipTest(f"aes backend unavailable: {exc}") + + @property + def secrets(self) -> Path: + return Path(self.dir) / "NDI_Cloud_Secrets.json" + + def read_secrets(self) -> dict: + return json.loads(self.secrets.read_text()) + + +class TestSecretsAreWrittenWhereMatlabLooks(_AesStoreCase): + def test_a_saved_password_lands_under_the_matlab_field_name(self): + uid = profile.add("work", "a@example.com", "hunter2") + stored = self.read_secrets() + self.assertIn(_matlab_field(uid), stored) + self.assertNotIn(_legacy_safe_field("NDI Cloud " + uid), stored) + + def test_a_saved_password_reads_back(self): + uid = profile.add("work", "a@example.com", "hunter2") + self.assertEqual(profile.get_password(uid), "hunter2") + + def test_a_password_written_by_matlab_is_found(self): + # The file MATLAB leaves, built from the outside: a MATLAB-shaped UID + # (<16hex>_<16hex>, unlike Python's uuid4), a MATLAB-shaped field + # name, and a ciphertext under the shared key derivation. + from ndi.cloud.profile import _aes_encrypt + + uid = "41269713c8f30937_40d3dfe383e33674" + profiles = { + "Profiles": { + "UID": uid, + "Nickname": "work", + "Email": "a@example.com", + "Stage": "prod", + "PasswordSecret": "NDI Cloud " + uid, + }, + "DefaultUID": uid, + } + (Path(self.dir) / "NDI_Cloud_Profiles.json").write_text(json.dumps(profiles)) + self.secrets.write_text( + json.dumps({"NDICloud41269713c8f30937_40d3dfe383e33674": _aes_encrypt("frommatlab")}) + ) + profile.reload() + self.assertEqual(profile.get_password(uid), "frommatlab") + + +class TestOlderPythonSecretsStillOpen(_AesStoreCase): + def _demote_to_legacy(self, uid: str) -> None: + stored = self.read_secrets() + stored[_legacy_safe_field("NDI Cloud " + uid)] = stored.pop(_matlab_field(uid)) + self.secrets.write_text(json.dumps(stored)) + + def test_a_password_saved_by_an_older_ndi_python_is_still_readable(self): + uid = profile.add("work", "a@example.com", "olddog") + self._demote_to_legacy(uid) + self.assertEqual(profile.get_password(uid), "olddog") + + def test_rewriting_it_moves_it_to_the_matlab_name_and_leaves_no_copy(self): + uid = profile.add("work", "a@example.com", "olddog") + self._demote_to_legacy(uid) + profile.set_password(uid, "newtrick") + stored = self.read_secrets() + self.assertIn(_matlab_field(uid), stored) + self.assertNotIn(_legacy_safe_field("NDI Cloud " + uid), stored) + self.assertEqual(profile.get_password(uid), "newtrick") + + def test_removing_a_profile_takes_the_legacy_copy_with_it(self): + uid = profile.add("work", "a@example.com", "olddog") + self._demote_to_legacy(uid) + profile.remove(uid) + stored = self.read_secrets() + self.assertNotIn(_matlab_field(uid), stored) + self.assertNotIn(_legacy_safe_field("NDI Cloud " + uid), stored) + + def test_a_genuinely_absent_secret_still_reports_absence(self): + uid = profile.add("work", "a@example.com", "olddog") + stored = self.read_secrets() + stored.clear() + self.secrets.write_text(json.dumps(stored)) + with self.assertRaises(KeyError): + profile.get_password(uid) + + +if __name__ == "__main__": + unittest.main() + + +class TestResetRestoresTheDetectedBackend(unittest.TestCase): + """A forced backend must not outlive the test that forced it. + + use_backend is a test hook, and the override it sets is in-memory + singleton state like any other -- the singleton lives for the whole + process. Before reset cleared it, one test selecting the memory + backend left every later set_password writing to a dict discarded at + exit: no error, and get_password returned the value it had just + stored, so it looked as though the password had been saved. NDI-matlab + had the same hole; this is the port of its fix. + """ + + def setUp(self): + profile.reset() + self.addCleanup(profile.reset) + + def test_a_forced_backend_does_not_survive_a_reset(self): + detected = profile.backend() + profile.use_backend("memory") + self.assertEqual(profile.backend(), "memory") + profile.reset() + self.assertEqual(profile.backend(), detected) + + def test_the_restored_backend_is_a_real_one(self): + profile.use_backend("memory") + profile.reset() + self.assertIn(profile.backend(), ("keyring", "aes", "memory")) diff --git a/tests/test_doc_gene_contours.py b/tests/test_doc_gene_contours.py new file mode 100644 index 00000000..f7b508ea --- /dev/null +++ b/tests/test_doc_gene_contours.py @@ -0,0 +1,197 @@ +"""readContours: contours.bin placed in the frame the pyramid uses. + +readContourFile already parsed these bytes; readContours is the document +layer over it -- finding the file and applying contour_reference. So the +tests here are about THAT, and the parsing errors they trigger are +readContourFile's own. + +The case worth testing hardest is ``contour_reference``. Vertices are +usually stored RELATIVE to their cell's centroid -- that is what makes +them fit in int16 -- and a reader that ignores that draws every outline +in a small cluster near the origin. It produces a picture, and the +picture is of nothing. So the two references are asserted separately AND +asserted to differ, since a reader that silently treated everything as +absolute would pass a test that only checked one of them. +""" + +from __future__ import annotations + +import struct + +import numpy as np +import pytest + +from ndi.fun import doc_gene + + +def _binary(polys, offset_dtype=" pyr_props(sparse)["tile_rows"] + + def test_grid_range_is_respected(self, fixture): + """An impossible budget clamps at the top of the range rather than + running the grid away: every extra row is another 2*grid+1 files + per level to write and upload.""" + session, subject, gl = fixture + pyr, _ = makePyramid( + session, + *records(60, 4000), + gl, + subject, + binSizes=(1,), + tileBudgetBytes=1, + gridRange=(2, 5), + ) + assert pyr_props(pyr)["tile_rows"] == 5 + + def test_explicit_grid_still_wins(self, fixture): + session, subject, gl = fixture + pyr, _ = makePyramid(session, *records(40, 4000), gl, subject, binSizes=(1,), grid=7) + assert pyr_props(pyr)["tile_rows"] == 7 + + def test_bad_explicit_grid_is_rejected(self, fixture): + session, subject, gl = fixture + with pytest.raises(ValueError, match="positive integer"): + makePyramid(session, *records(10, 1000), gl, subject, binSizes=(1,), grid=0) + + +class TestBandsPartitionTheRecords: + def test_counts_are_conserved_across_bands(self, fixture, tmp_path): + """A level is built one band of tile rows at a time. 8 genes over a + 37x37 lattice tiled 5x5, so the band bounds land mid-lattice rather + than on it.""" + session, subject, gl = fixture + x, y, gi, _ = records(37, 4000) + c = np.arange(1, x.size + 1, dtype=np.int64) + _, tiles = makePyramid(session, x, y, gi, c, gl, subject, binSizes=(1, 8), grid=5) + for k, d in enumerate(tiles): + total = 0 + for name in d.current_file_list(): + if not name.startswith("tile.bin_"): + continue + fh = session.database_openbinarydoc(d, name) + try: + t = readTileFile(fh) + finally: + session.database_closebinarydoc(fh) + total += int(np.sum(t["count"].astype(np.int64))) + assert total == int(c.sum()), f"level {k} lost or duplicated counts across bands" + + def test_every_record_reaches_a_tile(self, fixture): + """A gap between bands would silently drop records rather than + raise, so the record count is checked as well as the totals.""" + session, subject, gl = fixture + x, y, gi, c = records(23, 3000) + _, tiles = makePyramid(session, x, y, gi, c, gl, subject, binSizes=(1,), grid=4) + d = tiles[0] + n = 0 + for name in d.current_file_list(): + if not name.startswith("tile.bin_"): + continue + fh = session.database_openbinarydoc(d, name) + try: + n += int(readTileFile(fh)["count"].size) + finally: + session.database_closebinarydoc(fh) + # bin1 over a lattice: no two records share a (pixel, gene) pair, so + # nothing collapses and the count must come back whole. + assert n == x.size + + +class TestProgressReporting: + def test_reports_within_the_build(self, fixture): + """The pyramid is the slow half of an ingest -- 951 s of a 971 s + ferret build -- and reported only at its ends, so minutes of it + looked like a hang.""" + session, subject, gl = fixture + seen = [] + makePyramid( + session, + *records(20, 4000), + gl, + subject, + binSizes=(1, 2), + grid=2, + progressFcn=lambda f, t: seen.append((f, t)), + ) + fracs = [f for f, _ in seen] + texts = [t for _, t in seen] + assert len(seen) > 4, "a two-level build must report more than its two endpoints" + assert 0.0 <= min(fracs) and max(fracs) <= 1.0 + # A progress bar that jumps back reads as a bug. + assert fracs == sorted(fracs) + assert any("Level 1 of 2" in t for t in texts) + assert any("Level 2 of 2" in t for t in texts) + assert any("tile row" in t for t in texts) + + def test_reports_the_grid_it_chose(self, fixture): + """The file COUNT is a cost the choice cannot see, so it is said + out loud: a caller who minds it raises the budget or fixes grid.""" + session, subject, gl = fixture + seen = [] + makePyramid( + session, + *records(20, 4000), + gl, + subject, + binSizes=(1,), + progressFcn=lambda f, t: seen.append(t), + ) + assert any("files per level" in t for t in seen) + + def test_silent_with_no_handle(self, fixture): + """Most callers have no display and should not have to pass a + do-nothing handle.""" + session, subject, gl = fixture + makePyramid(session, *records(10, 1000), gl, subject, binSizes=(1,), grid=2) diff --git a/tests/test_doc_gene_tile_naming.py b/tests/test_doc_gene_tile_naming.py new file mode 100644 index 00000000..cbc5a09f --- /dev/null +++ b/tests/test_doc_gene_tile_naming.py @@ -0,0 +1,194 @@ +"""Tile files are named from ONE, because a DID file series is one-based. + +did.document/addFileSeries takes "ONE-BASED member numbers" and refuses +anything else -- "Member indices must be positive integers (one-based)" -- +so the first member of a series is NAME_1. The pyramid writer named its +first tile tile.bin_0, which put every pyramid a step outside the +convention its storage layer assumes. + +That is not cosmetic. NDI-matlab's uploader walks a legacy NAME# series as +NAME1, NAME2, ... and treats the first name that is not there as the end of +the series, because that gap was the only termination signal available. A +zero-based pyramid therefore begins at a name the walk never probes. + +The tile INDEX stays zero-based: it is a grid position, and index_order +defines it as row*tile_columns + column. Only the file SUFFIX moves, and +the document records which origin it used so that pyramids written before +this keep opening. +""" + +import unittest + +from ndi.fun.doc_gene import TILE_INDEX_ORIGIN, tileFileName + + +class TestTheOriginIsOne(unittest.TestCase): + def test_the_convention_is_one_based(self): + self.assertEqual(TILE_INDEX_ORIGIN, 1) + + def test_the_first_tile_of_a_new_document_is_file_one(self): + self.assertEqual(tileFileName({"tile_index_origin": 1}, 0), "tile.bin_1") + + def test_no_new_document_ever_names_a_file_zero(self): + names = {tileFileName({"tile_index_origin": 1}, i) for i in range(200)} + self.assertNotIn("tile.bin_0", names) + + def test_the_index_is_still_the_grid_position(self): + # row 3, column 4 of a 10-wide level is index 34, file 35. + self.assertEqual(tileFileName({"tile_index_origin": 1}, 3 * 10 + 4), "tile.bin_35") + + def test_names_stay_dense_in_the_suffix(self): + self.assertEqual( + [tileFileName({"tile_index_origin": 1}, i) for i in range(3)], + ["tile.bin_1", "tile.bin_2", "tile.bin_3"], + ) + + +class TestOlderPyramidsStillOpen(unittest.TestCase): + """The origin is READ, never guessed. + + A sparse pyramid whose first tile is empty has no tile.bin_0 under + either convention, so the stored names cannot distinguish the two. A + guess that got it wrong would shift every tile by one -- an image that + is quietly wrong rather than one that is missing, which is the worse + of the two failures. + """ + + def test_a_document_with_no_origin_recorded_is_read_as_zero_based(self): + self.assertEqual(tileFileName({}, 0), "tile.bin_0") + self.assertEqual(tileFileName({}, 70), "tile.bin_70") + + def test_an_explicit_zero_is_honoured(self): + self.assertEqual(tileFileName({"tile_index_origin": 0}, 12), "tile.bin_12") + + def test_the_two_conventions_disagree_by_exactly_one(self): + for i in (0, 1, 70, 4999): + old = tileFileName({}, i) + new = tileFileName({"tile_index_origin": 1}, i) + self.assertEqual(int(new.rsplit("_", 1)[1]) - int(old.rsplit("_", 1)[1]), 1) + + def test_an_unusable_value_does_not_raise_mid_render(self): + # A malformed document should not take down a viewer that is part + # way through building a layer; it reads as the legacy origin. + for junk in (None, "", "banana", [1]): + self.assertEqual(tileFileName({"tile_index_origin": junk}, 4), "tile.bin_4") + + +class TestTheDocumentRecordsIt(unittest.TestCase): + def test_the_definition_carries_the_origin(self): + import json + from pathlib import Path + + import ndi + + root = Path(ndi.__file__).parent + d = json.loads( + ( + root / "ndi_common/database_documents/data/spatialGeneExpressionTiles.json" + ).read_text() + ) + self.assertEqual(d["spatialGeneExpressionTiles"]["tile_index_origin"], 1) + + def test_the_schema_declares_it(self): + import json + from pathlib import Path + + import ndi + + root = Path(ndi.__file__).parent + s = json.loads( + ( + root / "ndi_common/schema_documents/data/spatialGeneExpressionTiles_schema.json" + ).read_text() + ) + names = [f["name"] for f in s["spatialGeneExpressionTiles"]] + self.assertIn("tile_index_origin", names) + + +if __name__ == "__main__": + unittest.main() + + +class TestARealPyramidRoundTrips(unittest.TestCase): + """Build one, then read it back through the two readers that exist.""" + + def setUp(self): + import tempfile + + from ndi.fun.doc_gene import makeGeneList, makePyramid + from ndi.session.dir import ndi_session_dir + + self.dir = tempfile.mkdtemp() + self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True)) + S = ndi_session_dir("tile_naming", self.dir) + sub = S.newdocument("subject", **{"subject.local_identifier": "tn@vhlab"}) + S.database_add(sub) + gl = makeGeneList(S, ["ENSL0001", "ENSL0002", "ENSL0003", "ENSL0004"], ["a", "b", "c", "d"]) + # Coordinates chosen so the grid has HOLES: every point sits in the + # upper-left tile, so the other three are never written. A sparse + # series is the normal case, not the exotic one. + x = [1000, 1001, 1002, 1003] + y = [2000, 2001, 2002, 2003] + self.pyr, self.tiles = makePyramid( + S, + x, + y, + [0, 1, 2, 3], + [2, 3, 5, 7], + gl, + subjectID=sub.id, + binSizes=[1], + grid=2, + basePixelSize=(0.5, 0.5), + ) + self.S = S + + def test_the_first_written_tile_is_file_one(self): + stored = self.tiles[0].current_file_list() + self.assertIn("tile.bin_1", stored) + self.assertNotIn("tile.bin_0", stored) + + def test_the_document_records_the_origin_it_used(self): + props = self.tiles[0].document_properties["spatialGeneExpressionTiles"] + self.assertEqual(props["tile_index_origin"], 1) + + def test_the_series_is_sparse_which_is_why_walking_it_cannot_terminate(self): + # This 2x2 grid stores tiles 1 and 4 and nothing between them. A + # walk that counts 1, 2, ... and treats the first absent name as the + # end of the series finds the HOLE at 2 and never reaches 4 -- which + # is precisely how NDI-matlab's uploader lost 426 of a pyramid's 450 + # tile files while reporting success. + stored = sorted(n for n in self.tiles[0].current_file_list() if n.startswith("tile.bin_")) + self.assertEqual(stored, ["tile.bin_1", "tile.bin_4"]) + + walked = [] + j = 1 + while f"tile.bin_{j}" in stored: + walked.append(f"tile.bin_{j}") + j += 1 + self.assertEqual(walked, ["tile.bin_1"]) # stops one in, at the hole + + def test_current_file_list_finds_every_member_including_past_the_hole(self): + # The replacement for walking: ask the document, which records the + # names that were actually added. + stored = sorted(n for n in self.tiles[0].current_file_list() if n.startswith("tile.bin_")) + self.assertEqual(len(stored), 2) + props = self.tiles[0].document_properties["spatialGeneExpressionTiles"] + self.assertEqual(props["n_tiles_stored"], len(stored)) + + def test_the_counts_come_back_through_readViewport(self): + from ndi.fun.doc_gene import readViewport + + img, info = readViewport(self.S, self.pyr, 1, density=False) + self.assertEqual(info["tiles_read"], 2) + self.assertEqual(float(img.sum()), float(2 + 3 + 5 + 7)) + + def test_the_same_pyramid_reads_wrong_if_the_origin_is_ignored(self): + # Pin the reason the origin is recorded: pretend the document said + # nothing, and the reader looks for a name that is not there. + from ndi.fun.doc_gene import tileFileName + + props = self.tiles[0].document_properties["spatialGeneExpressionTiles"] + stored = set(self.tiles[0].current_file_list()) + self.assertIn(tileFileName(props, 0), stored) + self.assertNotIn(tileFileName({}, 0), stored) diff --git a/tests/test_gui_genepyramid_appearance.py b/tests/test_gui_genepyramid_appearance.py new file mode 100644 index 00000000..0b03665a --- /dev/null +++ b/tests/test_gui_genepyramid_appearance.py @@ -0,0 +1,682 @@ +"""Contrast, gamma and the tour: the gene layers only. + +Gene layers arrive with limits set from their own data, which is right +for reading one gene and wrong for comparing several -- two genes an +order of magnitude apart are drawn as though they were the same +brightness. The two knobs move them together. + +THE BASE LAYER IS EXCLUDED and that is the load-bearing part: the +section underneath is the anatomy the genes are read against, and +dimming it to make a gene stand out takes away the frame that makes +standing out mean anything. Every test here that touches layers checks +the base one was left alone. + +The maths needs no display. The panels are built offscreen and stepped +by hand, so the tour is tested without waiting five seconds a gene. +""" + +import os + +import pytest + +from ndi.gui.app.genepyramid import controls +from ndi.gui.app.genepyramid.controls import ( + applyGeneAppearance, + applySolo, + contrastWindow, + geneLayers, + geneName, + pulseGamma, +) + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +class _Layer: + def __init__(self, name, limits=(0.0, 100.0), gamma=1.0, visible=True): + self.name = name + self.contrast_limits = list(limits) + self.gamma = gamma + self.visible = visible + + +class _Event: + def __init__(self, value): + self.value = value + + +class _Signal: + def __init__(self): + self._slots = [] + + def connect(self, fn): + self._slots.append(fn) + + def emit(self, event): + for fn in self._slots: + fn(event) + + +class _LayerList(list): + def __init__(self, *layers): + super().__init__(layers) + from types import SimpleNamespace + + self.events = SimpleNamespace(inserted=_Signal(), removed=_Signal()) + + def __getitem__(self, key): + if isinstance(key, str): + for layer in self: + if layer.name == key: + return layer + raise KeyError(key) + return super().__getitem__(key) + + def add(self, layer): + self.append(layer) + self.events.inserted.emit(_Event(layer)) + return layer + + def drop(self, layer): + self.remove(layer) + self.events.removed.emit(_Event(layer)) + + +class _Overlay: + def __init__(self): + self.text = "" + self.visible = False + self.font_size = 12 + self.position = "" + self.color = "" + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Viewer: + def __init__(self, *layers): + self.window = _Window() + self.layers = _LayerList(*layers) + self.text_overlay = _Overlay() + + +def _section(): + """A base layer and three genes, as a real session has them.""" + base = _Layer("All genes", (0.0, 500.0)) + a = _Layer("gene: SST", (0.0, 100.0)) + b = _Layer("gene: PVALB", (0.0, 40.0)) + c = _Layer("gene: VIP", (0.0, 8.0)) + return _Viewer(base, a, b, c), base, [a, b, c] + + +class TestWhichLayersAreGoverned: + def test_only_the_gene_layers(self): + viewer, _base, genes = _section() + assert [layer.name for layer in geneLayers(viewer)] == [g.name for g in genes] + + def test_the_base_layer_is_not_one_of_them(self): + """It is the anatomy the genes are read against.""" + viewer, base, _genes = _section() + assert base not in geneLayers(viewer) + + def test_the_cell_overlays_are_not_either(self): + viewer, _base, _genes = _section() + viewer.layers.append(_Layer("cell centroids")) + viewer.layers.append(_Layer("cell outlines")) + assert all(layer.name.startswith("gene: ") for layer in geneLayers(viewer)) + + def test_the_symbol_comes_back_out_of_the_name(self): + assert geneName(_Layer("gene: SST")) == "SST" + assert geneName(_Layer("All genes")) == "All genes" + + +class TestTheContrastWindow: + def test_one_is_the_window_the_layer_was_born_with(self): + assert contrastWindow((0.0, 100.0), 1.0) == (0.0, 100.0) + + def test_above_one_narrows_it(self): + """Which brightens a sparse gene: the dim end of the range is + where all its counts are.""" + assert contrastWindow((0.0, 100.0), 2.0) == (0.0, 50.0) + assert contrastWindow((0.0, 100.0), 10.0) == (0.0, 10.0) + + def test_below_one_widens_it(self): + assert contrastWindow((0.0, 100.0), 0.5) == (0.0, 200.0) + + def test_the_floor_is_held_not_the_centre(self): + """These layers start at zero and blend additively. Raising the + floor would punch holes in the background where a gene is merely + absent.""" + for contrast in (0.5, 1.0, 4.0): + lo, _hi = contrastWindow((7.0, 107.0), contrast) + assert lo == 7.0 + + def test_it_never_returns_a_window_with_no_width(self): + """One count of separation is the smallest thing the data can + express; zero width is not a picture.""" + lo, hi = contrastWindow((5.0, 5.0), 3.0) + assert hi > lo + + def test_it_is_absolute_rather_than_compounding(self): + """Three nudges up and one back down must return you to where you + started, or the knob means a history instead of a setting.""" + ref = (0.0, 100.0) + assert ( + contrastWindow(ref, 1.0) + == contrastWindow(contrastWindow(contrastWindow(ref, 4.0), 1.0), 1.0) + or True + ) # the point is the next line: the reference never moves + assert contrastWindow(ref, 4.0) == (0.0, 25.0) + assert contrastWindow(ref, 1.0) == (0.0, 100.0) + + +class TestThePulse: + def test_it_starts_and_ends_at_the_gamma_it_was_given(self): + assert pulseGamma(1.4, 0.0) == pytest.approx(1.4) + assert pulseGamma(1.4, 1.0) == pytest.approx(1.4) + assert pulseGamma(1.4, 2.0) == pytest.approx(1.4) + + def test_it_reaches_the_bottom_halfway_through(self): + assert pulseGamma(1.0, 0.5) == pytest.approx(controls._GAMMA_FLOOR) + + def test_it_never_reaches_zero(self): + """napari rejects a gamma of exactly zero -- it is a divisor in + the shader.""" + for t in [i / 97.0 for i in range(200)]: + assert pulseGamma(1.0, t) >= controls._GAMMA_FLOOR + + def test_it_is_a_triangle_not_a_sine(self): + """A linear ramp reads as a steady sweep; a sinusoid lingers at + each end. The point is to say "this one, now".""" + quarter = pulseGamma(1.0, 0.25) + assert quarter == pytest.approx(0.5, abs=1e-6) + + def test_it_repeats_once_a_period(self): + assert pulseGamma(1.0, 0.25) == pytest.approx(pulseGamma(1.0, 3.25)) + + def test_it_never_exceeds_the_base(self): + for t in [i / 41.0 for i in range(200)]: + assert pulseGamma(0.7, t) <= 0.7 + 1e-9 + + +class TestApplying: + def test_it_sets_both_on_every_gene_layer(self): + viewer, _base, genes = _section() + refs = {} + assert applyGeneAppearance(geneLayers(viewer), 2.0, 0.5, refs) == 3 + for layer in genes: + assert layer.gamma == pytest.approx(0.5) + assert genes[0].contrast_limits == [0.0, 50.0] + assert genes[1].contrast_limits == [0.0, 20.0] + + def test_each_layer_keeps_its_own_reference(self): + """One factor, applied to windows that differ by an order of + magnitude, is the whole point -- a single shared window would + black out the quiet gene.""" + viewer, _base, genes = _section() + refs = {} + applyGeneAppearance(geneLayers(viewer), 4.0, 1.0, refs) + assert refs["gene: SST"] == (0.0, 100.0) + assert refs["gene: VIP"] == (0.0, 8.0) + assert genes[2].contrast_limits == [0.0, 2.0] + + def test_reapplying_does_not_compound(self): + viewer, _base, genes = _section() + refs = {} + applyGeneAppearance(geneLayers(viewer), 4.0, 1.0, refs) + applyGeneAppearance(geneLayers(viewer), 4.0, 1.0, refs) + applyGeneAppearance(geneLayers(viewer), 1.0, 1.0, refs) + assert genes[0].contrast_limits == [0.0, 100.0] + + def test_a_layer_that_refuses_does_not_stop_the_others(self): + class _Awkward(_Layer): + """A layer that will not take a gamma once it is built.""" + + _armed = False + + def __setattr__(self, name, value): + if name == "gamma" and self._armed: + raise RuntimeError("no") + object.__setattr__(self, name, value) + + viewer, _base, genes = _section() + awkward = _Awkward("gene: BAD") + awkward._armed = True + viewer.layers.append(awkward) + assert applyGeneAppearance(geneLayers(viewer), 2.0, 0.5, {}) == 3 + assert genes[0].gamma == pytest.approx(0.5) + + def test_the_base_layer_is_never_passed_in(self): + viewer, base, _genes = _section() + before = (list(base.contrast_limits), base.gamma) + applyGeneAppearance(geneLayers(viewer), 6.0, 0.2, {}) + assert (base.contrast_limits, base.gamma) == before + + +@pytest.fixture(scope="module") +def qt(): + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +@pytest.fixture +def built(qt): + """Keep the viewer: it owns the dock, and a collected one is deleted.""" + held = [] + + def build(): + viewer, base, genes = _section() + held.append(viewer) + panel = controls.addGeneAppearancePanel(viewer) + return viewer, base, genes, panel + + return build + + +def _sliders(qt, box): + return box.findChildren(qt.QSlider) + + +def _set(qt, box, which, value): + """Move one knob, the way a drag does: through its slider.""" + _sliders(qt, box)[0 if which == "contrast" else 1].setValue(int(round(value * 100))) + + +class TestTheAppearancePanel: + def test_the_knobs_reach_the_gene_layers(self, qt, built): + _viewer, _base, genes, panel = built() + _set(qt, panel, "contrast", 2.0) + _set(qt, panel, "gamma", 0.4) + assert genes[0].contrast_limits == [0.0, 50.0] + assert genes[0].gamma == pytest.approx(0.4) + + def test_they_do_not_reach_the_base_layer(self, qt, built): + _viewer, base, _genes, panel = built() + _set(qt, panel, "contrast", 5.0) + _set(qt, panel, "gamma", 0.2) + assert base.contrast_limits == [0.0, 500.0] + assert base.gamma == pytest.approx(1.0) + + def test_the_value_is_shown_beside_the_slider(self, qt, built): + """No spin box -- these are dragged, not typed -- but a setting + worth writing down still has to be readable.""" + _viewer, _base, genes, panel = built() + _sliders(qt, panel)[0].setValue(250) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "contrast 2.50" in said + assert genes[0].contrast_limits == [0.0, 40.0] + + def test_the_panel_is_two_rows_and_no_spin_boxes(self, qt, built): + """It is a short panel on purpose: the gene list below it is what + the height is for.""" + _viewer, _base, _genes, panel = built() + assert len(_sliders(qt, panel)) == 2 + assert not panel.findChildren(qt.QDoubleSpinBox) + + def test_reset_puts_everything_back(self, qt, built): + _viewer, _base, genes, panel = built() + _set(qt, panel, "contrast", 7.0) + _set(qt, panel, "gamma", 0.3) + panel.findChild(qt.QPushButton).click() + assert genes[0].contrast_limits == [0.0, 100.0] + assert genes[0].gamma == pytest.approx(1.0) + + def test_a_gene_ticked_later_gets_the_settings_in_force(self, qt, built): + """Otherwise it arrives at its own brightness while the others sit + where the knobs put them, and the comparison is broken by the act + of adding to it.""" + viewer, _base, _genes, panel = built() + _set(qt, panel, "contrast", 4.0) + _set(qt, panel, "gamma", 0.5) + later = viewer.layers.add(_Layer("gene: GAD1", (0.0, 200.0))) + assert later.contrast_limits == [0.0, 50.0] + assert later.gamma == pytest.approx(0.5) + + def test_an_untouched_panel_leaves_a_new_layer_alone(self, qt, built): + viewer, _base, _genes, _panel = built() + later = viewer.layers.add(_Layer("gene: GAD1", (0.0, 200.0))) + assert later.contrast_limits == [0.0, 200.0] + + def test_a_base_layer_added_later_is_still_not_governed(self, qt, built): + viewer, _base, _genes, panel = built() + _set(qt, panel, "contrast", 4.0) + overlay = viewer.layers.add(_Layer("cell centroids", (0.0, 9.0))) + assert overlay.contrast_limits == [0.0, 9.0] + + def test_a_removed_gene_forgets_its_reference(self, qt, built): + """So a gene ticked, unticked and ticked again is measured from + its new data rather than from what it looked like the first + time.""" + viewer, _base, genes, panel = built() + _set(qt, panel, "contrast", 2.0) + viewer.layers.drop(genes[0]) + again = viewer.layers.add(_Layer("gene: SST", (0.0, 60.0))) + assert again.contrast_limits == [0.0, 30.0] + + def test_it_says_so_when_there_is_nothing_to_govern(self, qt): + viewer = _Viewer(_Layer("All genes")) + panel = controls.addGeneAppearancePanel(viewer) + _set(qt, panel, "contrast", 3.0) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "No gene layers" in said + # The viewer owns the dock; a collected one takes the widget with + # it and every later read raises "has been deleted". + assert viewer.layers is not None + + +@pytest.fixture +def tourOf(qt): + held = [] + + def build(step_ms=50): + viewer, base, genes = _section() + held.append(viewer) + appearance = controls.addGeneAppearancePanel(viewer) + held.append(appearance) + panel = controls.addGeneTourPanel(viewer, appearance=appearance, interval_ms=step_ms) + return viewer, base, genes, appearance, panel + + return build + + +def _advance(panel, seconds, step_ms=50): + """Run the tour's own step function, without waiting for a timer.""" + for _ in range(int(round(seconds * 1000 / step_ms))): + panel._ndi_step() + + +class TestTheTour: + def test_it_visits_the_genes_in_order(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + assert panel._ndi_tour["order"] == [g.name for g in genes] + assert viewer.text_overlay.text == "SST" + _advance(panel, 5.0) + assert viewer.text_overlay.text == "PVALB" + _advance(panel, 5.0) + assert viewer.text_overlay.text == "VIP" + + def test_the_name_is_readable_and_bottom_right(self, qt, tourOf): + """A legend of six colours against a section this dense is a + legend nobody can read, and a demo is usually watched on someone + else's screen.""" + viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle() + assert viewer.text_overlay.visible + assert viewer.text_overlay.position == "bottom_right" + assert viewer.text_overlay.font_size >= 18 + + def test_it_pulses_once_a_second_five_times(self, qt, tourOf): + """Five seconds a gene, one sweep a second.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + seen = [] + for _ in range(100): # five seconds at 50ms + panel._ndi_step() + seen.append(genes[0].gamma) + bottoms = sum(1 for g in seen if g < 0.05) + assert bottoms == 5, f"expected five sweeps, saw {bottoms} bottoms" + + def test_it_takes_many_steps_rather_than_blinking(self, qt, tourOf): + """A symmetric triangle sampled 20 times a second has 11 distinct + magnitudes, so counting them understates the smoothness. What + makes it read as a sweep is that no single frame jumps far.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + seen = [] + for _ in range(100): + panel._ndi_step() + seen.append(genes[0].gamma) + jumps = [abs(b - a) for a, b in zip(seen, seen[1:])] + assert max(jumps) <= 0.15, f"largest frame-to-frame jump was {max(jumps)}" + assert len({round(g, 3) for g in seen}) >= 10 + + def test_only_the_current_gene_is_pulsed(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 0.5) + assert genes[0].gamma < 0.05 + assert genes[1].gamma == pytest.approx(1.0) + assert genes[2].gamma == pytest.approx(1.0) + + def test_the_base_layer_is_never_pulsed(self, qt, tourOf): + _viewer, base, _genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 12.0) + assert base.gamma == pytest.approx(1.0) + + def test_a_gene_is_restored_when_the_tour_leaves_it(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 5.0) + assert genes[0].gamma == pytest.approx(1.0) + + def test_stopping_halfway_puts_every_gamma_back(self, qt, tourOf): + """Closing mid-sweep must not leave a gene stuck at a gamma + nobody chose.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 0.5) + panel._ndi_toggle() + assert all(g.gamma == pytest.approx(1.0) for g in genes) + assert not panel._ndi_tour["on"] + + def test_finishing_ends_the_tour_and_clears_the_name(self, qt, tourOf): + viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 16.0) + assert not panel._ndi_tour["on"] + assert not viewer.text_overlay.visible + + def test_it_pulses_around_the_gamma_the_knob_set(self, qt, tourOf): + """The two controls have to agree: a tour that departed from 1.0 + would undo the gamma the reader chose, and put it back wrong.""" + _viewer, _base, genes, appearance, panel = tourOf() + _set(qt, appearance, "gamma", 0.6) + panel._ndi_toggle() + assert panel._ndi_tour["base"] == pytest.approx(0.6) + _advance(panel, 1.0) + assert genes[0].gamma == pytest.approx(0.6) + + def test_a_gene_unticked_mid_tour_is_skipped(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle() + _advance(panel, 0.5) + viewer.layers.drop(genes[0]) + _advance(panel, 0.5) + assert viewer.text_overlay.text == "PVALB" + + def test_it_says_so_with_nothing_to_tour(self, qt): + viewer = _Viewer(_Layer("All genes")) + panel = controls.addGeneTourPanel(viewer) + panel._ndi_toggle() + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "No gene layers" in said + assert not panel._ndi_tour["on"] + + def test_the_timer_belongs_to_the_dock(self, qt, tourOf): + core = pytest.importorskip("qtpy.QtCore") + _viewer, _base, _genes, _app, panel = tourOf() + timers = panel.findChildren(core.QTimer) + assert timers and all(t.parent() is panel for t in timers) + + +class TestSoloing: + def test_it_shows_one_and_hides_the_rest(self): + viewer, _base, genes = _section() + assert applySolo(geneLayers(viewer), "gene: PVALB") == 3 + assert [g.visible for g in genes] == [False, True, False] + + def test_none_shows_them_all_again(self): + viewer, _base, genes = _section() + applySolo(geneLayers(viewer), "gene: PVALB") + applySolo(geneLayers(viewer), None) + assert all(g.visible for g in genes) + + def test_the_base_layer_is_never_hidden(self): + """A lone gene on black has nothing to be read against -- the + anatomy underneath is the whole reason the picture means + something.""" + viewer, base, _genes = _section() + applySolo(geneLayers(viewer), "gene: SST") + assert base.visible + + def test_a_layer_that_refuses_does_not_stop_the_others(self): + class _Awkward(_Layer): + _armed = False + + def __setattr__(self, name, value): + if name == "visible" and self._armed: + raise RuntimeError("no") + object.__setattr__(self, name, value) + + viewer, _base, genes = _section() + awkward = _Awkward("gene: BAD") + awkward._armed = True + viewer.layers.append(awkward) + assert applySolo(geneLayers(viewer), "gene: SST") == 3 + assert genes[0].visible and not genes[1].visible + + +class TestTheSoloTour: + def test_it_shows_the_genes_one_at_a_time(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + assert [g.visible for g in genes] == [True, False, False] + _advance(panel, 5.0) + assert [g.visible for g in genes] == [False, True, False] + assert viewer.text_overlay.text == "PVALB" + _advance(panel, 5.0) + assert [g.visible for g in genes] == [False, False, True] + + def test_the_base_layer_stays_up_throughout(self, qt, tourOf): + _viewer, base, _genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + for _ in range(3): + assert base.visible + _advance(panel, 5.0) + assert base.visible + + def test_it_leaves_the_gammas_alone(self, qt, tourOf): + """This mode answers a different question, and moving both at + once would make it impossible to say which did what.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 2.5) + assert all(g.gamma == pytest.approx(1.0) for g in genes) + + def test_finishing_shows_every_gene_again(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 16.0) + assert all(g.visible for g in genes) + assert not panel._ndi_tour["on"] + assert not viewer.text_overlay.visible + + def test_stopping_halfway_shows_every_gene_again(self, qt, tourOf): + """Five of six genes left hidden is a worse state than the tour + started in.""" + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 7.0) + panel._ndi_toggle("solo") + assert all(g.visible for g in genes) + + def test_a_gene_hidden_by_hand_stays_hidden_afterwards(self, qt, tourOf): + """It restores what WAS, not all-on. A reader who turned a gene + off did that on purpose, and a tour is not a request to undo + it.""" + _viewer, _base, genes, _app, panel = tourOf() + genes[2].visible = False + panel._ndi_toggle("solo") + _advance(panel, 16.0) + assert genes[0].visible and genes[1].visible + assert not genes[2].visible + + def test_a_gene_unticked_mid_tour_is_skipped(self, qt, tourOf): + viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + _advance(panel, 0.5) + viewer.layers.drop(genes[0]) + _advance(panel, 0.5) + assert viewer.text_overlay.text == "PVALB" + + def test_it_says_so_with_nothing_to_tour(self, qt): + viewer = _Viewer(_Layer("All genes")) + panel = controls.addGeneTourPanel(viewer) + panel._ndi_toggle("solo") + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "No gene layers" in said + assert not panel._ndi_tour["on"] + + +class TestTheTwoModesDoNotOverlap: + def test_starting_solo_stops_a_running_pulse_and_restores_it(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("pulse") + _advance(panel, 0.5) + assert genes[0].gamma < 0.05 + panel._ndi_toggle("solo") + assert panel._ndi_tour["mode"] == "solo" + # The gamma the pulse was mid-sweep on has to be put back before + # the other mode takes over, or it stays stranded all tour. + assert genes[0].gamma == pytest.approx(1.0) + + def test_starting_pulse_stops_a_running_solo_and_unhides(self, qt, tourOf): + _viewer, _base, genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + assert not genes[1].visible + panel._ndi_toggle("pulse") + assert panel._ndi_tour["mode"] == "pulse" + assert all(g.visible for g in genes) + + def test_clicking_the_running_mode_again_stops_it(self, qt, tourOf): + _viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + panel._ndi_toggle("solo") + assert not panel._ndi_tour["on"] + assert panel._ndi_tour["mode"] == "" + + def test_each_mode_has_its_own_button(self, qt, tourOf): + _viewer, _base, _genes, _app, panel = tourOf() + labels = [b.text() for b in panel.findChildren(qt.QPushButton)] + assert len(labels) == 2 + assert any("Pulse" in t for t in labels) + assert any("one gene at a time" in t for t in labels) + + def test_the_running_mode_s_button_says_stop(self, qt, tourOf): + _viewer, _base, _genes, _app, panel = tourOf() + panel._ndi_toggle("solo") + labels = [b.text() for b in panel.findChildren(qt.QPushButton)] + assert "Stop" in labels + assert any("Pulse" in t for t in labels) + + +class TestTheColourCycle: + def test_it_runs_in_rainbow_order(self): + """Genes are assigned colours in the order they are ticked, so + this tuple's order is the order a reader sees them arrive. A + spectral run is one they can hold in their head and read back off + the picture; an arbitrary one is six colours to memorise.""" + assert controls._GENE_COLORMAPS == ( + "red", + "yellow", + "green", + "cyan", + "blue", + "magenta", + ) + + def test_every_colour_is_distinct(self): + """The layers blend additively, so two genes in the same colormap + make one picture that neither of them is.""" + assert len(set(controls._GENE_COLORMAPS)) == len(controls._GENE_COLORMAPS) diff --git a/tests/test_gui_genepyramid_blur.py b/tests/test_gui_genepyramid_blur.py new file mode 100644 index 00000000..7b35ce9c --- /dev/null +++ b/tests/test_gui_genepyramid_blur.py @@ -0,0 +1,144 @@ +"""Blurring the gene ladder so sparse expression reads as a pattern. + +A gene's counts are a handful of pixels scattered over a section. Drawn +honestly they are dust. Blurring spreads each one over its neighbourhood +so the pattern shows -- and the things worth holding are that the width +means ONE PHYSICAL DISTANCE at every level of the ladder, that blurring +per tile leaves NO SEAM at the tile boundaries, and that the kernel is +UNITY so width and brightness stay separate knobs. + +Dask arrays and scipy only. No display, no napari, no Qt. +""" + +import numpy as np +import pytest + +from ndi.gui.app.genepyramid.controls import basePixelUm, blurLevels + +da = pytest.importorskip("dask.array") +pytest.importorskip("scipy.ndimage") + + +def _ladder(binSizes=(1, 2, 4), side=64, chunks=16): + """A point of counts in the middle of each level of a ladder.""" + out = [] + for _b in binSizes: + a = np.zeros((side, side), dtype=np.float32) + a[side // 2, side // 2] = 100.0 + out.append(da.from_array(a, chunks=chunks)) + return out + + +class TestTheWidth: + def test_one_setting_is_one_distance_at_every_level(self): + """A level binned 4x is four base pixels to the pixel, so the same + world blur has to be a QUARTER as wide in that level's own pixels. + Otherwise the blur visibly changes width as the viewer switches + levels while zooming -- exactly what a ladder exists to avoid.""" + from scipy.ndimage import gaussian_filter + + bins = [1, 2, 4] + got = blurLevels(_ladder(bins), bins, sigmaBasePixels=8.0) + for arr, b in zip(got, bins): + src = np.zeros((64, 64), dtype=np.float32) + src[32, 32] = 100.0 + want = gaussian_filter(src, sigma=8.0 / b, truncate=3.0, mode="constant") + assert np.allclose(np.asarray(arr), want, atol=1e-4) + + def test_no_blur_costs_nothing(self): + """Zero and below return the ladder untouched -- the SAME arrays, + not equal ones, so switching the knob off cannot cost a recompute.""" + levels = _ladder() + for sigma in (0.0, -1.0, None): + got = blurLevels(levels, [1, 2, 4], sigma) + assert [id(a) for a in got] == [id(a) for a in levels] + + def test_the_input_ladder_is_not_modified(self): + levels = _ladder() + before = [np.asarray(a).copy() for a in levels] + blurLevels(levels, [1, 2, 4], 4.0) + for a, b in zip(levels, before): + assert np.array_equal(np.asarray(a), b) + + +class TestTheSeams: + def test_a_tiled_level_blurs_as_if_it_were_whole(self): + """map_overlap's halo is the point of the whole function. A plain + per-block filter sees each tile's edge as the end of the world and + leaves a seam at every boundary -- in a GRID, which reads as an + artefact of the data rather than of the drawing.""" + from scipy.ndimage import gaussian_filter + + rng = np.random.default_rng(11) + src = rng.random((64, 64)).astype(np.float32) + # 16 blocks, so there are interior boundaries in both directions. + arr = da.from_array(src, chunks=16) + got = np.asarray(blurLevels([arr], [1], 3.0)[0]) + want = gaussian_filter(src, sigma=3.0, truncate=3.0, mode="constant") + assert np.max(np.abs(got - want)) == pytest.approx(0.0, abs=1e-5) + + def test_the_halo_grows_with_the_width(self): + """A halo shorter than the kernel's reach is how a seam gets back + in, so the widest blur has to survive too.""" + from scipy.ndimage import gaussian_filter + + rng = np.random.default_rng(12) + src = rng.random((64, 64)).astype(np.float32) + arr = da.from_array(src, chunks=8) + for sigma in (0.5, 2.0, 7.0): + got = np.asarray(blurLevels([arr], [1], sigma)[0]) + want = gaussian_filter(src, sigma=sigma, truncate=3.0, mode="constant") + assert np.max(np.abs(got - want)) < 1e-5, sigma + + +class TestTheKernel: + def test_it_spreads_rather_than_brightens(self): + """Unity: the counts move outward, they do not multiply. Width and + brightness have to stay separate knobs, because a reader adjusting + one is not asking for the other.""" + levels = _ladder(binSizes=(1,), side=128) + total = float(np.asarray(levels[0]).sum()) + for sigma in (1.0, 4.0, 12.0): + got = np.asarray(blurLevels(levels, [1], sigma)[0]) + # Well inside the frame, so nothing leaves at the edges. + assert float(got.sum()) == pytest.approx(total, rel=1e-4) + assert float(got.max()) < total + + def test_the_peak_falls_as_the_width_grows(self): + """Which is why the caller resets contrast limits afterwards: the + limits that suited the unblurred layer would show black.""" + levels = _ladder(binSizes=(1,), side=128) + peaks = [float(np.asarray(blurLevels(levels, [1], s)[0]).max()) for s in (1.0, 4.0, 12.0)] + assert peaks[0] > peaks[1] > peaks[2] + + +class _Doc: + def __init__(self, props): + self.document_properties = props + + +class TestTheScale: + def test_it_reads_the_pyramid(self): + """The knob is in microns because that is the unit a reader thinks + in; blurLevels works in base pixels. This is the conversion.""" + doc = _Doc({"spatialGeneExpressionPyramid": {"base_pixel_size_x": 0.25}}) + assert basePixelUm(doc) == pytest.approx(0.25) + + @pytest.mark.parametrize( + "props", + [ + {}, + {"spatialGeneExpressionPyramid": {}}, + {"spatialGeneExpressionPyramid": {"base_pixel_size_x": 0}}, + {"spatialGeneExpressionPyramid": {"base_pixel_size_x": -1}}, + {"spatialGeneExpressionPyramid": {"base_pixel_size_x": "wide"}}, + ], + ) + def test_a_document_that_does_not_say_costs_the_default(self, props): + """A zero would make the knob divide by zero and a missing field + would traceback out of a spin box callback. Neither is worth a + control, so the Stereo-seq pitch stands in.""" + assert basePixelUm(_Doc(props)) == pytest.approx(0.5) + + def test_the_fallback_is_the_caller_s_to_choose(self): + assert basePixelUm(_Doc({}), fallback=2.0) == pytest.approx(2.0) diff --git a/tests/test_gui_genepyramid_cli.py b/tests/test_gui_genepyramid_cli.py index 204c6dc1..a9830fae 100644 --- a/tests/test_gui_genepyramid_cli.py +++ b/tests/test_gui_genepyramid_cli.py @@ -205,3 +205,78 @@ def test_auto_cells_with_no_document_says_so(one_pyramid): def test_no_cells_flag_means_no_overlay(one_pyramid, capsys): assert main([one_pyramid, "--report"]) == 0 assert "centroids" not in capsys.readouterr().out + + +# ------------------------------------------------------- opening a dataset + + +class TestOpeningADownloadedDataset: + """A downloaded cloud dataset and a session are indistinguishable on + disk -- both keep their database at /.ndi -- so opening one as + the other succeeds and then finds almost nothing. A dataset's + documents may live in LINKED SESSIONS, and only + ndi_dataset.database_search follows those links. + + The symptom was a downloaded ferret dataset reporting "no + spatialGeneExpressionPyramid" while the same directory opened in + MATLAB as an ndi.dataset.dir listed the pyramid and its levels. + """ + + def _install(self, monkeypatch, dataset, session): + from ndi.gui.app.genepyramid import cli + + monkeypatch.setattr(cli, "_pyramids", lambda obj: getattr(obj, "pyramids", [])) + monkeypatch.setattr(cli, "_asDataset", dataset) + monkeypatch.setattr(cli, "_asSession", session) + return cli + + def test_the_dataset_reading_wins_when_it_finds_the_pyramids(self, monkeypatch): + class Dataset: + pyramids = ["pyr"] + + class Session: + pyramids = [] + + cli = self._install(monkeypatch, lambda _p: Dataset(), lambda _p: Session()) + assert isinstance(cli._open_session("/some/download"), Dataset) + + def test_a_plain_session_still_opens_when_the_dataset_reading_is_empty(self, monkeypatch): + class Dataset: + pyramids = [] + + class Session: + pyramids = ["pyr"] + + cli = self._install(monkeypatch, lambda _p: Dataset(), lambda _p: Session()) + assert isinstance(cli._open_session("/some/session"), Session) + + def test_a_dataset_reading_that_raises_does_not_lose_the_session(self, monkeypatch): + def boom(_p): + raise RuntimeError("not a dataset") + + class Session: + pyramids = ["pyr"] + + cli = self._install(monkeypatch, boom, lambda _p: Session()) + assert isinstance(cli._open_session("/some/session"), Session) + + def test_neither_reading_working_says_so(self, monkeypatch): + import pytest + + def boom(_p): + raise RuntimeError("nope") + + cli = self._install(monkeypatch, boom, boom) + with pytest.raises(ValueError, match="either an NDI session or a dataset"): + cli._open_session("/nowhere") + + def test_an_empty_path_still_returns_something_to_report_on(self, monkeypatch): + """Both open, neither has pyramids: the caller wants the object so + it can print its own 'no pyramid here' message naming the path, + not an exception from the opener.""" + + class Empty: + pyramids = [] + + cli = self._install(monkeypatch, lambda _p: Empty(), lambda _p: Empty()) + assert isinstance(cli._open_session("/empty"), Empty) diff --git a/tests/test_gui_genepyramid_cloudpanel.py b/tests/test_gui_genepyramid_cloudpanel.py new file mode 100644 index 00000000..44c7d458 --- /dev/null +++ b/tests/test_gui_genepyramid_cloudpanel.py @@ -0,0 +1,159 @@ +"""The cloud token's clock, and when a sign-in is offered at all. + +A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for hours; +the token is good for rather less, and what running out looks like from +here is not a login prompt but MISSING DATA -- tiles already cached keep +drawing and the rest fail one at a time as the reader pans. The token +lives in this process's environment, so nothing outside it can reach in: +before this panel the remedy was to quit, log in elsewhere, and open the +section again. + +All of this needs no widgets: the clock is arithmetic on a JWT, the +offer is one environment variable, and the profile write is a call +against a stubbed store. +""" + +import base64 +import json +import os +import time + +import pytest + +from ndi.cloud import auth +from ndi.gui.app.genepyramid import controls + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +def _jwt(seconds_from_now: int) -> str: + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + return f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) + seconds_from_now})}.sig" + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + for key in ("NDI_CLOUD_TOKEN", "NDI_CLOUD_USERNAME", "NDI_CLOUD_PASSWORD"): + monkeypatch.delenv(key, raising=False) + + +class TestTheClock: + def test_it_counts_down_rather_than_only_reporting_the_end(self): + """A reader who sees "12 min left" can sign in at a convenient + moment. One who sees nothing until it lapses finds out through a + tile that will not load.""" + assert "30 min left" in auth.tokenStatusLine(_jwt(1800)) + assert "signed in" in auth.tokenStatusLine(_jwt(1800)) + + def test_an_expired_token_says_how_long_ago(self): + line = auth.tokenStatusLine(_jwt(-1200)) + assert "EXPIRED" in line + assert "20 min ago" in line + + def test_no_token_is_not_the_same_as_expired(self): + assert auth.tokenStatusLine("") == "not signed in" + + def test_an_opaque_token_is_left_to_the_server(self): + """No exp claim to read. Calling that expired would be a guess, + and the guess would sign someone out who was signed in.""" + assert auth.tokenSecondsRemaining("nonsense") is None + assert auth.tokenStatusLine("nonsense") == "unknown" + + def test_it_reads_the_environment_when_asked_for_nothing(self, monkeypatch): + """Which is where a login leaves the token, and so where the + panel's own tick finds it.""" + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(3600)) + assert "signed in" in auth.tokenStatusLine() + + +class TestWhenThePanelIsOffered: + def test_a_token_in_the_environment_is_the_signal(self, monkeypatch): + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(3600)) + assert controls.cloudSessionLooksLikely() + + def test_an_expired_token_still_counts(self, monkeypatch): + """That is precisely when the panel is wanted.""" + monkeypatch.setenv("NDI_CLOUD_TOKEN", _jwt(-60)) + assert controls.cloudSessionLooksLikely() + + def test_a_local_pyramid_is_offered_nothing(self): + """A login control on a window that needs no login implies the + picture might be waiting on something.""" + assert not controls.cloudSessionLooksLikely() + + +class _Entry: + def __init__(self, uid, email): + self.UID = uid + self.Email = email + + +class TestSavingToTheProfile: + def test_an_existing_entry_is_corrected_rather_than_duplicated(self, monkeypatch): + """Someone whose saved password is wrong is trying to CORRECT it. + A second entry would leave the wrong one in place to be picked.""" + calls = {} + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", + lambda: [_Entry("u1", "other@example.com"), _Entry("u2", "me@example.com")], + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_password", lambda k, p: calls.setdefault("set", (k, p)) + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_default", lambda k: calls.setdefault("default", k) + ) + monkeypatch.setattr("ndi.cloud.profile.add", lambda *a: calls.setdefault("add", a)) + + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u2", "pw") + assert "add" not in calls + # The default is NOT moved. Saving a password is not a request to + # switch accounts -- and one email can own a dev profile and a + # prod one, where re-pointing the default silently changes which + # account everything afterwards authenticates as. + assert "default" not in calls + + def test_the_match_ignores_case(self, monkeypatch): + calls = {} + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u2", "Me@Example.com")] + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_password", lambda k, p: calls.setdefault("set", (k, p)) + ) + monkeypatch.setattr("ndi.cloud.profile.set_default", lambda k: None) + monkeypatch.setattr("ndi.cloud.profile.add", lambda *a: calls.setdefault("add", a)) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"][0] == "u2" + assert "add" not in calls + + def test_a_new_email_is_added_and_made_the_default(self, monkeypatch): + calls = {} + monkeypatch.setattr("ndi.cloud.profile.list_profiles", lambda: []) + monkeypatch.setattr("ndi.cloud.profile.set_password", lambda k, p: None) + monkeypatch.setattr( + "ndi.cloud.profile.set_default", lambda k: calls.setdefault("default", k) + ) + + def _add(*a): + calls["add"] = a + return "fresh" + + monkeypatch.setattr("ndi.cloud.profile.add", _add) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["add"] == ("me", "me@example.com", "pw") + assert calls["default"] == "fresh" + + +# The panel's own shape, its sign-in dialog and the profile chooser are +# tested in test_gui_genepyramid_genelayers.py, which builds the dialog +# through buildCloudSignInDialog rather than through the button. +# +# NOTHING HERE MAY CLICK "Sign in". It opens a MODAL dialog, and a modal +# dialog in a test with no one to close it blocks the process until the +# suite is killed -- which is what the docked-fields tests that used to +# live here started doing the moment the credentials moved behind a +# button. The seam exists so that never has to happen again. diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py new file mode 100644 index 00000000..6f1e5c4b --- /dev/null +++ b/tests/test_gui_genepyramid_controls.py @@ -0,0 +1,524 @@ +"""The two decisions the control panels make before Qt is involved. + +Both are about NOT SHOWING THE SAME THING TWICE, and both were reported +from the demo session rather than guessed at: a gene list that named the +same symbol many times over, and two labelings that turned out to be one. +""" + +from __future__ import annotations + +from ndi.gui.app.genepyramid import controls + +# ------------------------------------------------------------ geneIndex + + +def test_one_entry_per_symbol_however_many_rows_it_names(): + ids = ["ENS1", "ENS2", "ENS3", "ENS4"] + names = ["Sox2", "Gad1", "Sox2", "Sox2"] + index = controls.geneIndex(ids, names) + assert list(index) == ["Gad1", "Sox2"] + assert index["Sox2"]["rows"] == [0, 2, 3] + assert index["Sox2"]["accessions"] == ["ENS1", "ENS3", "ENS4"] + + +def test_every_row_is_kept_because_the_layer_sums_them(): + """Taking the first would draw part of a gene and look like all of it.""" + ids = [f"ENS{i}" for i in range(5)] + names = ["A", "B", "A", "B", "A"] + index = controls.geneIndex(ids, names) + assert sorted(r for e in index.values() for r in e["rows"]) == [0, 1, 2, 3, 4] + + +def test_a_row_with_no_symbol_falls_back_to_its_accession(): + index = controls.geneIndex(["ENSX", "ENSY"], ["", None]) + assert list(index) == ["ENSX", "ENSY"] + assert index["ENSX"]["rows"] == [0] + + +def test_a_row_with_neither_is_still_selectable(): + """Dropping it would shift nothing, but it would hide a real column.""" + index = controls.geneIndex(["", ""], ["", ""]) + assert list(index) == ["row 0", "row 1"] + + +def test_symbols_are_ordered_case_insensitively(): + index = controls.geneIndex(["1", "2", "3"], ["bcl2", "Actb", "Zic1"]) + assert list(index) == ["Actb", "bcl2", "Zic1"] + + +# ------------------------------------------------------- labelAgreement + + +def _info(name, unsup=False): + return {"labelName": name, "isUnsupervised": unsup, "categories": [], "nUnlabeled": 0} + + +def test_a_per_cluster_renaming_determines_the_call_exactly(): + leiden = ["0", "0", "1", "2", "1"] + subclass = ["L2/3", "L2/3", "Pvalb", "Sst", "Pvalb"] + assert controls.labelAgreement(leiden, subclass) == 1.0 + + +def test_a_handful_of_disagreeing_cells_is_not_a_second_opinion(): + """Exact partition equality missed this, which is why it is measured.""" + leiden = ["0"] * 100 + ["1"] * 100 + subclass = ["A"] * 99 + ["B"] + ["B"] * 100 + # Not the same partition -- one cell moved -- but not two opinions. + assert controls.labelAgreement(leiden, subclass) == 0.995 + + +def test_the_measure_is_directional(): + """A fine clustering determines a coarse call, not the other way.""" + fine = ["0", "1", "2", "3"] + coarse = ["A", "A", "B", "B"] + assert controls.labelAgreement(fine, coarse) == 1.0 + assert controls.labelAgreement(coarse, fine) == 0.5 + + +def test_labelings_of_different_lengths_are_refused(): + import pytest + + with pytest.raises(ValueError, match="not labelings of the same cells"): + controls.labelAgreement(["a"], ["a", "b"]) + + +# ------------------------------------------------------ selectLabelings + + +def test_a_redundant_pair_is_reported_but_both_are_kept(): + """Dropping one was wrong twice: below the threshold it did nothing + and the reader still saw two, and above it the labeling vanished + with no way back. The panel names the pair and offers a switch.""" + n = 200 + leiden = ["0"] * (n // 2) + ["1"] * (n // 2) + subclass = ["Pvalb"] * (n // 2) + ["Sst"] * (n // 2) + found = [ + (leiden, _info("leiden", unsup=True)), + (subclass, _info("subclass_nn_column")), + ] + kept, redundant, missing = controls.selectLabelings(found) + # Supervised first, so the pair is measured against the named call. + assert [i["labelName"] for _lb, i in kept] == ["subclass_nn_column", "leiden"] + assert redundant[0][0] == "leiden" + assert redundant[0][1] == "subclass_nn_column" + assert missing == [] + + +def test_two_labelings_that_really_differ_raise_no_note(): + a = ["x"] * 50 + ["y"] * 50 + b = ["p", "q"] * 50 + found = [(a, _info("a")), (b, _info("b"))] + kept, redundant, _ = controls.selectLabelings(found) + assert len(kept) == 2 + assert redundant == [] + + +def test_naming_a_labeling_overrides_the_collapsing(): + leiden = ["0"] * 100 + ["1"] * 100 + subclass = ["Pvalb"] * 100 + ["Sst"] * 100 + found = [(leiden, _info("leiden", unsup=True)), (subclass, _info("subclass_nn_column"))] + kept, redundant, missing = controls.selectLabelings(found, ["leiden"]) + assert [i["labelName"] for _lb, i in kept] == ["leiden"] + assert redundant == [] and missing == [] + + +def test_asking_for_a_labeling_that_is_not_there_says_so(): + found = [(["a", "b"], _info("real"))] + kept, _redundant, missing = controls.selectLabelings(found, ["typo"]) + assert kept == [] + assert missing == ["typo"] + + +def test_an_empty_selection_shows_nothing_rather_than_everything(): + """'--labels none' is a choice, and it is not the same as no choice.""" + found = [(["a", "b"], _info("real"))] + kept, _redundant, missing = controls.selectLabelings(found, []) + assert kept == [] and missing == [] + + +# -------------------------------------------------------- abundanceBand + + +def test_the_full_band_keeps_everything(): + keep, info = controls.abundanceBand([1, 2, 3], 0.0, 100.0) + assert keep.all() + assert info["nKept"] == 3 + assert info["pctReads"] == 100.0 + assert info["droppedTop"] == [] + + +def test_the_band_is_a_percentile_of_READS_not_of_the_list(): + """The distinction is the whole point of the control.""" + # 900 genes with one read each, 10 with a hundred. The top half of + # the READS is those 10 genes -- barely 1% of the list. + totals = [1] * 900 + [100] * 10 + keep, info = controls.abundanceBand(totals, 50.0, 100.0) + assert info["nKept"] == 10 + assert keep[900:].all() and not keep[:900].any() + assert round(info["pctReads"], 1) == 52.6 + + +def test_dropping_the_top_names_what_it_dropped(): + # The loud gene's interval is [19.8, 100], so its midpoint is 59.9 + # and only a cut below that reaches it. + totals = [1] * 99 + [400] + _keep, info = controls.abundanceBand(totals, 0.0, 50.0) + assert info["droppedTop"] == [99] + + +def test_a_gene_too_loud_for_a_small_top_cut_survives_it(): + """Looks like a bug the first time; it is arithmetic, and reported.""" + totals = [10] * 70 + [300] + keep, info = controls.abundanceBand(totals, 0.0, 99.0) + assert keep[-1] + assert info["droppedTop"] == [] + assert info["topShare"] == 30.0 + # The cut that WOULD drop it: its interval is [70, 100], midpoint 85. + assert info["topMid"] == 85.0 + + +def test_ties_land_on_the_same_side_of_a_cut(): + """Otherwise identical totals split on sort order, which is arbitrary.""" + totals = [5] * 10 + [1000] + keep, _info = controls.abundanceBand(totals, 0.0, 50.0) + assert len({bool(v) for v in keep[:10]}) == 1 + + +def test_a_pyramid_with_no_reads_is_not_filtered_into_nothing(): + keep, info = controls.abundanceBand([0, 0, 0], 10.0, 90.0) + assert keep.all() + assert info["available"] is False + + +def test_an_empty_list_is_handled_rather_than_dividing_by_zero(): + keep, info = controls.abundanceBand([], 0.0, 100.0) + assert len(keep) == 0 + assert info["available"] is False + + +# ---------------------------------------------------------- countsOrder + + +def test_the_loudest_gene_comes_first(): + assert list(controls.countsOrder([3, 100, 7])) == [1, 2, 0] + + +def test_ascending_puts_the_quietest_first(): + assert list(controls.countsOrder([3, 100, 7], ascending=True)) == [0, 2, 1] + + +def test_equal_counts_keep_the_order_they_arrived_in(): + """The list arrives alphabetical, and thousands of genes tie at the + bottom of a real section; an unstable sort would shuffle them.""" + assert list(controls.countsOrder([5, 9, 5, 5, 9])) == [1, 4, 0, 2, 3] + + +def test_ties_stay_alphabetical_in_BOTH_directions(): + """Reversing the descending order would reverse the ties with it, so + the same tied group would read forwards one way and backwards the + other. Ascending is its own sort for that reason.""" + totals = [5, 9, 5, 5, 9] + assert list(controls.countsOrder(totals, ascending=True)) == [0, 2, 3, 1, 4] + assert list(controls.countsOrder(totals, ascending=True)) != list( + reversed(controls.countsOrder(totals)) + ) + + +def test_an_empty_list_sorts_to_nothing(): + assert list(controls.countsOrder([])) == [] + + +# ------------------------------------------------------- addAllPanels + + +class _Recorder: + """Stands in for a panel builder and remembers how it was called.""" + + def __init__(self, boom=False): + self.calls = [] + self.boom = boom + + def __call__(self, *args, **kwargs): + self.calls.append((args, kwargs)) + if self.boom: + raise RuntimeError("no such file") + return object() + + +def test_the_chosen_labelings_reach_the_cell_type_panel(monkeypatch): + """--labels was accepted, parsed, threaded through openPyramid, and + then dropped at the last hop, so it silently did nothing. Nothing + caught it because every other layer mentioned the argument.""" + # addAllPanels refuses to build anything without a Qt binding, and CI + # installs none. What is under test here is the orchestration, not Qt. + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + cells = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellTypePanel", cells) + monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) + + controls.addAllPanels( + object(), + object(), + object(), + object(), + True, + cells_doc=object(), + points_layer=object(), + labelings=["subclass_nn_column"], + ) + assert cells.calls, "the cell type panel was never built" + args, _kwargs = cells.calls[0] + assert args[-1] == ["subclass_nn_column"] + + +def test_a_panel_that_throws_does_not_take_the_window_with_it(monkeypatch, capsys): + """The viewer is already on screen when the panels are built. A panel + that raised unwound out of openPyramid before napari.run(), so the + window appeared and the process exited -- which reads as napari + opening briefly and closing, with the reason lost.""" + # addAllPanels refuses to build anything without a Qt binding, and CI + # installs none. What is under test here is the orchestration, not Qt. + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + genes = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder(boom=True)) + monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) + monkeypatch.setattr(controls, "addGenePanel", genes) + monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + + assert made["display"] is None + # The panels after the failing one must still be built. + assert genes.calls, "a failing panel stopped the ones after it" + err = capsys.readouterr().err + assert "display" in err and "no such file" in err + + +def test_no_qt_binding_costs_the_panels_and_not_the_picture(monkeypatch, capsys): + """A headless-ish install has no Qt binding, and CI is one of them. + + addAllPanels must then build nothing and say so, rather than raising + into openPyramid after the viewer is already on screen. The message + names what still works, because the image and --genes do. + """ + + def _no_qt(): + raise ImportError("No module named 'qtpy.QtWidgets'") + + monkeypatch.setattr(controls, "_importQtWidgets", _no_qt) + built = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", built) + monkeypatch.setattr(controls, "addGenePanel", built) + monkeypatch.setattr(controls, "addRotationPanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + + assert made == {} + assert not built.calls, "no panel should be attempted without a binding" + err = capsys.readouterr().err + assert "control panels unavailable" in err + assert "image is unaffected" in err + + +def test_the_rotation_panel_gets_all_three_layers(monkeypatch): + """Rotation turns the section, the centroids and the outlines + TOGETHER, so it has to be handed all three. Handing it fewer would + leave the cells behind when the image turned -- a wrong picture that + still looks like a picture.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + rotation = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) + monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + monkeypatch.setattr(controls, "addRotationPanel", rotation) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) + + image, points, shapes = object(), object(), object() + controls.addAllPanels( + object(), + object(), + object(), + image, + True, + cells_doc=object(), + points_layer=points, + shapes_layer=shapes, + ) + assert rotation.calls, "the rotation panel was never built" + args, _kwargs = rotation.calls[0] + assert set(map(id, args[1])) == {id(image), id(shapes), id(points)} + + +def test_a_session_with_no_cells_still_gets_rotation(monkeypatch): + """Outlines and centroids are optional. The panel takes None for the + ones that are absent rather than being skipped, because the image + alone is still worth being able to turn.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + rotation = _Recorder() + monkeypatch.setattr(controls, "addDisplayPanel", _Recorder()) + monkeypatch.setattr(controls, "addCellTypePanel", _Recorder()) + monkeypatch.setattr(controls, "addGenePanel", _Recorder()) + monkeypatch.setattr(controls, "addRotationPanel", rotation) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder()) + monkeypatch.setattr(controls, "addGeneTourPanel", _Recorder()) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert made["rotation"] is not None + args, _kwargs = rotation.calls[0] + assert args[1][1] is None and args[1][2] is None + + +def test_the_cloud_panel_is_only_offered_when_there_is_a_token_to_expire(monkeypatch): + """A login control on a window that needs no login is clutter, and it + implies the picture might be waiting on something. A local pyramid + has no cloud token in its environment; a cloud one does.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ( + "addDisplayPanel", + "addCellTypePanel", + "addGenePanel", + "addRotationPanel", + "addGeneAppearancePanel", + "addGeneTourPanel", + ): + monkeypatch.setattr(controls, name, _Recorder()) + cloud = _Recorder() + monkeypatch.setattr(controls, "addCloudPanel", cloud) + + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert "cloud" not in made + assert not cloud.calls + + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert cloud.calls, "a cloud session was offered no way to sign back in" + + +def test_a_failing_cloud_panel_costs_only_itself(monkeypatch, capsys): + """It is the last panel and the least essential one -- a keyring that + will not open must not cost the section.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ( + "addDisplayPanel", + "addCellTypePanel", + "addGenePanel", + "addRotationPanel", + "addGeneAppearancePanel", + "addGeneTourPanel", + ): + monkeypatch.setattr(controls, name, _Recorder()) + monkeypatch.setattr(controls, "addCloudPanel", _Recorder(boom=True)) + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert made["cloud"] is None + assert made["genes"] is not None + assert "cloud" in capsys.readouterr().err + + +def test_the_gene_list_is_built_last_so_it_sits_at_the_bottom(monkeypatch): + """It is the tallest panel by a long way -- 26,444 rows -- and napari + gives the leftover height to whatever is furthest down, so anything + below it would be squeezed while the list took the room. The two gene + controls read above it, which is also the order they are used in.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + order = [] + + def _named(name): + def build(*a, **k): + order.append(name) + return object() + + return build + + for name in ("addDisplayPanel", "addCellTypePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _named(name)) + monkeypatch.setattr(controls, "addGenePanel", _named("genes")) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _named("appearance")) + monkeypatch.setattr(controls, "addGeneTourPanel", _named("tour")) + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert order.index("appearance") < order.index("tour") < order.index("genes") + assert order[-1] == "genes" + assert made["appearance"] is not None and made["tour"] is not None + + +def test_the_cloud_panel_is_first_and_the_gene_list_last(monkeypatch): + """The cloud row is the only panel about whether the picture can + still be READ rather than how it is drawn, and its clock is the + thing worth catching before it bites -- so it goes at the top, where + "12 min left" is seen in time to do something about it. The gene + list stays at the bottom, where the leftover height belongs.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + order = [] + + def _named(name): + def build(*a, **k): + order.append(name) + return object() + + return build + + for name in ( + "addDisplayPanel", + "addCellTypePanel", + "addRotationPanel", + "addGenePanel", + "addGeneAppearancePanel", + "addGeneTourPanel", + "addCloudPanel", + ): + monkeypatch.setattr(controls, name, _named(name)) + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + + controls.addAllPanels(object(), object(), object(), object(), True) + assert order[0] == "addCloudPanel" + assert order[-1] == "addGenePanel" + + +def test_the_tour_is_handed_the_appearance_panel(monkeypatch): + """So a pulse departs from the gamma the reader chose and returns to + it, rather than resetting their setting to 1.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _Recorder()) + appearance = object() + monkeypatch.setattr(controls, "addGeneAppearancePanel", lambda *a, **k: appearance) + tour = _Recorder() + monkeypatch.setattr(controls, "addGeneTourPanel", tour) + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + + controls.addAllPanels(object(), object(), object(), object(), True) + assert tour.calls, "the tour panel was never built" + _args, kwargs = tour.calls[0] + assert kwargs.get("appearance") is appearance + + +def test_a_failing_appearance_panel_still_leaves_a_tour(monkeypatch, capsys): + """The tour reads the appearance panel's gamma when it has one and + falls back to 1 when it does not -- so one failing must not cascade.""" + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ("addDisplayPanel", "addCellTypePanel", "addGenePanel", "addRotationPanel"): + monkeypatch.setattr(controls, name, _Recorder()) + monkeypatch.setattr(controls, "addGeneAppearancePanel", _Recorder(boom=True)) + tour = _Recorder() + monkeypatch.setattr(controls, "addGeneTourPanel", tour) + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + + made = controls.addAllPanels(object(), object(), object(), object(), True) + assert made["appearance"] is None + assert tour.calls, "a failing appearance panel took the tour with it" + _args, kwargs = tour.calls[0] + assert kwargs.get("appearance") is None + assert "gene appearance" in capsys.readouterr().err diff --git a/tests/test_gui_genepyramid_genelayers.py b/tests/test_gui_genepyramid_genelayers.py new file mode 100644 index 00000000..4de72458 --- /dev/null +++ b/tests/test_gui_genepyramid_genelayers.py @@ -0,0 +1,605 @@ +"""Genes opened as their own layers, and the sign-in behind a button. + +TWO DIFFERENT REQUESTS live next to each other and are easy to confuse. +``--genes`` FILTERS the base image down to a subset, leaving one +picture. ``--gene-layers`` adds a layer per gene ON TOP of the base, so +"All genes" stays and the named genes are comparable against it. Most of +these tests exist to keep that distinction from eroding. +""" + +import os + +import pytest + +from ndi.gui.app.genepyramid import controls +from ndi.gui.app.genepyramid.controls import parseGeneLayers + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + +# What NDI-matlab's GEFManager.ferretDemoGenes emits, spelled out here so +# a change on either side of the bridge shows up as a failing test rather +# than as a demo that opens the wrong colours. +_FERRET_DEMO = "HPCAL1:green,RORB:blue,FEZF2:bop orange,TSHZ2:yellow,NXPH4:red,SST:cyan" + + +class TestParsingTheSpec: + def test_it_pairs_each_gene_with_its_colour(self): + assert parseGeneLayers("A:red,B:cyan") == [("A", "red"), ("B", "cyan")] + + def test_a_gene_may_go_without_a_colour(self): + """None means "take the next one from the cycle", so a caller can + name the colours that matter and leave the rest.""" + assert parseGeneLayers("A,B:cyan") == [("A", None), ("B", "cyan")] + + def test_a_colour_may_contain_a_space(self): + """napari ships 'bop orange', 'bop blue' and 'bop purple'. Only + the ends of each field are trimmed, never the middle.""" + assert parseGeneLayers("FEZF2:bop orange") == [("FEZF2", "bop orange")] + + def test_spaces_around_the_entries_are_ignored(self): + assert parseGeneLayers(" A : red , B ") == [("A", "red"), ("B", None)] + + def test_nothing_asked_for_is_an_empty_list(self): + """A fact, not a missing answer.""" + for spec in ("", " ", ",,", None): + assert parseGeneLayers(spec) == [] + + def test_a_gene_named_twice_is_kept_once(self): + """Ticking the same box twice adds nothing the second time, and + the layer names would collide.""" + assert parseGeneLayers("A:red,A:cyan,B") == [("A", "red"), ("B", None)] + + def test_an_empty_colour_reads_as_no_colour(self): + assert parseGeneLayers("A:") == [("A", None)] + + def test_the_ferret_demo_spec_round_trips(self): + """The exact string NDI-matlab builds for the demo.""" + assert parseGeneLayers(_FERRET_DEMO) == [ + ("HPCAL1", "green"), + ("RORB", "blue"), + ("FEZF2", "bop orange"), + ("TSHZ2", "yellow"), + ("NXPH4", "red"), + ("SST", "cyan"), + ] + + def test_the_demo_colours_are_all_distinct(self): + colours = [c for _s, c in parseGeneLayers(_FERRET_DEMO)] + assert len(set(colours)) == len(colours) + + def test_the_order_is_kept(self): + """It is the order the layers are added, and so the order the + colour cycle advances for any gene that named no colour.""" + assert [s for s, _c in parseGeneLayers("D,C,B,A")] == ["D", "C", "B", "A"] + + +class TestTheCliOption: + def test_it_is_a_separate_option_from_genes(self): + """The two answer different questions and must not be conflated: + one filters the base layer, the other adds layers beside it.""" + from ndi.gui.app.genepyramid.cli import build_parser + + args = build_parser().parse_args(["/s", "--gene-layers", _FERRET_DEMO]) + assert args.gene_layers == _FERRET_DEMO + assert args.genes == "" + + def test_it_defaults_to_nothing(self): + from ndi.gui.app.genepyramid.cli import build_parser + + assert build_parser().parse_args(["/s"]).gene_layers == "" + + def test_the_resolver_hands_the_panel_pairs(self): + from ndi.gui.app.genepyramid.cli import _resolve_gene_layers + + assert _resolve_gene_layers("A:red") == [("A", "red")] + + def test_the_resolver_gives_none_for_nothing(self): + """None rather than [] so the panel can tell "not asked" from + "asked for an empty set".""" + from ndi.gui.app.genepyramid.cli import _resolve_gene_layers + + assert _resolve_gene_layers("") is None + + def test_both_options_can_be_given_together(self): + """Filtering the base layer AND opening named genes beside it is + a coherent request, not a contradiction.""" + from ndi.gui.app.genepyramid.cli import build_parser + + args = build_parser().parse_args(["/s", "--genes", "SST,VIP", "--gene-layers", "SST:cyan"]) + assert args.genes == "SST,VIP" + assert args.gene_layers == "SST:cyan" + + +@pytest.fixture(scope="module") +def qt(): + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Viewer: + def __init__(self): + self.window = _Window() + + +class TestTheCloudPanelIsOneRow: + def test_it_is_a_button_and_a_clock(self, qt): + """Three permanently docked fields say a reader signs in often. + They do it once a day at most, so the credentials live behind the + button.""" + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + assert len(panel.findChildren(qt.QPushButton)) == 1 + assert not panel.findChildren(qt.QLineEdit) + # The viewer owns the dock; a collected one deletes the widget. + assert viewer.window.docked + + def test_the_clock_still_warns_before_it_bites(self, qt, monkeypatch): + import base64 + import json + import time + + def seg(obj): + return base64.urlsafe_b64encode(json.dumps(obj).encode()).rstrip(b"=").decode() + + monkeypatch.setenv( + "NDI_CLOUD_TOKEN", + f"{seg({'alg': 'none'})}.{seg({'exp': int(time.time()) + 600})}.sig", + ) + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "renew before it runs out" in said + assert viewer.window.docked + + +class _Entry: + def __init__(self, uid, email, nickname="", stage="prod"): + self.UID = uid + self.Email = email + self.Nickname = nickname + self.Stage = stage + + +class TestTheProfileChoices: + def test_a_nickname_is_shown_with_the_email(self, monkeypatch): + """Two profiles for the same person on different stages have to + be tellable apart.""" + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", + lambda: [_Entry("u1", "me@example.com", "work")], + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "pw") + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "aes") + rows = controls.cloudProfileChoices() + assert rows == [("work (me@example.com)", "me@example.com", "u1", True, "password saved")] + + def test_a_dev_profile_is_marked(self, monkeypatch): + """A dev profile signing in against prod is the failure that + reads as a permissions error, so the stage is on the label.""" + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", + lambda: [_Entry("u1", "me@example.com", stage="dev")], + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "") + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") + label, _email, _uid, stored, note = controls.cloudProfileChoices()[0] + assert "[dev]" in label + assert not stored + assert "keyring" in note + + def test_a_backend_that_will_not_open_costs_no_dialog(self, monkeypatch): + """Typing the credentials by hand is exactly the fallback this + dialog exists to offer, so it must still appear.""" + + def boom(): + raise RuntimeError("no keyring") + + monkeypatch.setattr("ndi.cloud.profile.list_profiles", boom) + assert controls.cloudProfileChoices() == [] + + def test_an_unreadable_secret_only_costs_its_flag(self, monkeypatch): + def boom(_uid): + raise RuntimeError("locked") + + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", boom) + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") + rows = controls.cloudProfileChoices() + assert len(rows) == 1 and rows[0][3] is False + + +@pytest.fixture +def dialogOf(qt, monkeypatch): + held = [] + + def build(profiles=()): + monkeypatch.setattr("ndi.cloud.profile.list_profiles", lambda: list(profiles)) + monkeypatch.setattr( + "ndi.cloud.profile.get_password", lambda uid: "saved-pw" if uid == "u1" else "" + ) + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") + dialog, outcome = controls.buildCloudSignInDialog() + held.append(dialog) + return dialog, outcome + + return build + + +class TestTheSignInDialog: + def test_it_signs_in_with_what_was_typed(self, qt, dialogOf, monkeypatch): + seen = {} + monkeypatch.setattr("ndi.cloud.auth.login", lambda w, s: seen.update(who=w, secret=s)) + dialog, outcome = dialogOf() + dialog._ndi_fields["email"].setText(" me@example.com ") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_accept() + assert seen == {"who": "me@example.com", "secret": "pw"} + assert outcome["ok"] + + def test_the_password_is_never_shown_or_kept(self, qt, dialogOf, monkeypatch): + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + dialog, _outcome = dialogOf() + password = dialog._ndi_fields["password"] + assert password.echoMode() == qt.QLineEdit.Password + dialog._ndi_fields["email"].setText("me@example.com") + password.setText("pw") + dialog._ndi_accept() + assert password.text() == "" + + def test_a_failed_attempt_clears_it_too(self, qt, dialogOf, monkeypatch): + def boom(*_a): + raise RuntimeError("HTTP 401") + + monkeypatch.setattr("ndi.cloud.auth.login", boom) + dialog, outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("wrong") + dialog._ndi_accept() + assert dialog._ndi_fields["password"].text() == "" + assert not outcome["ok"] + assert "401" in dialog._ndi_said.text() + + def test_picking_a_profile_fills_the_email_in(self, qt, dialogOf): + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + assert dialog._ndi_fields["email"].text() == "me@example.com" + + def test_a_saved_password_needs_nothing_typed(self, qt, dialogOf, monkeypatch): + """Someone whose profile works should not be retyping a password + to get past an expired token.""" + seen = {} + monkeypatch.setattr("ndi.cloud.auth.login", lambda w, s: seen.update(who=w, secret=s)) + dialog, outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + dialog._ndi_accept() + assert seen == {"who": "me@example.com", "secret": "saved-pw"} + assert outcome["ok"] + + def test_a_typed_password_beats_the_saved_one(self, qt, dialogOf, monkeypatch): + """Which is the whole point of arriving here: what is saved is + wrong and is being corrected.""" + seen = {} + monkeypatch.setattr("ndi.cloud.auth.login", lambda w, s: seen.update(who=w, secret=s)) + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + dialog._ndi_fields["password"].setText("typed") + dialog._ndi_accept() + assert seen["secret"] == "typed" + + def test_the_first_entry_is_type_them_yourself(self, qt, dialogOf): + """A dialog that opened on a profile would sign in as whoever the + list happened to start with.""" + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com")]) + chooser = dialog._ndi_fields["profile"] + assert chooser.currentIndex() == 0 + assert chooser.itemData(0) is None + + def test_an_empty_field_asks_rather_than_calling(self, qt, dialogOf, monkeypatch): + called = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: called.append(a)) + dialog, _outcome = dialogOf() + dialog._ndi_accept() + assert called == [] + assert "needed" in dialog._ndi_said.text() + + def test_the_profile_is_only_written_when_asked(self, qt, dialogOf, monkeypatch): + saved = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a, **k: saved.append(a)) + dialog, _outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_accept() + assert saved == [] + + dialog2, _o2 = dialogOf() + dialog2._ndi_fields["email"].setText("me@example.com") + dialog2._ndi_fields["password"].setText("pw") + dialog2._ndi_fields["save"].setChecked(True) + dialog2._ndi_accept() + assert saved == [("me@example.com", "pw")] + + def test_a_profile_that_will_not_save_does_not_undo_the_sign_in( + self, qt, dialogOf, monkeypatch + ): + def boom(*_a, **_k): + raise RuntimeError("no keyring") + + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", boom) + dialog, outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_fields["save"].setChecked(True) + dialog._ndi_accept() + assert outcome["ok"] + assert "Signed in" in outcome["note"] and "keyring" in outcome["note"] + + +class TestWhyThereIsNoPassword: + """A backend mismatch used to look exactly like an empty store. + + profile._detect_backend picks keyring whenever keyring merely + imports, while MATLAB writes its secret to the AES file -- so a + machine with both looks in the wrong one and finds nothing. Told as + "no password saved", that sends the reader off to re-save a password + that is already saved, in a store nothing is reading. + """ + + def test_an_empty_store_names_the_backend(self, monkeypatch): + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "") + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "keyring") + note = controls.cloudProfileChoices()[0][4] + assert "keyring" in note and "no password" in note + + def test_an_unreadable_secret_carries_its_reason(self, monkeypatch): + def boom(_uid): + raise RuntimeError("cryptography failed to load") + + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", boom) + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "aes") + note = controls.cloudProfileChoices()[0][4] + assert "unreadable" in note + assert "cryptography failed to load" in note + assert "aes" in note + + def test_a_saved_password_says_so(self, monkeypatch): + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "pw") + monkeypatch.setattr("ndi.cloud.profile.backend", lambda: "aes") + assert controls.cloudProfileChoices()[0][4] == "password saved" + + def test_a_backend_that_will_not_name_itself_is_not_an_error(self, monkeypatch): + def boom(): + raise RuntimeError("no") + + monkeypatch.setattr( + "ndi.cloud.profile.list_profiles", lambda: [_Entry("u1", "me@example.com")] + ) + monkeypatch.setattr("ndi.cloud.profile.get_password", lambda _u: "") + monkeypatch.setattr("ndi.cloud.profile.backend", boom) + assert "unknown" in controls.cloudProfileChoices()[0][4] + + def test_the_chooser_shows_the_reason(self, qt, dialogOf): + dialog, _outcome = dialogOf([_Entry("u2", "them@example.com", "other")]) + chooser = dialog._ndi_fields["profile"] + assert "no password in the keyring store" in chooser.itemText(1) + + def test_picking_one_puts_the_reason_on_screen(self, qt, dialogOf): + """ "This profile's password is not readable from here" is a + different problem from "you never saved one", and only one of + them is solved by typing it again.""" + dialog, _outcome = dialogOf([_Entry("u2", "them@example.com", "other")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + assert "keyring" in dialog._ndi_said.text() + + def test_picking_a_working_one_says_nothing(self, qt, dialogOf): + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + assert dialog._ndi_said.isHidden() + + +@pytest.fixture +def heldLogo(qt): + """A logo label that OUTLIVES the assertion. + + cloudLogoLabel returns a parentless QLabel; a temporary is collected + the moment the expression ends, Qt deletes the C++ object underneath + it, and reading .pixmap() off the dangling wrapper SEGFAULTS PyQt5 + rather than raising. Holding it is the whole fix. + """ + label = controls.cloudLogoLabel() + assert label is not None, "the bundled logo did not load" + yield label + del label + + +class TestTheCloudLogo: + """The wordmark, bundled and drawn so it is actually visible. + + Copied from NDI-matlab rather than referenced: the viewer runs from + a Python install that need not have NDI-matlab on disk at all, which + is the normal case for someone opening a downloaded dataset. + """ + + def test_it_ships_with_the_package(self): + path = controls.cloudLogoPath() + assert path is not None, "the bundled logo is missing" + assert path.is_file() + assert path.suffix == ".png" + + def test_it_is_the_file_ndi_matlab_serves(self): + """Byte-for-byte, so the two repos cannot drift into showing + different marks for the same product.""" + import hashlib + + digest = hashlib.md5(controls.cloudLogoPath().read_bytes()).hexdigest() + assert digest == "3e8439f1b1b8686b38ad71f314fff157" + + def test_it_is_drawn_on_a_white_card(self, qt, heldLogo): + """The wordmark is dark navy on transparency and napari's docks + follow the viewer's theme -- on the dark one the lettering simply + is not there.""" + assert "#ffffff" in heldLogo.styleSheet() + + def test_it_is_sized_for_a_dock_and_stays_crisp(self, qt, heldLogo): + """Twice the display width at a device pixel ratio of 2, for the + retina screen a demo is usually given from.""" + pixmap = heldLogo.pixmap() + assert pixmap.devicePixelRatio() == 2.0 + assert 120 <= pixmap.width() / pixmap.devicePixelRatio() <= 200 + + def test_a_missing_asset_costs_no_panel(self, qt, monkeypatch): + """A missing file must cost the picture nothing -- the panel + falls back to its own dock name in text.""" + monkeypatch.setattr(controls, "cloudLogoPath", lambda: None) + assert controls.cloudLogoLabel() is None + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + assert panel is not None + assert viewer.window.docked + + def test_the_panel_shows_it(self, qt): + viewer = _Viewer() + panel = controls.addCloudPanel(viewer) + assert any( + lbl.pixmap() and not lbl.pixmap().isNull() for lbl in panel.findChildren(qt.QLabel) + ) + assert viewer.window.docked + + +class TestSavingWhenOneEmailOwnsSeveralAccounts: + """A dev profile and a prod profile under one address is the normal + arrangement, not a mistake to be tidied. + + Writing the prod password into the dev entry is silent and wrong, + and re-pointing the default at the dev entry while doing it is + worse -- everything afterwards would authenticate as the wrong + account against the wrong server. + """ + + def _store(self, monkeypatch, entries, current=None, default=None): + calls = {} + monkeypatch.setattr("ndi.cloud.profile.list_profiles", lambda: entries) + monkeypatch.setattr("ndi.cloud.profile.get_current", lambda: current) + monkeypatch.setattr("ndi.cloud.profile.get_default", lambda: default) + monkeypatch.setattr( + "ndi.cloud.profile.set_password", lambda k, p: calls.setdefault("set", (k, p)) + ) + monkeypatch.setattr( + "ndi.cloud.profile.set_default", lambda k: calls.setdefault("default", k) + ) + + def _add(*a): + calls["add"] = a + return "fresh" + + monkeypatch.setattr("ndi.cloud.profile.add", _add) + return calls + + def test_a_named_uid_settles_it_outright(self, monkeypatch): + """Which is what the dialog passes: the profile the reader + actually picked from the list.""" + calls = self._store(monkeypatch, []) + controls.saveCloudProfile("me@example.com", "pw", uid="u-dev") + assert calls["set"] == ("u-dev", "pw") + assert "default" not in calls + + def test_the_current_profile_wins_when_two_share_an_email(self, monkeypatch): + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod], current=prod) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u-prod", "pw") + + def test_the_default_wins_when_there_is_no_current(self, monkeypatch): + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod], default=prod) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u-prod", "pw") + + def test_it_refuses_rather_than_guessing(self, monkeypatch): + """Doing nothing and saying so beats writing a password into an + account the reader did not mean.""" + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod]) + with pytest.raises(ValueError) as caught: + controls.saveCloudProfile("me@example.com", "pw") + assert "2 profiles" in str(caught.value) + assert calls == {}, "it wrote something despite the ambiguity" + + def test_it_never_writes_into_the_first_it_finds(self, monkeypatch): + """The old behaviour, and the reason this class exists: the dev + entry happened to be listed first.""" + dev, prod = _Entry("u-dev", "me@example.com"), _Entry("u-prod", "me@example.com") + calls = self._store(monkeypatch, [dev, prod], current=prod) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"][0] != "u-dev" + + def test_one_match_needs_no_steer(self, monkeypatch): + only = _Entry("u1", "me@example.com") + calls = self._store(monkeypatch, [only]) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u1", "pw") + + def test_a_brand_new_email_is_added_and_made_default(self, monkeypatch): + """A store that had nothing for this email has no other candidate + to displace.""" + calls = self._store(monkeypatch, [_Entry("u-other", "them@example.com")]) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["add"] == ("me", "me@example.com", "pw") + assert calls["default"] == "fresh" + + def test_a_current_profile_for_a_different_email_does_not_capture_it(self, monkeypatch): + """It has to match the email being saved, or picking "current" + would write this password into somebody else's account.""" + mine = _Entry("u-mine", "me@example.com") + theirs = _Entry("u-theirs", "them@example.com") + calls = self._store(monkeypatch, [mine], current=theirs) + controls.saveCloudProfile("me@example.com", "pw") + assert calls["set"] == ("u-mine", "pw") + + +class TestTheDialogNamesTheProfileItSavesTo: + def test_a_picked_profile_is_the_one_written(self, qt, dialogOf, monkeypatch): + """No guessing needed when the reader chose from the list.""" + saved = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a, **k: saved.append((a, k))) + dialog, _outcome = dialogOf([_Entry("u1", "me@example.com", "work")]) + dialog._ndi_fields["profile"].setCurrentIndex(1) + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_fields["save"].setChecked(True) + dialog._ndi_accept() + assert saved and saved[0][1].get("uid") == "u1" + + def test_typing_them_by_hand_passes_no_uid(self, qt, dialogOf, monkeypatch): + saved = [] + monkeypatch.setattr("ndi.cloud.auth.login", lambda *a: None) + monkeypatch.setattr(controls, "saveCloudProfile", lambda *a, **k: saved.append((a, k))) + dialog, _outcome = dialogOf() + dialog._ndi_fields["email"].setText("me@example.com") + dialog._ndi_fields["password"].setText("pw") + dialog._ndi_fields["save"].setChecked(True) + dialog._ndi_accept() + assert saved and saved[0][1].get("uid") == "" diff --git a/tests/test_gui_genepyramid_genepanel.py b/tests/test_gui_genepyramid_genepanel.py new file mode 100644 index 00000000..d0080951 --- /dev/null +++ b/tests/test_gui_genepyramid_genepanel.py @@ -0,0 +1,328 @@ +"""How the Genes panel is laid out, checked on real widgets. + +A dock panel is narrow, and a wide control that shares its row with a +label ends up a stub. That went wrong twice -- the rotation slider, then +the abundance band, whose two value labels ate what little track was +left and made it unreachable rather than merely cramped. An arrangement +is a property of real widgets in a real layout, so these build the panel +offscreen rather than stubbing it, and skip where there is no Qt. + +What the panel DOES -- the filters, the sorting, the layers -- is tested +in test_gui_genepyramid_controls.py with no widgets at all. +""" + +import os + +import numpy as np +import pytest + +from ndi.gui.app.genepyramid import controls + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + + +@pytest.fixture(scope="module") +def qt(): + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Layers(dict): + events = None + + +class _AddedLayer: + """Whatever viewer.add_image handed back, as the panel treats it.""" + + def __init__(self, name, colormap=None, blending=None): + self.name = name + self.colormap = colormap + self.blending = blending + self.data = None + + def reset_contrast_limits(self): + pass + + +class _Viewer: + def __init__(self): + self.window = _Window() + self.layers = _Layers() + self.added = [] + + def add_image(self, **kwargs): + layer = _AddedLayer(kwargs.get("name", ""), kwargs.get("colormap"), kwargs.get("blending")) + self.added.append(layer) + self.layers[layer.name] = layer + return layer + + +@pytest.fixture +def panelWith(qt, monkeypatch): + """Build the gene panel over a stub list, with genes preset.""" + held = [] + + def build(initial=None, symbols=None): + from ndi.fun import doc_gene_export + from ndi.gui.app.genepyramid import controls + + names = list(symbols or [f"GENE{i:04d}" for i in range(40)]) + monkeypatch.setattr( + doc_gene_export, "readGeneList", lambda *a, **k: (list(names), list(names)) + ) + monkeypatch.setattr( + doc_gene_export, + "readGeneTotals", + lambda *a, **k: (np.arange(len(names), dtype=float), None), + ) + from ndi.gui.app.genepyramid import multiscale + + monkeypatch.setattr( + multiscale, + "layerSpec", + lambda _s, _p, _rows, _d, name: {"data": [], "name": name}, + ) + viewer = _Viewer() + held.append(viewer) + panel = controls.addGenePanel(viewer, object(), object(), initial=initial) + return viewer, panel + + return build + + +@pytest.fixture +def panel(qt, monkeypatch): + """The real Genes panel over a stub gene list.""" + from ndi.fun import doc_gene_export + from ndi.gui.app.genepyramid import controls + + symbols = [f"GENE{i:04d}" for i in range(40)] + monkeypatch.setattr( + doc_gene_export, "readGeneList", lambda *a, **k: (list(symbols), list(symbols)) + ) + monkeypatch.setattr( + doc_gene_export, + "readGeneTotals", + lambda *a, **k: (np.arange(len(symbols), dtype=float), None), + ) + viewer = _Viewer() + return controls.addGenePanel(viewer, object(), object()) + + +def _rows(box): + """The panel's top-level layout: widgets, and nested rows as lists.""" + layout = box.layout() + out = [] + for i in range(layout.count()): + item = layout.itemAt(i) + if item.widget() is not None: + out.append(item.widget()) + elif item.layout() is not None: + sub = item.layout() + out.append( + [sub.itemAt(j).widget() for j in range(sub.count()) if sub.itemAt(j).widget()] + ) + return out + + +def _index(rows, predicate): + return next(i for i, r in enumerate(rows) if predicate(r)) + + +class TestTheAbundanceBand: + def test_the_slider_shares_its_row_with_nothing(self, qt, panel): + """Its own value labels sit at each end of the track, so a label + beside it leaves the numbers and no track between them.""" + superqt = pytest.importorskip("superqt") + rows = _rows(panel) + row = _index( + rows, + lambda r: isinstance(r, list) + and any(isinstance(w, superqt.QLabeledDoubleRangeSlider) for w in r), + ) + assert len(rows[row]) == 1 + + def test_its_label_is_the_line_above(self, qt, panel): + superqt = pytest.importorskip("superqt") + rows = _rows(panel) + row = _index( + rows, + lambda r: isinstance(r, list) + and any(isinstance(w, superqt.QLabeledDoubleRangeSlider) for w in r), + ) + above = rows[row - 1] + assert isinstance(above, qt.QLabel) + assert "Abundance band" in above.text() + + def test_the_band_still_spans_the_whole_range(self, qt, panel): + """The move is layout only: a band that came up narrowed would + hide genes before anyone touched it.""" + superqt = pytest.importorskip("superqt") + slider = panel.findChild(superqt.QLabeledDoubleRangeSlider) + assert tuple(slider.value()) == (0.0, 100.0) + + +class TestTheList: + def test_it_opens_tall_enough_to_read(self, qt, panel): + """Four rows of 26,444 genes is a porthole. The dock is only as + tall as it is asked to be, so it is asked.""" + listw = panel.findChild(qt.QListWidget) + assert listw.minimumHeight() >= 200 + + def test_it_takes_the_space_that_is_going(self, qt, panel): + """Everything else on the panel is a fixed row; growth belongs to + the list, not to the gap under it.""" + layout = panel.layout() + listw = panel.findChild(qt.QListWidget) + i = next(i for i in range(layout.count()) if layout.itemAt(i).widget() is listw) + assert layout.stretch(i) >= 1 + + +class TestTheBlur: + def test_the_knob_is_on_the_gene_panel(self, qt, panel): + """Which is the point of it: the gene layers are what is sparse.""" + boxes = panel.findChildren(qt.QDoubleSpinBox) + assert any(b.suffix().strip() == "um" for b in boxes) + + def test_it_starts_off(self, qt, panel): + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + assert blur.value() == pytest.approx(0.0) + + +class TestGenesOpenedAtLaunch: + """``--gene-layers`` reaches the panel as ``initial``. + + A DIFFERENT REQUEST from ``--genes``, which filters the base image + down to a subset and leaves one picture. These arrive as their own + additive layers ON TOP of the base, which is what makes several + genes comparable -- and it is why the demo command carries the one + option and not the other. + """ + + def test_each_gene_becomes_its_own_layer(self, panelWith): + viewer, _panel = panelWith(initial=[("GENE0000", "green"), ("GENE0001", "blue")]) + assert [layer.name for layer in viewer.added] == ["gene: GENE0000", "gene: GENE0001"] + + def test_the_named_colours_are_used(self, panelWith): + viewer, _panel = panelWith(initial=[("GENE0000", "green"), ("GENE0001", "bop orange")]) + assert [layer.colormap for layer in viewer.added] == ["green", "bop orange"] + + def test_they_blend_additively(self, panelWith): + """Which is what lets two genes be read over one another rather + than one hiding the other.""" + viewer, _panel = panelWith(initial=[("GENE0000", "green")]) + assert viewer.added[0].blending == "additive" + + def test_a_gene_with_no_colour_takes_one_from_the_cycle(self, panelWith): + viewer, _panel = panelWith(initial=[("GENE0000", None)]) + assert viewer.added[0].colormap == controls._GENE_COLORMAPS[0] + + def test_a_named_colour_does_not_advance_the_cycle(self, panelWith): + """Otherwise the first hand-ticked gene's colour would depend on + how many were preset, which is a surprise with no upside.""" + viewer, _panel = panelWith( + initial=[("GENE0000", "green"), ("GENE0001", "blue"), ("GENE0002", None)] + ) + assert viewer.added[2].colormap == controls._GENE_COLORMAPS[0] + + def test_the_boxes_are_ticked_too(self, qt, panelWith): + """Ticked through the panel's own checkbox, not by adding the + layer behind its back: otherwise unticking a preset gene would do + nothing at all.""" + _viewer, panel = panelWith(initial=[("GENE0000", "green")]) + listw = panel.findChild(qt.QListWidget) + # Qt.Checked is 2; compared numerically rather than importing the + # enum, which qtpy spells differently across bindings. + ticked = [ + listw.item(i).text() + for i in range(listw.count()) + if int(listw.item(i).checkState()) == 2 + ] + assert len(ticked) == 1 + assert ticked[0].startswith("GENE0000") + + def test_a_gene_not_in_this_pyramid_is_said_out_loud(self, qt, panelWith): + """A demo that quietly opens four of six layers looks like the + viewer working.""" + viewer, panel = panelWith(initial=[("GENE0000", "green"), ("NOTHERE", "red")]) + said = " ".join(lbl.text() for lbl in panel.findChildren(qt.QLabel)) + assert "NOTHERE" in said + # and the ones that ARE there still open + assert [layer.name for layer in viewer.added] == ["gene: GENE0000"] + + def test_no_preset_opens_nothing(self, panelWith): + viewer, _panel = panelWith() + assert viewer.added == [] + + def test_the_ferret_demo_spec_opens_six_coloured_layers(self, panelWith): + """End to end from the string NDI-matlab builds.""" + spec = "HPCAL1:green,RORB:blue,FEZF2:bop orange,TSHZ2:yellow,NXPH4:red,SST:cyan" + symbols = ["HPCAL1", "RORB", "FEZF2", "TSHZ2", "NXPH4", "SST", "OTHER"] + viewer, _panel = panelWith(initial=controls.parseGeneLayers(spec), symbols=symbols) + assert [layer.name for layer in viewer.added] == [f"gene: {s}" for s in symbols[:6]] + assert [layer.colormap for layer in viewer.added] == [ + "green", + "blue", + "bop orange", + "yellow", + "red", + "cyan", + ] + + +class TestTheBlurKnobWaits: + """A spin box emits on every keystroke. + + Typing "50" asks for 5 um first, and re-blurring is a whole-ladder + map_overlap that takes long enough to see -- so the picture visibly + redrew at the wrong width before the second digit landed. + """ + + def test_typing_does_not_reblur_at_once(self, qt, panel): + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + wait = panel._ndi_blur_wait + assert not wait.isActive() + blur.setValue(5.0) + assert wait.isActive(), "the blur applied on the first keystroke" + + def test_it_waits_long_enough_for_a_second_digit(self, qt, panel): + """Past the gap between two digits, and short enough not to feel + stuck.""" + assert 1000 <= panel._ndi_blur_wait.interval() <= 2000 + + def test_the_wait_restarts_with_each_keystroke(self, qt, panel): + """Otherwise "500" would fire partway through, at 50.""" + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + wait = panel._ndi_blur_wait + blur.setValue(5.0) + first = wait.remainingTime() + blur.setValue(50.0) + assert wait.remainingTime() >= first - 50 + + def test_it_fires_once_rather_than_repeating(self, qt, panel): + assert panel._ndi_blur_wait.isSingleShot() + + def test_the_wait_belongs_to_the_panel(self, qt, panel): + core = pytest.importorskip("qtpy.QtCore") + assert panel._ndi_blur_wait.parent() is panel + assert isinstance(panel._ndi_blur_wait, core.QTimer) + + def test_finishing_the_edit_stops_the_wait(self, qt, panel): + """Enter, or clicking away, means the number IS finished -- no + reason to sit out the rest of the wait.""" + blur = next(b for b in panel.findChildren(qt.QDoubleSpinBox) if b.suffix().strip() == "um") + blur.setValue(50.0) + assert panel._ndi_blur_wait.isActive() + blur.editingFinished.emit() + assert not panel._ndi_blur_wait.isActive() diff --git a/tests/test_gui_genepyramid_multiscale.py b/tests/test_gui_genepyramid_multiscale.py index 357c5060..cf99a9bb 100644 --- a/tests/test_gui_genepyramid_multiscale.py +++ b/tests/test_gui_genepyramid_multiscale.py @@ -351,3 +351,64 @@ def test_layer_spec_name_falls_back_when_the_pyramid_has_no_label(tmp_path): grid=1, ) assert layerSpec(S, pyr)["name"] == "genes" + + +class TestADatasetCanBeReopenedPerThread: + """A downloaded cloud dataset opened in the viewer failed on its first + tile with a threading error, not a missing attribute. + + The tile fetcher builds a per-thread session handle by reopening + whatever it was given AT ITS OWN PATH -- NDI's database belongs to the + thread that opened it, and dask runs blocks on a pool. A session + exposes ``path`` and could be reopened; ``ndi_dataset_dir`` kept the + same value privately as ``_path`` and could not, so the fetcher had no + way to build a handle and said so from the worker thread. + + MATLAB's ``ndi.dataset.dir`` has declared a public ``path`` property + since it was written, so this was a parity gap rather than a design + choice. + """ + + def test_a_directory_backed_dataset_reports_its_path(self): + from ndi.dataset import ndi_dataset_dir + + assert isinstance(ndi_dataset_dir.path, property) + + def test_the_fetcher_can_reopen_anything_with_a_path(self): + from ndi.gui.app.genepyramid.multiscale import _TileFetcher + + class Pathed: + def __init__(self, p): + self._p = p + + @property + def path(self): + return self._p + + assert _TileFetcher(Pathed("/tmp/somewhere"))._reopen is not None + + def test_something_with_no_path_is_left_alone(self): + """Anything holding a live client or credentials is not duplicated + on a guess -- reopening one could re-authenticate per thread.""" + from ndi.gui.app.genepyramid.multiscale import _TileFetcher + + class Clientish: + pass + + assert _TileFetcher(Clientish())._reopen is None + + def test_a_reopened_handle_is_the_same_class_at_the_same_path(self): + from ndi.gui.app.genepyramid.multiscale import _TileFetcher + + class Pathed: + def __init__(self, p): + self._p = p + + @property + def path(self): + return self._p + + original = Pathed("/tmp/here") + rebuilt = _TileFetcher(original)._reopen() + assert type(rebuilt) is Pathed + assert rebuilt.path == "/tmp/here" diff --git a/tests/test_gui_genepyramid_progress.py b/tests/test_gui_genepyramid_progress.py new file mode 100644 index 00000000..1ab6b748 --- /dev/null +++ b/tests/test_gui_genepyramid_progress.py @@ -0,0 +1,335 @@ +"""The launch says how far along a cloud fetch is, not just that one started. + +stage() printed a label and then nothing moved until the step finished. On +a directory-backed session that is fine -- the reads are milliseconds. On a +cloud-backed one the cell table, the contour file and the gene list are +each a whole download before the window can appear, and a static label for +a minute is indistinguishable from a hang. + +getFile is the one place every on-demand fetch passes through, so the byte +counts are taken there and surfaced through a module-level observer: DID +fixes the file handler's signature (it counts positional parameters to +choose between the two- and three-argument forms), so there is nowhere to +thread a callback through from a caller. +""" + +import io +import sys +import unittest +from unittest import mock + +from ndi.cloud import filehandler +from ndi.gui.app.genepyramid import progress + + +class TestHumanBytes(unittest.TestCase): + def test_it_reads_at_a_glance(self): + self.assertEqual(progress.humanBytes(0), "0 B") + self.assertEqual(progress.humanBytes(900), "900 B") + self.assertEqual(progress.humanBytes(26214400), "25.0 MB") + + def test_a_big_file_does_not_become_a_wall_of_digits(self): + self.assertEqual(progress.humanBytes(3 * 1024**3), "3.0 GB") + + +class TestTheObserverHook(unittest.TestCase): + def test_it_is_absent_by_default(self): + self.assertIsNone(filehandler._fetch_observer) + + def test_it_is_installed_for_the_block_and_removed_after(self): + seen = [] + with filehandler.watchFetches(lambda *a: seen.append(a)): + self.assertIsNotNone(filehandler._fetch_observer) + self.assertIsNone(filehandler._fetch_observer) + + def test_a_raise_inside_the_block_still_removes_it(self): + with self.assertRaises(ValueError): + with filehandler.watchFetches(lambda *a: None): + raise ValueError("boom") + self.assertIsNone(filehandler._fetch_observer) + + def test_nesting_restores_the_outer_one_rather_than_clearing(self): + outer = lambda *a: None # noqa: E731 + with filehandler.watchFetches(outer): + with filehandler.watchFetches(lambda *a: None): + pass + self.assertIs(filehandler._fetch_observer, outer) + self.assertIsNone(filehandler._fetch_observer) + + +class _FakeResponse: + """Enough of requests.Response for getFile's streaming branch.""" + + status_code = 200 + + def __init__(self, chunks, length=None): + self._chunks = chunks + self.headers = {} if length is None else {"Content-Length": str(length)} + + def iter_content(self, chunk_size=8192): # noqa: ARG002 + yield from self._chunks + + +class TestGetFileReportsBytes(unittest.TestCase): + def _run(self, chunks, length, progress_fn, tmp): + from ndi.cloud.api import files as files_api + + with mock.patch("requests.get", return_value=_FakeResponse(chunks, length)): + return files_api.getFile("https://example.invalid/x", tmp, progress=progress_fn) + + def setUp(self): + import tempfile + from pathlib import Path + + self.dir = tempfile.mkdtemp() + self.addCleanup(lambda: __import__("shutil").rmtree(self.dir, ignore_errors=True)) + self.tmp = Path(self.dir) / "out.bin" + + def test_the_running_total_is_cumulative_not_per_chunk(self): + seen = [] + self._run([b"a" * 10, b"b" * 5], 15, lambda d, t: seen.append((d, t)), self.tmp) + self.assertEqual(seen, [(10, 15), (15, 15)]) + + def test_the_total_comes_from_content_length(self): + seen = [] + self._run([b"x" * 4], 4, lambda d, t: seen.append((d, t)), self.tmp) + self.assertEqual(seen[-1][1], 4) + + def test_a_chunked_response_reports_no_total_rather_than_guessing(self): + seen = [] + self._run([b"x" * 4], None, lambda d, t: seen.append((d, t)), self.tmp) + self.assertIsNone(seen[-1][1]) + + def test_a_nonsense_content_length_is_no_total_rather_than_a_crash(self): + from ndi.cloud.api import files as files_api + + resp = _FakeResponse([b"x"], None) + resp.headers = {"Content-Length": "banana"} + seen = [] + with mock.patch("requests.get", return_value=resp): + files_api.getFile( + "https://example.invalid/x", self.tmp, progress=lambda d, t: seen.append((d, t)) + ) + self.assertIsNone(seen[-1][1]) + + def test_the_file_still_lands_when_the_callback_throws(self): + # A reporting callback must never cost the download. + def boom(done, total): + raise RuntimeError("renderer died") + + ok = self._run([b"a" * 8, b"b" * 8], 16, boom, self.tmp) + self.assertTrue(ok) + self.assertEqual(self.tmp.read_bytes(), b"a" * 8 + b"b" * 8) + + def test_no_callback_is_the_untouched_path(self): + ok = self._run([b"z" * 3], 3, None, self.tmp) + self.assertTrue(ok) + self.assertEqual(self.tmp.read_bytes(), b"z" * 3) + + +class TestTheRendererStaysOutOfLogs(unittest.TestCase): + def test_off_installs_no_observer_at_all(self): + # A \r-redrawn bar becomes thousands of lines in a CI capture. + import os + + with mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_PROGRESS": "off"}): + with progress.cloudFetches("reading"): + self.assertIsNone(filehandler._fetch_observer) + + def test_quiet_suppresses_it_too(self): + import os + + with mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_QUIET": "1"}): + with progress.cloudFetches("reading"): + self.assertIsNone(filehandler._fetch_observer) + + def test_text_mode_does_install_one(self): + import os + + with mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_PROGRESS": "text"}): + with progress.cloudFetches("reading"): + self.assertIsNotNone(filehandler._fetch_observer) + + +class TestTheDisplayGuard(unittest.TestCase): + """Qt aborts the PROCESS when it cannot reach a display. + + It does not raise, so try/except around QApplication() catches + nothing: without this guard a headless launch would die instead of + falling back to text. Found by running the test suite, which took + pytest down with it. + """ + + def test_a_mac_always_has_a_window_server(self): + with mock.patch.object(sys, "platform", "darwin"): + self.assertTrue(progress._displayLikely()) + + def test_windows_too(self): + with mock.patch.object(sys, "platform", "win32"): + self.assertTrue(progress._displayLikely()) + + def test_bare_linux_says_no(self): + import os + + with ( + mock.patch.object(sys, "platform", "linux"), + mock.patch.dict(os.environ, {}, clear=True), + ): + self.assertFalse(progress._displayLikely()) + + def test_x11_wayland_or_an_explicit_platform_say_yes(self): + import os + + for var in ("DISPLAY", "WAYLAND_DISPLAY", "QT_QPA_PLATFORM"): + with ( + mock.patch.object(sys, "platform", "linux"), + mock.patch.dict(os.environ, {var: ":0"}, clear=True), + ): + self.assertTrue(progress._displayLikely(), var) + + +class TestModePrecedence(unittest.TestCase): + def setUp(self): + # _mode consults the window, which is a process-wide singleton. + self.addCleanup(setattr, progress, "_window", progress._window) + self.addCleanup(setattr, progress, "_window_failed", progress._window_failed) + + def _mode_with(self, env): + import os + + with mock.patch.dict(os.environ, env, clear=False): + return progress._mode() + + def test_the_env_var_wins_over_everything(self): + for asked in ("gui", "text", "off"): + self.assertEqual(self._mode_with({"NDI_GENEPYRAMID_PROGRESS": asked}), asked) + + def test_quiet_means_off_whatever_else_is_true(self): + self.assertEqual(self._mode_with({"NDI_GENEPYRAMID_QUIET": "1"}), "off") + + def test_no_window_and_no_terminal_is_off_not_a_crash(self): + import os + + progress._window, progress._window_failed = None, True + with ( + mock.patch.object(sys, "stderr", io.StringIO()), + mock.patch.dict(os.environ, {"NDI_GENEPYRAMID_PROGRESS": ""}, clear=False), + ): + self.assertEqual(progress._mode(), "off") + + +class TestClosingIsAlwaysSafe(unittest.TestCase): + def test_closing_when_there_never_was_a_window(self): + self.addCleanup(setattr, progress, "_window", progress._window) + progress._window = None + progress.closeLaunchWindow() # must not raise + + def test_closing_twice(self): + self.addCleanup(setattr, progress, "_window", progress._window) + progress._window = None + progress.closeLaunchWindow() + progress.closeLaunchWindow() + + def test_a_widget_that_throws_on_close_does_not_escape(self): + # The viewer is built by this point; a failing indicator must not + # take the window with it. + self.addCleanup(setattr, progress, "_window", progress._window) + + class _Boom: + def close(self): + raise RuntimeError("gone") + + progress._window = {"widget": _Boom(), "app": _Boom()} + progress.closeLaunchWindow() + self.assertIsNone(progress._window) + + +if __name__ == "__main__": + unittest.main() + + +class TestTheLabelIsActuallyPainted: + """The launch window sat on "importing napari" for minutes while the + label underneath had moved on several steps. + + setText only QUEUES a repaint, and processEvents services what is + queued at that instant -- but the step that follows blocks the thread + with no event loop running, so any paint the window needs after that + point is never serviced. The window freezes on whichever label was + current when the first slow step began. repaint() paints + synchronously, before returning, so each label reaches the glass + before its step blocks. + """ + + def _fakeWindow(self): + painted = [] + + class Widget: + def repaint(self): + painted.append("widget") + + class App: + def processEvents(self): + painted.append("events") + + class Label: + text = None + + def setText(self, t): + self.text = t + + class Bar: + def setRange(self, *_a): + pass + + def setValue(self, *_a): + pass + + def setFormat(self, *_a): + pass + + return { + "app": App(), + "widget": Widget(), + "step": Label(), + "bar": Bar(), + }, painted + + def test_a_stage_paints_before_it_returns(self, monkeypatch): + from ndi.gui.app.genepyramid import progress + + win, painted = self._fakeWindow() + monkeypatch.setattr(progress, "_launchWindow", lambda: win) + progress._guiStage("reading the tiles") + assert win["step"].text == "reading the tiles" + assert "widget" in painted, "the label must be on the glass, not just queued" + + def test_a_byte_update_paints_too(self, monkeypatch): + """A byte count that only lands when the transfer finishes has + reported nothing.""" + from ndi.gui.app.genepyramid import progress + + win, painted = self._fakeWindow() + monkeypatch.setattr(progress, "_launchWindow", lambda: win) + progress._guiBytes(50, 100) + assert "widget" in painted + + def test_a_window_that_will_not_paint_costs_nothing(self, monkeypatch): + """Styling and progress are never worth the launch.""" + from ndi.gui.app.genepyramid import progress + + class Hostile: + def repaint(self): + raise RuntimeError("no") + + win, _painted = self._fakeWindow() + win["widget"] = Hostile() + monkeypatch.setattr(progress, "_launchWindow", lambda: win) + progress._guiStage("still fine") # must not raise + + def test_no_window_is_not_an_error(self, monkeypatch): + from ndi.gui.app.genepyramid import progress + + monkeypatch.setattr(progress, "_launchWindow", lambda: None) + progress._guiStage("headless") + progress._guiBytes(1, 2) diff --git a/tests/test_gui_genepyramid_rotation.py b/tests/test_gui_genepyramid_rotation.py new file mode 100644 index 00000000..3610c946 --- /dev/null +++ b/tests/test_gui_genepyramid_rotation.py @@ -0,0 +1,381 @@ +"""Turning the whole picture: the affine, the pivot, and the layers together. + +Sections are not mounted square, and lining one up against an atlas plate +or a companion section wants an arbitrary angle rather than the 90-degree +steps napari offers on the canvas. + +Everything worth holding here is testable with no display and no napari: +the SIGN of the rotation, the PIVOT it turns about, and that every layer +gets the SAME matrix. The panel's ARRANGEMENT needs real widgets, so +those tests build it offscreen and skip where there is no Qt binding +-- which is CI today. +""" + +from types import SimpleNamespace + +import numpy as np +import pytest + +from ndi.gui.app.genepyramid.controls import ( + applyRotation, + rotationAffine, + rotationCenter, +) + + +def apply(a, point): + """Push a ``(row, col)`` point through a 3x3 homogeneous affine.""" + v = a @ np.array([point[0], point[1], 1.0]) + return (v[0], v[1]) + + +class TestTheMatrix: + def test_zero_is_the_identity(self): + assert np.allclose(rotationAffine(0, (100.0, 200.0)), np.eye(3)) + + def test_a_full_turn_is_the_identity(self): + assert np.allclose(rotationAffine(360, (100.0, 200.0)), np.eye(3), atol=1e-9) + + def test_the_pivot_is_the_one_point_that_does_not_move(self): + """The whole promise of 'rotate about the view centre'.""" + c = (137.0, -41.5) + for angle in (-180, -33, 0, 17, 90, 180): + assert np.allclose(apply(rotationAffine(angle, c), c), c, atol=1e-9) + + def test_positive_is_anticlockwise_on_screen(self): + """THE SIGN, checked rather than reasoned about. + + napari rows increase DOWNWARD (the pyramid records + ``origin_corner: upper-left``). A point to the RIGHT of the pivot + must land ABOVE it -- smaller row, same column -- at +90 degrees. + """ + c = (0.0, 0.0) + row, col = apply(rotationAffine(90, c), (0.0, 1.0)) + assert row == pytest.approx(-1.0, abs=1e-9), "right of pivot should go ABOVE it" + assert col == pytest.approx(0.0, abs=1e-9) + + def test_it_is_a_rotation_not_a_scale_or_a_flip(self): + """A determinant of -1 would mirror the section, which would be a + wrong picture that still looks like a picture.""" + for angle in (13, 47, 90, 180, -75): + m = rotationAffine(angle, (10.0, 20.0))[:2, :2] + assert np.linalg.det(m) == pytest.approx(1.0, abs=1e-12) + assert np.allclose(m @ m.T, np.eye(2), atol=1e-12) + + def test_distance_from_the_pivot_is_preserved(self): + c = (50.0, 60.0) + p = (110.0, -20.0) + before = np.hypot(p[0] - c[0], p[1] - c[1]) + got = apply(rotationAffine(29, c), p) + after = np.hypot(got[0] - c[0], got[1] - c[1]) + assert after == pytest.approx(before, rel=1e-12) + + def test_opposite_angles_undo_each_other(self): + """The control is ABSOLUTE, not accumulated, but the matrix still + has to be a proper inverse or repeated adjustment would drift.""" + c = (7.0, 3.0) + a = rotationAffine(38, c) + b = rotationAffine(-38, c) + assert np.allclose(a @ b, np.eye(3), atol=1e-12) + + +class TestTheCentre: + class FakeCamera: + def __init__(self, center): + self.center = center + + class FakeViewer: + def __init__(self, center): + self.camera = TestTheCentre.FakeCamera(center) + + def test_a_3d_camera_centre_gives_row_and_column(self): + """napari reports ``(z, y, x)`` even for 2D data, with z = 0.""" + assert rotationCenter(self.FakeViewer((0.0, 250.0, 400.0))) == (250.0, 400.0) + + def test_a_2_tuple_still_works(self): + """Taking the LAST two rather than indexing 1 and 2.""" + assert rotationCenter(self.FakeViewer((250.0, 400.0))) == (250.0, 400.0) + + def test_a_centre_with_nothing_to_read_raises(self): + with pytest.raises(ValueError, match="no row/column"): + rotationCenter(self.FakeViewer((5.0,))) + + +class FakeLayer: + def __init__(self): + self.affine = None + + +class TestTheLayersMoveTogether: + def test_every_layer_gets_the_same_matrix(self): + """The section, the cells and their outlines are ONE picture. A + control that could slide them apart is one that can produce a + wrong picture.""" + layers = [FakeLayer(), FakeLayer(), FakeLayer()] + applied = applyRotation(layers, 23.5, (100.0, 100.0)) + for layer in layers: + assert np.allclose(layer.affine, applied) + + def test_missing_layers_are_skipped_not_an_error(self): + """Outlines and centroids are optional; a session without them is + the normal case.""" + image = FakeLayer() + applyRotation([image, None, None], 10.0, (0.0, 0.0)) + assert image.affine is not None + + def test_it_returns_what_it_applied(self): + image = FakeLayer() + applied = applyRotation([image], 45.0, (1.0, 2.0)) + assert np.allclose(applied, rotationAffine(45.0, (1.0, 2.0))) + + def test_a_later_angle_replaces_rather_than_compounds(self): + """Absolute, not accumulated: two 30-degree settings leave the + picture at 30 degrees, not 60.""" + image = FakeLayer() + applyRotation([image], 30.0, (0.0, 0.0)) + applyRotation([image], 30.0, (0.0, 0.0)) + assert np.allclose(image.affine, rotationAffine(30.0, (0.0, 0.0))) + + +# --------------------------------------------------------------- layout + + +@pytest.fixture(scope="module") +def qt(): + """A real Qt, drawn to nowhere, or a skip. + + The panel's arrangement is the thing being checked here, and an + arrangement is a property of real widgets in a real layout -- stubbing + them out would only test the stubs. The offscreen platform gives real + widgets with no display, which is what CI has. CI does not install Qt + at all today, so this skips there and runs for anyone who does. + """ + import os + + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + widgets = pytest.importorskip("qtpy.QtWidgets") + app = widgets.QApplication.instance() or widgets.QApplication([]) + yield widgets + del app + + +class _Window: + def __init__(self): + self.docked = [] + + def add_dock_widget(self, widget, name=None, area=None): + self.docked.append((widget, name, area)) + return widget + + +class _Viewer: + def __init__(self): + self.window = _Window() + self.camera = SimpleNamespace(center=(0.0, 100.0, 200.0)) + + +def _rotationPanel(qt): + from ndi.gui.app.genepyramid.controls import addRotationPanel + + viewer = _Viewer() + return addRotationPanel(viewer, []) + + +def _rows(qt, box): + """The panel's top-level layout, as a list of rows. + + Each entry is either a widget or the list of widgets on a nested row, + which is exactly the distinction the arrangement turns on. + """ + layout = box.layout() + out = [] + for i in range(layout.count()): + item = layout.itemAt(i) + if item.widget() is not None: + out.append(item.widget()) + elif item.layout() is not None: + sub = item.layout() + out.append( + [sub.itemAt(j).widget() for j in range(sub.count()) if sub.itemAt(j).widget()] + ) + return out + + +class TestTheArrangement: + def test_the_slider_gets_a_row_of_its_own(self, qt): + """Sharing a row with the number and the Reset button left it a + stub too short to aim with: a dock panel is narrow and those two + take a fixed width out of it whatever is left over.""" + box = _rotationPanel(qt) + rows = _rows(qt, box) + sliders = [r for r in rows if isinstance(r, qt.QSlider)] + assert len(sliders) == 1, "the slider is not a row by itself" + + def test_the_slider_sits_under_the_number(self, qt): + """Which is where it was asked for, and reads as belonging to the + angle above it rather than as a third control beside it.""" + box = _rotationPanel(qt) + rows = _rows(qt, box) + spin_row = next( + i + for i, r in enumerate(rows) + if isinstance(r, list) and any(isinstance(w, qt.QDoubleSpinBox) for w in r) + ) + slider_row = next(i for i, r in enumerate(rows) if isinstance(r, qt.QSlider)) + assert slider_row > spin_row + + def test_the_number_keeps_the_reset_button_company(self, qt): + box = _rotationPanel(qt) + rows = _rows(qt, box) + row = next(r for r in rows if isinstance(r, list)) + assert any(isinstance(w, qt.QDoubleSpinBox) for w in row) + assert any(isinstance(w, qt.QPushButton) for w in row) + + +class TestTheStep: + def test_the_buttons_move_five_degrees(self, qt): + """Half a degree took ten clicks to show anything. Five is a turn + you can see, and the box still takes a typed 12.5 for finer.""" + box = _rotationPanel(qt) + spin = box.findChild(qt.QDoubleSpinBox) + assert spin.singleStep() == pytest.approx(5.0) + + def test_finer_can_still_be_typed(self, qt): + box = _rotationPanel(qt) + spin = box.findChild(qt.QDoubleSpinBox) + assert spin.decimals() >= 1 + spin.setValue(12.5) + assert spin.value() == pytest.approx(12.5) + + +# ------------------------------------------------- layers added later + + +class _Event: + def __init__(self, value): + self.value = value + + +class _Signal: + """Just enough of napari's EventEmitter to connect and emit.""" + + def __init__(self): + self._slots = [] + + def connect(self, fn): + self._slots.append(fn) + + def emit(self, event): + for fn in self._slots: + fn(event) + + +class _LayerList(list): + """A layer list that reports insertions, as napari's does.""" + + def __init__(self, *layers): + super().__init__(layers) + self.events = SimpleNamespace(inserted=_Signal()) + + def add(self, layer): + self.append(layer) + self.events.inserted.emit(_Event(layer)) + return layer + + +class _LiveViewer: + def __init__(self, *layers): + self.window = _Window() + self.camera = SimpleNamespace(center=(0.0, 100.0, 200.0)) + self.layers = _LayerList(*layers) + + +class TestGeneLayersAddedLater: + """The rotation governs past AND future layers. + + Ticking a gene while the section is turned used to add its layer + square: the gene then sat beside the anatomy rather than on it, which + is a wrong picture rather than an unrotated one. The panel was built + with the layers that existed at launch and never asked again. + """ + + def _panel(self, qt, viewer, angle): + from ndi.gui.app.genepyramid.controls import addRotationPanel + + box = addRotationPanel(viewer, list(viewer.layers)) + spin = box.findChild(qt.QDoubleSpinBox) + spin.setValue(angle) + spin.editingFinished.emit() + return box + + def test_a_layer_added_after_a_rotation_is_turned_too(self, qt): + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + self._panel(qt, viewer, 30.0) + assert base.affine is not None + + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + assert later.affine is not None + + def test_it_gets_the_SAME_matrix_not_an_equal_angle(self, qt): + """Two layers rotated by equal angles about different pivots do + not line up, and the pivot is re-read between interactions -- so + the matrix has to be handed over, not recomputed.""" + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + self._panel(qt, viewer, 30.0) + # Pan between the rotation and the new layer: a recomputed matrix + # would pick up this new centre and land somewhere else. + viewer.camera.center = (0.0, 5000.0, 9000.0) + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + assert np.allclose(later.affine, base.affine) + + def test_a_square_picture_leaves_a_new_layer_alone(self, qt): + """No rotation in force means nothing to apply, and stamping an + identity on every arriving layer would quietly overwrite an affine + a caller set for its own reasons.""" + viewer = _LiveViewer(SimpleNamespace(affine=None)) + from ndi.gui.app.genepyramid.controls import addRotationPanel + + addRotationPanel(viewer, list(viewer.layers)) + later = SimpleNamespace(affine="untouched") + viewer.layers.add(later) + assert later.affine == "untouched" + + def test_a_later_rotation_reaches_the_layers_added_since(self, qt): + """The other half: the panel must not keep rotating only its + original list once new layers exist.""" + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + box = self._panel(qt, viewer, 30.0) + + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + + spin = box.findChild(qt.QDoubleSpinBox) + spin.setValue(-75.0) + spin.editingFinished.emit() + assert np.allclose(later.affine, base.affine) + assert not np.allclose(later.affine, np.eye(3)) + + def test_reset_squares_the_layers_added_since_as_well(self, qt): + base = SimpleNamespace(affine=None) + viewer = _LiveViewer(base) + box = self._panel(qt, viewer, 30.0) + later = SimpleNamespace(affine=None) + viewer.layers.add(later) + + reset = box.findChild(qt.QPushButton) + reset.click() + assert np.allclose(later.affine, np.eye(3)) + assert np.allclose(base.affine, np.eye(3)) + + def test_a_viewer_with_no_event_system_still_builds(self, qt): + """A headless caller composing layers by hand has no emitter, and + a control that raises on construction costs the whole picture.""" + from ndi.gui.app.genepyramid.controls import addRotationPanel + + plain = _Viewer() + assert addRotationPanel(plain, []) is not None