diff --git a/.github/workflows/build-lightsheet-fixture.yml b/.github/workflows/build-lightsheet-fixture.yml index 7338cb81..fcaab7fb 100644 --- a/.github/workflows/build-lightsheet-fixture.yml +++ b/.github/workflows/build-lightsheet-fixture.yml @@ -21,7 +21,7 @@ on: ndi_matlab_ref: description: 'NDI-matlab branch/tag/SHA that carries lightsheet_blob_cloud_roundtrip' required: true - default: 'claude/lightsheet-zarr-ndi-viewer-djp5vk' + default: 'main' concurrency: group: build-lightsheet-fixture @@ -45,12 +45,10 @@ jobs: path: NDI-python # The blob-fixture routine (ndi.test.cloud.lightsheet_blob_cloud_roundtrip) - # is not on NDI-matlab main yet -- it lives on the sibling - # claude/lightsheet-zarr-ndi-viewer-djp5vk branch. `ndi_matlab_ref` - # picks which ref to check out (default: that branch). Fall back - # to main only when the requested ref no longer exists on the - # remote, and fail loudly rather than silently use a main that - # cannot resolve the routine. + # is on NDI-matlab main since VH-Lab/NDI-matlab#979. `ndi_matlab_ref` + # picks which ref to check out (default: main), so a fixture can still + # be built from a branch that changes the routine. Fall back to main + # only when the requested ref no longer exists on the remote. - name: Check out NDI-matlab (from ndi_matlab_ref) uses: actions/checkout@v4 with: @@ -139,11 +137,11 @@ jobs: % them up: on stdout for the log, and via % GITHUB_OUTPUT so a follow-up job can consume them. fprintf('CLOUD_DATASET_ID=%s\n', info.cloudDatasetId); - fprintf('DATASET_NAME=%s\n', info.remoteDatasetName); + fprintf('DATASET_NAME=%s\n', info.cloudDatasetName); fid = fopen(getenv("GITHUB_OUTPUT"), "a"); fprintf(fid, "cloud_dataset_id=%s\n", info.cloudDatasetId); - fprintf(fid, "dataset_name=%s\n", info.remoteDatasetName); + fprintf(fid, "dataset_name=%s\n", info.cloudDatasetName); fclose(fid); - name: Write summary diff --git a/.github/workflows/test-cloud-api.yml b/.github/workflows/test-cloud-api.yml index e0b3b98a..9d7f6ef5 100644 --- a/.github/workflows/test-cloud-api.yml +++ b/.github/workflows/test-cloud-api.yml @@ -8,6 +8,18 @@ on: # Allow manual runs from the Actions tab workflow_dispatch: + # Run on pull requests that touch the pyramid loader, its cloud + # fixture, or the workflow itself. Kept path-filtered so a PR that + # only edits, say, gene-pyramid code does not burn cloud API calls + # and bill the two shared test accounts on every push. + pull_request: + paths: + - 'src/ndi/pyramid/**' + - 'tests/test_pyramid_loader_integration.py' + - 'tests/fixtures/lightsheet_cloud_fixture.json' + - 'tests/test_cloud_*.py' + - '.github/workflows/test-cloud-api.yml' + concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true @@ -49,4 +61,9 @@ jobs: # All live cloud test files, not just test_cloud_live.py -- the # others were previously exercised only by ci.yml, so they had no # coverage on days when nobody opened a PR. - pytest tests/test_cloud_*.py -v --tb=short + # + # test_pyramid_loader_integration.py points at the permanent + # lightsheet blob fixture (see tests/fixtures/lightsheet_cloud_fixture.json, + # built by build-lightsheet-fixture.yml). It runs under the + # same account-and-env matrix as the rest of the cloud tests. + pytest tests/test_cloud_*.py tests/test_pyramid_loader_integration.py -v --tb=short diff --git a/pyproject.toml b/pyproject.toml index 7a2b2516..7e993dac 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -70,6 +70,16 @@ dependencies = [ # default. keyring is preferred when present but stays optional, # since it is the OS-keychain path rather than the portable one. "cryptography>=41.0", + # dask[array] powers ndi.pyramid.ImagePyramidLoader -- the + # napari-independent multiscale loader. It used to live under + # the napari extra because only napari-driven viewers touched + # it, but the loader is now a first-class ndi API that + # matplotlib, Jupyter and (eventually) a MATLAB comparison + # harness consume without napari. That makes dask a runtime + # dependency of the pyramid subpackage. It stays pinned in the + # napari extra as well (as a redundant floor), so an old lock + # file that pinned dask through napari does not break. + "dask[array]>=2023.1", ] [project.optional-dependencies] @@ -82,10 +92,12 @@ dependencies = [ gui = [ "PySide6>=6.5", ] -# napari, and the dask it drives the tile ladder through. An EXTRA for the +# napari is the tile viewer over the pyramid loader. An EXTRA for the # same reason gui is: a headless install should not pull in Qt and OpenGL. # ndi.gui.app.genepyramid.viewer imports through require_napari(), so the -# failure mode for a missing toolkit is a sentence naming this extra. +# failure mode for a missing toolkit is a sentence naming this extra. The +# loader's own dask requirement moved to project.dependencies once the +# loader graduated out of the viewer namespace into ndi.pyramid. napari = [ "napari[all]>=0.4.19", "dask[array]>=2023.1", @@ -161,6 +173,7 @@ addopts = "-v --tb=short --ignore=tests/symmetry" markers = [ "slow: marks tests as slow (deselect with '-m \"not slow\"')", "symmetry: marks cross-language symmetry tests (run with 'pytest tests/symmetry/')", + "cloud: marks live-cloud tests (skipped when NDI_CLOUD_USERNAME / NDI_CLOUD_PASSWORD are missing)", ] [tool.black] diff --git a/src/ndi/cloud/api/files.py b/src/ndi/cloud/api/files.py index 669e86ef..fca513f2 100644 --- a/src/ndi/cloud/api/files.py +++ b/src/ndi/cloud/api/files.py @@ -10,11 +10,14 @@ from __future__ import annotations +import logging import re import time from pathlib import Path from typing import Annotated, Any, Literal +_module_logger = logging.getLogger(__name__) + from pydantic import SkipValidation, validate_call from ..client import APIResponse, CloudClient, _auto_client @@ -279,6 +282,7 @@ def getFile( timeout: int = 120, *, progress=None, + error_out: dict | None = None, ) -> bool: """Download a file from a presigned URL. @@ -292,6 +296,13 @@ def getFile( ``timeout``, and it is called from whichever thread is downloading -- a caller that renders it is responsible for its own locking. + ``error_out``, if given, is populated on FAILURE (return value + ``False``) with the HTTP details -- currently ``{"status": int, + "body": str}`` -- so a caller who needs the status code (e.g. to + invalidate a cached signed URL that returned S3 403) does not have + to re-download or infer from a log line. It is not touched on + success. Kept opt-in so getFile's happy-path signature is unchanged. + WHY HERE. This is the one place every on-demand fetch passes through: the cell table, the contour file, the gene list and every pyramid tile all arrive by fetch_cloud_file -> getFile. Reporting anywhere else @@ -349,6 +360,9 @@ def getFile( url[:80], body[:200], ) + if error_out is not None: + error_out["status"] = int(resp.status_code) + error_out["body"] = body[:200] return False @@ -728,7 +742,18 @@ def getSignedURLSetAll( "pages": 0, } cursor = "" + walk_started = time.monotonic() for _ in range(max_pages): + page_started = time.monotonic() + _module_logger.info( + "getSignedURLSetAll: fetching page %d for document %s (series=%r, " + "so far %d uids, %.1fs elapsed)", + merged["pages"] + 1, + document_id, + file_series, + merged["pageCount"], + page_started - walk_started, + ) page = getSignedURLSet( dataset_id, document_id, @@ -738,6 +763,7 @@ def getSignedURLSetAll( id_namespace=id_namespace, client=client, ) + page_dt = time.monotonic() - page_started files = page.get("files", {}) if hasattr(page, "get") else {} if isinstance(files, dict): merged["files"].update(files) @@ -752,6 +778,14 @@ def getSignedURLSetAll( if expires_at: merged["expiresAt"] = expires_at merged["pages"] += 1 + _module_logger.info( + "getSignedURLSetAll: page %d returned %d uids in %.2fs " "(total so far %d/%d)", + merged["pages"], + len(files) if isinstance(files, dict) else 0, + page_dt, + merged["pageCount"], + merged["totalCount"] or -1, + ) next_cursor = page.get("nextCursor", "") if hasattr(page, "get") else "" if not next_cursor: @@ -766,6 +800,300 @@ def getSignedURLSetAll( raise SignedURLSetMaxPagesReached(merged) +# --------------------------------------------------------------------------- +# Signed-URL-set-job family (async) +# +# The paged getSignedURLSetAll walks the /signed-url-set route with cursor +# pagination and takes ~25 s per 500-uid page. For a document with 121k +# members that is 100+ minutes to prime the batch cache. The async job path +# builds the whole uid -> URL map server-side, gzips it to S3, and hands +# back one presigned URL to download the blob: +# +# createSignedURLSetJob -> POST /.../signed-url-set-jobs returns jobId +# waitForSignedURLSetJob -> GET /signed-url-set-jobs/{jobId} poll to 'ready' +# getSignedURLSetResult -> download + parse the gzipped result blob +# +# See NDI-matlab#952, NDI-matlab#1009, NDI-python#206. +# --------------------------------------------------------------------------- + +# Terminal states reported by the signed-URL-set job service. Non-terminal +# states are 'queued' and 'running'; the mirror is intentional so callers +# see a consistent poll vocabulary across the bulk-upload and file-tier +# jobs. +_TERMINAL_SIGNED_URL_SET_STATES = ("ready", "failed") + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def createSignedURLSetJob( + dataset_id: CloudId, + document_id: NonEmptyStr, + *, + file_series: str = "", + id_namespace: Literal["cloud", "ndi"] = "cloud", + client: _Client = None, +) -> dict[str, Any]: + """Kick off an async job that builds a document's signed URL set. + + POSTs to /datasets/{datasetId}/.../signed-url-set-jobs. The server signs + every file the document references, gzips the resulting UID -> signed + URL map and writes it to S3, then hands back a job id. Poll with + :func:`getSignedURLSetJob` (or :func:`waitForSignedURLSetJob`) until the + job reaches state ``"ready"``, then read the map with + :func:`getSignedURLSetResult`. + + Preferred over :func:`getSignedURLSetAll` for documents with thousands + of members: one API call per document instead of one per 500-uid page. + + Args: + dataset_id: The cloud dataset id. + document_id: The document id. Namespace controlled by *id_namespace*. + file_series: If set, restrict the job to one file series' members. + id_namespace: ``"cloud"`` (default) sends the mongo ``_id`` to the + by-``_id`` route; ``"ndi"`` sends ``data.base.id`` to the + ndi-documents route. NDI's own callers hold NDI ids and must + pass ``"ndi"``; passing one as ``"cloud"`` is a 404. See + NDI-matlab#968. + client: Authenticated cloud client (auto-created if omitted). + + Returns: + Dict with (at least) ``jobId``, ``datasetId``, ``documentId``, + ``statusUrl``, ``pollAfterSec``. Pass ``jobId`` to the wait + helpers below. + + MATLAB equivalent: +cloud/+api/+files/createSignedURLSetJob.m + """ + if id_namespace == "ndi": + endpoint = "/datasets/{datasetId}/ndi-documents/{ndiDocumentId}/signed-url-set-jobs" + path_params = {"datasetId": dataset_id, "ndiDocumentId": document_id} + else: + endpoint = "/datasets/{datasetId}/documents/{documentId}/signed-url-set-jobs" + path_params = {"datasetId": dataset_id, "documentId": document_id} + + # The MATLAB implementation appends fileSeries as a query parameter on + # the URL; the client's post() takes path params, not query params, so + # bake it into the endpoint template here. + if file_series: + endpoint = endpoint + "?fileSeries={fileSeries}" + path_params["fileSeries"] = file_series + + # MATLAB posts an empty JSON object rather than a zero-byte body so the + # gateway does not reject the POST. Mirror that here. + return client.post(endpoint, json={}, **path_params) + + +class SignedURLSetJobFailed(RuntimeError): + """A signed-URL-set job reached state ``"failed"``. + + The ``.status`` attribute carries the last status dict from the server, + so a caller can inspect the ``error`` field. + """ + + def __init__(self, status: dict[str, Any]): + err = status.get("error", "") if isinstance(status, dict) else "" + super().__init__( + f"signed-URL-set job {status.get('jobId', '?')} failed: {err}" + if err + else f"signed-URL-set job {status.get('jobId', '?')} failed" + ) + self.status = status + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def getSignedURLSetJob( + job_id: NonEmptyStr, + *, + client: _Client = None, +) -> dict[str, Any]: + """GET /signed-url-set-jobs/{jobId} -- one poll of a signed-URL-set job. + + Returns a dict with (at least) ``jobId``, ``datasetId``, ``documentId``, + ``state``, ``createdAt``, ``startedAt``, ``completedAt``, + ``heartbeatAt``, ``signedCount``, ``totalCount``. When ``state`` is + ``"ready"`` the payload also carries ``resultUrl`` (a 24 h presigned + GET for the gzipped JSON blob), ``resultByteSize``, ``expiresAt``, and + ``filesExpireAt``. When ``state`` is ``"failed"`` it carries + ``error``. Non-terminal states are ``"queued"`` and ``"running"``. + + MATLAB equivalent: +cloud/+api/+files/getSignedURLSetJob.m + """ + return client.get("/signed-url-set-jobs/{jobId}", jobId=job_id) + + +@_auto_client +@validate_call(config=VALIDATE_CONFIG) +def waitForSignedURLSetJob( + job_id: NonEmptyStr, + *, + timeout: float = 300.0, + initial_interval: float = 3.0, + max_interval: float = 30.0, + backoff_factor: float = 2.0, + client: _Client = None, +) -> dict[str, Any]: + """Poll a signed-URL-set job until it finishes or times out. + + Repeatedly calls :func:`getSignedURLSetJob` at exponentially growing + intervals until the job reaches a terminal state (``"ready"`` or + ``"failed"``) or the overall timeout elapses. A transient API failure + is NOT treated as terminal -- a gateway blip would otherwise be + mistaken for a dead job, mirroring :func:`waitForBulkUpload` and + :func:`waitForFileTierJob`. + + Args: + job_id: The signed-URL-set job identifier from + :func:`createSignedURLSetJob`. + timeout: Overall deadline in seconds. Default 300. + initial_interval: First sleep between polls (s). Default 3. + max_interval: Cap on the per-poll sleep (s). Default 30. + backoff_factor: Multiplier applied after each poll. Default 2. + + Returns: + The last status dict from the server. On timeout, the returned + dict has ``state='timeout'`` and ``elapsed`` set to the + wall-clock seconds spent polling. + + MATLAB equivalent: +cloud/+api/+files/waitForSignedURLSetJob.m + """ + start = time.monotonic() + interval = initial_interval + last: Any = None + poll = 0 + while True: + elapsed = time.monotonic() - start + try: + status = getSignedURLSetJob(job_id, client=client) + last = status + state = status.get("state", "") if hasattr(status, "get") else "" + signed = status.get("signedCount") if hasattr(status, "get") else None + total = status.get("totalCount") if hasattr(status, "get") else None + _module_logger.info( + "waitForSignedURLSetJob: jobId=%s poll #%d state=%r signed=%s/%s " + "(elapsed %.1fs)", + job_id, + poll, + state, + signed, + total, + elapsed, + ) + if state in _TERMINAL_SIGNED_URL_SET_STATES: + return status + except Exception as exc: + # A failed poll is not a failed job; ride out gateway blips. + _module_logger.info( + "waitForSignedURLSetJob: jobId=%s poll #%d failed (%s: %s); " "will retry", + job_id, + poll, + type(exc).__name__, + exc, + ) + poll += 1 + if elapsed + interval > timeout: + payload: dict[str, Any] + if last is not None and hasattr(last, "data") and isinstance(last.data, dict): + payload = dict(last.data) + elif isinstance(last, dict): + payload = dict(last) + else: + payload = {} + payload["state"] = "timeout" + payload["elapsed"] = time.monotonic() - start + return payload + time.sleep(interval) + interval = min(interval * backoff_factor, max_interval) + + +@validate_call +def getSignedURLSetResult( + result_url: NonEmptyStr, + *, + timeout: int = 120, +) -> dict[str, Any]: + """Download and parse the blob a ready signed-URL-set job produced. + + A ready job's status carries a ``resultUrl``: a presigned 24 h GET for + a gzipped JSON body of shape:: + + { jobId, datasetId, documentId, generatedAt, fileCount, + files: {uid: url, ...} } + + This fetches that blob and returns a dict with (at least) the ``files`` + map. Whether the body arrives gzipped or already inflated by the + transport is decided from the gzip magic number rather than assumed -- + that mirrors the MATLAB implementation and covers both S3 responses + (raw bytes) and any gateway that auto-inflates on the way through. + + The URL is presigned, so no Authorization header is added; adding one + makes S3 reject the request. + + Args: + result_url: The ``resultUrl`` a ready :func:`getSignedURLSetJob` + response carries. + timeout: HTTP timeout in seconds for the download. Default 120. + + Returns: + Dict with (at least) ``files`` (a ``dict[str, str]`` mapping file + uid to presigned download URL). Also carries ``fileCount``, + ``generatedAt`` and any other fields the server included, so the + signer can hand the whole payload back to + :class:`ndi.cloud.batch_signed_url.BatchSignedUrlLookup` without + reshaping. + + Raises: + RuntimeError: If the download failed or the body could not be + parsed as JSON. + + MATLAB equivalent: +cloud/+api/+files/getSignedURLSetResult.m + """ + import gzip + import json + + assert_safe_transfer_url(result_url, what="signed-URL-set result URL") + + resp = _download_session().get(result_url, timeout=timeout) + if resp.status_code != 200: + raise RuntimeError( + f"signed-URL-set result download failed (HTTP {resp.status_code}): " + f"{resp.text[:200]}" + ) + + raw = resp.content + # Decide gzip vs. inflated by the magic number rather than the + # Content-Encoding header, because requests auto-decompresses on some + # Content-Encoding values and leaves others alone. The MATLAB port + # takes the same approach for the same reason. + if len(raw) >= 2 and raw[0] == 0x1F and raw[1] == 0x8B: + try: + body = gzip.decompress(raw).decode("utf-8") + except OSError as exc: + raise RuntimeError( + f"signed-URL-set result blob could not be decompressed: {exc}" + ) from exc + else: + body = raw.decode("utf-8") + + try: + data = json.loads(body) + except json.JSONDecodeError as exc: + raise RuntimeError(f"signed-URL-set result blob is not valid JSON: {exc}") from exc + + if not isinstance(data, dict): + raise RuntimeError( + f"signed-URL-set result blob decoded to a {type(data).__name__}, " + "expected an object with a 'files' field." + ) + + files = data.get("files") + if not isinstance(files, dict): + raise RuntimeError( + f"signed-URL-set result 'files' arrived as a " f"{type(files).__name__}, not a dict." + ) + + return data + + @_auto_client @validate_call(config=VALIDATE_CONFIG) def getFileCollectionUploadURL( diff --git a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml index 8b6131b6..03adcde0 100644 --- a/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/api/ndi_matlab_python_bridge.yaml @@ -1137,6 +1137,20 @@ functions: elapses. On timeout the returned dict has state='timeout' and elapsed=. + - name: waitForFileUpload + matlab_path: "+ndi/+cloud/+api/+files/waitForFileUpload.m" + matlab_last_sync_hash: "eb583289" + status: porting_deferred + decision_log: > + Added to NDI-matlab by VH-Lab/NDI-matlab#1027: polls NDI Cloud until + single-file uploads are recorded (uploaded=true) or a timeout passes, + so a PUT that S3 accepted but the upload-processing lambda has not yet + recorded is not mistaken for a missing file. MATLAB's uploadSingleFile + and serial uploadFilesForDatasetDocuments now wait through it. Not yet + ported: NDI-python's single-file upload path does not wait, and its + live tests poll listFiles themselves. Port alongside the next change + to ndi.cloud.upload's single-file path. + - name: waitForAllBulkUploads matlab_path: "+ndi/+cloud/+api/+files/waitForAllBulkUploads.m" matlab_last_sync_hash: "2566fe4d" @@ -1730,9 +1744,10 @@ not_yet_ported: # +ndi/+cloud/+api/+files -- signed-URL-set family # # A coherent group of six MATLAB functions added by NDI-matlab's - # signed-URL-set work (PRs #967, #969, #970 and preceding). Together they - # cover the /signed-url-set (paged) and /signed-url-set-jobs (async) - # routes the cloud exposes for bulk pre-signed downloads: + # signed-URL-set work (PRs #967, #969, #970 and preceding, and #1009 + # which stabilized the async job path). Together they cover the + # /signed-url-set (paged) and /signed-url-set-jobs (async) routes the + # cloud exposes for bulk pre-signed downloads: # # getSignedURLSet - one page of a document's UID -> signed URL map # getSignedURLSetAll - walk every page and merge into a single map @@ -1742,26 +1757,36 @@ not_yet_ported: # waitForSignedURLSetJob- poll with exponential backoff until terminal # getSignedURLSetResult - GET + parse the blob a ready job produced # - # This is what fetching a 28,000-member file-series document over the - # cloud requires (VH-Lab/NDI-matlab#952) -- one API call per document - # instead of one per member. Python currently has no signed-URL-set - # support at all (grep -r "signed_url_set\|SignedURLSet" src/ndi/ is - # empty), so the honest status for all six is not_yet_ported. Recorded - # together with one shared decision_log because there is no case-by-case - # judgement to make: the family lands as a unit or not at all. + # This is what fetching a document with tens or hundreds of thousands of + # members over the cloud requires (VH-Lab/NDI-matlab#952) -- one API + # call per document instead of one per member. # - # Filed as Waltham-Data-Science/ndi-python#206. + # STATUS. Ported as of NDI-python#206. The four async-job functions were + # added to src/ndi/cloud/api/files.py alongside the paged pair, and + # batch_signed_url._default_signer now submits a job via + # createSignedURLSetJob -> waitForSignedURLSetJob -> + # getSignedURLSetResult instead of walking pages. For a 121k-uid + # document the walk took 100+ minutes at ~25 s per 500-uid page; the + # async job builds the whole map server-side and returns it in one + # blob. The paged pair is still exported as an API entry point (a + # smaller document can still walk it), but no NDI code path uses it any + # longer. Filed as Waltham-Data-Science/ndi-python#206. # ======================================================================= - name: createSignedURLSetJob matlab_path: "+ndi/+cloud/+api/+files/createSignedURLSetJob.m" - matlab_last_sync_hash: "5804cd80" - status: porting_deferred + python_path: "ndi/cloud/api/files.py" + matlab_last_sync_hash: "0a2cdec" + status: ported decision_log: > - Part of the signed-URL-set family (#206). Python has no - signed-URL-set support yet, so createSignedURLSetJob has no Python - counterpart. Ports as a unit with the other five entries under this - heading; see the shared note above. + Ported by NDI-python#206 as part of the signed-URL-set family. POSTs + to /datasets/{d}/documents/{doc}/signed-url-set-jobs (or the + ndi-documents route when id_namespace='ndi', which is what NDI's own + callers use). Optional file_series is folded into the endpoint as a + query parameter; MATLAB does the same via + matlab.net.QueryParameter. Returns the accepted-job envelope + ({jobId, datasetId, documentId, statusUrl, pollAfterSec}) so the + caller can hand jobId to waitForSignedURLSetJob. - name: getSignedURLSet matlab_path: "+ndi/+cloud/+api/+files/getSignedURLSet.m" @@ -1789,9 +1814,12 @@ not_yet_ported: no nextCursor or empty nextCursor ends the walk; a cursor that equals the one just sent raises SignedURLSetCursorDidNotAdvance; running past max_pages raises SignedURLSetMaxPagesReached with the - partial merged dict on the exception. The async job path - (createSignedURLSetJob + waitForSignedURLSetJob) is still deferred - because batch_signed_url does not use it. + partial merged dict on the exception. Still exported as an API entry + point but no longer the batch_signed_url signer: at ~25 s per + 500-uid page a 121k-member document took 100+ minutes here, so the + default signer now goes through the async job path + (createSignedURLSetJob + waitForSignedURLSetJob + + getSignedURLSetResult). 2026-10-12: synced to NDI-matlab 8fa6ea5e (NDI-matlab#1025): pages' fileInfo lists ({uid, uploaded, size, crc32}, ndi-cloud-node#162) are concatenated into merged["fileInfo"]. getSignedURLSet passes fileInfo @@ -1799,30 +1827,43 @@ not_yet_ported: - name: getSignedURLSetJob matlab_path: "+ndi/+cloud/+api/+files/getSignedURLSetJob.m" - matlab_last_sync_hash: "cd11b198" - status: porting_deferred + python_path: "ndi/cloud/api/files.py" + matlab_last_sync_hash: "0a2cdec" + status: ported decision_log: > - Part of the signed-URL-set family (#206). Poll one signed-URL-set - job's state via GET /signed-url-set-jobs/{jobId}. Python has no - counterpart; ports as a unit, see the shared note above. + Ported by NDI-python#206 as part of the signed-URL-set family. GETs + /signed-url-set-jobs/{jobId}. Returns the job's status payload + verbatim, including state (one of 'queued', 'running', 'ready', + 'failed'), signedCount/totalCount, and -- on 'ready' -- resultUrl, + resultByteSize, expiresAt and filesExpireAt. - name: getSignedURLSetResult matlab_path: "+ndi/+cloud/+api/+files/getSignedURLSetResult.m" - matlab_last_sync_hash: "4b5c0478" - status: porting_deferred + python_path: "ndi/cloud/api/files.py" + matlab_last_sync_hash: "0a2cdec" + status: ported decision_log: > - Part of the signed-URL-set family (#206). Downloads and parses the - gzipped blob a ready job produced (a UID -> signed URL map plus - count and generation timestamp). Python has no counterpart; ports - as a unit, see the shared note above. + Ported by NDI-python#206 as part of the signed-URL-set family. + Downloads the presigned resultUrl a ready job carries, detects gzip + vs. inflated body from the magic number (0x1f 0x8b) rather than + Content-Encoding -- some transports auto-inflate, and MATLAB takes + the same approach for the same reason -- and returns the parsed + dict verbatim ({files: {uid -> url}, fileCount, generatedAt, ...}) + so the batch_signed_url signer can hand the whole payload through + without reshaping. MATLAB wraps files in a containers.Map; Python's + json.loads keeps arbitrary string keys as a dict, so no key-recovery + pass is needed. - name: waitForSignedURLSetJob matlab_path: "+ndi/+cloud/+api/+files/waitForSignedURLSetJob.m" - matlab_last_sync_hash: "cd11b198" - status: porting_deferred + python_path: "ndi/cloud/api/files.py" + matlab_last_sync_hash: "0a2cdec" + status: ported decision_log: > - Part of the signed-URL-set family (#206). Polls - getSignedURLSetJob with exponentially growing intervals until the - job reaches a terminal state ('ready' or 'failed') or the overall - timeout elapses. Python has no counterpart; ports as a unit, see - the shared note above. + Ported by NDI-python#206 as part of the signed-URL-set family. + Polls getSignedURLSetJob at exponentially growing intervals until + the job reaches state 'ready' or 'failed', or the overall timeout + elapses. Mirrors waitForBulkUpload/waitForFileTierJob: on timeout + returns the last status dict with state='timeout' and elapsed set, + and a transient API failure is NOT treated as terminal so a gateway + blip does not mark the job dead. diff --git a/src/ndi/cloud/batch_signed_url.py b/src/ndi/cloud/batch_signed_url.py index de56edcc..c0ef528b 100644 --- a/src/ndi/cloud/batch_signed_url.py +++ b/src/ndi/cloud/batch_signed_url.py @@ -8,10 +8,15 @@ many members -- 28,000 in the lightsheet-pyramid case that motivates this (VH-Lab/NDI-matlab#952) -- it is one API round trip per member. -The batch path: one call to /signed-url-set (walked with getSignedURLSetAll) -returns the whole document's uid -> URL map. Subsequent uids in the same -(dataset, document [, series]) scope resolve from the cached map without -another round trip. +The batch path: submit an async signed-URL-set job (createSignedURLSetJob +-> waitForSignedURLSetJob -> getSignedURLSetResult) that builds the whole +document's uid -> URL map server-side and hands it back in one gzipped +blob. Subsequent uids in the same (dataset, document [, series]) scope +resolve from the cached map without another round trip. The paged +getSignedURLSetAll walk is still supported as an API entry point but the +default signer no longer uses it -- for a 121k-uid document the walk +takes 100+ minutes at ~25 s per 500-uid page (NDI-matlab#952, +NDI-matlab#1009). Empty return values are legitimate answers, not errors: the batch call may fail (network, auth, timeout) or the map it returns may not name this uid @@ -26,7 +31,7 @@ import threading import time import warnings -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any @@ -46,12 +51,59 @@ # MATLAB counterpart's testFailingSignerReturnsEmpty. DEFAULT_FAILURE_TTL_SECONDS = 60 -# 500 ms: pause before re-fetching a scope whose first answer was a +# Exponential backoff for re-fetching a scope whose previous answer was a # partial map -- the scope was populated, but it did not name the uid the # caller asked about. On some cloud environments the batch endpoint lags -# briefly behind ``waitForAllBulkUploads``, and the second call names the -# whole set. See Waltham-Data-Science/NDI-python#309. -DEFAULT_PARTIAL_MAP_RETRY_SECONDS = 0.5 +# briefly behind ``waitForAllBulkUploads``, and a later call names the +# whole set. Three retries at 1 s / 3 s / 9 s cover the User-1-prod tail +# observed on Waltham-Data-Science/NDI-python#320: one 0.5 s retry was not +# enough to bridge the settle between bulk-upload extraction and the +# signed-URL-set index catching up. Each retry costs one signer call for +# the whole scope, not one per uid; once the map settles every subsequent +# uid resolves from the fresh cache entry. See +# Waltham-Data-Science/NDI-python#309 and #320. +DEFAULT_PARTIAL_MAP_RETRY_DELAYS: tuple[float, ...] = (1.0, 3.0, 9.0) + +# Deprecated name kept for callers that still pass ``partial_map_retry_seconds`` +# explicitly (a single scalar means "one retry after this many seconds"). +# New code should use ``partial_map_retry_delays`` instead. +DEFAULT_PARTIAL_MAP_RETRY_SECONDS = DEFAULT_PARTIAL_MAP_RETRY_DELAYS[0] + +# Backoff schedule for :func:`_default_signer` when the batch endpoint call +# raises. One transient TLS or connection error at start of run used to +# mark the scope failed and drop every uid in it to per-member +# getFileDetails -- O(N) API calls on a series that should have cost one. +# Retrying the batch itself keeps the O(1) fast path across residential +# network blips. See Waltham-Data-Science/NDI-python#322 (and the parallel +# VH-Lab/NDI-matlab#1010). Only the default production signer retries; +# a caller-injected signer is left as-is so tests keep control of counts. +DEFAULT_BATCH_RETRY_DELAYS: tuple[float, ...] = (1.0, 4.0, 16.0) + + +class BatchScopeUnreachable(RuntimeError): + """The async signed-URL-set job path could not answer for a scope. + + Raised by :func:`_default_signer` after every retry attempt has failed + -- ``createSignedURLSetJob``, ``waitForSignedURLSetJob`` and + ``getSignedURLSetResult`` between them have not produced a usable map + for this (dataset, document, series) scope. + + Why raise rather than return ``(False, ...)`` and let the caller fall + back to per-member ``getFileDetails``? A napari viewport loads dozens + of chunks per frame, and per-member fallback for a 100k-member series + is thousands of API calls per zoom -- unusable UX. Worse, that + fallback works "well enough" that the underlying bug (the async job + endpoint returned 404 / timed out / drifted from the schema) stays + hidden while the viewer feels merely slow. Raising forces the real + error to the surface so we see and fix it, and never silently ship a + bad path. + + The partial-map case is separate and does NOT raise: a batch that + successfully returned a map but did not happen to name a particular + uid still hits the per-uid fallback in ``fetch_cloud_file``, because + that is a data-drift case rather than a broken endpoint. Only real + failures of the async job path raise here. + """ @dataclass @@ -64,10 +116,17 @@ class _CacheEntry: @dataclass class _FailureEntry: - """A scope's most recent batch failure, with the uid that saw it.""" + """A scope's most recent batch failure, with the uid that saw it. + + ``unreachable`` distinguishes a real async-job failure (surfaced as + :class:`BatchScopeUnreachable`, no per-uid fallback) from an + injected-signer failure or a partial-map miss (per-uid fallback OK). + """ at: float uid: str + unreachable: bool = False + cause: str = "" @dataclass @@ -112,33 +171,183 @@ class Stats: partial_map_retries: int = 0 -# The default signer walks every page. Injected for tests. +# Default timeout for the async signed-URL-set job. The server has to sign +# every member and gzip the resulting map, so a 121k-uid document is not a +# short wait -- 15 min gives it room while still surfacing a truly stuck +# job. The batch cache's caller (fetch_cloud_file) falls back per-member if +# the wait times out, so this is a "give up on the fast path" deadline, +# not a "give up on the read" one. +DEFAULT_JOB_TIMEOUT_SECONDS = 15 * 60 + + +# The default signer submits the async job, waits for it to reach 'ready', +# and reads the gzipped result blob. Injected for tests. def _default_signer( dataset_id: str, document_id: str, *, file_series: str = "", client: CloudClient | None = None, + retry_delays: tuple[float, ...] = DEFAULT_BATCH_RETRY_DELAYS, + sleep: Callable[[float], None] = time.sleep, + job_timeout: float = DEFAULT_JOB_TIMEOUT_SECONDS, ) -> tuple[bool, dict[str, Any]]: - """Fetch the whole scope through the paged getSignedURLSetAll call. - - Returns ``(True, answer)`` on success, ``(False, {})`` on any failure -- - the batch is best-effort and its caller's fallback is what keeps the - read correct. + """Fetch the whole scope through the async signed-URL-set-job path. + + Submits a ``createSignedURLSetJob`` for the (dataset, document, series) + scope, waits for the job to reach state ``"ready"`` (or ``"failed"``, + or the overall ``job_timeout``), and downloads and parses the gzipped + result blob. Returns ``(True, answer)`` on success and + ``(False, {"__error__": ...})`` on any failure -- the batch is + best-effort and its caller's fallback keeps the read correct. + + Preferred over the paged ``getSignedURLSetAll`` walk: for a document + with 121k members the walk takes 100+ minutes at ~25 s per 500-uid + page, whereas the async job builds the whole map server-side and + returns it in one blob (NDI-matlab#952, NDI-matlab#1009). + + A raise anywhere in ``create -> wait -> read`` is retried with the + delays in ``retry_delays`` before it is reported as a failure. This is + what keeps one transient TLS blip on a residential connection from + cascading to the O(N) per-member fallback: the batch itself gets a few + more chances at ~20 s of wall clock in the worst case before its + caller is told the scope is unreachable. See + Waltham-Data-Science/NDI-python#322. """ from .api import files as files_api - try: - answer = files_api.getSignedURLSetAll( - dataset_id, - document_id, - id_namespace="ndi", - file_series=file_series, - client=client, - ) - except Exception as exc: # noqa: BLE001 - reported as a miss reason - return False, {"__error__": f"{type(exc).__name__}: {exc}"} - return True, answer + attempts = len(retry_delays) + 1 + last_exc: Exception | None = None + scope_started = time.monotonic() + for attempt in range(attempts): + try: + logger.info( + "batch signed-URL job: submitting createSignedURLSetJob for scope " + "(%s, %s, series=%r) attempt %d/%d", + dataset_id, + document_id, + file_series, + attempt + 1, + attempts, + ) + job = files_api.createSignedURLSetJob( + dataset_id, + document_id, + id_namespace="ndi", + file_series=file_series, + client=client, + ) + job_id = job.get("jobId", "") if hasattr(job, "get") else "" + if not job_id: + raise RuntimeError(f"createSignedURLSetJob returned no jobId (payload: {job!r})") + logger.info( + "batch signed-URL job: submitted jobId=%s, waiting for terminal state " + "(timeout %.0fs)", + job_id, + job_timeout, + ) + + status = files_api.waitForSignedURLSetJob( + job_id, + timeout=job_timeout, + client=client, + ) + state = status.get("state", "") if hasattr(status, "get") else "" + logger.info( + "batch signed-URL job: jobId=%s reached state=%r after %.1fs total", + job_id, + state, + time.monotonic() - scope_started, + ) + if state == "failed": + err = status.get("error", "") if hasattr(status, "get") else "" + raise RuntimeError( + f"signed-URL-set job {job_id} failed: {err}" + if err + else f"signed-URL-set job {job_id} failed" + ) + if state == "timeout": + elapsed = ( + status.get("elapsed", job_timeout) if hasattr(status, "get") else job_timeout + ) + raise RuntimeError( + f"signed-URL-set job {job_id} did not finish within " + f"{elapsed:.0f}s (state after wait: {state!r})" + ) + if state != "ready": + raise RuntimeError( + f"signed-URL-set job {job_id} ended in unexpected state " + f"{state!r} (expected 'ready')" + ) + + result_url = status.get("resultUrl", "") if hasattr(status, "get") else "" + if not result_url: + raise RuntimeError( + f"signed-URL-set job {job_id} was ready but carried no resultUrl" + ) + logger.info( + "batch signed-URL job: jobId=%s ready, fetching result blob", + job_id, + ) + + answer = files_api.getSignedURLSetResult(result_url) + # Carry the URL-expiration fields over from the JOB STATUS + # into the RESULT payload. The status is the only place the + # server puts filesExpireAt / expiresAt; the result blob + # itself does not include them (see getSignedURLSetResult's + # docstring for the blob shape). Without this merge the + # disk cache's save() correctly refuses to persist a + # payload it cannot age-check, and the on-disk cache stays + # empty even after a full successful signing -- exactly + # what we saw on the first fresh-machine run. Non-string / + # missing values are dropped so an accidental None does + # not overwrite a good value we might learn to fill in + # elsewhere later. + if isinstance(answer, dict): + for expiry_key in ("filesExpireAt", "expiresAt"): + value = status.get(expiry_key, "") if hasattr(status, "get") else "" + if isinstance(value, str) and value and expiry_key not in answer: + answer[expiry_key] = value + n_files = 0 + if isinstance(answer, dict) and isinstance(answer.get("files"), dict): + n_files = len(answer["files"]) + logger.info( + "batch signed-URL job: jobId=%s delivered %d uid->URL entries " + "for scope (%s, %s, series=%r) in %.1fs total", + job_id, + n_files, + dataset_id, + document_id, + file_series, + time.monotonic() - scope_started, + ) + return True, answer + except Exception as exc: # noqa: BLE001 - reported by BatchScopeUnreachable + last_exc = exc + if attempt < attempts - 1: + delay = retry_delays[attempt] + logger.info( + "batch signed-URL fetch attempt %d/%d for scope " + "(%s, %s, series=%r) failed (%s: %s); retrying in %.1fs", + attempt + 1, + attempts, + dataset_id, + document_id, + file_series, + type(exc).__name__, + exc, + delay, + ) + sleep(delay) + # Every attempt raised. Surface the last cause instead of returning + # (False, ...) -- see BatchScopeUnreachable's docstring for why a + # silent per-uid fallback would just hide the bug. + raise BatchScopeUnreachable( + f"async signed-URL-set job failed for scope " + f"({dataset_id!r}, {document_id!r}, file_series={file_series!r}) " + f"after {attempts} attempts. Last cause: " + f"{type(last_exc).__name__}: {last_exc}" + ) from last_exc class BatchSignedUrlLookup: @@ -160,19 +369,42 @@ def __init__( signer: Callable[..., tuple[bool, dict[str, Any]]] | None = None, ttl_seconds: float = DEFAULT_TTL_SECONDS, failure_ttl_seconds: float = DEFAULT_FAILURE_TTL_SECONDS, - partial_map_retry_seconds: float = DEFAULT_PARTIAL_MAP_RETRY_SECONDS, + partial_map_retry_delays: tuple[float, ...] | None = None, + partial_map_retry_seconds: float | None = None, sleep: Callable[[float], None] | None = None, + disk_cache: bool = False, ) -> None: self._signer = signer or _default_signer self._ttl_seconds = ttl_seconds self._failure_ttl_seconds = failure_ttl_seconds - self._partial_map_retry_seconds = partial_map_retry_seconds + # partial_map_retry_seconds (scalar, deprecated) reduces to a + # one-element schedule, kept for callers/tests that pin the old + # single-retry contract. The scalar names the WAIT before the + # single retry, so ``0`` still fires one retry (with no wait), + # matching the pre-schedule behavior. + if partial_map_retry_delays is not None: + self._partial_map_retry_delays: tuple[float, ...] = tuple(partial_map_retry_delays) + elif partial_map_retry_seconds is not None: + self._partial_map_retry_delays = (partial_map_retry_seconds,) + else: + self._partial_map_retry_delays = DEFAULT_PARTIAL_MAP_RETRY_DELAYS # Injected only for tests: the retry has a real wait, and a test # that runs it 20 times should not pay 10 seconds for it. self._sleep = sleep or time.sleep + # Disk cache off by default: a scripted signer in a test suite + # would otherwise persist fake URLs to ~/.ndi/signed-url-cache + # and hit them on the next test's first lookup. Production + # callers (the DID chunk-download path) turn it on explicitly + # via ``get_default()``, which is where reopens actually pay. + # See :mod:`ndi.cloud.signed_url_disk_cache`. + self._disk_cache = disk_cache self._cache: dict[str, _CacheEntry] = {} self._failed_scopes: dict[str, _FailureEntry] = {} - self._retried_scopes: set[str] = set() + # Per-scope count of partial-map retries already spent. Bounded + # by ``len(self._partial_map_retry_delays)``: once exhausted, a + # subsequent uid miss on the scope surfaces as a data-drift miss + # without another signer call. + self._scope_retry_attempts: dict[str, int] = {} self._warned: set[str] = set() self._stats = Stats() # A batch fetch is IO-bound and slow, and download handlers can be @@ -219,6 +451,17 @@ def lookup( del self._cache[cache_key] entry = None + if entry is None and self._disk_cache: + # Consult the on-disk cache before running the async + # signed-URL-set job. On hit, populate the in-memory + # cache so every uid in the scope resolves without + # another disk read. On miss (no file, corrupt file, + # expired-or-within-safety-buffer), fall through to the + # signer as before. + entry = self._try_disk_cache( + cache_key, cloud_dataset_id, ndi_document_id, series_name + ) + if entry is None: # A scope that just failed is not retried for every uid # after it. Without this, a batch endpoint that cannot @@ -233,6 +476,17 @@ def lookup( if failure is not None: if (now - failure.at) >= self._failure_ttl_seconds: del self._failed_scopes[cache_key] + elif failure.unreachable: + # The async job path was strict-mode-failed for + # this scope within the TTL. Re-raise fast rather + # than running another 4-attempt retry cycle for + # every next uid the viewport asks about. The + # cause carried on the entry preserves what the + # first raise said. + raise BatchScopeUnreachable( + f"scope {cache_key!r} was unreachable " + f"{now - failure.at:.1f}s ago: {failure.cause}" + ) elif failure.uid != uid: self._stats.uid_misses += 1 self._warn_once(cache_key) @@ -261,23 +515,31 @@ def lookup( # scope), or a lagged index (the file IS in the scope on the # server, but the batch endpoint's map has not caught up yet). # Only the second is worth a retry, and this side has no way - # to distinguish them a priori -- so retry ONCE per scope, - # sleep briefly first, and if the second answer still does not - # name the uid, take that as data drift and warn. + # to distinguish them a priori -- so retry with a bounded + # exponential-backoff schedule (default 1 s / 3 s / 9 s), + # sleep between attempts, and if the last answer still does + # not name the uid, take that as data drift and warn. # - # Bounded and per-scope: a data-drift miss costs one extra - # signer call for the whole scope, not one per uid. A lagged - # index that recovers replaces the cache entry for every - # subsequent uid in the same scope, so a whole series' worth - # of misses becomes a whole series' worth of hits from one - # retry. See Waltham-Data-Science/NDI-python#309. - if cache_key not in self._retried_scopes: - self._retried_scopes.add(cache_key) - if self._partial_map_retry_seconds > 0: - self._sleep(self._partial_map_retry_seconds) + # Bounded and per-scope: a data-drift miss costs at most + # len(delays) extra signer calls for the whole scope, not + # one per uid. A lagged index that recovers replaces the + # cache entry for every subsequent uid in the same scope, + # so a whole series' worth of misses becomes a whole + # series' worth of hits from one retry. Three waves cover + # the User-1-prod tail after ``waitForAllBulkUploads`` + # returns; on a User-2-prod-shaped environment the first + # retry lands and the rest never fire. See + # Waltham-Data-Science/NDI-python#309 and #320. + attempts_spent = self._scope_retry_attempts.get(cache_key, 0) + while attempts_spent < len(self._partial_map_retry_delays): + delay = self._partial_map_retry_delays[attempts_spent] + if delay > 0: + self._sleep(delay) # Drop the stale entry so _fetch_scope replaces it fresh. self._cache.pop(cache_key, None) self._stats.partial_map_retries += 1 + attempts_spent += 1 + self._scope_retry_attempts[cache_key] = attempts_spent refreshed = self._fetch_scope( cache_key, cloud_dataset_id, @@ -317,10 +579,152 @@ def clear(self) -> None: with self._lock: self._cache.clear() self._failed_scopes.clear() - self._retried_scopes.clear() + self._scope_retry_attempts.clear() self._warned.clear() self._stats = Stats() + def prefetch_scope( + self, + cloud_dataset_id: str, + ndi_document_id: str, + series_name: str, + *, + client: CloudClient | None = None, + ) -> bool: + """Populate the cache for one scope without asking about a specific uid. + + Seeds the cache so a subsequent :meth:`lookup` on any uid in the + scope resolves without another signer call. Safe to call from a + background thread; the same lock that guards :meth:`lookup` + serialises the fetch so a concurrent lookup on the same scope + does not stampede the endpoint. + + The partial-map retry is NOT run here on purpose: prefetch's + job is to seed the cache, and the retry only helps when a + specific uid is expected to be present. If the map that arrives + is partial, the first lookup that misses will drive its own + retry through the existing bounded schedule. + + Args: + cloud_dataset_id: The dataset id. + ndi_document_id: The NDI document id. Empty returns False + immediately: nothing to batch against. + series_name: ``""`` for a whole-document scope, or a file + series name to scope the batch. + client: Passed to the signer (default signer only). + + Returns: + True when a cache entry now exists for the scope (either + just populated, or already fresh); False otherwise. + """ + if not ndi_document_id: + return False + + cache_key = f"{cloud_dataset_id}/{ndi_document_id}/{series_name}" + now = time.monotonic() + + with self._lock: + entry = self._cache.get(cache_key) + if entry is not None and (now - entry.fetched_at) < self._ttl_seconds: + return True + # Drop stale entry so _fetch_scope replaces it. + if entry is not None: + del self._cache[cache_key] + + # Consult the on-disk cache before running the async signed- + # URL-set job -- the whole point of the disk cache is that + # a scope warmed by a previous process should not have to + # re-sign 100k URLs at every viewer launch. Mirrors what + # ``lookup`` does. ``_try_disk_cache`` filters entries whose + # URLs would expire within the safety buffer, so a hit here + # is safe to hand to callers. See + # :mod:`ndi.cloud.signed_url_disk_cache`. + if self._disk_cache: + entry = self._try_disk_cache( + cache_key, cloud_dataset_id, ndi_document_id, series_name + ) + if entry is not None: + return True + + entry = self._fetch_scope( + cache_key, + cloud_dataset_id, + ndi_document_id, + series_name, + uid="", + client=client, + ) + return entry is not None + + def start_prefetch( + self, + scopes: Sequence[tuple[str, str, str]], + *, + client: CloudClient | None = None, + ) -> threading.Thread: + """Spawn a daemon thread that prefetches ``scopes`` in order. + + Each scope is ``(cloud_dataset_id, ndi_document_id, series_name)``. + Scopes with an empty ``ndi_document_id`` are skipped (nothing to + batch against). A :class:`BatchScopeUnreachable` from one scope + is logged and the thread continues with the next -- the point + of prefetch is to warm the cache best-effort, not to fail loud. + + Returns the started thread so tests can join it; production + callers can fire and forget. + """ + scopes_list = list(scopes) + + def _run() -> None: + logger.info( + "signed-URL prefetch: warming %d scope(s) in background", + len(scopes_list), + ) + for cloud_dataset_id, ndi_document_id, series_name in scopes_list: + if not ndi_document_id: + logger.info( + "signed-URL prefetch: skipping scope " + "(%s, , series=%r) -- nothing to batch against", + cloud_dataset_id, + series_name, + ) + continue + started = time.monotonic() + try: + ok = self.prefetch_scope( + cloud_dataset_id, + ndi_document_id, + series_name, + client=client, + ) + except BatchScopeUnreachable as exc: + logger.warning( + "signed-URL prefetch: scope (%s, %s, series=%r) unreachable " + "after %.1fs: %s -- continuing with the next scope", + cloud_dataset_id, + ndi_document_id, + series_name, + time.monotonic() - started, + exc, + ) + continue + logger.info( + "signed-URL prefetch: scope (%s, %s, series=%r) %s in %.1fs", + cloud_dataset_id, + ndi_document_id, + series_name, + "cached" if ok else "failed", + time.monotonic() - started, + ) + + thread = threading.Thread( + target=_run, + name="ndi-signed-url-prefetch", + daemon=True, + ) + thread.start() + return thread + # ------------------------------------------------------------------ # Internal # ------------------------------------------------------------------ @@ -335,7 +739,14 @@ def _fetch_scope( *, client: CloudClient | None, ) -> _CacheEntry | None: - """Populate the cache for one scope. None on any failure.""" + """Populate the cache for one scope. None on any failure. + + :class:`BatchScopeUnreachable` from the signer is NOT caught -- it + means the async job path itself is broken and per-uid fallback + would just hide the real error. Every other exception from an + injected signer is still reported as a miss reason so existing + tests keep working. + """ failure_reason = "" try: ok, answer = self._signer( @@ -344,6 +755,21 @@ def _fetch_scope( file_series=series_name, client=client, ) + except BatchScopeUnreachable as exc: + # Real failure of the async signed-URL-set path. Record the + # scope as unreachable so subsequent lookups in the same scope + # fast-fail (re-raising a fresh BatchScopeUnreachable) instead + # of running another full retry cycle every time napari asks + # for a chunk. Then propagate so the caller sees the cause. + self._stats.last_failure_reason = f"async job path unreachable: {exc}" + self._stats.uid_misses += 1 + self._failed_scopes[cache_key] = _FailureEntry( + at=time.monotonic(), + uid=uid, + unreachable=True, + cause=str(exc), + ) + raise except Exception as exc: # noqa: BLE001 - reported as a miss reason ok = False answer = {} @@ -372,6 +798,69 @@ def _fetch_scope( self._cache[cache_key] = entry self._stats.last_map_size = len(entry.files) self._stats.last_map_uids = list(entry.files.keys()) + + # Persist to disk when the caller has opted in AND the payload + # carries a server-signed expiry we can age-check off of. A + # payload with neither filesExpireAt nor expiresAt is not + # cacheable -- the disk cache refuses to invent a TTL, on + # purpose (see :func:`signed_url_disk_cache.save`). Best-effort: + # any I/O failure is swallowed so a full disk does not break a + # read that just succeeded. + if self._disk_cache: + try: + from . import signed_url_disk_cache + + signed_url_disk_cache.save(cloud_dataset_id, ndi_document_id, series_name, answer) + except Exception: # noqa: BLE001 - best-effort + logger.debug( + "signed-URL disk cache save raised for scope %r; ignored", + cache_key, + exc_info=True, + ) + return entry + + def _try_disk_cache( + self, + cache_key: str, + cloud_dataset_id: str, + ndi_document_id: str, + series_name: str, + ) -> _CacheEntry | None: + """Load one scope from the on-disk cache, or return None. + + Populates the in-memory cache on hit so subsequent uids in the + same scope don't re-read the file. A miss is silent -- the + caller falls through to the signer, which is the shape a fresh + or expired-cache first open takes. + """ + try: + from . import signed_url_disk_cache + + disk = signed_url_disk_cache.load(cloud_dataset_id, ndi_document_id, series_name) + except Exception: # noqa: BLE001 - never fail a read on disk I/O + logger.debug( + "signed-URL disk cache load raised for scope %r; ignored", + cache_key, + exc_info=True, + ) + return None + if not disk: + return None + files = disk.get("files") + if not isinstance(files, dict) or not files: + return None + entry = _CacheEntry(files=dict(files), fetched_at=time.monotonic()) + self._cache[cache_key] = entry + self._stats.last_map_size = len(entry.files) + self._stats.last_map_uids = list(entry.files.keys()) + logger.info( + "signed-URL disk cache: served scope (%s, %s, series=%r) " + "with %d uids -- signer not called", + cloud_dataset_id, + ndi_document_id, + series_name, + len(entry.files), + ) return entry def _warn_once(self, cache_key: str) -> None: @@ -432,12 +921,25 @@ def _describe_failure(ok: bool, answer: Any) -> str: def get_default() -> BatchSignedUrlLookup: - """The process-wide default cache. Created on first use.""" + """The process-wide default cache. Created on first use. + + Constructed with the on-disk cache enabled: this is the production + reopen path. A scientist reopening the same lightsheet the next + morning must not pay the ~85 min signed-URL-set job again -- the + first open persists the scope to ``~/.ndi/signed-url-cache/`` (or + wherever ``NDI_SIGNED_URL_CACHE_DIR`` points), the reopen reads it + back and never touches the signer. See + :mod:`ndi.cloud.signed_url_disk_cache`. + + Tests inject their own :class:`BatchSignedUrlLookup` -- disk cache + off by default there -- so scripted-signer suites don't persist + fake URLs into the user's real cache directory. + """ global _DEFAULT_LOOKUP if _DEFAULT_LOOKUP is None: with _DEFAULT_LOCK: if _DEFAULT_LOOKUP is None: - _DEFAULT_LOOKUP = BatchSignedUrlLookup() + _DEFAULT_LOOKUP = BatchSignedUrlLookup(disk_cache=True) return _DEFAULT_LOOKUP diff --git a/src/ndi/cloud/client.py b/src/ndi/cloud/client.py index a2f6917c..6e953566 100644 --- a/src/ndi/cloud/client.py +++ b/src/ndi/cloud/client.py @@ -12,12 +12,15 @@ import functools import json +import logging import random import re import time from typing import Any from urllib.parse import quote as _url_quote +logger = logging.getLogger(__name__) + from .config import CloudConfig from .exceptions import ( CloudAPIError, @@ -180,6 +183,12 @@ class CloudClient: def __init__(self, config: CloudConfig): self.config = config + # A hand-built ``CloudClient(CloudConfig(token=...))`` uses that + # token as given and does not refresh against env credentials on + # 401/403 -- the caller told us what token to use. ``from_env()`` + # (and any explicit login helper we add) sets this True after a + # successful ``authenticate()``. + self._can_reauth = False try: import requests except ImportError as exc: @@ -286,6 +295,7 @@ def _request( retryable = method.upper() in self.RETRY_METHODS attempt = 0 + reauth_tried = False while True: attempt += 1 try: @@ -304,16 +314,71 @@ def _request( and isinstance(exc, transient_exceptions) and attempt < self.MAX_ATTEMPTS ): - time.sleep(self._retry_delay(attempt)) + delay = self._retry_delay(attempt) + logger.info( + "cloud request %s %s: attempt %d/%d hit %s " "(%s); retrying in %.1fs", + method, + endpoint, + attempt, + self.MAX_ATTEMPTS, + type(exc).__name__, + exc, + delay, + ) + time.sleep(delay) continue raise CloudAPIError(f"Request failed{self._attempt_note(attempt)}: {exc}") from exc + # Token expired mid-run? Refresh once and retry. A long-running + # operation (e.g. polling a signed-URL-set job for tens of + # minutes) can outlive the token TTL, and without this every + # subsequent request 403s forever until the outer timeout + # fires -- observed on a 1-hour signing job that broke at + # ~10 min. One-shot per request: a second 401/403 after + # reauthentication is a real auth problem (bad creds, revoked + # key) and belongs at the surface. + if resp.status_code in (401, 403) and not reauth_tried and self._can_reauth: + logger.info( + "cloud request %s %s: got HTTP %d (token likely expired); " + "reauthenticating and retrying", + method, + endpoint, + resp.status_code, + ) + try: + self._reauthenticate() + except Exception as exc: # noqa: BLE001 - report and surface original 4xx + logger.info( + "cloud request %s %s: reauthenticate failed (%s: %s); " + "letting the original HTTP %d surface", + method, + endpoint, + type(exc).__name__, + exc, + resp.status_code, + ) + else: + if self.config.token: + headers["Authorization"] = f"Bearer {self.config.token}" + reauth_tried = True + continue + if ( retryable and resp.status_code in self.RETRY_STATUSES and attempt < self.MAX_ATTEMPTS ): - time.sleep(self._retry_delay(attempt)) + delay = self._retry_delay(attempt) + logger.info( + "cloud request %s %s: attempt %d/%d got HTTP %d; " "retrying in %.1fs", + method, + endpoint, + attempt, + self.MAX_ATTEMPTS, + resp.status_code, + delay, + ) + time.sleep(delay) continue break @@ -429,7 +494,34 @@ def from_env(cls) -> CloudClient: config = CloudConfig.from_env() config.token, config.org_id = authenticate(config) - return cls(config) + client = cls(config) + # We just obtained this token ourselves, so a mid-run 401/403 + # (typical after the token's TTL expires) can be answered by + # calling ``authenticate`` again with the same credentials. + client._can_reauth = True + return client + + def _reauthenticate(self) -> None: + """Refresh this client's token in place from the same credentials. + + Called by :meth:`_request` when a 401/403 comes back mid-operation + (typically because the current token's TTL has expired during a + long-running poll or download). Uses the same credential-resolution + path as :meth:`from_env`, so a valid ``NDI_CLOUD_USERNAME`` / + ``NDI_CLOUD_PASSWORD`` pair yields a fresh bearer token; a + pre-baked ``NDI_CLOUD_TOKEN`` that is itself expired will still + fail, and the retry surfaces the auth error to the caller. + + Mutates ``self.config.token`` in place so any other caller holding + the same client keeps working. + """ + from .auth import authenticate + + fresh_config = CloudConfig.from_env() + token, org_id = authenticate(fresh_config) + self.config.token = token + if org_id: + self.config.org_id = org_id def __repr__(self) -> str: return f"CloudClient(api_url={self.config.api_url!r})" diff --git a/src/ndi/cloud/filehandler.py b/src/ndi/cloud/filehandler.py index cbe228d8..f6a3380d 100644 --- a/src/ndi/cloud/filehandler.py +++ b/src/ndi/cloud/filehandler.py @@ -292,6 +292,13 @@ def _fetch_manifest( what we want here). Otherwise, call :func:`fetch_cloud_file` directly, matching what the read-side handler does when a manifest is asked for by uid. + + We bypass the batch signed-URL lookup here. The batch is scoped + per-document, and a series with N members has an N-URL scope: for a + lightsheet OME-Zarr level with 100k+ chunks that is a page walk of + 50-100 minutes for the ONE URL a manifest fetch actually needs. A + single-file fetch through :func:`getFileDetails` is the right shape + at O(1). See Waltham-Data-Science/NDI-python#322 / issue TBD. """ source_path = f"{NDIC_SCHEME}{cloud_dataset_id}/{manifest_uid}" if custom_file_handler is not None: @@ -309,7 +316,10 @@ def _fetch_manifest( source_path, dest_path, client=client, - ndi_document_id=document_id, + # Empty ndi_document_id skips the batch lookup and goes straight to + # getFileDetails for the one uid. Batch here would ask the server + # for every other file in the document too. + ndi_document_id="", series_name="", ) @@ -385,6 +395,22 @@ def watchFetches(observer): _fetch_observer = previous +#: Per-thread timing of the last :func:`fetch_cloud_file` on that thread. +_last_fetch = threading.local() + + +def lastFetchTiming() -> dict | None: + """Timing of this thread's most recent :func:`fetch_cloud_file`, or None. + + ``{"sign_s", "batch", "transfer_s", "bytes", "ok"}``: seconds spent + getting a signed URL (and whether it came from the batch cache), seconds + spent downloading, and bytes written. Per thread, so a fetcher worker + reads back the fetch it just made and no other. Used by the lightsheet + viewer's debug log to say where a slow tile's time went. + """ + return getattr(_last_fetch, "timing", None) + + def fetch_cloud_file( ndic_uri: str, target_path: str | Path, @@ -430,7 +456,12 @@ def fetch_cloud_file( if client is None: client = get_or_create_cloud_client() + import time as _time + + _last_fetch.timing = None + t_sign = _time.monotonic() download_url = "" + url_came_from_batch = False if ndi_document_id: # Batch path first. Empty means the batch could not answer for this # uid; fall back per uid so the read still succeeds. @@ -444,6 +475,7 @@ def fetch_cloud_file( file_uid, client=client, ) + url_came_from_batch = bool(download_url) if not download_url: details = getFileDetails(dataset_id, file_uid, client=client) @@ -453,6 +485,8 @@ def fetch_cloud_file( raise CloudError(f"No downloadUrl in file details for {ndic_uri}. Response: {details}") + sign_s = _time.monotonic() - t_sign + # Stream download to temp file, then atomic rename target = Path(target_path) target.parent.mkdir(parents=True, exist_ok=True) @@ -460,20 +494,41 @@ def fetch_cloud_file( logger.debug("Fetching cloud file %s -> %s", ndic_uri, target) observer = _fetch_observer + # error_out is only needed when the URL came from the batch disk cache + # and we might have to invalidate a stale scope on S3 403. Passing it + # unconditionally would force every mocked getFile in the test suite + # to accept the new keyword. + error_out: dict | None = {} if url_came_from_batch else None + call_kwargs: dict = {"timeout": 300} + if error_out is not None: + call_kwargs["error_out"] = error_out + t_get = _time.monotonic() if observer is None: - success = getFile(download_url, tmp_path, timeout=300) + success = getFile(download_url, tmp_path, **call_kwargs) else: observer("start", ndic_uri, 0, None) try: success = getFile( download_url, tmp_path, - timeout=300, progress=lambda done, total: observer("chunk", ndic_uri, done, total), + **call_kwargs, ) finally: observer("done", ndic_uri, 0, None) + try: + nbytes = tmp_path.stat().st_size if success else 0 + except OSError: + nbytes = 0 + _last_fetch.timing = { + "sign_s": sign_s, + "batch": url_came_from_batch, + "transfer_s": _time.monotonic() - t_get, + "bytes": nbytes, + "ok": bool(success), + } + if success: tmp_path.rename(target) logger.debug("Cloud file cached: %s", target) @@ -481,6 +536,28 @@ def fetch_cloud_file( else: # Clean up partial download tmp_path.unlink(missing_ok=True) + + # An S3 403 on a URL that WAS served from the disk cache says + # the cached scope has gone stale (token revoked, or the object + # rotated). Drop the scope so the next read refetches -- + # otherwise we'd 403 our way through every subsequent uid in + # the same scope. Best-effort; a failed forget is not fatal + # to the CloudError we're raising. Mirrors NDI-matlab's own + # 403-invalidation hook in didsqlite.download_file_from_cloud. + if url_came_from_batch and error_out is not None and error_out.get("status") == 403: + try: + from . import signed_url_disk_cache + + signed_url_disk_cache.forget(dataset_id, ndi_document_id, series_name) + except Exception: # noqa: BLE001 - best-effort + logger.debug( + "signed-URL disk cache forget raised for scope " "(%s, %s, series=%r); ignored", + dataset_id, + ndi_document_id, + series_name, + exc_info=True, + ) + from .exceptions import CloudError raise CloudError(f"Failed to download file from {ndic_uri}") @@ -958,6 +1035,22 @@ def download_file_from_cloud( ndi_document_id = str(context.get("documentId", "") or "") series_name = str(context.get("seriesName", "") or "") + # A single-file fetch (a series manifest, or any doc-level attachment) + # arrives here with seriesName="", because there is no member being + # asked for. Going through the per-document batch scope to answer one + # uid is pure loss: for a lightsheet-scale pyramid document that scope + # names 15k+ files, so the batch endpoint spends 60-90 s on a signed- + # URL set the caller has no use for, delaying the ONE URL that + # download actually needs by more than a minute. Same reasoning that + # ``_fetch_manifest`` above uses for the internal manifest fetch: for + # a single uid, the direct ``getFileDetails`` path is O(1) and wins. + # (VH-Lab/NDI-matlab#1010's Python analog. When the caller does need + # a whole-doc scope -- an ordinary series member fetch that follows + # with seriesName="chunk.bin" -- the batch fires there and the cost + # amortizes across every member.) + if not series_name: + ndi_document_id = "" + fetch_cloud_file( uri, dest_path, diff --git a/src/ndi/cloud/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/ndi_matlab_python_bridge.yaml index 47731d6b..5a92e800 100644 --- a/src/ndi/cloud/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/ndi_matlab_python_bridge.yaml @@ -924,7 +924,7 @@ functions: - name: uploadFilesForDatasetDocuments matlab_path: "+ndi/+cloud/+sync/+internal/uploadFilesForDatasetDocuments.m" - matlab_last_sync_hash: "4e45d96e" + matlab_last_sync_hash: "d94973c1" python_path: "ndi/cloud/upload.py" input_arguments: - name: org_id @@ -948,6 +948,14 @@ functions: decision_log: > MATLAB places this in +sync/+internal. Python places it in upload.py. Exact name match. + 2026-10-13: examined against NDI-matlab d94973c1 on + claude/lightsheet-zarr-ndi-viewer-djp5vk: main's 9c21eb61 + (VH-Lab/NDI-matlab#1027) merged with that branch's verbose upload count + (VH-Lab/NDI-matlab#979, merged as d292984f). + The serial path now uploads every file with waitForUpload=false and + then waits once, through waitForFileUpload, for NDI Cloud to record + them all. Not ported: Python has no waitForFileUpload yet (recorded + as porting_deferred in ndi/cloud/api); Python's uploads do not wait. 2026-09-25: NDI-matlab 4e45d96e added verbose stage prints and a listFiles-not-yet-uploaded diagnostic. Python code is silent in the same way it was; no behavioural change. 2026-09-15: added a ``dataset`` kwarg. On MATLAB, +sync/+internal/ @@ -967,7 +975,7 @@ functions: - name: uploadSingleFile matlab_path: "+ndi/+cloud/uploadSingleFile.m" - matlab_last_sync_hash: "8fa6ea5e" + matlab_last_sync_hash: "9c21eb61" python_path: "ndi/cloud/upload.py" input_arguments: - name: dataset_id @@ -996,6 +1004,11 @@ functions: - name: error_message type_python: "str" decision_log: > + 2026-10-13: examined against NDI-matlab 9c21eb61 (VH-Lab/NDI-matlab#1027). + MATLAB gained a waitForUpload option (default true) that polls + waitForFileUpload until NDI Cloud records the upload. Not ported yet: + Python returns once the PUT succeeds; see waitForFileUpload + (porting_deferred) in ndi/cloud/api. Exact match. 2026-10-12: synced to NDI-matlab 8fa6ea5e (claude/file-crc32-lookup, NDI-matlab#1025): 'checksum' (default true) computes the file's CRC-32 @@ -1041,7 +1054,7 @@ functions: # --- filehandler.py functions --- - name: updateFileInfoForRemoteFiles matlab_path: "+ndi/+cloud/+sync/+internal/updateFileInfoForRemoteFiles.m" - matlab_last_sync_hash: "9467a0a7" + matlab_last_sync_hash: "a84130a7" python_path: "ndi/cloud/filehandler.py" input_arguments: - name: doc_props @@ -1470,7 +1483,7 @@ functions: # ============================================================================= # Not-yet-ported MATLAB functions # ============================================================================= -not_yet_ported: +porting_deferred: # ======================================================================= # +ndi/+cloud/+download/+internal -- internals of the download loop @@ -1480,7 +1493,7 @@ not_yet_ported: # without a live dataset (NDI-matlab#962, #963, #970). Python's cloud # download path is in cloud/download.py and does not currently mirror # these boundaries -- the whole loop is one function -- so recording - # them as not_yet_ported is the honest status. They land as a group + # them as porting_deferred is the honest status. They land as a group # with the batch-presign work referenced by # +sync/+internal/reconstructSeriesIngestLocations and by the # signed-URL-set family in the api YAML (issue #206), because none of @@ -1501,7 +1514,7 @@ not_yet_ported: - name: batchSignedUrlLookup matlab_path: "+ndi/+cloud/+download/+internal/batchSignedUrlLookup.m" python_path: "ndi/cloud/batch_signed_url.py" - matlab_last_sync_hash: "9120c002" + matlab_last_sync_hash: "02880062" status: ported decision_log: > PORTED for NDI-python#262. Turns one-call-per-uid into one call per @@ -1527,10 +1540,6 @@ not_yet_ported: NEXT uid in a sweep is suppressed, a caller re-asking for the same uid gets a fresh attempt. - Skips the async job path (createSignedURLSetJob + - waitForSignedURLSetJob) since the batch helper does not use it; - those remain porting_deferred in the api YAML. - NDI-matlab 9120c002 adds a one-shot partial-map retry: when a cached scope answers with a URL but a subsequent uid in that scope misses, drop the entry, pause, and re-fetch once @@ -1539,6 +1548,85 @@ not_yet_ported: (_partial_map_retry_seconds / STATS.partial_map_retries in ndi/cloud/batch_signed_url.py), so this is a hash-only bump. + NDI-python#206: _default_signer switched from getSignedURLSetAll + (paged, ~25 s per 500-uid page) to the async signed-URL-set-job + family (createSignedURLSetJob -> waitForSignedURLSetJob -> + getSignedURLSetResult). For a 121k-uid document the paged walk + took 100+ minutes; the async job builds the whole map server-side + and returns it in one gzipped blob. The signer contract is + unchanged -- still returns (True, {"files": {uid: url}, ...}) on + success and (False, {"__error__": ...}) on failure -- and the + transient-error retry (NDI-python#322) still wraps the whole + three-step exchange. + + NDI-matlab#1010 follow-on (MATLAB commit 27661a3): the constructor + gains ``disk_cache: bool = False`` and get_default() constructs + with ``disk_cache=True``, wiring the persistent on-disk cache + module below. ``ttlSeconds`` is still the in-memory freshness + cap; the disk cache is age-checked against the SERVER's + filesExpireAt with a 30 min safety buffer. + + - name: signedUrlDiskCache + matlab_path: "+ndi/+cloud/+download/+internal/signedUrlDiskCache.m" + python_path: "ndi/cloud/signed_url_disk_cache.py" + matlab_last_sync_hash: "988ec74" + status: ported + decision_log: > + PORTED for VH-Lab/NDI-matlab#1010 follow-on / Waltham-Data-Science/ + NDI-python#322. Persistent (on-disk) layer beneath + batchSignedUrlLookup's in-process cache: a scientist reopening the + same lightsheet-scale dataset over a day pays the ~85 min async + signed-URL-set-job cost ONCE. + + The two implementations write byte-for-byte identical files: a + cache dir written by MATLAB is readable by Python and vice + versa. Portability is the whole point -- a mixed lab must not pay + the sign cost twice. + + Shared on-disk contract (documented at the module top of both + files): + - Layout: //.json.gz for + whole-doc scopes, //_.json.gz + for scoped ones. Empty seriesName maps to the whole-doc + filename explicitly so the two spellings can never collide. + - Filename escaping: bytes outside [A-Za-z0-9._-] become %XX + (uppercase hex); the escaped string above 96 characters is + replaced with hash- so a pathologically long name + never overflows a filesystem's 255-byte name limit. + - Format: standard gzip (RFC 1952), single member, UTF-8 JSON + body with schemaVersion=1 and the fields datasetId, + documentId, seriesName, cachedAt, filesExpireAt, expiresAt, + generatedAt, fileCount, files. + - TTL: authoritative filesExpireAt (falls back to expiresAt), + minus a 30 min safety buffer (overridable via + NDI_SIGNED_URL_CACHE_SAFETY_SECONDS). A payload with neither + is un-cacheable; the cache refuses to invent a TTL. + - Env override: NDI_SIGNED_URL_CACHE_DIR points cache_dir() + anywhere. Default ~/.ndi/signed-url-cache, mode 0700 on POSIX. + + Departures from the MATLAB implementation (all internal-only, no + on-disk difference): + - Standard-library gzip + os.replace instead of MATLAB's Java + GZIPInputStream/GZIPOutputStream + java.nio.file.Files.move + dance. Same atomic-rename semantics, cleaner one-liner. + - json.load handles digit-prefixed uid keys natively, so the + MATLAB signedURLFileMap workaround (recover keys from raw + payload because jsondecode renames them) has no counterpart + here. + - SHA-1 via hashlib rather than java.security.MessageDigest. + - forget()'s idempotence is expressed via Path.unlink(missing_ok=True) + rather than an isfile() guard, but the caller-visible + contract is the same: silent no-op on a missing scope. + + The read-path 403 invalidation hook: ndi.cloud.filehandler. + fetch_cloud_file consults the batch cache when the caller has a + document id; on a subsequent S3 403 during the actual getFile + the scope is forgotten from disk so the next lookup refetches -- + mirroring NDI-matlab's own hook in + didsqlite.download_file_from_cloud. getFile grew an opt-in + error_out: dict | None keyword to surface the HTTP status; the + status code, not a stringly-typed message, drives the hook. + - name: buildGenericFileDownloadList matlab_path: "+ndi/+cloud/+download/+internal/buildGenericFileDownloadList.m" matlab_last_sync_hash: "65aeeeb" @@ -1625,7 +1713,6 @@ not_yet_ported: - name: resolveOrganizationId matlab_path: "+ndi/+cloud/+internal/resolveOrganizationId.m" matlab_last_sync_hash: "f847d7c4" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" status: porting_deferred decision_log: > MATLAB helper introduced with NDI-matlab#979: given diff --git a/src/ndi/cloud/signed_url_disk_cache.py b/src/ndi/cloud/signed_url_disk_cache.py new file mode 100644 index 00000000..0b740581 --- /dev/null +++ b/src/ndi/cloud/signed_url_disk_cache.py @@ -0,0 +1,469 @@ +""" +ndi.cloud.signed_url_disk_cache - Persistent (on-disk) signed-URL cache. + +MATLAB equivalent: + +ndi/+cloud/+download/+internal/signedUrlDiskCache.m + (VH-Lab/NDI-matlab commit 27661a3 on the same branch) + +This is the disk-backed layer that sits beneath the in-process cache +inside :class:`ndi.cloud.batch_signed_url.BatchSignedUrlLookup`. It exists +so a working scientist reopening the same dataset over a day pays the +~85-minute async signed-URL-set-job cost ONCE and reads from disk on +every subsequent viewer open. NDI cloud infrastructure, not DID: DID's +file-bytes cache is a separate, complementary concern. + +Scope key +--------- +The unit the cache stores and evicts is one ``(dataset_id, document_id, +series_name)`` tuple -- the same tuple ``BatchSignedUrlLookup`` uses. An +empty ``series_name`` marks a whole-document scope. + +Payload +------- +What :func:`save` writes and :func:`load` returns is a dict with the +fields the ``getSignedURLSetResult`` answer already carries: + + files dict[str, str]: uid -> pre-signed download URL. + filesExpireAt ISO-8601 UTC string; the server's authoritative expiry + for the signed URLs. Preferred. + expiresAt ISO-8601 UTC string; older paging-path field. Used + when filesExpireAt is not present. + generatedAt ISO-8601 UTC string; when the server built the map. + Optional, informational. + fileCount int; how many uids the server reported. Optional. + +On-disk contract +---------------- +The MATLAB and Python implementations write the SAME bytes: a scientist +can hand a cache dir written by MATLAB to a Python viewer and see +zero-cost reopens, and vice versa. + +Location: //.json.gz (whole-doc) + //_.json.gz (scoped) + +``cache_dir`` defaults to ``~/.ndi/signed-url-cache``, overridable via +``NDI_SIGNED_URL_CACHE_DIR``. The base directory is created with +mode 0700 on POSIX because a signed URL is a bearer token for 24 h; +Windows relies on the default per-user profile ACL. + +Filename escaping for ``series_name``: any byte outside +``[A-Za-z0-9._-]`` becomes ``%XX`` (uppercase hex). A resulting string +longer than 96 characters is replaced with ``hash-`` (45 +chars). MATLAB does the same, byte-for-byte. + +Format: Standard gzip framing (RFC 1952), single member, UTF-8 + JSON body. Unzipping yields a human-readable JSON object:: + + { + "schemaVersion": 1, + "datasetId": "", + "documentId": "", + "seriesName": "" or "", + "cachedAt": "YYYY-MM-DDTHH:MM:SSZ", + "filesExpireAt": "YYYY-MM-DDTHH:MM:SSZ" or "", + "expiresAt": "YYYY-MM-DDTHH:MM:SSZ" or "", + "generatedAt": "YYYY-MM-DDTHH:MM:SSZ" or "", + "fileCount": N, + "files": { "": "", ... } + } + +TTL and safety buffer +--------------------- +The cache does not invent a TTL. It reads the authoritative +``filesExpireAt`` (or ``expiresAt``) from the payload; a payload with +neither is treated as un-cacheable (:func:`save` is a no-op) and, on +load, as an immediate miss. A URL that would expire in less than the +safety buffer (default 30 min, override with +``NDI_SIGNED_URL_CACHE_SAFETY_SECONDS``) is also treated as a miss -- +we never hand a caller a URL likely to 403 before they can use it. + +Concurrency +----------- +Writes go to a per-writer temp file next to the target and are moved +into place with :func:`os.replace`, which is atomic on both POSIX +(``rename(2)``) and Windows (``MoveFileEx MOVEFILE_REPLACE_EXISTING``). +Two viewers signing the same scope concurrently is fine: whichever +atomic move lands last wins, both readers see a consistent file +afterwards. There is no cross-process lock; the contract is +last-writer-wins, not exclusive. + +Invalidation +------------ +:func:`forget` removes the file for one scope. The caller does this +from the chunk-download path when it sees an S3 403 -- the cached URL +was in the map but the object rotated or the token was revoked, so +drop the scope and let the next lookup refetch. Failing to remove the +file is not fatal. + +Miss / error semantics +---------------------- +:func:`load` returns ``None`` on ANY of: no cache file, unreadable file, +corrupt gzip, invalid JSON, missing timestamp, expired-or-within-buffer. +The caller has no need to distinguish -- every one of them means +"refetch from the server". + +:func:`save` swallows every I/O error: a full disk, permission denied +or a race with another process must never break the read that just +succeeded. The in-memory cache still works; the next viewer open pays +one signer call and tries again. +""" + +from __future__ import annotations + +import gzip +import hashlib +import json +import logging +import os +import re +import secrets +import stat +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +logger = logging.getLogger(__name__) + + +SCHEMA_VERSION = 1 +DEFAULT_SAFETY_SECONDS = 1800 # 30 min; task-declared floor. +MAX_ESCAPED_SERIES_LEN = 96 # Beyond this we hash the series name. + +_ISO_FMT = "%Y-%m-%dT%H:%M:%SZ" + +_SAFE_BYTE_RE = re.compile(rb"[A-Za-z0-9._-]") + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def cache_dir() -> Path: + """Absolute path to the on-disk cache root. + + Honors ``NDI_SIGNED_URL_CACHE_DIR``; otherwise + ``~/.ndi/signed-url-cache``. Creates the directory the first time + it is asked for, with mode 0700 on POSIX so a file that is legibly + a bearer token is not world-readable. + """ + override = os.environ.get("NDI_SIGNED_URL_CACHE_DIR", "") + if override: + base = Path(override) + else: + base = Path.home() / ".ndi" / "signed-url-cache" + if not base.exists(): + try: + base.mkdir(parents=True, exist_ok=True) + _chmod_user_only(base) + except OSError: + # Best-effort: BatchSignedUrlLookup treats a missing cache + # dir as "no disk cache available", so a failure to create + # here is not fatal to the read. + pass + return base + + +def load(dataset_id: str, document_id: str, series_name: str = "") -> dict[str, Any] | None: + """Read one scope's cached signed-URL set, or ``None`` on any miss. + + Returns a dict (fields: files, filesExpireAt, expiresAt, + generatedAt, fileCount) on cache hit. Returns ``None`` on any of: + no cache file, corrupt file, JSON parse error, + expired-or-within-safety-buffer. + + The safety buffer is 30 min by default, overridable via + ``NDI_SIGNED_URL_CACHE_SAFETY_SECONDS``. A URL that would expire + inside that window is a miss so the caller refetches rather than + handing bytes to a viewer that then 403s. + """ + file_path = _cache_path(dataset_id, document_id, series_name) + if not file_path.is_file(): + return None + + try: + with gzip.open(file_path, "rb") as fh: + raw = fh.read() + body = raw.decode("utf-8") + data = json.loads(body) + except (OSError, ValueError, json.JSONDecodeError): + return None + + if not isinstance(data, dict): + return None + + files = data.get("files") + if not isinstance(files, dict): + return None + + # Enforce the safety-buffer floor against the authoritative server + # timestamp. A payload with no timestamp is a miss: better to + # refetch than hand out a URL we can't age-check. + expiry_str = _read_string(data, "filesExpireAt") or _read_string(data, "expiresAt") + if not expiry_str: + return None + expiry = _parse_iso_utc(expiry_str) + if expiry is None: + return None + remaining = (expiry - _now_utc()).total_seconds() + if remaining < _safety_seconds(): + return None + + return { + "files": {str(k): str(v) for k, v in files.items()}, + "filesExpireAt": _read_string(data, "filesExpireAt"), + "expiresAt": _read_string(data, "expiresAt"), + "generatedAt": _read_string(data, "generatedAt"), + "fileCount": _read_int(data, "fileCount", default=len(files)), + } + + +def save( + dataset_id: str, + document_id: str, + series_name: str, + payload: dict[str, Any], +) -> None: + """Persist a scope's signed-URL set. Best-effort, atomic. + + Writes the payload to the scope's cache path via a temp file + + :func:`os.replace`. A payload without a ``filesExpireAt`` / + ``expiresAt`` is NOT written -- the cache cannot age-check + something it has no timestamp for, and we refuse to invent a TTL. + + Any I/O failure (missing dir, permission denied, disk full, + another process racing us) is swallowed silently: the in-memory + cache still works, so the read the caller just finished still + succeeded. + """ + if not isinstance(payload, dict): + return + files = payload.get("files") + if not isinstance(files, dict) or not files: + # Empty maps are legal ("scope exists, has zero members") but + # so is the "endpoint gave us nothing back" case; we cannot + # tell them apart, so err on the side of not persisting. + # A caller who wants to persist an empty scope should still be + # able to next-lookup with no penalty. + if not isinstance(files, dict): + return + + files_expire_at = _read_string(payload, "filesExpireAt") + expires_at = _read_string(payload, "expiresAt") + if not files_expire_at and not expires_at: + # Refuse to persist an un-age-checkable payload. This is a + # CALLER bug, not an environmental issue -- a signer that hands + # us a payload with neither expiry field means the disk cache + # will silently stay empty forever, which is exactly how the + # Waltham-Data-Science/NDI-python#320 first-fresh-machine run + # spent minutes signing 121k URLs and left no trace to reuse. + # WARN rather than raise: the surrounding read has already + # succeeded and we don't want to break it, but the noise is + # what turns "the cache mysteriously never fills" into "here + # is the exact scope and the exact reason". Silent on I/O + # errors (missing dir, permission, disk full) below -- those + # are legitimate best-effort skips, not callee bugs. + logger.warning( + "signed-URL disk cache: refusing to persist scope " + "(dataset=%s, document=%s, series=%r): payload carries " + "neither 'filesExpireAt' nor 'expiresAt', so the cache " + "cannot age-check it. This is a signer bug -- the URL " + "expiration must be carried on the payload passed to " + "save(). Cache stays empty for this scope.", + dataset_id, + document_id, + series_name, + ) + return + + generated_at = _read_string(payload, "generatedAt") + file_count = _read_int(payload, "fileCount", default=len(files)) + + file_path = _cache_path(dataset_id, document_id, series_name) + try: + file_path.parent.mkdir(parents=True, exist_ok=True) + except OSError: + return + + body = { + "schemaVersion": SCHEMA_VERSION, + "datasetId": str(dataset_id), + "documentId": str(document_id), + "seriesName": str(series_name), + "cachedAt": _iso_now(), + "filesExpireAt": files_expire_at, + "expiresAt": expires_at, + "generatedAt": generated_at, + "fileCount": int(file_count), + # dict insertion order maps to JSON key order; digit-prefixed + # uids are legal JSON object keys, so nothing here renames them. + "files": {str(k): str(v) for k, v in files.items()}, + } + try: + text = json.dumps(body, ensure_ascii=False, separators=(",", ":")) + except (TypeError, ValueError): + return + + # Per-writer temp path in the SAME directory so os.replace is a + # rename() rather than a cross-device copy+unlink. The random + # suffix means two writers on the same scope in the same process + # don't collide on their own temp file. + tmp_path = file_path.parent / (file_path.name + f".tmp.{os.getpid()}.{secrets.token_hex(4)}") + try: + with gzip.open(tmp_path, "wb") as gz: + gz.write(text.encode("utf-8")) + os.replace(tmp_path, file_path) + except OSError: + # Any I/O failure: try to clean up the temp file, then swallow. + try: + tmp_path.unlink(missing_ok=True) + except OSError: + pass + + +def forget(dataset_id: str, document_id: str, series_name: str = "") -> None: + """Remove one scope's cache file. Best-effort. + + Called from the chunk-download path when an S3 403 says the cached + URL is no longer valid. If the file is not present -- or the delete + itself fails -- it is not an error: the next lookup will refetch. + """ + file_path = _cache_path(dataset_id, document_id, series_name) + try: + file_path.unlink(missing_ok=True) + except OSError: + # Best-effort: a failed delete is not fatal to the caller who + # is already recovering from a 403. + pass + + +# --------------------------------------------------------------------------- +# Internal +# --------------------------------------------------------------------------- + + +def _safety_seconds() -> float: + """Read the safety buffer, honoring the env-var override.""" + override = os.environ.get("NDI_SIGNED_URL_CACHE_SAFETY_SECONDS", "") + if override: + try: + v = float(override) + if v >= 0: + return v + except ValueError: + pass + return DEFAULT_SAFETY_SECONDS + + +def _cache_path(dataset_id: str, document_id: str, series_name: str) -> Path: + """The absolute path this (ds, doc, series) scope lands at. + + Empty ``series_name`` maps to a whole-document file so + ``.json.gz`` and ``_.json.gz`` can never collide. + NDI-matlab uses exactly the same convention. + """ + base = cache_dir() / str(dataset_id) + if not series_name: + return base / f"{document_id}.json.gz" + escaped = _escape_series_name(str(series_name)) + return base / f"{document_id}_{escaped}.json.gz" + + +def _escape_series_name(raw: str) -> str: + """Percent-encode a series name for filesystem safety. + + Any byte outside ``[A-Za-z0-9._-]`` becomes ``%XX`` (uppercase + hex). Above :data:`MAX_ESCAPED_SERIES_LEN` fall back to + ``hash-`` so a pathologically long name never overflows + a filesystem's 255-byte name limit. + """ + bytes_ = raw.encode("utf-8") + parts: list[str] = [] + for b in bytes_: + if _SAFE_BYTE_RE.fullmatch(bytes([b])): + parts.append(chr(b)) + else: + parts.append(f"%{b:02X}") + s = "".join(parts) + if len(s) > MAX_ESCAPED_SERIES_LEN: + digest = hashlib.sha1(bytes_, usedforsecurity=False).hexdigest() + s = f"hash-{digest}" + return s + + +def _chmod_user_only(path: Path) -> None: + """Set the cache root to user-only permissions on POSIX. + + A signed URL is effectively a bearer token for 24 h. Windows + relies on the default per-user profile ACL for ``~/.ndi/``, which + is roughly equivalent -- and ``os.chmod`` there is largely a no-op. + """ + if os.name != "posix": + return + try: + os.chmod(path, stat.S_IRWXU) # 0o700 + except OSError: + # Best-effort: an unusual filesystem (SMB mount, WSL under Win) + # can refuse chmod but still work. The cache should not fail + # solely because we could not tighten the mode. + pass + + +def _now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def _iso_now() -> str: + return _now_utc().strftime(_ISO_FMT) + + +_ISO_FRACTION_RE = re.compile(r"\.\d+") + + +def _parse_iso_utc(text: str) -> datetime | None: + """Parse a server-emitted UTC timestamp. + + Handles ``YYYY-MM-DDTHH:MM:SSZ`` and ``YYYY-MM-DDTHH:MM:SS.SSSZ``, + the two shapes the ``getSignedURLSetJob`` ready-state payload has + been observed to use. A trailing ``+00:00`` or ``-00:00`` is + equivalent to ``Z``. Anything else is a parse failure and returns + ``None``. + """ + if not text: + return None + trimmed = text.strip() + if trimmed.endswith("Z"): + trimmed = trimmed[:-1] + elif trimmed.endswith("+00:00") or trimmed.endswith("-00:00"): + trimmed = trimmed[:-6] + trimmed = trimmed.replace(" ", "T") + # Strip fractional seconds if present -- our own writer does not + # emit them, but the server sometimes does, and we do not need + # millisecond precision to age-check a 24 h URL. + trimmed = _ISO_FRACTION_RE.sub("", trimmed) + try: + parsed = datetime.strptime(trimmed, "%Y-%m-%dT%H:%M:%S") + except ValueError: + return None + return parsed.replace(tzinfo=timezone.utc) + + +def _read_string(data: dict[str, Any], name: str) -> str: + """Read a field as a string, "" for missing / null / non-string.""" + value = data.get(name) + if value is None: + return "" + if isinstance(value, str): + return value + return str(value) + + +def _read_int(data: dict[str, Any], name: str, *, default: int) -> int: + """Read a field as an int, ``default`` for missing / non-numeric.""" + value = data.get(name) + if value is None: + return default + try: + return int(value) + except (TypeError, ValueError): + return default diff --git a/src/ndi/cloud/sync/ndi_matlab_python_bridge.yaml b/src/ndi/cloud/sync/ndi_matlab_python_bridge.yaml index ec7a155e..6e49d170 100644 --- a/src/ndi/cloud/sync/ndi_matlab_python_bridge.yaml +++ b/src/ndi/cloud/sync/ndi_matlab_python_bridge.yaml @@ -709,6 +709,25 @@ not_yet_ported: MATLAB path resolver. Python handles this inside SyncIndex.read() and SyncIndex.write() methods. + - name: fetchManifest + matlab_path: "+ndi/+cloud/+sync/+internal/fetchManifest.m" + matlab_last_sync_hash: "a84130a7" + status: ported_differently + python_path: "ndi/cloud/filehandler.py" + decision_log: > + MATLAB extracts fetchManifest as a standalone helper so the + "resolve one known uid without going through the per-document + batch signed-URL cache" path can be exercised in isolation. + Python has the same behavior baked directly into + ndi.cloud.filehandler._fetch_manifest (commit 5ab9c04: the + single-manifest fetch bypasses BatchSignedUrlLookup and calls + getFileDetails on the manifest uid). The single-uid direct + path is what both languages need to avoid the ~12 h penalty of + walking a lightsheet document's entire batch to answer one + question -- see VH-Lab/NDI-matlab#1010. No separate + fetch_manifest module in Python: it stays inline with the + caller that needs the manifest bytes. + - name: uploadedDocumentIds matlab_path: "+ndi/+cloud/+sync/+internal/uploadedDocumentIds.m" matlab_last_sync_hash: "29546720" diff --git a/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml b/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml index b6545fc4..6fae12ca 100644 --- a/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml +++ b/src/ndi/fun/doc_lightsheet/ndi_matlab_python_bridge.yaml @@ -32,7 +32,6 @@ functions: - name: fromOMEZarr matlab_path: "+ndi/+fun/+doc/+lightsheet/fromOMEZarr.m" matlab_last_sync_hash: "5c9e8a83" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" status: porting_deferred decision_log: >- MATLAB probes an OME-Zarr store with ndr.format.omezarr.listPyramids, @@ -59,8 +58,7 @@ functions: - name: makePyramid matlab_path: "+ndi/+fun/+doc/+lightsheet/makePyramid.m" - matlab_last_sync_hash: "5c9e8a83" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" + matlab_last_sync_hash: "04ce18f9" status: porting_deferred decision_log: >- The worker fromOMEZarr calls; writes one lightsheetZarrPyramid @@ -99,11 +97,42 @@ functions: follow: this function has no Python counterpart yet, and the Python viewer only reads the levels MATLAB wrote. The Python port must use the 32 MB default when it lands. + 2026-10-12: examined against 04ce18f9 on + claude/lightsheet-zarr-ndi-viewer-djp5vk (was 5c9e8a83). Adds + levelSources (per level path: 'omezarr' -- the unchanged default -- + 'smartspim' slices, or 'reduce' from an earlier level's stored + chunks) and channelNames, for fromSPIM. Default behaviour is + unchanged; the Python port must carry levelSources when it lands. + + - name: fromSPIM + matlab_path: "+ndi/+fun/+doc/+lightsheet/fromSPIM.m" + matlab_last_sync_hash: "668beb13" + status: porting_deferred + decision_log: >- + Builds a lightsheetZarrPyramid straight from a SmartSPIM + acquisition's stitched All_Channels slices (no spim2zarr step): + level 0 from ndr.format.smartspim, mean and max ladders built by + block-reducing each level just written, through makePyramid's + levelSources. Deferred with the rest of the ingest side (the + Python package is a viewer today); planned to port after the + MATLAB version is validated, on top of a makePyramid port, using + ndr-python's smartspim All_Channels reader. + 2026-10-12: examined against 668beb13 (was 04ce18f9): preallocation of + the ladder arrays only; no behaviour change, nothing to port. + + - name: reduceBlocks + matlab_path: "+ndi/+fun/+doc/+lightsheet/reduceBlocks.m" + matlab_last_sync_hash: "04ce18f9" + status: porting_deferred + decision_log: >- + Vectorized per-axis block mean/max with + ndr.format.omezarr.reduce's semantics (trailing partial block + included), used by fromSPIM to build reduced levels. Ports with + fromSPIM; in numpy it is a reshape + mean/max per axis. - name: chooseTileShape matlab_path: "+ndi/+fun/+doc/+lightsheet/chooseTileShape.m" matlab_last_sync_hash: "5c9e8a83" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" status: porting_deferred decision_log: >- Pure function that picks an OME-Zarr chunk shape landing near a @@ -133,7 +162,6 @@ functions: - name: makeSourceFile matlab_path: "+ndi/+fun/+doc/+lightsheet/makeSourceFile.m" matlab_last_sync_hash: "1af66bb" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" status: porting_deferred decision_log: >- MATLAB creates the fileReference document the lightsheet documents' @@ -148,7 +176,6 @@ functions: - name: chooseLevel matlab_path: "+ndi/+fun/+doc/+lightsheet/chooseLevel.m" matlab_last_sync_hash: "5ab84f3" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" status: porting_deferred decision_log: >- Given a level table and a target per-axis voxel size, returns the @@ -159,7 +186,6 @@ functions: - name: levelTable matlab_path: "+ndi/+fun/+doc/+lightsheet/levelTable.m" matlab_last_sync_hash: "1af66bb" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" status: porting_deferred decision_log: >- Joins a lightsheetZarrPyramid and its lightsheetZarrLevel children @@ -175,7 +201,6 @@ functions: - name: viewCommand matlab_path: "+ndi/+fun/+doc/+lightsheet/viewCommand.m" matlab_last_sync_hash: "5ab84f3" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" status: porting_deferred decision_log: >- Pure command builder that ndi.gui.app.LightsheetZarrManager calls @@ -190,8 +215,7 @@ functions: thin. - name: view matlab_path: "+ndi/+fun/+doc/+lightsheet/view.m" - matlab_last_sync_hash: "a7ba07a" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" + matlab_last_sync_hash: "2f9b0eaf" status: porting_deferred decision_log: >- MATLAB caller that pairs viewCommand + SYSTEM: builds the @@ -203,3 +227,44 @@ functions: stays interactive. No Python-side counterpart is needed because the Python side IS the launcher; this is the MATLAB-side entry point. Ports with the other entries. + 2026-10-12: examined against 2f9b0eaf on + claude/lightsheet-zarr-ndi-viewer-djp5vk (was a7ba07a). A viewer + launch now goes through launchViewer (NDI_LIGHTSHEET_DEBUG=1, output + appended to lightsheet_YYYYMMDDHHMMSS.txt in NDI's temp folder), and + view gains 'debug' and 'logFile' options; --list mode is unchanged. + Nothing on the Python side to follow: the Python side is the + launched process, and it already honours NDI_LIGHTSHEET_DEBUG. + + - name: launchViewer + matlab_path: "+ndi/+fun/+doc/+lightsheet/launchViewer.m" + matlab_last_sync_hash: "2f9b0eaf" + status: matlab_only + decision_log: >- + MATLAB-side launch of the Python viewer: runs a viewCommand line + through SYSTEM with NDI_LIGHTSHEET_DEBUG=1 and stdout/stderr + appended to lightsheet_YYYYMMDDHHMMSS.txt in + ndi.common.PathConstants.TempFolder, so a viewer started from + MATLAB (no terminal) leaves a log. No Python counterpart: from + Python the viewer is called directly (napariViewLightsheet or + ndi.gui.app.lightsheetZarr.viewer.openPyramid) with its output on + the caller's own stderr, and the in-viewer log buffer + (ndi.gui.feedback) already keeps it for problem reports. + + - name: viewerLogFile + matlab_path: "+ndi/+fun/+doc/+lightsheet/viewerLogFile.m" + matlab_last_sync_hash: "2f9b0eaf" + status: matlab_only + decision_log: >- + Pure helper of launchViewer: the timestamped log path + lightsheet_YYYYMMDDHHMMSS.txt under NDI's temp folder. Not + needed in Python for the same reason as launchViewer -- nothing in + Python launches the viewer as a detached child that needs a log. + + - name: viewerShellCommand + matlab_path: "+ndi/+fun/+doc/+lightsheet/viewerShellCommand.m" + matlab_last_sync_hash: "2f9b0eaf" + status: matlab_only + decision_log: >- + Pure helper of launchViewer: the POSIX / cmd.exe line that sets + NDI_LIGHTSHEET_DEBUG, appends output to the log and detaches. Not + needed in Python for the same reason as launchViewer. diff --git a/src/ndi/gui/app/genepyramid/controls.py b/src/ndi/gui/app/genepyramid/controls.py index 6b2049ae..af27da93 100644 --- a/src/ndi/gui/app/genepyramid/controls.py +++ b/src/ndi/gui/app/genepyramid/controls.py @@ -32,6 +32,7 @@ "addCellTypePanel", "addAllPanels", "addCloudPanel", + "addReportPanel", "cloudSignInDialog", "buildCloudSignInDialog", "cloudProfileChoices", @@ -1827,7 +1828,7 @@ def cloudLogoLabel(parent=None): return label -def addCloudPanel(viewer) -> Any: +def addCloudPanel(viewer, report_collect=None) -> Any: """A clock on the cloud token, and a button that renews it. A VIEWER OUTLIVES ITS TOKEN. Someone reading a section works for @@ -1844,6 +1845,10 @@ def addCloudPanel(viewer) -> Any: and the time left. The credentials live behind the button, in a dialog, rather than as three permanently docked fields for something a reader does once a day at most. + + REPORT_COLLECT, when given, adds the shared "Report a problem..." + button (:func:`ndi.gui.feedback.reportButton`): this is where a reader + is signed in, which is what sending a report needs. """ from qtpy.QtCore import QTimer from qtpy.QtWidgets import QHBoxLayout, QLabel, QPushButton, QVBoxLayout, QWidget @@ -1876,6 +1881,8 @@ def addCloudPanel(viewer) -> Any: result.setWordWrap(True) result.hide() outer.addWidget(result) + if report_collect is not None: + outer.addWidget(_reportButton(box, report_collect)) outer.addStretch() def _tick(): @@ -1915,6 +1922,35 @@ def _open(): return box +def _reportButton(parent, report_collect): + from ...feedback import reportButton + + return reportButton( + parent, + app="genepyramid-viewer", + collect=report_collect, + title="Report a problem with the gene expression viewer", + ) + + +def addReportPanel(viewer, report_collect) -> Any: + """Just the "Report a problem..." button, for a session with no cloud + panel to carry it. + + A purely local section has no token and so no NDI Cloud panel, but its + reader can still hit a problem. Sending needs a sign-in the dialog + will ask for; Save to file and Copy work without one. + """ + from qtpy.QtWidgets import QVBoxLayout, QWidget + + box = QWidget() + outer = QVBoxLayout(box) + outer.addWidget(_reportButton(box, report_collect)) + outer.addStretch() + _dockScrollable(viewer, box, name="Feedback") + return box + + def cloudSignInDialog(parent=None): """Show the sign-in dialog and sign in. Returns ``(signed_in, note)``. @@ -2561,6 +2597,7 @@ def addAllPanels( shapes_layer=None, labelings=None, gene_layers=None, + report_collect=None, ): """Every panel the session supports, skipping what it has no data for. @@ -2616,7 +2653,16 @@ def _build(name, fn): # at a convenient moment instead of discovering it through a tile # that will not load. if cloudSessionLooksLikely(): - made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) + if report_collect is None: + made["cloud"] = _build("cloud", lambda: addCloudPanel(viewer)) + else: + made["cloud"] = _build( + "cloud", lambda: addCloudPanel(viewer, report_collect=report_collect) + ) + elif report_collect is not None: + # No cloud panel to carry the report button; a local reader can + # still hit a problem, so it gets a panel of its own. + made["report"] = _build("report", lambda: addReportPanel(viewer, report_collect)) made["display"] = _build( "display", lambda: addDisplayPanel(viewer, session, pyr_doc, image_layer, density) ) diff --git a/src/ndi/gui/app/genepyramid/progress.py b/src/ndi/gui/app/genepyramid/progress.py index 492389b5..0e75641c 100644 --- a/src/ndi/gui/app/genepyramid/progress.py +++ b/src/ndi/gui/app/genepyramid/progress.py @@ -166,6 +166,12 @@ def observer(event, _uri, done, total): _window = None _window_failed = False +#: Set once closeLaunchWindow has run. Without it, the first progress +#: call after the close (``note("viewer running ...")`` is one) goes +#: through ``_mode()`` -> ``_launchWindow()``, finds no window, and +#: builds a FRESH one stuck on "starting ..." that nothing ever closes. +_window_closed = False + def _mode() -> str: """Which indicator to use: ``"gui"``, ``"text"`` or ``"off"``.""" @@ -221,7 +227,7 @@ def _launchWindow(): harmless where they are already the default (Qt6 deprecates both). """ global _window, _window_failed - if _window is not None or _window_failed: + if _window is not None or _window_failed or _window_closed: return _window try: from qtpy.QtCore import Qt @@ -324,8 +330,9 @@ def closeLaunchWindow() -> None: a progress window that outlives the thing it was reporting on is worse than none, because it makes a finished launch look stuck. """ - global _window + global _window, _window_closed win, _window = _window, None + _window_closed = True if win is None: return with contextlib.suppress(Exception): diff --git a/src/ndi/gui/app/genepyramid/viewer.py b/src/ndi/gui/app/genepyramid/viewer.py index 1b56a4fc..f5e37f61 100644 --- a/src/ndi/gui/app/genepyramid/viewer.py +++ b/src/ndi/gui/app/genepyramid/viewer.py @@ -35,6 +35,50 @@ def require_napari(): return napari +def _reportCollector(viewer, session, pyr_doc, gene_rows, density): + """A zero-argument callable returning ``buildReport`` extras. + + The canvas and every layer (via :func:`ndi.gui.feedback.napariState`), + which section and gene selection is open, and the recent log -- what + a report needs that a reader cannot be expected to type. + """ + + def _docId(doc): + if doc is None: + return None + ident = getattr(doc, "id", None) + return str(ident() if callable(ident) else ident) + + def collect() -> dict: + from ...feedback import installLogBuffer, napariState + + state = napariState(viewer) + try: + state["section"] = { + "pyramid": _docId(pyr_doc), + "density": bool(density), + "gene_rows": None if gene_rows is None else len(list(gene_rows)), + } + except Exception as exc: # noqa: BLE001 + state["section"] = f"" + extra: dict[str, Any] = {"state": state, "documentId": _docId(pyr_doc)} + try: + checker = getattr(session, "is_in_cloud", None) + if callable(checker): + in_cloud, cloud_id = checker() + if in_cloud and cloud_id: + extra["datasetId"] = str(cloud_id) + except Exception: # noqa: BLE001 + pass + try: + extra["log"] = installLogBuffer().text() + except Exception: # noqa: BLE001 + pass + return {k: v for k, v in extra.items() if v is not None} + + return collect + + def openPyramid( session, pyr_doc, @@ -104,6 +148,16 @@ def openPyramid( """ from .progress import closeLaunchWindow, note, stage + # Keep the recent log in memory from launch on, so "Report a + # problem..." can attach it -- a viewer opened from MATLAB has no + # terminal to copy it from. + try: + from ...feedback import installLogBuffer + + installLogBuffer() + except Exception: # noqa: BLE001 - reporting is never worth the launch + pass + with stage("starting napari"): napari = require_napari() @@ -198,6 +252,7 @@ def openPyramid( shapes_layer=shapes, labelings=labelings, gene_layers=gene_layers, + report_collect=_reportCollector(viewer, session, pyr_doc, gene_rows, density), ) # The launch window has reported everything it can: the viewer is diff --git a/src/ndi/gui/app/lightsheetZarr/cli.py b/src/ndi/gui/app/lightsheetZarr/cli.py index 342f8049..11795667 100644 --- a/src/ndi/gui/app/lightsheetZarr/cli.py +++ b/src/ndi/gui/app/lightsheetZarr/cli.py @@ -252,8 +252,31 @@ def _pick_reduction(session: Any, pyramid_doc: Any, requested: str | None) -> st return available[0] +def _configure_logging() -> None: + """Set up INFO-level stderr logging when the root has no handlers. + + Without this, every ``logger.info(...)`` in ``ndi.cloud.*`` is silently + dropped -- the progress traces we rely on (async signed-URL-set-job + submit/wait/read, batch retry decisions, per-page walk timings) never + reach the terminal, and a stuck viewer looks identical to a healthy + one. Only configure when nothing else has: an embedding host (a + notebook, a downstream CLI) that has already set up logging keeps + control. + """ + import logging + + if logging.getLogger().handlers: + return + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stderr, + ) + + def main(argv: Sequence[str] | None = None) -> int: args = build_parser().parse_args(argv) + _configure_logging() try: session = _open_session(args.session) diff --git a/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml b/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml index 849e37ca..0e6b3a5e 100644 --- a/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/app/lightsheetZarr/ndi_matlab_python_bridge.yaml @@ -35,8 +35,7 @@ functions: # ----------------------------------------------------------------------- - name: LightsheetZarrManager matlab_path: "+ndi/+gui/+app/LightsheetZarrManager.m" - matlab_last_sync_hash: "60a79cf0" - matlab_branch: "claude/lightsheet-zarr-ndi-viewer-djp5vk" + matlab_last_sync_hash: "fb10169b" status: matlab_only decision_log: >- matlab_only: napari is Python-only, so the manager class stays @@ -53,6 +52,21 @@ functions: lightsheetZarrLevel documents rather than nonexistent parent fields. Still matlab_only; no Python counterpart to update. + 2026-10-12: examined against 2f9b0eaf (was 60a79cf0). This range + brings in the 32 MB tile-budget and main merges plus 2f9b0eaf, where + the View dialog launches through ndi.fun.doc.lightsheet.launchViewer + (NDI_LIGHTSHEET_DEBUG=1, output kept in + lightsheet_YYYYMMDDHHMMSS.txt in NDI's temp folder, path shown in + the status line) and the private detach helper is removed. Still + matlab_only; the launched Python viewer already honours + NDI_LIGHTSHEET_DEBUG. + + 2026-10-13: examined against fb10169b (was 2f9b0eaf). The range adds + the main merge (CRC32 file support) and fb10169b, where the View + dialog's command preview shows the exact logged shell line + (viewerShellCommand with the log named when the dialog opens) and + Launch reuses that log file. Still matlab_only. + # ----------------------------------------------------------------------- # CLI (napariViewLightsheet). # ----------------------------------------------------------------------- diff --git a/src/ndi/gui/app/lightsheetZarr/progress.py b/src/ndi/gui/app/lightsheetZarr/progress.py index c51eeb62..148732d3 100644 --- a/src/ndi/gui/app/lightsheetZarr/progress.py +++ b/src/ndi/gui/app/lightsheetZarr/progress.py @@ -188,6 +188,12 @@ def observer(event, _uri, done, total): _window = None _window_failed = False +#: Set once closeLaunchWindow has run. Without it, the first progress +#: call after the close (``note("viewer running ...")`` is one) goes +#: through ``_mode()`` -> ``_launchWindow()``, finds no window, and +#: builds a FRESH one stuck on "starting ..." that nothing ever closes. +_window_closed = False + def _mode() -> str: """Which indicator to use: ``"gui"``, ``"text"`` or ``"off"``.""" @@ -231,7 +237,7 @@ def _launchWindow(): QApplication ourselves, which napari then reuses. """ global _window, _window_failed - if _window is not None or _window_failed: + if _window is not None or _window_failed or _window_closed: return _window try: from qtpy.QtCore import Qt @@ -331,8 +337,9 @@ def closeLaunchWindow() -> None: ``processEvents`` at the end flushes whatever the close and destroy queued so the desktop actually repaints without us. """ - global _window + global _window, _window_closed win, _window = _window, None + _window_closed = True if win is None: return widget = win["widget"] diff --git a/src/ndi/gui/app/lightsheetZarr/refresh_hint.py b/src/ndi/gui/app/lightsheetZarr/refresh_hint.py index 87437206..c8a07325 100644 --- a/src/ndi/gui/app/lightsheetZarr/refresh_hint.py +++ b/src/ndi/gui/app/lightsheetZarr/refresh_hint.py @@ -9,116 +9,236 @@ finds the freshly-cached fine data. Kept in :mod:`ndi.gui.app.lightsheetZarr` (not in :mod:`ndi.pyramid`) -because the Qt dependency belongs on the viewer side. A -matplotlib-driven caller would build its own equivalent -- a -callback that redraws the current figure -- and the loader stays -uninvolved either way. +because the Qt dependency belongs on the viewer side. """ from __future__ import annotations import sys -import threading - - -class RefreshHint: - """Debounce ``layer.refresh()`` calls triggered by async fetches. - - An async fine-fetch fires this on completion. Many hundreds of - chunks in flight would swamp napari's slicer with refresh events, - so this collects them into one refresh per ``debounce_ms`` - window. - """ - - def __init__(self, viewer, layers, debounce_ms: int = 250): - self._viewer = viewer - self._layers = list(layers) - self._debounce_ms = debounce_ms - self._lock = threading.Lock() - self._timer = None - self._QTimer = None - try: - from qtpy.QtCore import QTimer - - self._QTimer = QTimer - except ImportError: # pragma: no cover - Qt required for napari - return - # The QTimer lives on the main thread; construct it lazily - # on first use so we do not touch Qt at import time. - - def __call__(self, _path=None) -> None: - """Called from the completion of an async fetch. Not on Qt thread.""" - if self._QTimer is None: - return - # QTimer.singleShot is safe from any thread; the callback - # runs on the Qt main thread. - try: - self._QTimer.singleShot(self._debounce_ms, self._fire) - except Exception: # noqa: BLE001 - pass - def _fire(self) -> None: - # napari 0.5 has three different ways to force a re-slice - # depending on whether the async slicer is on, and they do - # not overlap in every version. Try them in order of least - # invasive to most; whichever one napari actually reacts to - # is what makes the swap from coarse to fine visible. - # Under debug, print each attempt so we can see which one - # napari finally responded to. - from ndi.pyramid.upsample_fallback import _fallback_debug - - debug = _fallback_debug() - for layer in self._layers: - emitted = [] + +def _tryImportQt(): + """Return the Qt classes we need or ``None`` headlessly.""" + try: + from qtpy.QtCore import QEvent, QObject, QPoint, QPointF, Qt, Signal + from qtpy.QtCore import QTimer as _QTimer + from qtpy.QtGui import QWheelEvent + from qtpy.QtWidgets import QApplication + except ImportError: # pragma: no cover - Qt required for napari + return None + return { + "QObject": QObject, + "Qt": Qt, + "Signal": Signal, + "QTimer": _QTimer, + "QApplication": QApplication, + "QPoint": QPoint, + "QPointF": QPointF, + "QEvent": QEvent, + "QWheelEvent": QWheelEvent, + } + + +_qt = _tryImportQt() + + +if _qt is not None: + _QObject = _qt["QObject"] + _Qt = _qt["Qt"] + _Signal = _qt["Signal"] + _QTimer = _qt["QTimer"] + _QApplication = _qt["QApplication"] + _QPoint = _qt["QPoint"] + _QPointF = _qt["QPointF"] + _QWheelEvent = _qt["QWheelEvent"] + + class _RefreshHintImpl(_QObject): + """QObject that marshals fetch completions into a main-thread refresh. + + The ``_request`` signal is wired to ``_onRequest`` with + ``Qt.QueuedConnection``, so emitting it from any background + thread posts a queued event to the main-thread event loop. + ``_onRequest`` (re)starts a single debounce timer; when the + burst of fetch completions quiets down for ``debounce_ms``, + the timer fires ``_fire`` once. + + ``_fire`` has to do more than ``layer.refresh()``: observed + on this user's napari 0.9.1 + PyQt6, calling ``refresh()`` or + emitting ``events.set_data()`` -- or even reassigning + ``viewer.camera.zoom`` from Python -- does NOT trigger the + async slicer to re-run against the newly-cached fine chunks. + Only an actual mouse wheel event over the canvas does. So we + post a synthetic ``QWheelEvent`` to the vispy canvas widget, + which is the one Qt path we have proof works on this build. + """ + + _request = _Signal() + + def __init__(self, viewer, layers, debounce_ms: int): + super().__init__() + self._viewer = viewer + self._layers = list(layers) + self._debounce_ms = debounce_ms + app = _QApplication.instance() + if app is not None: + try: + self.moveToThread(app.thread()) + except Exception: # noqa: BLE001 + pass + # One timer, reused: QTimer.start() restarts it, so + # a burst of __call__s collapses into a single _fire. + self._timer = None # built lazily on main thread + self._request.connect(self._onRequest, _Qt.QueuedConnection) + + # Entry point from any thread (fetch worker calls us). + def __call__(self, _path=None) -> None: + from ndi.pyramid.upsample_fallback import bump as _bump_stat + + _bump_stat("refresh_hints_called") try: - # 1. Public API: refresh() -- best case, napari - # re-slices from current view. Some versions only - # redraw the cached slice, which is why we do more. - layer.refresh() - emitted.append("refresh") - except Exception: # noqa: BLE001 - pass + self._request.emit() + _bump_stat("refresh_hints_scheduled") + except Exception as exc: # noqa: BLE001 + print( + f"[lightsheet] refresh-hint emit failed: " f"{type(exc).__name__}: {exc}", + file=sys.stderr, + flush=True, + ) + + def _onRequest(self) -> None: + # On the Qt main thread now; safe to touch QTimer. + if self._timer is None: + self._timer = _QTimer(self) + self._timer.setSingleShot(True) + self._timer.timeout.connect(self._fire) + # Restart the debounce window. If a hundred fetches + # complete in 50 ms, this gets restarted a hundred + # times and _fire runs exactly ONCE, ~250 ms after the + # last one. + self._timer.start(self._debounce_ms) + + def _fire(self) -> None: + from ndi.pyramid.upsample_fallback import _fallback_debug + from ndi.pyramid.upsample_fallback import bump as _bump_stat + + _bump_stat("refresh_hints_fired") + debug = _fallback_debug() + + # 1. The in-API calls. Cheap, no-op on this user's + # napari but harmless and the right thing to do on + # napari builds where it works. + for layer in self._layers: + try: + layer.refresh() + except Exception: # noqa: BLE001 + pass + try: + events = getattr(layer, "events", None) + if events is not None: + set_data = getattr(events, "set_data", None) + if set_data is not None: + set_data() + except Exception: # noqa: BLE001 + pass + + # 2. The real nudge: post a synthetic QWheelEvent to + # the vispy canvas. The user's log shows programmatic + # camera.zoom writes don't re-slice but real mouse + # wheels do, so we send what the kernel-level wheel + # handler would post. Two notches cancel out so the + # zoom level does not visibly change. + posted = self._postCancellingWheelPair() + + if debug: + print( + f"[lightsheet] refresh-hint fired: " + f"refresh()+set_data() on {len(self._layers)} layer(s), " + f"synth-wheel posted={'yes' if posted else 'no'}", + file=sys.stderr, + flush=True, + ) + + def _findCanvasWidget(self): + """Return the Qt widget to post wheel events to, or None. + + napari's internal layout changes between versions; try a + few known attribute paths and fall back to the viewer + window's central widget. + """ + viewer = self._viewer try: - # 2. Fire the set_data event by hand. napari's - # async slicer listens to this; it is what - # `layer.data = layer.data` would emit, without - # re-assigning the array. - events = getattr(layer, "events", None) - if events is not None: - set_data = getattr(events, "set_data", None) - if set_data is not None: - set_data() - emitted.append("events.set_data") + window = getattr(viewer, "window", None) + qt_viewer = getattr(window, "_qt_viewer", None) or getattr( + window, "qt_viewer", None + ) + if qt_viewer is None: + return None + canvas = getattr(qt_viewer, "canvas", None) + if canvas is None: + return None + native = getattr(canvas, "native", None) or getattr(canvas, "_backend", None) + if native is None: + return canvas # may itself be a QWidget + return native except Exception: # noqa: BLE001 - pass + return None + + def _postCancellingWheelPair(self) -> bool: + """Post wheel-up then wheel-down to the canvas. + + Returns True if events were posted. The two notches + cancel so the user does not see a zoom flicker, but + napari's view machinery sees two real camera events + and re-slices accordingly. + """ + widget = self._findCanvasWidget() + if widget is None: + return False try: - # 3. Private force-reload -- present on napari - # >= 0.4.18 to invalidate the slice cache and - # trigger a fresh compute. - reload = getattr(layer, "reload", None) or getattr(layer, "_reload_async", None) - if callable(reload): - reload() - emitted.append("reload") - except Exception: # noqa: BLE001 - pass - if debug and emitted: + app = _QApplication.instance() + w = widget.width() if hasattr(widget, "width") else 400 + h = widget.height() if hasattr(widget, "height") else 300 + pos_f = _QPointF(w / 2, h / 2) + global_pos = widget.mapToGlobal(pos_f.toPoint()) + for dy in (120, -120): + ev = _QWheelEvent( + pos_f, + _QPointF(global_pos), + _QPoint(0, 0), + _QPoint(0, dy), + _Qt.NoButton, + _Qt.NoModifier, + _Qt.NoScrollPhase, + False, + ) + app.postEvent(widget, ev) + return True + except Exception as exc: # noqa: BLE001 print( - f"[lightsheet] refresh-hint fired on {getattr(layer, 'name', '?')!r}: " - f"{', '.join(emitted)}", + f"[lightsheet] synth-wheel post failed: " f"{type(exc).__name__}: {exc}", file=sys.stderr, flush=True, ) + return False + + RefreshHint = _RefreshHintImpl + +else: + + class RefreshHint: # type: ignore[no-redef] + """Headless no-op shim for environments without Qt.""" + + def __init__(self, *args, **kwargs): + pass + def __call__(self, _path=None) -> None: + return None -def refreshHintFor(viewer, layers, debounce_ms: int = 250) -> RefreshHint | None: - """Factory: build a debounced refresh hint or None when Qt is missing. - Returns None when the layers list is empty (nothing to refresh) - or when qtpy cannot be imported (headless test, no display). - """ +def refreshHintFor(viewer, layers, debounce_ms: int = 250): + """Build a debounced refresh hint, or None when Qt is unavailable.""" if not layers: return None - hint = RefreshHint(viewer, layers, debounce_ms=debounce_ms) - if hint._QTimer is None: + if _qt is None: return None - return hint + return RefreshHint(viewer, layers, debounce_ms=debounce_ms) diff --git a/src/ndi/gui/app/lightsheetZarr/viewer.py b/src/ndi/gui/app/lightsheetZarr/viewer.py index 7d9599f5..58552a63 100644 --- a/src/ndi/gui/app/lightsheetZarr/viewer.py +++ b/src/ndi/gui/app/lightsheetZarr/viewer.py @@ -18,33 +18,148 @@ import sys import threading import time +from datetime import datetime from typing import Any +def _ts() -> str: + """Return the current wall clock as ``HH:MM:SS.mmm``. + + A wall-clock timestamp on every ``[lightsheet]`` line lets a user + quantify the actual UX -- how long the signed-URL job took, how + long between the first tile request and the first tile back, how + long a level swap sat on a black screen -- without cross-referencing + the ``ndi.cloud.*`` INFO lines that already carry timestamps. + + Sub-second precision matters when a tile fetch finishes in + ~0.5-2 s: a second-only stamp collapses adjacent lines into the + same second and makes progression look stepped. + """ + return datetime.now().strftime("%H:%M:%S.%f")[:-3] + + +def _ls(msg: str, *, file=None) -> None: + """Print ``[HH:MM:SS.mmm lightsheet] msg`` to stderr, flushed. + + Named so a caller can write ``_ls(f"tiles: {n}/{m}")`` instead of + building the full prefix by hand. Pass ``file=`` to route to a + stream other than stderr (the fetch counter uses this). + """ + print(f"{_ts()} [lightsheet] {msg}", file=file or sys.stderr, flush=True) + + +def _cloud_dataset_id(session: Any) -> str: + """Discover the remote NDI Cloud dataset id from an opened session. + + ``_open_session`` in ``cli.py`` may hand back either an + ``ndi.session.dir`` or an ``ndi.dataset.dir``; only the dataset + exposes ``is_in_cloud()``, which returns ``(in_cloud, id)`` off + of the ``dataset_remote`` document written the first time the + dataset was uploaded. Everything else (plain session, non-cloud + open, older reader) reads as ``""`` -- no cloud context, no + prefetch to do. + """ + checker = getattr(session, "is_in_cloud", None) + if not callable(checker): + return "" + try: + in_cloud, cloud_id = checker() + except Exception: # noqa: BLE001 - a bad probe is never fatal + return "" + if not in_cloud: + return "" + return str(cloud_id or "") + + class _NapariStatusReporter: - """Push a tile-loading status message into napari's status bar. - - Called from the fetch counter's main-thread-safe surfaces. Napari's - status bar is a Qt widget, so a background thread calling - ``viewer.status = ...`` would need main-thread dispatch; the - counter's ``_maybe_print`` runs on whichever thread fired the - fetch event. Rather than adding QThread machinery here, we keep - a reference to the viewer and to the last string, and only - push on the main thread via a QTimer.singleShot(0, ...). + """Push a tile-loading status message into napari's status bar + and -- when fetches are in flight -- a large overlay on the canvas. + + Status bar alone was easy to miss: on the user's zoom-and-wait + cases, nothing in the canvas said "loading", so they read a + coarse-upsampled placeholder as "this is the final image." The + overlay is big and lives on top of the picture, so a fetch in + flight is unambiguous. + + Cross-thread safe: push() is called from the fetch counter, which + runs on worker threads (the heartbeat, the fetch handlers). + Napari's status bar and text overlay are Qt widgets and must only + be written from the main thread. A QTimer.singleShot started from + a worker thread has no event dispatcher there and does nothing + useful (and spams "QBasicTimer::start: current thread's event + dispatcher has already been destroyed"), so we use a Qt signal: + signals emitted from a worker thread cross to the owning thread + via a queued connection, which is exactly the main-thread + delivery we need. The signal lives on a QObject (``_Bridge``) + moved to the main thread at construction time. """ def __init__(self, viewer): self._viewer = viewer + self._overlay_configured = False + self._bridge = None + try: + from qtpy.QtCore import QCoreApplication, QObject, Qt, Signal + + class _Bridge(QObject): + update = Signal(str, str) + + def __init__(self, outer): + super().__init__() + self._outer = outer + self.update.connect(self._on_update, Qt.QueuedConnection) + + def _on_update(self, message, overlay_text): + self._outer._apply(message, overlay_text) + + self._bridge = _Bridge(self) + app = QCoreApplication.instance() + if app is not None: + self._bridge.moveToThread(app.thread()) + except Exception: + self._bridge = None - def push(self, message: str) -> None: - # QTimer.singleShot is thread-safe; the callback runs on the - # Qt main thread, which is what viewer.status expects. + def _apply(self, message: str, overlay_text: str) -> None: + """Runs on the main thread (queued-signal callback).""" + v = self._viewer try: - from qtpy.QtCore import QTimer - except ImportError: + v.status = message + except Exception: + pass + overlay = getattr(v, "text_overlay", None) + if overlay is None: return - v = self._viewer - QTimer.singleShot(0, lambda: setattr(v, "status", message)) + if not self._overlay_configured: + try: + overlay.font_size = 18 + overlay.position = "top_left" + try: + overlay.color = "yellow" + except Exception: + pass + self._overlay_configured = True + except Exception: + pass + try: + if overlay_text: + overlay.text = overlay_text + overlay.visible = True + else: + overlay.visible = False + except Exception: + pass + + def push(self, message: str, *, overlay: str = "") -> None: + """Called from any thread; delivery hops to main via a signal. + + OVERLAY is the canvas text; empty hides the overlay. + """ + if self._bridge is None: + return + try: + self._bridge.update.emit(message, str(overlay or "")) + except Exception: + pass class _FetchCounter: @@ -78,12 +193,15 @@ class _FetchCounter: def __init__( self, - out=sys.stderr, + out=None, interval: float = 0.5, idle_interval: float = 5.0, status_reporter=None, ): self._lock = threading.Lock() + # None: resolve sys.stderr at print time (see _ls), so a log tee + # installed after import -- the feedback report's log buffer -- + # still sees the counter's lines. self._out = out self._interval = interval self._idle_interval = idle_interval @@ -99,6 +217,12 @@ def __init__( self._last_duration: float | None = None self._last_bytes: int | None = None self._session_start = time.monotonic() + # Tiles the CURRENT VIEW needs vs has on disk, set from the + # main-thread poll in viewtiles.attachViewTally. None until the + # first poll, or when napari does not expose enough to tally. + self._view_ready: int | None = None + self._view_needed: int | None = None + self._view_since: float | None = None self._stop = threading.Event() self._heartbeat = threading.Thread( target=self._heartbeat_loop, @@ -118,6 +242,29 @@ def set_status_reporter(self, reporter) -> None: with self._lock: self._status_reporter = reporter + def set_view_tally(self, ready: int | None, needed: int | None) -> None: + """Record how many of the current view's tiles are on disk. + + Logs once when a view finishes loading, with how long it took + from the first poll that found it incomplete. + """ + with self._lock: + now = time.monotonic() + was_complete = self._view_needed is not None and self._view_ready == self._view_needed + self._view_ready, self._view_needed = ready, needed + if ready is None or needed is None: + self._view_since = None + elif ready < needed: + if self._view_since is None or was_complete: + self._view_since = now + elif self._view_since is not None: + _ls( + f"view complete: {needed} tile(s) in {now - self._view_since:.1f}s", + file=self._out, + ) + self._view_since = None + self._maybe_print(force=True) + def __call__(self, event, uri, done, total): with self._lock: now = time.monotonic() @@ -153,30 +300,71 @@ def _maybe_print(self, force: bool = False) -> None: if self._last_duration is not None: mb = (self._last_bytes or 0) / (1024.0 * 1024.0) last = f"; last fetch {self._last_duration:.2f}s, {mb:.1f} MB" - stderr_line = ( - f"[lightsheet] tiles: {self._done} loaded / {self._started} requested " - f"(in flight: {in_flight}){last}" + ready, needed = self._view_ready, self._view_needed + view = "" + if needed is not None: + view = f"view: {ready}/{needed} tiles ready; " + _ls( + f"{view}tiles: {self._done} loaded / {self._started} requested " + f"(in flight: {in_flight}){last}", + file=self._out, ) - print(stderr_line, file=self._out, flush=True) if self._status_reporter is not None: - status_msg = ( - f"lightsheet: {self._done}/{self._started} tiles" + parts = [] + if needed is not None: + parts.append(f"view {ready}/{needed} tiles ready") + parts.append( + f"{self._done}/{self._started} fetched" + (f" ({in_flight} in flight)" if in_flight else "") - + (f" | last {self._last_duration:.2f}s" if self._last_duration else "") ) - self._status_reporter.push(status_msg) + if self._last_duration: + parts.append(f"last {self._last_duration:.2f}s") + status_msg = "lightsheet: " + " | ".join(parts) + if needed is not None and ready < needed: + overlay = f"Loading view: {ready} / {needed} tiles" + if in_flight: + overlay += f" ({in_flight} in flight)" + elif in_flight: + # View is complete (or untallied); what is still + # moving is prefetch or a neighbouring region. + overlay = f"Loading tiles ... ({in_flight} in flight)" + else: + overlay = "" + self._status_reporter.push(status_msg, overlay=overlay) def _heartbeat_loop(self) -> None: while not self._stop.wait(self._idle_interval): with self._lock: idle = time.monotonic() - self._last_activity if self._started == 0: - print( - "[lightsheet] no tiles requested yet by napari; " - "waiting for first slice ...", - file=self._out, - flush=True, - ) + # _started counts cloud HTTPS fetches via + # watchFetches. On a local-disk pyramid that's + # permanently zero -- but the slicer may well be + # running against the local-disk fallback reader. + # Check that activity before claiming the slicer + # is dead, so we don't tell the user "no tiles + # requested" when they're staring at ~200 + # fallback-reader calls per minute. + try: + from ndi.pyramid.upsample_fallback import ( + totalReaderActivity as _tra, + ) + + fallback_n = _tra() + except Exception: # noqa: BLE001 + fallback_n = 0 + if fallback_n == 0: + _ls( + "no tiles requested yet by napari; waiting for first slice ...", + file=self._out, + ) + else: + _ls( + f"local-disk slicer active: {fallback_n} fallback-reader call(s). " + "(No cloud fetches on a local pyramid, so the fetch counter stays " + "at zero -- that is normal here.)", + file=self._out, + ) elif self._done < self._started and idle >= self._idle_interval: self._maybe_print(force=True) @@ -192,14 +380,13 @@ def stop(self) -> None: # slice. Whichever it is, silence would hide it. started = self._started in_flight = self._started - self._done - print( - f"[lightsheet] fetch summary: 0 tiles fetched in {wall:.1f}s " + _ls( + f"fetch summary: 0 tiles fetched in {wall:.1f}s " f"(started={started}, in_flight={in_flight}). " "napari either drew only sparse/fill regions, or the async " "slicer never dispatched -- try dragging the Z slider or " "zooming in to force a slice compute.", file=self._out, - flush=True, ) return n = len(self._durations) @@ -210,15 +397,45 @@ def stop(self) -> None: mean_mb = (total_bytes / n) / (1024.0 * 1024.0) total_mb = total_bytes / (1024.0 * 1024.0) throughput = (total_bytes / wall) / (1024.0 * 1024.0) if wall > 0 else 0.0 - print( - f"[lightsheet] fetch summary: {n} tiles, {total_mb:.1f} MB total, " + _ls( + f"fetch summary: {n} tiles, {total_mb:.1f} MB total, " f"{wall:.1f}s wall, mean {mean_dt:.2f}s/tile ({mean_mb:.1f} MB), " f"min {min_dt:.2f}s, max {max_dt:.2f}s, throughput {throughput:.1f} MB/s", file=self._out, - flush=True, ) +def _quiet_stale_chunk_errors() -> None: + """Drop napari error notifications caused by StaleChunkRead. + + Idempotent; any napari that does not route slice errors through + ``notification_manager.receive_error`` is left alone (the release + then shows as a notification, which is noisy but harmless). + """ + try: + from napari.utils.notifications import notification_manager + + from ndi.pyramid.multiscale import StaleChunkRead + except Exception: # noqa: BLE001 + return + original = notification_manager.receive_error + if getattr(original, "_ndi_quiet_stale", False): + return + + def receive_error(exc_type, exc_value, tb, *args, **kwargs): + e = exc_value + seen = 0 + while e is not None and seen < 10: + if isinstance(e, StaleChunkRead): + return None + e = e.__cause__ or e.__context__ + seen += 1 + return original(exc_type, exc_value, tb, *args, **kwargs) + + receive_error._ndi_quiet_stale = True # type: ignore[attr-defined] + notification_manager.receive_error = receive_error + + def require_napari(): """Return the napari module or raise a helpful ImportError.""" try: @@ -317,6 +534,16 @@ def openPyramid( # picks GUI on macOS/Windows and where a display is set. from . import progress + # Keep the last few thousand log lines in memory from here on, so a + # "Report a problem" from the NDI Cloud panel can attach them -- a + # viewer launched from MATLAB has no terminal to copy them from. + try: + from ndi.gui.feedback import installLogBuffer + + installLogBuffer() + except Exception: # noqa: BLE001 - reporting is never worth the launch + pass + progress.note("opening lightsheet pyramid ...") napari = require_napari() @@ -351,9 +578,26 @@ def openPyramid( specs = loader.specs fetcher = loader.fetcher - # Warm the worker pool now, off the main thread. The one-off - # session-open cost then overlaps with napari.Viewer()'s Qt startup - # rather than serialising against the first fetch. + # napari reads through dask, whose thread pool defaults to one thread + # per CPU -- sized for computation, but a chunk read is seconds of + # waiting on the network. Widen it to the fetcher's worker count so + # napari can keep every worker busy. + try: + from ndi.pyramid.multiscale import configureDaskPool + + n_pool = configureDaskPool(getattr(fetcher, "maxWorkers", 1)) + print( + f"[lightsheet] fetch pool: {getattr(fetcher, 'workerLimit', 1)} tiles in " + f"flight (up to {getattr(fetcher, 'maxWorkers', 1)}), dask pool {n_pool} threads", + file=sys.stderr, + flush=True, + ) + except Exception as exc: # noqa: BLE001 - the default pool still works + print(f"[lightsheet] could not widen the dask pool ({exc})", file=sys.stderr) + + # Start the worker threads now, off the main thread, so the first + # fetch does not pay for it. Workers open no session up front: chunks + # resolve session-free, and a worker opens one only if it must. with progress.stage("connecting to image store"): try: fetcher.warm() @@ -374,9 +618,39 @@ def openPyramid( flush=True, ) + # Eagerly warm the signed-URL cache for every level's chunk.bin + # series. Each async signed-URL-set job takes 20-80 s server-side, + # so a 5-level pyramid otherwise pays that wall time the first + # time the user zooms into each level; running the jobs in the + # background while level 0 renders hides those waits behind the + # moment the user is already looking at level 0. + try: + cloud_dataset_id = _cloud_dataset_id(session) + if cloud_dataset_id: + loader.startSignedUrlPrefetch(cloud_dataset_id) + else: + print( + "[lightsheet] signed-URL prefetch skipped: no cloud dataset id " + "on this session (local-only open?)", + file=sys.stderr, + flush=True, + ) + except Exception as exc: # noqa: BLE001 - a prefetch failure is never fatal + print( + f"[lightsheet] signed-URL prefetch: could not start ({exc})", + file=sys.stderr, + flush=True, + ) + with progress.stage("opening image viewer"): viewer = napari.Viewer() + # A slice for a view the user has zoomed past is released with + # StaleChunkRead (see _ChunkFetcher.setView) so napari's one slicing + # thread can move on to the current view. napari reports a failed + # slice as an error notification; this one is deliberate. + _quiet_stale_chunk_errors() + # Now that a viewer exists, wire the counter to napari's status # bar so users see tile progress inside the viewer window rather # than having to watch the terminal. @@ -395,6 +669,49 @@ def openPyramid( for spec in specs: added_layers.append(viewer.add_image(**spec)) + # Set physical units on the dimension sliders. Without this napari + # labels the slider in "pixels" and reports cursor position in + # voxels, so switching multiscale levels appears to the user as + # the slider range changing underneath them. With world units set, + # the slider and status bar read microns (or whatever the level + # declares) and stay stable across level swaps. Pulled from the + # pyramid's finest level since every level shares the same + # axes_order and the same world units. + try: + level0_props = loader.docs[0].document_properties["lightsheetZarrLevel"] + axes = str(level0_props.get("axes_order", "tczyx")) + unit_str = str(level0_props.get("voxel_size_units", "micrometer")) + except Exception: # noqa: BLE001 - fall back to the napari default on any shape surprise + axes = "" + unit_str = "" + if axes: + dim_labels = tuple(axes) + try: + viewer.dims.axis_labels = dim_labels + except Exception: + pass + # viewer.dims.units wants pint.Unit instances in current napari + # (strings get rejected by the pydantic validator). Build them + # via pint when it is installed; skip units silently otherwise + # -- labels are already set and are the main thing users read. + # Spatial axes get the pyramid's unit, non-spatial (c, t) stay + # dimensionless. + try: + import pint + + ureg = pint.UnitRegistry() + spatial_unit = ureg.Unit(unit_str) if unit_str else ureg.Unit("") + dim_units = tuple( + ureg.Unit("") if ax.lower() in ("c", "t") else spatial_unit for ax in axes + ) + viewer.dims.units = dim_units + except Exception as exc: # noqa: BLE001 - a units set failure is never fatal + print( + f"[lightsheet] viewer.dims.units set skipped ({exc!s}); labels only.", + file=sys.stderr, + flush=True, + ) + # Install the debounced napari refresh hint via the loader's # public hook. Every async fine-chunk fetch that completes calls # this hint; the hint coalesces a burst of arrivals into one @@ -529,6 +846,74 @@ def openPyramid( # all and we lose only the extra line. _subscribe_level_change(viewer) + # Camera-nudge workaround for the vispy/napari slicer wedge + # observed on macOS: after launch, zoom events sometimes do not + # trigger a re-slice, so a user has to zoom-out-then-in several + # times before napari asks the fetcher for finer tiles. Call + # layer.refresh() ourselves on every camera event (debounced), + # which forces napari's slicer to compile a new slice request. + # Observed to help only partially on the Maddie dataset; the + # explicit Refresh View button below is the user-facing lever. + # Silent no-op if the camera doesn't expose the expected events + # (older napari) or if NDI_LIGHTSHEET_NUDGE=0 is set. + _attach_camera_nudge(viewer, added_layers) + + # Manual Refresh View button in a right-dock widget. Toggles + # layer.visible off/on and calls refresh() -- the sequence the + # user discovered unsticks napari's slicer when zoom alone does + # not. Users click this when regions still show coarse-level + # data after a zoom-in. Hidden-by-default via + # NDI_LIGHTSHEET_REFRESH_BUTTON=0 for a scripted viewer where + # the panel chrome is unwanted. + _attach_refresh_button(viewer, added_layers) + + # NDI Cloud sign-in panel, same shape as the gene-pyramid + # viewer uses. Shown only when a cloud token is in the + # environment -- a purely local pyramid has no reason for a + # login control. The lightsheet viewer carries a "beta" badge + # under the wordmark because this surface is still rough on + # the dataset-shape edges called out in the README. + _attach_cloud_panel( + viewer, + fetcher=fetcher, + report_collect=_reportCollector( + viewer, session, pyramid_doc, loader, counter, added_layers + ), + ) + + # Tiles needed for the current view vs tiles on disk, polled on a + # main-thread timer and fed to the counter, so the status bar and + # overlay say "37 / 120" rather than only "12 in flight". Skipped + # in the modes where layer.data is not the full level ladder + # (single-level, viewport clip): there the level index napari + # reports does not line up with the level documents. + # Each env var is read with the parser its own feature uses. + from ndi.gui.app.lightsheetZarr.viewport import _env_true as _clip_env_true + from ndi.pyramid.multiscale import _env_true as _level_env_true + + if not ( + _level_env_true("NDI_LIGHTSHEET_SINGLE_LEVEL") + or _clip_env_true("NDI_LIGHTSHEET_VIEWPORT_CLIP") + ): + try: + from ndi.gui.app.lightsheetZarr.viewtiles import attachViewTally + + view_timer = attachViewTally( # noqa: F841 - parented to the window + viewer, added_layers, loader.levelMeta, fetcher, counter + ) + except Exception as exc: # noqa: BLE001 - a tally is never worth the viewer + print( + f"[lightsheet] view tile tally: could not start ({exc})", + file=sys.stderr, + flush=True, + ) + else: + print( + "[lightsheet] view tile tally: off in single-level / viewport-clip mode", + file=sys.stderr, + flush=True, + ) + # Launch window is closed BEFORE napari.run() because napari's # event loop blocks the main thread and our Qt window can't # repaint during it -- a frozen progress bar next to napari looks @@ -621,9 +1006,592 @@ def _report_loaded(layer) -> None: ) +# Weak reference holder so the nudge QTimer doesn't get garbage +# collected the moment _attach_camera_nudge returns. Keyed by viewer +# id so a second viewer in the same process gets its own timer. The +# timer is Qt-owned once started, but Python GC can still cut it loose +# if nothing holds a reference. +_NUDGE_TIMERS: dict[int, Any] = {} + + +def _attach_camera_nudge(viewer, layers) -> None: + """Force a layer.refresh() on every camera event (debounced). + + Workaround for a vispy/napari slicer wedge observed on macOS: after + launch, zoom and pan events sometimes do not trigger a re-slice, so + napari sits on a stale frame while the user zooms. The symptom in + the fetch log is "no tiles requested yet by napari" that never + clears, with vispy spamming "QBasicTimer::start: current thread's + event dispatcher has already been destroyed" from the slicer. + Calling layer.refresh() ourselves forces napari to compile a new + slice request on the main thread, which does fire, so a single + zoom gesture becomes enough to kick a level swap. + + Debounced 120 ms so a pinch-zoom or wheel-zoom settling through + many micro-events refreshes once at rest rather than per event. + Silent no-op when the camera doesn't expose the expected events + (older napari) or when ``NDI_LIGHTSHEET_NUDGE=0`` turns it off. + """ + if not layers: + return + if os.environ.get("NDI_LIGHTSHEET_NUDGE", "1").strip().lower() in ("0", "false", "off"): + print( + "[lightsheet] camera-nudge: disabled via NDI_LIGHTSHEET_NUDGE", + file=sys.stderr, + flush=True, + ) + return + + try: + from qtpy.QtCore import QTimer + except ImportError: + return + + try: + timer = QTimer() + timer.setSingleShot(True) + timer.setInterval(120) + except Exception: + return + + def _do_refresh(): + for layer in layers: + try: + layer.refresh() + except Exception: + pass + + try: + timer.timeout.connect(_do_refresh) + except Exception: + return + + def _on_camera(_event): + try: + timer.start() + except Exception: + pass + + camera = getattr(viewer, "camera", None) + events = getattr(camera, "events", None) + if events is None: + return + hooked = [] + for name in ("zoom", "center", "angles"): + emitter = getattr(events, name, None) + if emitter is None: + continue + try: + emitter.connect(_on_camera) + hooked.append(name) + except Exception: + pass + if not hooked: + return + + # Keep the timer (and its slot connections) alive for the viewer's + # lifetime; a plain local goes out of scope at return and Qt + # cannot keep a dangling PyQt QTimer running without a reference. + _NUDGE_TIMERS[id(viewer)] = timer + + print( + f"[lightsheet] camera-nudge: on, hooked camera.events.{'/'.join(hooked)} " + "(set NDI_LIGHTSHEET_NUDGE=0 to disable)", + file=sys.stderr, + flush=True, + ) + + +# Weak reference holder so the refresh-button widget doesn't get GC'd +# the moment _attach_refresh_button returns. Qt owns the dock widget +# once docked, but the magicgui wrapper is a Python object and will +# be collected without this. +_REFRESH_BUTTONS: dict[int, Any] = {} + + +def _attach_refresh_button(viewer, layers) -> None: + """Dock a Refresh View button on the right. + + Users click it when the view shows coarse-level data in regions + that should have refined after a zoom-in. The handler zooms the + camera out by 20% and then back in 100 ms later -- the sequence + a user discovered unsticks napari's slicer on the Maddie dataset + (toggling layer.visible or calling layer.refresh() alone does + not; only a real camera event does). A visibility toggle still + runs as a secondary nudge in case the camera change is coalesced + out. + + Hidden via NDI_LIGHTSHEET_REFRESH_BUTTON=0 for scripted / headless + viewers. Falls back to a silent no-op if magicgui isn't installed. + """ + if not layers: + return + if os.environ.get("NDI_LIGHTSHEET_REFRESH_BUTTON", "1").strip().lower() in ( + "0", + "false", + "off", + ): + return + + try: + from magicgui.widgets import Container, PushButton + except ImportError: + print( + "[lightsheet] refresh button: magicgui not installed; skipping " + "(pip install magicgui)", + file=sys.stderr, + flush=True, + ) + return + + try: + from qtpy.QtCore import QTimer + except ImportError: + QTimer = None # type: ignore[assignment] + + def _do_refresh(): + import time as _time + + t0 = _time.monotonic() + + # Phase 1: zoom out by 20%. This is the piece that actually + # unsticks the slicer -- a real camera event forces napari + # to re-slice. The 20% factor is big enough that napari + # doesn't coalesce it with the "set it back" that follows, + # and small enough that it looks like a brief flash on screen + # rather than a jarring zoom out. + orig_zoom = None + try: + orig_zoom = float(viewer.camera.zoom) + viewer.camera.zoom = orig_zoom * 0.8 + except Exception as exc: # noqa: BLE001 + print( + f"[lightsheet] refresh view: zoom-out failed ({exc})", + file=sys.stderr, + flush=True, + ) + + # Phase 2 (100 ms later): zoom back in, and -- as a belt-and- + # suspenders fallback -- toggle layer.visible off then on. + # The hide/show pair tears down and rebuilds each layer's + # vispy node, which is what the manual workaround users + # discovered first did. + def _restore_and_toggle(): + if orig_zoom is not None: + try: + viewer.camera.zoom = orig_zoom + except Exception: + pass + hidden: list = [] + for layer in layers: + try: + if getattr(layer, "visible", False): + layer.visible = False + hidden.append(layer) + except Exception: + pass + + def _unhide(): + for layer in hidden: + try: + layer.visible = True + except Exception: + pass + for layer in layers: + try: + layer.refresh() + except Exception: + pass + dt = _time.monotonic() - t0 + print( + f"[lightsheet] refresh view: zoom-nudge + toggled " + f"{len(hidden)} layer(s), took {dt * 1000:.0f} ms", + file=sys.stderr, + flush=True, + ) + + if QTimer is not None: + QTimer.singleShot(50, _unhide) + else: + _unhide() + + if QTimer is not None: + QTimer.singleShot(100, _restore_and_toggle) + else: + _restore_and_toggle() + + try: + button = PushButton(text="Refresh view") + button.clicked.connect(_do_refresh) + container = Container(widgets=[button], labels=False) + except Exception as exc: + print( + f"[lightsheet] refresh button: construction failed ({exc})", + file=sys.stderr, + flush=True, + ) + return + + try: + viewer.window.add_dock_widget(container, area="right", name="Refresh") + except Exception as exc: + print( + f"[lightsheet] refresh button: could not dock ({exc})", + file=sys.stderr, + flush=True, + ) + return + + _REFRESH_BUTTONS[id(viewer)] = container + print( + "[lightsheet] refresh button: docked (click when regions still show " + "coarse data after a zoom)", + file=sys.stderr, + flush=True, + ) + + +# Weak reference holder for the cloud panel widget. Same reason as +# the refresh button: Qt owns the dock once added, but the Python +# wrapper and its slot connections need a strong ref. +_CLOUD_PANELS: dict[int, Any] = {} + + +def _reportCollector(viewer, session, pyramid_doc, loader, counter, layers): + """A zero-argument callable returning ``buildReport`` extras. + + Everything a "the view is stuck / slow / wrong" report needs that + the user cannot be expected to type: which pyramid, which level and + region napari is drawing, how far the view's tiles have got, what + the fetcher has done, and the recent log. Each probe is guarded on + its own: a report about a broken viewer must not fail to build + because the viewer is broken. + """ + + def collect() -> dict: + state: dict[str, Any] = {} + + def probe(key, fn): + try: + state[key] = fn() + except Exception as exc: # noqa: BLE001 + state[key] = f"" + + probe( + "pyramid", + lambda: { + "id": ( + str(pyramid_doc.id()) + if callable(getattr(pyramid_doc, "id", None)) + else str(getattr(pyramid_doc, "id", "")) + ), + "levels": [ + { + k: d.document_properties["lightsheetZarrLevel"].get(k) + for k in ("shape", "chunks", "chunk_grid", "dtype", "codec", "axes_order") + } + for d in loader.docs + ], + }, + ) + probe( + "view", + lambda: { + "ready": counter._view_ready, + "needed": counter._view_needed, + "dims_point": [float(x) for x in viewer.dims.point], + "ndisplay": int(viewer.dims.ndisplay), + "camera_zoom": float(viewer.camera.zoom), + "camera_center": [float(x) for x in viewer.camera.center], + }, + ) + probe( + "layers", + lambda: [ + { + "name": layer.name, + "visible": bool(layer.visible), + "data_level": int(getattr(layer, "data_level", -1)), + "corner_pixels": [[int(v) for v in row] for row in layer.corner_pixels], + "loaded": bool(getattr(layer, "loaded", True)), + } + for layer in layers + ], + ) + probe( + "fetches", + lambda: { + "requested": counter._started, + "loaded": counter._done, + "in_flight": counter._started - counter._done, + "last_fetch_s": counter._last_duration, + }, + ) + probe("loader", loader.stats) + + extra: dict[str, Any] = {"state": state} + try: + extra["datasetId"] = _cloud_dataset_id(session) or None + except Exception: # noqa: BLE001 + pass + try: + doc_id = ( + pyramid_doc.id() if callable(getattr(pyramid_doc, "id", None)) else pyramid_doc.id + ) + extra["documentId"] = str(doc_id) + except Exception: # noqa: BLE001 + pass + try: + from ndi.gui.feedback import installLogBuffer + + extra["log"] = installLogBuffer().text() + except Exception: # noqa: BLE001 + pass + return {k: v for k, v in extra.items() if v is not None} + + return collect + + +def _tilesInFlightRow(fetcher, QHBoxLayout, QLabel, QComboBox): + """A "Tiles in flight" drop-down that changes the fetch limit live. + + More tiles at once hides per-request latency, until the line is full: + past that, every tile in flight shares the same bandwidth and each one + arrives later, including the ones in view. The right number depends on + the connection, so the reader picks it. + """ + from ndi.pyramid.multiscale import WORKER_CHOICES + + row = QHBoxLayout() + label = QLabel("Tiles in flight") + combo = QComboBox() + choices = [n for n in WORKER_CHOICES if n <= fetcher.maxWorkers] + current = fetcher.workerLimit + if current not in choices: + choices = sorted(set(choices) | {current}) + for n in choices: + combo.addItem(str(n), n) + combo.setCurrentIndex(choices.index(current)) + tip = ( + "How many tiles download at once. More hides network latency;\n" + "too many on a slow line just share the bandwidth, so every\n" + "tile -- including the ones in view -- arrives later. Try 8 on\n" + "~100 Mb/s, 32-64 on a fast connection. Takes effect now." + ) + label.setToolTip(tip) + combo.setToolTip(tip) + + def _changed(_index): + n = fetcher.setWorkers(int(combo.currentData())) + print(f"[lightsheet] tiles in flight: {n}", file=sys.stderr, flush=True) + + combo.currentIndexChanged.connect(_changed) + row.addWidget(label) + row.addWidget(combo) + row.addStretch(1) + return row + + +def _attach_cloud_panel(viewer, report_collect=None, fetcher=None) -> None: + """Dock an NDI Cloud sign-in panel with a BETA badge. + + Reuses the gene-pyramid viewer's addCloudPanel contract -- same + shape, same auth plumbing, so a user who has signed in to that + viewer sees the same profile list and the same token clock + here. We don't call addCloudPanel directly because we want a + small "BETA" badge immediately under the wordmark, which the + gene-pyramid viewer does not carry; building the panel locally + is less fragile than monkey-patching after dock. + + Silent no-op when the environment has no cloud token (same + cloudSessionLooksLikely rule the gene viewer uses) or when + NDI_LIGHTSHEET_CLOUD_PANEL=0 opts it out. A purely local + pyramid has no reason for a login control; a login panel on a + local-only window implies the picture might be waiting on + something, which it isn't. + """ + if os.environ.get("NDI_LIGHTSHEET_CLOUD_PANEL", "1").strip().lower() in ("0", "false", "off"): + return + + try: + from ndi.gui.app.genepyramid.controls import ( + cloudLogoLabel, + cloudSessionLooksLikely, + cloudSignInDialog, + ) + except ImportError as exc: + print( + f"[lightsheet] cloud panel: gene-pyramid controls unavailable ({exc})", + file=sys.stderr, + flush=True, + ) + return + + # Always show the NDI Cloud panel, even on a local-only dataset: + # the wordmark and BETA badge identify what this viewer is, and + # a user who later wants cloud access needs an obvious place to + # sign in. cloudSessionLooksLikely is still consulted to tune + # the clock copy (an unsigned/local session reads "not signed + # in" rather than a token-expiry clock), but it no longer gates + # the panel. + has_cloud = False + try: + has_cloud = bool(cloudSessionLooksLikely()) + except Exception: + pass + + try: + from qtpy.QtCore import Qt, QTimer + from qtpy.QtWidgets import ( + QComboBox, + QHBoxLayout, + QLabel, + QPushButton, + QVBoxLayout, + QWidget, + ) + except ImportError: + return + + try: + from ndi.cloud import auth + except ImportError as exc: + print( + f"[lightsheet] cloud panel: ndi.cloud.auth unavailable ({exc})", + file=sys.stderr, + flush=True, + ) + return + + box = QWidget() + outer = QVBoxLayout(box) + + logo = cloudLogoLabel(box) + if logo is not None: + outer.addWidget(logo) + + beta = QLabel("BETA") + beta.setAlignment(Qt.AlignLeft | Qt.AlignVCenter) + beta.setStyleSheet( + "QLabel {" + " color: #ffffff;" + " background: #c9372c;" + " font-weight: 700;" + " font-size: 10px;" + " letter-spacing: 2px;" + " padding: 2px 8px;" + " border-radius: 3px;" + "}" + ) + beta.setToolTip( + "The lightsheet viewer is in beta. First-paint and\n" + "zoom-refinement can be janky on large pyramids;\n" + "use the Refresh View button if a region stays coarse." + ) + beta_row = QHBoxLayout() + beta_row.setContentsMargins(0, 2, 0, 6) + beta_row.addWidget(beta, 0, Qt.AlignLeft) + beta_row.addStretch(1) + outer.addLayout(beta_row) + + if report_collect is not None: + from ndi.gui.feedback import reportButton + + outer.addWidget( + reportButton( + box, + app="lightsheetZarr-viewer", + collect=report_collect, + title="Report a problem with the lightsheet viewer", + ) + ) + + signin = QPushButton("Sign in...") + signin.setToolTip( + "Sign in to NDI Cloud, so tiles that are not already\n" + "downloaded keep loading. The token lives in this\n" + "process, so signing in anywhere else does not reach\n" + "this window." + ) + clock = QLabel("") + clock.setWordWrap(True) + row = QHBoxLayout() + row.addWidget(signin) + row.addWidget(clock, 1) + outer.addLayout(row) + + result = QLabel("") + result.setWordWrap(True) + result.hide() + outer.addWidget(result) + + if fetcher is not None and hasattr(fetcher, "setWorkers"): + outer.addLayout(_tilesInFlightRow(fetcher, QHBoxLayout, QLabel, QComboBox)) + outer.addStretch() + + def _tick(): + # Local-only session: show "not signed in" rather than a stale + # token clock. tokenStatusLine()/tokenSecondsRemaining() are + # safe to call but their output can be misleading when no cloud + # pyramid is loaded, so we prefer an explicit copy. + if not has_cloud: + clock.setText("Not signed in. Sign in to access NDI Cloud pyramids.") + return + left = auth.tokenSecondsRemaining() + line = auth.tokenStatusLine() + if left is not None and 0 < left < 900: + clock.setText(f"{line} -- renew before it runs out.") + elif left is not None and left <= 0: + clock.setText(f"{line}. Undownloaded tiles will fail until you sign in.") + else: + clock.setText(line) + + def _open(): + ok, note = cloudSignInDialog(box) + if note: + result.setText(note) + result.show() + elif ok: + result.hide() + _tick() + + signin.clicked.connect(_open) + + # Parented to the widget so the timer stops when the dock + # goes. A free timer would fire at a deleted label and take + # the process with it. + timer = QTimer(box) + timer.setInterval(30_000) + timer.timeout.connect(_tick) + timer.start() + _tick() + + try: + viewer.window.add_dock_widget(box, area="right", name="NDI Cloud") + except Exception as exc: + print( + f"[lightsheet] cloud panel: could not dock ({exc})", + file=sys.stderr, + flush=True, + ) + return + + _CLOUD_PANELS[id(viewer)] = box + print( + "[lightsheet] cloud panel: docked (NDI Cloud sign-in + beta badge)", + file=sys.stderr, + flush=True, + ) + + def _attach_level_selector(viewer, layers, per_layer_levels, per_layer_scales): """Dock a "Resolution" selector that swaps each layer's level. + No-op when the layers are already native multiscale layers: + napari then picks the level itself from the current zoom and + swapping ``layer.data`` out from under it would corrupt the + MultiScaleData wrapper. + Swaps ``layer.data`` AND ``layer.scale`` together so world coordinates stay locked when the pixel dimensions change. No new fetches happen on swap -- the graphs are pre-built during @@ -651,6 +1619,10 @@ def _attach_level_selector(viewer, layers, per_layer_levels, per_layer_scales): """ if not layers or not per_layer_levels: return None, [] + # Multiscale layers manage their own level selection; a dock-widget + # swap would fight napari. + if any(getattr(layer, "multiscale", False) for layer in layers): + return None, [] max_levels = max((len(lst) for lst in per_layer_levels), default=0) if max_levels < 2: return None, [] # Nothing to switch between. @@ -678,11 +1650,7 @@ def _picker(level: str = default_label): idx = labels.index(level) t0 = time.monotonic() elapsed_start = t0 - session_start - print( - f"[lightsheet] level selector: request {level} at t=+{elapsed_start:.1f}s", - file=sys.stderr, - flush=True, - ) + _ls(f"level selector: request {level} at t=+{elapsed_start:.1f}s") for layer, arrays, scales in zip(layers, per_layer_levels, per_layer_scales): if not arrays or idx >= len(arrays): continue @@ -718,17 +1686,13 @@ def _picker(level: str = default_label): layer.refresh() except Exception: # noqa: BLE001 pass - print( - f"[lightsheet] layer {layer.name!r} scale/data swapped " - f"at t=+{time.monotonic() - session_start:.1f}s", - file=sys.stderr, - flush=True, + _ls( + f" layer {layer.name!r} scale/data swapped " + f"at t=+{time.monotonic() - session_start:.1f}s" ) - print( - f"[lightsheet] level selector: request completed in " - f"{time.monotonic() - t0:.2f}s (napari now re-slicing)", - file=sys.stderr, - flush=True, + _ls( + f"level selector: request completed in {time.monotonic() - t0:.2f}s " + "(napari now re-slicing)" ) try: diff --git a/src/ndi/gui/app/lightsheetZarr/viewport.py b/src/ndi/gui/app/lightsheetZarr/viewport.py index aff99883..3bccac48 100644 --- a/src/ndi/gui/app/lightsheetZarr/viewport.py +++ b/src/ndi/gui/app/lightsheetZarr/viewport.py @@ -84,6 +84,17 @@ def attach_viewport_clip( return None if not layers or not per_layer_levels: return None + # Native multiscale layers manage their own level + viewport + # choice; swapping layer.data from the clip would corrupt the + # MultiScaleData wrapper. Clipping is a single-level-mode tool. + if any(getattr(layer, "multiscale", False) for layer in layers): + print( + "[lightsheet] viewport clip: skipped (layers are native " + "multiscale; napari handles level selection)", + file=sys.stderr, + flush=True, + ) + return None try: clip = ViewportClip(viewer, layers, per_layer_levels, per_layer_scales, picker, labels) diff --git a/src/ndi/gui/app/lightsheetZarr/viewtiles.py b/src/ndi/gui/app/lightsheetZarr/viewtiles.py new file mode 100644 index 00000000..e8fc3e08 --- /dev/null +++ b/src/ndi/gui/app/lightsheetZarr/viewtiles.py @@ -0,0 +1,291 @@ +"""How many tiles does the current view need, and how many are here? + +The fetch counter in :mod:`.viewer` sees fetches only as napari asks +for them, so it can say "12 in flight" but not "12 in flight, 83 to +go". The total IS knowable: napari draws one level per layer +(``layer.data_level``) over one region of it (``layer.corner_pixels``) +at one slice of the non-displayed axes (``viewer.dims.point``). The +level documents give the chunk grid and the list of stored chunks, so +the set of chunk files covering that region -- minus the sparse ones +that resolve to fill_value with no fetch -- is the set the view needs. +A chunk is "ready" once the fetcher has it on local disk. + +Split in two so the arithmetic is testable without napari: + +* :func:`chunksInBox` -- pure: per-axis inclusive data bounds in, the + stored ``chunk.bin_#`` names covering them out. +* :func:`layerBox` / :func:`viewTally` -- read napari state and feed + the pure half. Anything napari does not expose the way we expect + returns None, and the viewer falls back to the in-flight count. + +:func:`attachViewTally` polls on a main-thread QTimer (napari layer +state is not safe to read from the fetcher threads) and hands each +change to the fetch counter, which owns the status bar and overlay. +""" + +from __future__ import annotations + +from typing import Any + +__all__ = ["chunksInBox", "layerBox", "viewChunks", "viewTally", "attachViewTally"] + +#: A view whose box spans more chunk positions than this is not +#: tallied (it would stall the main thread on every tick). napari's 3D +#: mode draws the coarsest level, which stays far below it. +MAX_POSITIONS = 50_000 + +#: Poll interval for the main-thread timer, in milliseconds. +POLL_MS = 400 + +#: Why the last :func:`viewTally` returned None. A silent fallback to +#: the in-flight count looks exactly like a viewer without this +#: feature, so the poll logs this whenever it changes. +lastReason = "" + + +def _why(reason: str): + """Record why the view cannot be tallied; return None.""" + global lastReason + lastReason = reason + return None + + +def chunksInBox(meta: dict, lo: list[int], hi: list[int]) -> list[str] | None: + """Stored chunk names covering the inclusive data box [LO, HI]. + + LO and HI have one entry per axis of the level's chunk grid. Bounds + are clipped to the grid; a box past the edge just covers fewer + chunks. Returns None when the box spans more than + :data:`MAX_POSITIONS` chunk positions. + """ + chunks = meta["chunks"] + grid = meta["chunk_grid"] + stored = meta.get("stored") + n = len(grid) + if len(lo) != n or len(hi) != n: + return None + + ranges = [] + count = 1 + for a in range(n): + first = max(0, int(lo[a]) // chunks[a]) + last = min(grid[a] - 1, int(hi[a]) // chunks[a]) + if last < first: + return [] + ranges.append(range(first, last + 1)) + count *= last - first + 1 + if count > MAX_POSITIONS: + return None + + strides = [1] * n + for a in range(n - 2, -1, -1): + strides[a] = strides[a + 1] * grid[a + 1] + + names: list[str] = [] + + def walk(axis: int, linear: int) -> None: + if axis == n: + name = f"chunk.bin_{linear + 1:d}" + if stored is None or name in stored: + names.append(name) + return + for i in ranges[axis]: + walk(axis + 1, linear + i * strides[axis]) + + walk(0, 0) + return names + + +def _gridAxisMap(meta: dict, layer_ndim: int) -> tuple[list[int], int] | None: + """Map layer axes onto grid axes. + + Returns ``(layer_to_grid, c_axis)``: the grid axis for each layer + axis, and the grid's channel axis (-1 when there is none). The + layer arrays are the level arrays with the channel axis taken out, + so a ``tczyx`` grid backs a ``tzyx`` layer. Layer axes line up + with the TRAILING grid axes; any leading grid axis the layer does + not carry (a singleton ``t``) is pinned to 0 by the caller. + """ + n = len(meta["chunk_grid"]) + axes = meta.get("axes") or ("tczyx" if n == 5 else "") + c = axes.index("c") if len(axes) == n and "c" in axes else -1 + if n == layer_ndim: + # Nothing was stripped: the layer IS the grid. + return list(range(n)), -1 + spatial = [a for a in range(n) if a != c] + if len(spatial) < layer_ndim: + return None + return spatial[-layer_ndim:], c + + +def layerBox(layer, viewer, meta_levels: list[dict], channel: int = 0): + """``(level_index, lo, hi)`` for what napari is drawing of LAYER. + + LO/HI are inclusive data bounds on the level's chunk grid. None + when napari does not expose what we need. + """ + # The level index only lines up with META_LEVELS on a native + # multiscale layer over the full ladder (not single-level mode). + if not getattr(layer, "multiscale", False): + return _why(f"layer {getattr(layer, 'name', '?')!r} is not multiscale") + try: + if len(layer.data) != len(meta_levels): + return _why( + f"layer has {len(layer.data)} level(s) but the pyramid has " + f"{len(meta_levels)} level document(s)" + ) + level = int(layer.data_level) + meta = meta_levels[level] + corners = layer.corner_pixels + ndim = int(layer.ndim) + # Layer axes are the LAST ndim world axes. + offset = int(viewer.dims.ndim) - ndim + displayed = [int(d) - offset for d in viewer.dims.displayed] + point = list(viewer.dims.point)[-ndim:] + data_pt = list(layer.world_to_data(point)) + factors = layer.downsample_factors[level] + except Exception as exc: # noqa: BLE001 - any napari shape surprise -> unknown + return _why(f"could not read napari layer state ({type(exc).__name__}: {exc})") + + mapping = _gridAxisMap(meta, ndim) + if mapping is None: + return _why( + f"cannot map a {ndim}-D layer onto a {len(meta['chunk_grid'])}-D chunk " + f"grid (axes_order={meta.get('axes')!r})" + ) + layer_to_grid, c_axis = mapping + + n = len(meta["chunk_grid"]) + lo = [0] * n + hi = [0] * n + for j in range(ndim): + g = layer_to_grid[j] + if j in displayed: + lo[g] = int(corners[0][j]) + hi[g] = int(corners[1][j]) + else: + # world_to_data maps to level-0 data; scale down to LEVEL. + idx = int(float(data_pt[j]) / float(factors[j])) + lo[g] = hi[g] = idx + if c_axis >= 0: + lo[c_axis] = hi[c_axis] = int(channel) + return level, lo, hi + + +def viewChunks(viewer, layers, meta_levels: list[dict]) -> dict | None: + """Stored chunks the current view needs: ``{(doc_id, name): doc}``. + + Keyed the way :class:`ndi.pyramid.multiscale._ChunkFetcher` keys its + cache, so the result can be handed to ``fetcher.setView``. Counts + each chunk once even when two channel layers share it. Invisible + layers draw nothing and are skipped. None when the view cannot be + worked out (``lastReason`` says why). + """ + if not meta_levels: + return _why("no level chunk metadata from the loader") + wanted: dict[tuple, Any] = {} + for c, layer in enumerate(layers): + if not getattr(layer, "visible", True): + continue + box = layerBox(layer, viewer, meta_levels, channel=c) + if box is None: + return None + level, lo, hi = box + names = chunksInBox(meta_levels[level], lo, hi) + if names is None: + return _why(f"view spans more than {MAX_POSITIONS:,} chunk positions") + doc = meta_levels[level]["doc"] + doc_id = getattr(doc, "id", str(doc)) + for name in names: + wanted[(doc_id, name)] = doc + return wanted + + +def _countReady(wanted: dict, fetcher) -> int: + ready = 0 + for (_doc_id, name), doc in wanted.items(): + try: + if fetcher.chunkPathIfCached(doc, name) is not None: + ready += 1 + except Exception: # noqa: BLE001 - unreachable scope: not ready + pass + return ready + + +def viewTally(viewer, layers, meta_levels: list[dict], fetcher) -> tuple[int, int] | None: + """``(ready, needed)`` stored chunks for the current view, or None.""" + wanted = viewChunks(viewer, layers, meta_levels) + if wanted is None: + return None + return _countReady(wanted, fetcher), len(wanted) + + +def attachViewTally(viewer, layers, meta_levels: list[dict], fetcher, counter): + """Poll :func:`viewTally` on a main-thread timer; report to COUNTER. + + COUNTER gets ``set_view_tally(ready, needed)`` on every change and + ``set_view_tally(None, None)`` when the view cannot be tallied. + Returns the QTimer (kept alive by its parent window), or None when + Qt or the window is not available. + """ + import sys + + def log(msg: str) -> None: + print(f"[lightsheet] view tile tally: {msg}", file=sys.stderr, flush=True) + + try: + from qtpy.QtCore import QTimer + except Exception as exc: # noqa: BLE001 + log(f"off -- no Qt timer ({exc})") + return None + + parent = None + try: + window = viewer.window + parent = getattr(window, "_qt_window", None) + except Exception: # noqa: BLE001 + pass + + last = {"value": object(), "reason": "", "keys": object()} + + # The coarsest level is never released as stale: napari's + # auto-contrast samples it whatever the view is. + coarsest = meta_levels[-1]["doc"] if meta_levels else None + never_stale = [getattr(coarsest, "id", str(coarsest))] if coarsest is not None else [] + set_view = getattr(fetcher, "setView", None) + + def tick() -> None: + try: + wanted = viewChunks(viewer, layers, meta_levels) + value = None if wanted is None else (_countReady(wanted, fetcher), len(wanted)) + except Exception as exc: # noqa: BLE001 - never take the viewer down + wanted = None + value = _why(f"tally raised {type(exc).__name__}: {exc}") + # Hand the view to the fetcher only when it CHANGES: the change + # time is what decides whether a waiting read is stale. + keys = None if wanted is None else frozenset(wanted) + if set_view is not None and keys != last["keys"]: + last["keys"] = keys + try: + set_view(wanted, never_stale_doc_ids=never_stale) + except Exception as exc: # noqa: BLE001 + log(f"could not prioritise the view ({exc})") + if value is None and lastReason != last["reason"]: + last["reason"] = lastReason + log(f"cannot count this view -- {lastReason}") + if value is not None: + last["reason"] = "" + if value == last["value"]: + return + last["value"] = value + if value is None: + counter.set_view_tally(None, None) + else: + counter.set_view_tally(*value) + + timer = QTimer(parent) + timer.setInterval(POLL_MS) + timer.timeout.connect(tick) + timer.start() + log(f"on ({len(meta_levels)} level(s), polling every {POLL_MS} ms)") + return timer diff --git a/src/ndi/gui/component/ndi_matlab_python_bridge.yaml b/src/ndi/gui/component/ndi_matlab_python_bridge.yaml index 9cfed52d..31fc9fe8 100644 --- a/src/ndi/gui/component/ndi_matlab_python_bridge.yaml +++ b/src/ndi/gui/component/ndi_matlab_python_bridge.yaml @@ -498,7 +498,7 @@ classes: - name: ndi_gui_component_ProgressBarWindow type: class matlab_path: "+ndi/+gui/+component/ProgressBarWindow.m" - matlab_last_sync_hash: "b367283b" + matlab_last_sync_hash: "2b23dd30" python_path: "ndi/gui/component/ProgressBarWindow.py" python_class: "ndi_gui_component_ProgressBarWindow" inherits: "matlab.apps.AppBase" @@ -521,6 +521,7 @@ classes: Porting a silent mode is a design decision for a maintainer rather than something to invent while dating a hash. 2026-09-25: NDI-matlab b367283b added a per-bar Timeout name-value on addBar so a long-running bar can opt out of the global auto-timeout sweep. No Python UI counterpart; no behavioural change here. + 2026-09-28: NDI-matlab 2b23dd30 relaxed the addBar Timeout argument-block size constraint (dropped `(1,:)`, which was rejecting `duration.empty` under some MATLAB releases). Pure MATLAB argument-validation adjustment; no behavioural change here. NDI-python#295: the MATLAB properties this port does not expose under MATLAB's name are enumerated in tests/bridge_property_allowlist.yaml, diff --git a/src/ndi/gui/feedback.py b/src/ndi/gui/feedback.py new file mode 100644 index 00000000..765c5a97 --- /dev/null +++ b/src/ndi/gui/feedback.py @@ -0,0 +1,629 @@ +"""Report a problem from inside a viewer. + +A beta viewer's problems mostly go unreported: asking a user to find a +log, copy versions and write an email is too much. This module turns +that into one dialog -- the user says what they were doing, the viewer +attaches what it knows, and the report goes to NDI Cloud through +:func:`ndi.cloud.api.feedback.submitFeedback` (the user is already +signed in for the tiles). Save-to-file and Copy cover a user who is +not signed in, or offline. + +Pieces, each usable on its own: + +* :class:`LogBuffer` -- keeps the last lines written to stderr (and to + the ``logging`` root) so a report can carry them without the user + ever having seen a terminal. +* :func:`scrub` -- strips credentials before anything leaves the + machine. Viewer logs DO carry them: signed S3 URLs embed a signature + and session token, and auth lines can carry a bearer token. +* :func:`buildReport` -- the ``submitFeedback`` keyword arguments, + scrubbed and cut to the server's size limits. +* :func:`sendReport` / :func:`saveReport` -- deliver it. +* :class:`FeedbackDialog` -- the Qt dialog (imported lazily). + +The payload field names are a contract with NDI-matlab and the server +(Waltham-Data-Science/ndi-cloud-node#158). +""" + +from __future__ import annotations + +import collections +import json +import logging +import os +import platform +import re +import sys +import threading +from collections.abc import Callable +from typing import Any + +__all__ = [ + "LogBuffer", + "installLogBuffer", + "scrub", + "buildReport", + "sendReport", + "saveReport", + "describeSendError", + "FeedbackDialog", + "openFeedbackDialog", + "reportButton", + "napariState", +] + +#: Server limits (characters / JSON bytes). Reports are cut to fit, not +#: refused: a truncated log is worth more than no report. +MAX_DESCRIPTION = 20_000 +MAX_LOG = 500_000 +MAX_TRACEBACK = 50_000 +MAX_ENVIRONMENT_JSON = 20_000 +MAX_STATE_JSON = 200_000 + +#: Lines the log buffer keeps. Generous: a stuck-view report wants the +#: minutes before the user gave up, and the log is cut to MAX_LOG anyway. +LOG_LINES = 5_000 + +_REDACTED = "[REDACTED]" + +# Order matters: whole signed-URL query strings first, then the bits +# that can appear on their own. +_SCRUB_PATTERNS: list[tuple[re.Pattern, str]] = [ + # AWS signed-URL credentials (query params, any case). + ( + re.compile( + r"(?i)\b(X-Amz-(?:Signature|Credential|Security-Token)|Signature|" + r"AWSAccessKeyId|x-amz-security-token)=[^&\s\"'<>]+" + ), + rf"\1={_REDACTED}", + ), + # Authorization headers / bearer tokens. + (re.compile(r"(?i)\b(Bearer)\s+[A-Za-z0-9._~+/=-]+"), rf"\1 {_REDACTED}"), + # JWTs anywhere (header.payload.signature, base64url). + (re.compile(r"\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]+"), _REDACTED), + # key=value / "key": "value" for anything named like a secret. + ( + re.compile( + r"(?i)([\"']?\b[\w-]*(?:token|password|passwd|secret|api[_-]?key)[\w-]*[\"']?" + r"\s*[:=]\s*[\"']?)([^\s\"',&}]+)" + ), + rf"\1{_REDACTED}", + ), +] + + +def scrub(text: str) -> str: + """Remove credentials from TEXT. Safe on any string.""" + if not text: + return text + for pattern, repl in _SCRUB_PATTERNS: + text = pattern.sub(repl, text) + return text + + +def _scrubValue(value: Any) -> Any: + """Scrub every string inside a JSON-like value; drop secret-named keys.""" + if isinstance(value, str): + return scrub(value) + if isinstance(value, dict): + out = {} + for k, v in value.items(): + key = str(k) + if re.search(r"(?i)token|password|passwd|secret|api[_-]?key", key): + out[key] = _REDACTED + else: + out[key] = _scrubValue(v) + return out + if isinstance(value, (list, tuple)): + return [_scrubValue(v) for v in value] + if value is None or isinstance(value, (bool, int, float)): + return value + return scrub(str(value)) + + +# ------------------------------------------------------------------ log buffer + + +class LogBuffer: + """The last :data:`LOG_LINES` lines written to stderr and to logging. + + Installed as a tee in front of ``sys.stderr``: everything still + reaches the real stream, and a copy of each complete line lands in + a ring buffer. A ``\\r``-redrawn progress line keeps only its last + state. Thread-safe; fetcher threads print too. + """ + + def __init__(self, max_lines: int = LOG_LINES): + self._lines: collections.deque[str] = collections.deque(maxlen=max_lines) + self._partial = "" + self._lock = threading.Lock() + + def feed(self, text: str) -> None: + if not text: + return + with self._lock: + data = self._partial + text + parts = data.split("\n") + self._partial = parts.pop() + for line in parts: + line = line.rsplit("\r", 1)[-1] + if line.strip(): + self._lines.append(line) + # A long run of \r redraws never ends in \n; keep only the + # current state so it cannot grow without bound. + self._partial = self._partial.rsplit("\r", 1)[-1][-10_000:] + + def text(self) -> str: + with self._lock: + lines = list(self._lines) + if self._partial.strip(): + lines.append(self._partial) + return "\n".join(lines) + + +class _Tee: + """sys.stderr stand-in: write through, and copy into a LogBuffer.""" + + def __init__(self, stream, buffer: LogBuffer): + self._stream = stream + self._buffer = buffer + + def write(self, text): + try: + self._buffer.feed(text) + except Exception: # noqa: BLE001 - the log must never break printing + pass + return self._stream.write(text) + + def flush(self): + return self._stream.flush() + + def __getattr__(self, name): + return getattr(self._stream, name) + + +class _BufferHandler(logging.Handler): + def __init__(self, buffer: LogBuffer): + super().__init__(level=logging.INFO) + self._buffer = buffer + self.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")) + + def emit(self, record): + try: + self._buffer.feed(self.format(record) + "\n") + except Exception: # noqa: BLE001 + pass + + +_installed: LogBuffer | None = None + + +def installLogBuffer() -> LogBuffer: + """Start buffering stderr and logging for reports. Idempotent. + + Logging records reach the buffer through their own handler, since + a logging StreamHandler holds the stderr it was built with and would + bypass the tee. A record that ALSO prints to stderr after the tee is + installed can appear twice; harmless in a diagnostic log. + """ + global _installed + if _installed is not None: + return _installed + buf = LogBuffer() + if not isinstance(sys.stderr, _Tee): + sys.stderr = _Tee(sys.stderr, buf) + logging.getLogger().addHandler(_BufferHandler(buf)) + _installed = buf + return buf + + +# ------------------------------------------------------------------ report + + +def _versions() -> dict[str, Any]: + env: dict[str, Any] = { + "os": f"{platform.system()} {platform.release()}", + "platform": platform.platform(), + "machine": platform.machine(), + "python": sys.version.split()[0], + } + try: + import ndi + + env["ndi"] = str(getattr(ndi, "__version__", "?")) + except Exception: # noqa: BLE001 + pass + for mod in ("napari", "numpy", "dask", "qtpy"): + m = sys.modules.get(mod) + if m is not None: + env[mod] = str(getattr(m, "__version__", "?")) + try: + import qtpy + + env["qtApi"] = str(getattr(qtpy, "API_NAME", "?")) + env["qt"] = str(getattr(qtpy, "QT_VERSION", "?")) + except Exception: # noqa: BLE001 + pass + return env + + +def _envFlags(prefixes: tuple[str, ...]) -> dict[str, str]: + return {k: v for k, v in sorted(os.environ.items()) if k.startswith(prefixes)} + + +def _capJson(value: dict, limit: int, what: str) -> dict: + """VALUE if its JSON fits LIMIT bytes, else a note saying it was cut.""" + try: + raw = json.dumps(value, default=str) + except Exception as exc: # noqa: BLE001 + return {"_note": f"{what} could not be serialised ({exc})"} + if len(raw.encode("utf-8")) <= limit: + return json.loads(raw) + return { + "_note": f"{what} was {len(raw):,} bytes, over the {limit:,} limit; truncated", + "_head": raw[: limit // 2], + } + + +def _tail(text: str | None, limit: int) -> str | None: + if text is None or len(text) <= limit: + return text + marker = f"[... {len(text) - limit:,} earlier characters cut ...]\n" + return marker + text[-(limit - len(marker)) :] + + +def buildReport( + app: str, + category: str, + description: str, + *, + appVersion: str | None = None, + datasetId: str | None = None, + documentId: str | None = None, + state: dict | None = None, + log: str | None = None, + traceback: str | None = None, + envPrefixes: tuple[str, ...] = ("NDI_",), +) -> dict[str, Any]: + """Keyword arguments for ``submitFeedback``, scrubbed and size-capped. + + ``environment`` is filled in here (OS, Python, NDI and GUI library + versions, and the ``NDI_*`` environment flags -- with any secret- + named ones redacted). Everything the user did not type is scrubbed + of credentials; the description is the user's own words and is + sent as typed (cut to the limit). + """ + if appVersion is None: + appVersion = f"NDI-python {_versions().get('ndi', '?')}" + environment = _versions() + environment["flags"] = _envFlags(envPrefixes) + report: dict[str, Any] = { + "app": app, + "category": category, + "description": (description or "")[:MAX_DESCRIPTION], + "appVersion": appVersion, + "environment": _capJson(_scrubValue(environment), MAX_ENVIRONMENT_JSON, "environment"), + } + if datasetId: + report["datasetId"] = str(datasetId) + if documentId: + report["documentId"] = str(documentId) + if state is not None: + report["state"] = _capJson(_scrubValue(state), MAX_STATE_JSON, "state") + if log: + report["log"] = _tail(scrub(log), MAX_LOG) + if traceback: + report["traceback"] = _tail(scrub(traceback), MAX_TRACEBACK) + return report + + +def sendReport(report: dict[str, Any], *, client=None) -> str: + """Submit REPORT; return the stored report's id. Raises on failure.""" + from ndi.cloud.api.feedback import submitFeedback + + kwargs = dict(report) + if client is not None: + kwargs["client"] = client + b, feedback_id, _status, _url = submitFeedback(**kwargs) + return str(feedback_id) + + +def describeSendError(exc: BaseException) -> str: + """One sentence a user can act on, for a failed send.""" + try: + from ndi.cloud.exceptions import CloudAPIError, CloudAuthError + except Exception: # noqa: BLE001 + CloudAPIError = CloudAuthError = () # type: ignore[assignment] + if CloudAuthError and isinstance(exc, CloudAuthError): + return ( + "Not signed in to NDI Cloud. Sign in, then send again -- or save the report to a file." + ) + status = getattr(exc, "status_code", None) + if CloudAPIError and isinstance(exc, CloudAPIError) and status is not None: + if status == 429: + return "Too many reports this hour. Save it to a file and send it later." + if status == 413: + return "The report is too large for the server. Save it to a file instead." + if status == 404: + return "This NDI Cloud server does not accept reports yet. Save it to a file instead." + return f"NDI Cloud refused the report (HTTP {status}). Save it to a file instead." + return f"Could not send the report ({type(exc).__name__}: {exc}). Save it to a file instead." + + +def saveReport(report: dict[str, Any], path: str) -> None: + """Write REPORT as JSON to PATH (schema as sent to the server).""" + from ndi.cloud.api.feedback import FEEDBACK_SCHEMA_VERSION + + body = {"schemaVersion": FEEDBACK_SCHEMA_VERSION, **report} + with open(path, "w", encoding="utf-8") as f: + json.dump(body, f, indent=2, default=str) + + +# ------------------------------------------------------------------ dialog + + +def FeedbackDialog( # noqa: N802 - a class factory; Qt is imported lazily + parent=None, + *, + app: str, + collect: Callable[[], dict[str, Any]], + title: str = "Report a problem", + send: Callable[[dict[str, Any]], str] = sendReport, +): + """Build the report dialog. COLLECT returns the non-text fields. + + COLLECT is called when the dialog opens (and again on Send/Save so + the attached state is current) and returns ``buildReport`` keyword + arguments other than app/category/description: ``state``, ``log``, + ``datasetId``, ``documentId``, ``traceback``, ... + """ + from qtpy.QtCore import Qt, QTimer + from qtpy.QtWidgets import ( + QApplication, + QComboBox, + QDialog, + QFileDialog, + QHBoxLayout, + QLabel, + QPlainTextEdit, + QPushButton, + QToolButton, + QVBoxLayout, + ) + + class _Dialog(QDialog): + CATEGORIES = [("Something is broken", "bug"), ("Slow", "performance"), ("Other", "other")] + + def __init__(self): + super().__init__(parent) + self.setWindowTitle(title) + self.resize(560, 480) + self.feedbackID: str | None = None + self._future = None + + lay = QVBoxLayout(self) + intro = QLabel( + "What were you doing, and what did you see? The viewer attaches " + "its own diagnostics (versions, what it was loading, recent log " + "lines) with sign-in tokens and signed links removed." + ) + intro.setWordWrap(True) + lay.addWidget(intro) + + row = QHBoxLayout() + row.addWidget(QLabel("Kind of problem:")) + self.category = QComboBox() + for label, value in self.CATEGORIES: + self.category.addItem(label, value) + row.addWidget(self.category, 1) + lay.addLayout(row) + + self.description = QPlainTextEdit() + self.description.setPlaceholderText( + "e.g. I zoomed into the cortex at Z 1921 and the view stayed blurry " + "for over a minute." + ) + lay.addWidget(self.description, 2) + + self.toggle = QToolButton() + self.toggle.setText("Show what will be sent") + self.toggle.setCheckable(True) + self.toggle.setToolButtonStyle(Qt.ToolButtonTextBesideIcon) + self.toggle.setArrowType(Qt.RightArrow) + lay.addWidget(self.toggle, 0, Qt.AlignLeft) + self.preview = QPlainTextEdit() + self.preview.setReadOnly(True) + self.preview.hide() + lay.addWidget(self.preview, 3) + self.toggle.toggled.connect(self._togglePreview) + + self.status = QLabel("") + self.status.setWordWrap(True) + self.status.setTextInteractionFlags(Qt.TextSelectableByMouse) + lay.addWidget(self.status) + + buttons = QHBoxLayout() + self.saveButton = QPushButton("Save to file...") + self.copyButton = QPushButton("Copy") + buttons.addWidget(self.saveButton) + buttons.addWidget(self.copyButton) + buttons.addStretch(1) + self.cancelButton = QPushButton("Close") + self.sendButton = QPushButton("Send") + self.sendButton.setDefault(True) + buttons.addWidget(self.cancelButton) + buttons.addWidget(self.sendButton) + lay.addLayout(buttons) + + self.description.textChanged.connect(self._sync) + self.sendButton.clicked.connect(self._send) + self.saveButton.clicked.connect(self._save) + self.copyButton.clicked.connect(self._copy) + self.cancelButton.clicked.connect(self.reject) + + self._poll = QTimer(self) + self._poll.setInterval(100) + self._poll.timeout.connect(self._checkSend) + self._sync() + + # -- report + def report(self) -> dict[str, Any]: + try: + extra = dict(collect() or {}) + except Exception as exc: # noqa: BLE001 - a broken probe is itself a finding + extra = {"state": {"_collectError": f"{type(exc).__name__}: {exc}"}} + return buildReport( + app, + self.category.currentData(), + self.description.toPlainText().strip() or "(no description)", + **extra, + ) + + def _togglePreview(self, on): + self.toggle.setArrowType(Qt.DownArrow if on else Qt.RightArrow) + self.toggle.setText("Hide what will be sent" if on else "Show what will be sent") + if on: + self.preview.setPlainText(json.dumps(self.report(), indent=2, default=str)) + self.preview.setVisible(on) + + def _sync(self): + has_text = bool(self.description.toPlainText().strip()) + busy = self._future is not None + self.sendButton.setEnabled(has_text and not busy and self.feedbackID is None) + + # -- deliver + def _send(self): + from concurrent.futures import ThreadPoolExecutor + + rep = self.report() + self.status.setText("Sending ...") + pool = ThreadPoolExecutor(max_workers=1, thread_name_prefix="ndi-feedback") + self._future = pool.submit(send, rep) + pool.shutdown(wait=False) + self._sync() + self._poll.start() + + def _checkSend(self): + fut = self._future + if fut is None or not fut.done(): + return + self._poll.stop() + self._future = None + exc = fut.exception() + if exc is None: + self.feedbackID = fut.result() + self.status.setText(f"Sent -- thank you. Report id: {self.feedbackID}") + self.cancelButton.setText("Done") + else: + self.status.setText(describeSendError(exc)) + self._sync() + + def _save(self): + path, _ = QFileDialog.getSaveFileName( + self, "Save problem report", "ndi-problem-report.json", "JSON (*.json)" + ) + if not path: + return + try: + saveReport(self.report(), path) + self.status.setText(f"Saved to {path}") + except Exception as exc: # noqa: BLE001 + self.status.setText(f"Could not save ({exc})") + + def _copy(self): + QApplication.clipboard().setText(json.dumps(self.report(), indent=2, default=str)) + self.status.setText("Report copied to the clipboard.") + + return _Dialog() + + +def openFeedbackDialog(parent=None, **kwargs): + """Build and show a :func:`FeedbackDialog` (non-modal); return it.""" + dlg = FeedbackDialog(parent, **kwargs) + dlg.show() + dlg.raise_() + return dlg + + +def reportButton(parent, *, app: str, collect: Callable[[], dict[str, Any]], title: str): + """A "Report a problem..." QPushButton that opens the dialog. + + One definition for every viewer, so the wording, the tooltip and the + dialog stay the same wherever the button appears. The open dialog is + kept on the button (it is non-modal and would otherwise be collected). + """ + from qtpy.QtWidgets import QPushButton + + button = QPushButton("Report a problem...", parent) + button.setToolTip( + "Tell us what went wrong. The viewer attaches its own\n" + "diagnostics (versions, what it was showing, recent log\n" + "lines) with sign-in tokens and signed links removed,\n" + "and you can see exactly what will be sent." + ) + + def _open(): + button._ndi_dialog = openFeedbackDialog(parent, app=app, collect=collect, title=title) + + button.clicked.connect(_open) + return button + + +def napariState(viewer) -> dict[str, Any]: + """What a napari viewer is showing: camera, dims and every layer. + + Viewer-agnostic, so each viewer's collector starts from the same + picture of the canvas and adds its own specifics. Each part is + guarded: a report about a broken viewer must still build. + """ + state: dict[str, Any] = {} + + def probe(key, fn): + try: + state[key] = fn() + except Exception as exc: # noqa: BLE001 + state[key] = f"" + + probe( + "canvas", + lambda: { + "dims_point": [float(x) for x in viewer.dims.point], + "ndisplay": int(viewer.dims.ndisplay), + "camera_zoom": float(viewer.camera.zoom), + "camera_center": [float(x) for x in viewer.camera.center], + }, + ) + + def _layers(): + out = [] + for layer in viewer.layers: + entry: dict[str, Any] = { + "name": str(layer.name), + "type": type(layer).__name__, + "visible": bool(layer.visible), + } + if getattr(layer, "multiscale", False): + entry["n_levels"] = len(layer.data) + entry["data_level"] = int(layer.data_level) + entry["corner_pixels"] = [[int(v) for v in row] for row in layer.corner_pixels] + try: + entry["loaded"] = bool(layer.loaded) + except Exception: # noqa: BLE001 + pass + try: + entry["n_items"] = ( + len(layer.data) + if type(layer).__name__ + in ( + "Points", + "Shapes", + ) + else entry.get("n_items") + ) + except Exception: # noqa: BLE001 + pass + out.append({k: v for k, v in entry.items() if v is not None}) + return out + + probe("layers", _layers) + return state diff --git a/src/ndi/pyramid/loader.py b/src/ndi/pyramid/loader.py index 5798bd15..4cbce6a3 100644 --- a/src/ndi/pyramid/loader.py +++ b/src/ndi/pyramid/loader.py @@ -24,9 +24,55 @@ from __future__ import annotations +import logging +import os import threading from typing import Any +logger = logging.getLogger(__name__) + + +def _docHasChunkBinSeries(doc: Any) -> bool: + """True iff DOC's file metadata mentions the ``chunk.bin`` series. + + Named at module scope so it stays testable and so the prefetch + path stays honest about what "no chunk.bin" means: skip. Checks + the DID ``files.series_info`` shape first (the current writer), + and falls back to any ``file_info`` entry whose name starts with + ``chunk.bin`` (older ingests that wrote each member into + ``file_info`` directly). Missing or malformed metadata reads + as "no series", which is safe -- we just don't prefetch for + that doc. + """ + props = getattr(doc, "document_properties", None) or {} + if not isinstance(props, dict): + return False + files = props.get("files") + if not isinstance(files, dict): + return False + + raw_series = files.get("series_info") + series_entries: list[dict] + if isinstance(raw_series, dict): + series_entries = [raw_series] + elif isinstance(raw_series, list): + series_entries = [e for e in raw_series if isinstance(e, dict)] + else: + series_entries = [] + for entry in series_entries: + if str(entry.get("name", "")) == "chunk.bin": + return True + + raw_file_info = files.get("file_info") + if isinstance(raw_file_info, list): + for entry in raw_file_info: + if not isinstance(entry, dict): + continue + name = str(entry.get("name", "")) + if name == "chunk.bin" or name.startswith("chunk.bin_"): + return True + return False + class ImagePyramidLoader: """A lightsheetZarrPyramid, ready to be read. @@ -121,6 +167,19 @@ def fetcher(self): self.build() return self._fetcher + @property + def levelMeta(self) -> list[dict]: + """Per-level chunk geometry, finest first. + + Each entry has ``doc``, ``axes`` (the level's ``axes_order``, + lower case), ``chunks``, ``chunk_grid`` and ``stored`` (the set + of ``chunk.bin_#`` names the level holds, or None when the + document has no file list). The viewer uses it to count the + chunks the current view needs. + """ + self.build() + return list(getattr(self._fetcher, "_level_meta", None) or []) + @property def docs(self) -> list: """The level documents, sorted finest-first. @@ -168,6 +227,64 @@ def startPrefetch(self, level: int = -1) -> threading.Thread | None: reduction=self.reduction, ) + def startSignedUrlPrefetch( + self, + cloud_dataset_id: str, + *, + client: Any = None, + ) -> threading.Thread | None: + """Prefetch signed-URL scopes for every level document, in background. + + Each level's ``chunk.bin`` file series takes 20-80 s server-side + to sign, and the user hits that wait the first time they zoom + into a new level. Kicking off the async signed-URL-set jobs + for every level while the initial level-0 view is loading + hides those waits behind a moment the user was already + watching -- when they later zoom, the URLs are cached. + + Sequential inside one background thread on purpose (see the + design note in the task that added this): parallelizing across + scopes would need a lock refactor and 5 levels at ~1 minute + each still fit inside the initial-render wall time. + + Args: + cloud_dataset_id: The remote NDI Cloud dataset id. Empty + means "no cloud context"; the call is a no-op. + client: Passed to the signer (default signer only). + + Returns: + The background thread on success, ``None`` when disabled + or when there is nothing to prefetch. Off with + ``NDI_LIGHTSHEET_PREFETCH_SIGNED_URLS=0``. + """ + if os.environ.get("NDI_LIGHTSHEET_PREFETCH_SIGNED_URLS", "1").strip().lower() in ( + "0", + "false", + "off", + ): + logger.info("signed-URL prefetch disabled via NDI_LIGHTSHEET_PREFETCH_SIGNED_URLS") + return None + if not cloud_dataset_id: + return None + + self.build() + + scopes: list[tuple[str, str, str]] = [] + for doc in self.docs: + doc_id = getattr(doc, "id", "") or "" + if not doc_id: + continue + if not _docHasChunkBinSeries(doc): + continue + scopes.append((cloud_dataset_id, doc_id, "chunk.bin")) + + if not scopes: + return None + + from ndi.cloud.batch_signed_url import get_default + + return get_default().start_prefetch(scopes, client=client) + # ------------------------------------------------------------------ hooks def registerRefreshHint(self, hint) -> None: @@ -186,9 +303,36 @@ def registerRefreshHint(self, hint) -> None: self.build() fetcher = self._fetcher ctx = getattr(fetcher, "_fallback_context", None) + import os + import sys as _sys + + debug = os.environ.get("NDI_LIGHTSHEET_DEBUG", "").strip().lower() in ( + "1", + "true", + "on", + "yes", + ) if ctx is None: + if debug: + print( + "[lightsheet] registerRefreshHint: SKIPPED -- fetcher has " + "no _fallback_context (fallback env off, or single-level " + "pyramid). Napari will not be auto-nudged when fine " + "chunks arrive; the user must trigger a camera event.", + file=_sys.stderr, + flush=True, + ) return ctx["refresh_hint_slot"][0] = hint + if debug: + print( + f"[lightsheet] registerRefreshHint: installed " + f"{'hint' if hint is not None else 'None (no-op)'}; " + f"future async fine fetches will " + f"{'nudge napari to re-slice' if hint is not None else 'NOT nudge napari'}.", + file=_sys.stderr, + flush=True, + ) # ------------------------------------------------------------------ stats diff --git a/src/ndi/pyramid/multiscale.py b/src/ndi/pyramid/multiscale.py index d811ea0c..f3fd1af2 100644 --- a/src/ndi/pyramid/multiscale.py +++ b/src/ndi/pyramid/multiscale.py @@ -31,10 +31,55 @@ from __future__ import annotations +import heapq +import itertools import os import threading +import time from typing import Any +from ndi.cloud.batch_signed_url import BatchScopeUnreachable + + +class StaleChunkRead(Exception): + """A blocking chunk read was released because the view moved on. + + napari computes slices on ONE thread, so a slice for a view the user + has already zoomed past holds up the slice for where they stopped + until every one of its tiles has arrived. When the viewer reports + (:meth:`_ChunkFetcher.setView`) that a waiting chunk is no longer in + view, the read raises this instead of waiting. The fetch itself is + NOT cancelled: it stays queued at low priority and lands in the + local cache, so zooming back costs nothing. + + Raised, not answered with fill_value: napari's opportunistic dask + cache would keep a fill block under the chunk's task key, and the + region would stay blank after the real data arrived. + """ + + +# Returned by _ChunkFetcher._resolveDirect when the session-free path cannot +# answer (no manifest on disk yet, no cloud location, not a series member) and +# the caller should go through the session's database_openbinarydoc instead. +_FALLBACK = object() + + +class _Job: + """One queued or running chunk fetch, shared by everyone waiting on it.""" + + __slots__ = ("key", "doc", "filename", "started", "done", "path", "exc", "callbacks") + + def __init__(self, key, doc, filename): + self.key = key + self.doc = doc + self.filename = filename + self.started = False + self.done = threading.Event() + self.path: str | None = None + self.exc: BaseException | None = None + self.callbacks: list = [] + + # --------------------------------------------------------------------------- # depends_on / doc discovery @@ -138,34 +183,72 @@ class _ChunkFetcher: resolving is fetching. Eager resolution of every chunk of every level then downloads the whole pyramid before napari draws a pixel. - This class is the middle ground: dedicated fetcher threads each own - their own re-opened session. Delayed tasks call ``chunkPath(doc, name)`` + This class is the middle ground: dedicated fetcher threads resolve + chunks off the caller's thread. Delayed tasks call ``chunkPath(doc, name)`` from whichever thread they run on; the fetcher dispatches through its pool and returns the resolved local path. Resolved paths are memoised per (doc_id, filename) so a re-render never reaches the fetch threads again -- a cached hit is one ``os.path.exists`` from any thread. - ``workers`` DEFAULTS TO 1: one session opened once, exactly as if a - separate process held it. Cloud fetches are latency-bound, so raising - it buys overlap; it also costs another whole session per worker. + SESSION-FREE MEMBER RESOLUTION. A chunk is a file series member, and + resolving one needs no database at all: the level document is already + in memory, its manifest names each member's uid, and DID's + ``cached_path_for_uid`` finds a member on disk with no SQL (it is + documented as safe from any thread). A member that is not on disk is + fetched through NDI's cloud handler -- the same + ``download_file_from_cloud`` DID itself would call, with the same + context -- and lands in DID's file cache under its uid, exactly where + ``open_doc`` would have put it. So workers are plain threads, and a + worker only opens a session of its own (lazily, on first need) when a + chunk has to go the long way: the manifest is not on disk yet, or the + document records no cloud location. ``NDI_LIGHTSHEET_DIRECT=0`` turns + the session-free path off. + + ``workers`` defaults to 1 here; the viewer passes + :func:`_default_workers`. Cloud fetches are latency-bound, so many + workers buy overlap, and with session-free resolution an extra worker + costs one idle thread. """ - def __init__(self, session, workers: int = 1): + def __init__(self, session, workers: int = 1, max_workers: int | None = None): self._session = session self._owner = threading.get_ident() self._reopen = self._reopener(session) - self._workers = max(1, int(workers)) + # max_workers threads are started; at most `workers` of them fetch + # at once. The limit can change while the viewer runs (setWorkers): + # threads past it just wait, so lowering it never interrupts a + # fetch and raising it takes effect on the next job. + self._limit = max(1, int(workers)) + self._workers = max(self._limit, int(max_workers or 0)) + self._running = 0 self._local = threading.local() - self._pool = None self._lock = threading.Lock() self._paths: dict[tuple[str, str], str] = {} - # Async prefetch bookkeeping. When the upsample-fallback reader - # kicks off a background fetch, this maps the chunk key to the - # list of on_complete callbacks that fire when the pool worker - # finishes. Sharing the map de-duplicates: the same chunk - # requested twice enqueues one task and both callers are - # notified when it lands. - self._in_flight: dict[tuple[str, str], list] = {} + # Work queue. A heap of (tier, -seq, job): tier 0 is a chunk in + # the current view (see setView), tier 1 everything else, and + # within a tier the NEWEST request goes first -- the user is + # looking at what they asked for last, not at the levels they + # zoomed through on the way. One _Job per chunk key, shared by + # every waiter and callback, so a chunk is never fetched twice. + # A re-request pushes a fresh entry; stale heap entries for a + # started job are skipped on pop. + self._cv = threading.Condition() + self._heap: list = [] + self._jobs: dict[tuple[str, str], _Job] = {} + self._seq = itertools.count() + self._threads: list[threading.Thread] = [] + self._threads_ready = 0 + self._closed = False + # The current view's chunk keys, set from the viewer's main- + # thread poll. None means unknown: nothing is prioritised and + # no read is ever released as stale. + self._view_keys: frozenset | None = None + self._view_time = 0.0 + # Chunks of these docs are never released as stale: napari's + # auto-contrast samples the coarsest level at launch, whatever + # the view is. + self._never_stale_docs: set[str] = set() + self._stale_released = 0 # Per-fetch timing bucketed by outcome. Set by _resolve on every # call, including memo hits, so a caller can tell whether a # slow tile is "cloud is slow" vs "local cache lookup is slow" @@ -174,6 +257,48 @@ def __init__(self, session, workers: int = 1): self._resolve_times_cache: list[float] = [] self._resolve_times_fetch: list[float] = [] self._resolve_none: int = 0 + # Systemic failure of the batch signed-URL path. Once one background + # prefetch surfaces a BatchScopeUnreachable, every chunk in the + # affected scope will fail identically -- and returning fill_value + # for all of them would silently render the whole canvas as zeros, + # exactly the pattern the loader was written to prevent. We hold + # the first raise here so subsequent lookups (chunkPathIfCached, + # chunkPath) can surface it instead of hiding behind the fallback. + # See Waltham-Data-Science/NDI-python#322 and the strict-mode + # rationale in ndi.cloud.batch_signed_url.BatchScopeUnreachable. + self._unreachable_exc: BatchScopeUnreachable | None = None + # Session-free member resolution (see the class docstring). The + # file directory is read once, here, on the owner thread; the + # per-document and per-series caches hold only in-memory data + # and manifest uid lists. + self._direct = os.environ.get("NDI_LIGHTSHEET_DIRECT", "1").strip().lower() not in ( + "0", + "false", + "off", + ) + self._file_dir = self._fileDir(session) + self._did_docs: dict[str, Any] = {} + self._series: dict[tuple[str, str], tuple] = {} + self._direct_hits = 0 + self._direct_fetches = 0 + self._fallbacks = 0 + + @staticmethod + def _fileDir(session) -> str | None: + """DID's per-database file directory, or None if not reachable. + + A session exposes its database directly; a dataset holds one + through its inner session. + """ + for holder in (session, getattr(session, "_session", None)): + database = getattr(holder, "database", None) if holder is not None else None + try: + path = database.binary_path if database is not None else None + except Exception: # noqa: BLE001 - not DID-backed + path = None + if path: + return str(path) + return None @staticmethod def _reopener(session): @@ -190,22 +315,118 @@ def _reopener(session): cls = type(session) return lambda: cls(path) - def _initThread(self): - self._local.session = self._reopen() - - def _ensurePool(self): - if self._pool is not None or self._reopen is None: - return self._pool - with self._lock: - if self._pool is None: - from concurrent.futures import ThreadPoolExecutor - - self._pool = ThreadPoolExecutor( - max_workers=self._workers, - thread_name_prefix="ndi-lightsheet", - initializer=self._initThread, + def _ensureWorkers(self) -> bool: + """Start the fetcher threads once. False when the session cannot + be reopened for another thread (then everything runs inline).""" + if self._reopen is None and not self._direct: + return False + with self._cv: + if self._threads or self._closed: + return not self._closed + for i in range(self._workers): + t = threading.Thread( + target=self._workerLoop, + name=f"ndi-lightsheet_{i}", + daemon=True, ) - return self._pool + self._threads.append(t) + t.start() + return True + + def _workerLoop(self) -> None: + # No session here: most chunks resolve without one, and a worker + # that needs one opens it on first use (_threadSession). + with self._cv: + self._threads_ready += 1 + self._cv.notify_all() + while True: + with self._cv: + job = None + while job is None: + while (not self._heap or self._running >= self._limit) and not self._closed: + self._cv.wait() + if self._closed: + return + tier, nseq, cand = heapq.heappop(self._heap) + if cand.started: + continue + if tier == 0 and (self._view_keys is None or cand.key not in self._view_keys): + # Was in view when queued, is not any more. + heapq.heappush(self._heap, (1, nseq, cand)) + continue + cand.started = True + self._running += 1 + job = cand + self._run(job) + + def _run(self, job: _Job) -> None: + """Resolve one job on this thread and settle everyone waiting.""" + path = None + try: + path = self._resolve(job.doc, job.filename) + except BatchScopeUnreachable as exc: + # Systemic scope failure. Remember it so the next + # chunkPathIfCached / chunkPath surfaces it instead of + # letting the upsample fallback quietly return a canvas + # full of zeros. + with self._lock: + if self._unreachable_exc is None: + self._unreachable_exc = exc + job.exc = exc + except BaseException as exc: # noqa: BLE001 - handed to the waiter + job.exc = exc + if path is not None: + with self._lock: + self._paths[job.key] = path + with self._cv: + self._jobs.pop(job.key, None) + callbacks, job.callbacks = job.callbacks, [] + self._running -= 1 + self._cv.notify_all() + job.path = path + job.done.set() + for cb in callbacks: + try: + cb(path) + except Exception: # noqa: BLE001 - a bad callback is not fatal + pass + + def _enqueue(self, doc, filename, on_complete=None) -> _Job: + """Queue a fetch (or bump an already-queued one to newest).""" + key = (getattr(doc, "id", str(doc)), filename) + with self._cv: + job = self._jobs.get(key) + if job is None: + job = _Job(key, doc, filename) + self._jobs[key] = job + if on_complete is not None: + job.callbacks.append(on_complete) + if not job.started: + tier = 0 if self._view_keys is not None and key in self._view_keys else 1 + heapq.heappush(self._heap, (tier, -next(self._seq), job)) + self._cv.notify() + return job + + @property + def workerLimit(self) -> int: + """How many chunks may be fetched at once right now.""" + return self._limit + + @property + def maxWorkers(self) -> int: + """The most :meth:`setWorkers` will allow: the threads started.""" + return self._workers + + def setWorkers(self, n: int) -> int: + """Change how many chunks are fetched at once; returns the new limit. + + Clamped to 1..:attr:`maxWorkers`. Takes effect immediately for + queued work; fetches already running finish undisturbed. + """ + with self._cv: + self._limit = max(1, min(int(n), self._workers)) + self._cv.notify_all() + return self._limit def warm(self) -> None: """Build the session handles now, off the calling thread. @@ -214,19 +435,119 @@ def warm(self) -> None: one-off session-open cost lands while the user is staring at an empty canvas rather than at the first pan. """ - pool = self._ensurePool() - if pool is None: + if not self._ensureWorkers(): + return + with self._cv: + while self._threads_ready < self._workers and not self._closed: + self._cv.wait() + + def setView(self, keys, never_stale_doc_ids=None) -> None: + """Tell the fetcher which chunks the current view needs. + + KEYS maps ``(doc_id, filename)`` to the level document (None: + the view is unknown). Chunks in view jump the queue -- and are + queued now if nobody has asked yet, so they are already on the + way while napari is still finishing an older slice. A blocking + read for a chunk that has dropped OUT of view is released with + :class:`StaleChunkRead`; its fetch stays queued at low priority. + """ + with self._cv: + if never_stale_doc_ids is not None: + self._never_stale_docs = set(never_stale_doc_ids) + if keys is None: + self._view_keys = None + self._cv.notify_all() + return + self._view_keys = frozenset(keys) + self._view_time = time.monotonic() + # Re-rank what is queued: anything now in view to tier 0. + for key in self._view_keys: + job = self._jobs.get(key) + if job is not None and not job.started: + heapq.heappush(self._heap, (0, -next(self._seq), job)) + self._cv.notify_all() + # Queue what the view needs and nobody has asked for yet. + if not self._threads and not self._ensureWorkers(): return - for f in [pool.submit(lambda: None) for _ in range(self._workers)]: - f.result() + for key, doc in keys.items(): + known = self._paths.get(key) + if known is not None and os.path.exists(known): + continue + self._enqueue(doc, key[1]) + + def _isStale(self, job: _Job, since: float) -> bool: + view = self._view_keys + return ( + view is not None + and self._view_time > since + and job.key not in view + and job.key[0] not in self._never_stale_docs + ) + + def staleReleased(self) -> int: + """How many blocking reads were released as stale so far.""" + return self._stale_released + + def _threadSession(self): + """The session this thread may use, opening one on first need. + + The owner thread uses the caller's session. A worker reopens its + own the first time a chunk cannot be resolved session-free. A + session that cannot be reopened (no path) is used as-is, as before: + that is the directory-less case the inline fallback serves. + """ + if threading.get_ident() == self._owner or self._reopen is None: + return self._session + s = getattr(self._local, "session", None) + if s is None: + s = self._reopen() + self._local.session = s + return s def _resolve(self, doc, filename) -> str | None: - s = getattr(self._local, "session", None) or self._session import time as _time t0 = _time.monotonic() + if self._direct: + try: + direct = self._resolveDirect(doc, filename) + except BatchScopeUnreachable: + with self._stats_lock: + self._resolve_none += 1 + raise + except Exception as exc: # noqa: BLE001 - same contract as below + with self._stats_lock: + self._resolve_none += 1 + if os.environ.get("NDI_LIGHTSHEET_DEBUG"): + import sys + + print( + f"[lightsheet] chunk fetch failed for {filename}: " + f"{type(exc).__name__}: {exc}", + file=sys.stderr, + ) + return None + if direct is not _FALLBACK: + self._recordResolve(filename, _time.monotonic() - t0, direct) + return direct + with self._stats_lock: + self._fallbacks += 1 try: + s = self._threadSession() fh = s.database_openbinarydoc(doc, filename) + except BatchScopeUnreachable: + # A systemic failure of the batch signed-URL path is not a + # transient blip we can paper over with fill_value: every + # chunk in the scope will fail identically, and swallowing + # this one turns the whole canvas into silent zeros -- the + # exact "silent fetch failure" pattern this loader was + # written to prevent. Propagate so callers (viewer, dask + # compute, integration test) see the actual cause. See + # Waltham-Data-Science/NDI-python#322 and the strict-mode + # rationale on BatchScopeUnreachable itself. + with self._stats_lock: + self._resolve_none += 1 + raise except Exception as exc: with self._stats_lock: self._resolve_none += 1 @@ -241,29 +562,212 @@ def _resolve(self, doc, filename) -> str | None: return None try: path = getattr(fh, "fullpathfilename", None) - dt = _time.monotonic() - t0 - # database_openbinarydoc has two happy paths: a local-cache - # hit (near instant, microseconds; no cloud roundtrip) and - # a cloud fetch (tens of ms to seconds). Split the buckets - # so a summary later can say how many of each we saw and - # what each one costs. - # - # 50 ms is comfortably above local disk latency and well - # under any credible cloud RTT; picking a threshold rather - # than routing on internal state keeps the classifier - # independent of DID's evolving fast paths. - with self._stats_lock: - if dt < 0.05: - self._resolve_times_cache.append(dt) - else: - self._resolve_times_fetch.append(dt) - return str(path) if path else None + path = str(path) if path else None + self._recordResolve(filename, _time.monotonic() - t0, path) + return path finally: try: s.database_closebinarydoc(fh) except Exception: pass + def _recordResolve(self, filename, dt: float, path) -> None: + """Bucket one resolve's time; under debug, say what a fetch cost. + + database_openbinarydoc and the session-free path both have two + happy paths: a local hit (microseconds) and a cloud fetch (tens of + ms to seconds). 50 ms is comfortably above local disk latency and + well under any credible cloud round trip; a threshold keeps the + classifier independent of either path's internals. + """ + with self._stats_lock: + if dt < 0.05: + self._resolve_times_cache.append(dt) + else: + self._resolve_times_fetch.append(dt) + if dt >= 0.05 and os.environ.get("NDI_LIGHTSHEET_DEBUG"): + import sys + + from ndi.cloud.filehandler import lastFetchTiming + + t = lastFetchTiming() + if t: + size = t.get("bytes", 0) or 0 + xfer = t.get("transfer_s", 0.0) or 0.0 + rate = f", {size / xfer / 1e6:.1f} MB/s" if xfer > 0 and size else "" + detail = ( + f"sign {t.get('sign_s', 0.0):.2f}s " + f"({'batch' if t.get('batch') else 'per-file'}), " + f"transfer {xfer:.2f}s, {size / 1e6:.2f} MB{rate}" + ) + else: + detail = "no cloud timing recorded" + print( + f"[lightsheet] fetched {filename} in {dt:.2f}s: {detail} " + f"(thread {threading.current_thread().name})", + file=sys.stderr, + flush=True, + ) + + # ------------------------------------------------------------ direct path + + def _resolveDirect(self, doc, filename): + """Resolve a series member with no database, or return _FALLBACK. + + Local first, through DID's ``cached_path_for_uid`` (global file + cache, then this database's file directory -- the order + ``open_doc`` uses). A member that is not on disk is fetched through + the cloud handler into DID's file cache. Returns the local path, + None for a member the manifest records as absent (sparse; what + ``open_doc`` reports as NoSuchMember), or _FALLBACK when this path + cannot answer and the session must. + """ + from did.file import cached_path_for_uid + + found = self._seriesFor(doc, filename) + if found is None: + return _FALLBACK + stem, index, uids, manifest_location, doc_id = found + if index > len(uids) or not uids[index - 1]: + return None + member_uid = str(uids[index - 1]) + roots = [self._file_dir] if self._file_dir else [] + local = cached_path_for_uid(member_uid, additional_roots=roots) + if local: + with self._stats_lock: + self._direct_hits += 1 + return str(local) + if not manifest_location: + return _FALLBACK + path = self._fetchMember(doc_id, filename, stem, member_uid, manifest_location) + with self._stats_lock: + self._direct_fetches += 1 + return path + + def _seriesFor(self, doc, filename): + """(stem, index, member uids, manifest ndic location, doc id), or None. + + Reads only the document in memory and the manifest on disk. The + manifest's uid list is read once per (document, series) and kept; + a manifest that is not on disk yet is not remembered as missing, + so once the session path has fetched it the next call finds it. + """ + try: + from did.document import Document as DIDDocument + from did.file import cached_path_for_uid, read_series_manifest + except ImportError: + return None + doc_id = getattr(doc, "id", None) + if not doc_id: + return None + did_doc = self._did_docs.get(doc_id) + if did_doc is None: + props = getattr(doc, "document_properties", None) + if not isinstance(props, dict): + return None + did_doc = DIDDocument(props) + self._did_docs[doc_id] = did_doc + stem, index = did_doc.series_member_of(filename) + if not stem or index is None or index < 1: + return None + entry = self._series.get((doc_id, stem)) + if entry is None: + roots = [self._file_dir] if self._file_dir else [] + manifest_path = None + for uid in did_doc.file_uids(stem): + manifest_path = cached_path_for_uid(uid, additional_roots=roots) + if manifest_path: + break + if not manifest_path: + return None + try: + uids = [ + str(u) if u else "" for u in read_series_manifest(manifest_path).get("uids", []) + ] + except Exception: # noqa: BLE001 - let the session path report it + return None + entry = (uids, _ndicLocation(did_doc.document_properties, stem)) + self._series[(doc_id, stem)] = entry + uids, location = entry + return stem, index, uids, location, doc_id + + def _fetchMember(self, doc_id, filename, stem, member_uid, manifest_location) -> str: + """Fetch one member through the cloud handler into DID's file cache. + + The same call DID's ``open_doc`` makes -- the manifest's location + as the source and the member in the context, so the handler fetches + the member's own object through the batch signed-URL cache -- into + a unique scratch file in DID's temp folder, then moved into the + global file cache under the member's uid. A cache that cannot take + it (unwritable, or a uid of the wrong width) leaves the file in + this database's file directory instead; either way + ``cached_path_for_uid`` finds it next time, in this process or the + next, and so does DID. + """ + import tempfile + + from did.common import PathConstants + + from ndi.cloud.filehandler import download_file_from_cloud + + temp_dir = PathConstants().temppath + os.makedirs(temp_dir, exist_ok=True) + fd, scratch = tempfile.mkstemp(dir=temp_dir, prefix=f"{member_uid}.", suffix=".part") + os.close(fd) + os.remove(scratch) + try: + download_file_from_cloud( + scratch, + manifest_location, + { + "documentId": doc_id, + "filename": filename, + "seriesName": stem, + "uid": member_uid, + "mode": "open", + }, + ) + if not os.path.isfile(scratch): + raise FileNotFoundError( + f"the cloud handler produced no file for {filename} (uid {member_uid})" + ) + return self._keepFetched(scratch, member_uid) + except BaseException: + try: + os.remove(scratch) + except OSError: + pass + raise + + def _keepFetched(self, scratch: str, member_uid: str) -> str: + """Move a fetched member to where cached_path_for_uid will find it.""" + import struct + + try: + from did.common import get_cache + + cache = get_cache() + if len(member_uid) == cache.file_name_characters: + if cache.is_file(member_uid): + existing = cache.full_path(member_uid) + if os.path.isfile(existing): + # Another fetch of the same uid won the race; same bytes. + os.remove(scratch) + return existing + cache.remove_file(member_uid) + cache.add_file(scratch, member_uid) + if os.path.exists(scratch): + os.remove(scratch) + return cache.full_path(member_uid) + except (OSError, ValueError, struct.error): + pass + if self._file_dir: + os.makedirs(self._file_dir, exist_ok=True) + dest = os.path.join(self._file_dir, member_uid) + os.replace(scratch, dest) + return dest + return scratch + def stats_summary(self) -> str: """One-line summary of resolve timings for the last session. @@ -282,10 +786,13 @@ def stats_summary(self) -> str: mean_cache_ms = 1000.0 * (sum(self._resolve_times_cache) / n_cache if n_cache else 0.0) mean_fetch_s = sum(self._resolve_times_fetch) / n_fetch if n_fetch else 0.0 max_fetch_s = max(self._resolve_times_fetch) if n_fetch else 0.0 + n_fallback = self._fallbacks return ( f"resolves: {n_cache} cache-hits (mean {mean_cache_ms:.1f}ms), " f"{n_fetch} cloud-fetches (mean {mean_fetch_s:.2f}s, max " - f"{max_fetch_s:.2f}s), {n_none} failures" + f"{max_fetch_s:.2f}s), {n_none} failures, " + f"{self._stale_released} stale reads released, " + f"{n_fallback} via a session ({self._limit} of {self._workers} workers)" ) def reset_stats(self) -> None: @@ -293,6 +800,9 @@ def reset_stats(self) -> None: self._resolve_times_cache.clear() self._resolve_times_fetch.clear() self._resolve_none = 0 + self._direct_hits = 0 + self._direct_fetches = 0 + self._fallbacks = 0 def chunkPath(self, doc, filename) -> str | None: """Resolve one chunk file to a local path, fetching it if remote. @@ -304,6 +814,8 @@ def chunkPath(self, doc, filename) -> str | None: ``os.path.exists`` from any thread. Files can be evicted, so a stale hit that fails on read should be dropped with :meth:`forget`. """ + if self._unreachable_exc is not None: + raise self._unreachable_exc key = (getattr(doc, "id", str(doc)), filename) known = self._paths.get(key) if known is not None and os.path.exists(known): @@ -342,7 +854,16 @@ def chunkPathIfCached(self, doc, filename) -> str | None: upsampled coarse data" WITHOUT waiting on a cloud round trip. A miss means "we do not have it now"; whether we ever will is a separate question the async prefetch answers. + + Raises :class:`BatchScopeUnreachable` if a prior background + prefetch established that the batch signed-URL path is dead + for this fetcher. The upsample fallback would otherwise + quietly return a zero block for every chunk in the affected + scope; surfacing the exception here forces the actual cause + out through ``.compute()``. """ + if self._unreachable_exc is not None: + raise self._unreachable_exc key = (getattr(doc, "id", str(doc)), filename) known = self._paths.get(key) if known is not None and os.path.exists(known): @@ -354,48 +875,35 @@ def prefetchAsync(self, doc, filename, on_complete=None) -> None: The upsample-fallback reader returns coarse-upsampled data immediately for a chunk that is not on disk; this asks the - pool to go fetch the real thing so a later re-slice can - find it via :meth:`chunkPathIfCached`. On completion (or + fetcher threads to go fetch the real thing so a later re-slice + can find it via :meth:`chunkPathIfCached`. On completion (or failure), ``on_complete`` is called with the resolved path or None -- callers use that to trigger a napari refresh so the user is not stuck looking at blurred data forever. De-duplicates by memo key: the same (doc, filename) already - in flight or already cached does not enqueue a second task. + queued or already cached does not fetch a second time. Errors are absorbed; the reader will retry on the next re-slice and eventually get real data or a permanent fallback. """ key = (getattr(doc, "id", str(doc)), filename) - with self._lock: - if key in self._paths and os.path.exists(self._paths[key]): - # Already cached -- no need to enqueue. Fire the - # completion callback synchronously so the caller - # can react uniformly whether the answer was cheap - # or expensive. - if on_complete is not None: - try: - on_complete(self._paths[key]) - except Exception: # noqa: BLE001 - pass - return - if key in self._in_flight: - if on_complete is not None: - self._in_flight[key].append(on_complete) - return - self._in_flight[key] = [on_complete] if on_complete is not None else [] - - pool = self._ensurePool() - if pool is None: - # No pool means this session cannot be reopened, and - # the fetcher is single-threaded. Fall back to a - # blocking resolve on the caller's thread -- it is the - # rare directory-backed case and does not need async. - path = self._resolve(doc, filename) - self._settle_in_flight(key, path) + known = self._paths.get(key) + if known is not None and os.path.exists(known): + # Already cached -- fire the completion callback + # synchronously so the caller can react uniformly whether + # the answer was cheap or expensive. + if on_complete is not None: + try: + on_complete(known) + except Exception: # noqa: BLE001 + pass return - - def _run(): + if not self._ensureWorkers(): + # No workers means this session cannot be reopened, and + # the fetcher is single-threaded. Fall back to a blocking + # resolve on the caller's thread -- it is the rare + # directory-backed case and does not need async. try: path = self._resolve(doc, filename) except Exception: # noqa: BLE001 @@ -403,25 +911,18 @@ def _run(): if path is not None: with self._lock: self._paths[key] = path - self._settle_in_flight(key, path) - - pool.submit(_run) - - def _settle_in_flight(self, key, path) -> None: - """Drain and call every pending completion callback for one key.""" - with self._lock: - callbacks = self._in_flight.pop(key, []) - for cb in callbacks: - try: - cb(path) - except Exception: # noqa: BLE001 - a bad callback is not fatal - pass + if on_complete is not None: + try: + on_complete(path) + except Exception: # noqa: BLE001 + pass + return + self._enqueue(doc, filename, on_complete=on_complete) def _fetch(self, doc, filename) -> str | None: if threading.get_ident() == self._owner: return self._resolve(doc, filename) - pool = self._ensurePool() - if pool is None: + if not self._ensureWorkers(): raise RuntimeError( f"Cannot fetch {filename!r} from thread " f"{threading.current_thread().name}: this session cannot be " @@ -430,23 +931,92 @@ def _fetch(self, doc, filename) -> str | None: f"with dask's synchronous scheduler, or pass a session that " f"exposes a path." ) - return pool.submit(self._resolve, doc, filename).result() + since = time.monotonic() + job = self._enqueue(doc, filename) + # Wait, but keep checking whether the view has moved on: a + # slice for a view the user zoomed past must not hold up the + # slice for where they stopped (napari slices on one thread). + while not job.done.wait(0.1): + if self._isStale(job, since): + with self._lock: + self._stale_released += 1 + raise StaleChunkRead(f"{filename} left the view before it arrived") + if job.exc is not None: + raise job.exc + return job.path def close(self) -> None: - if self._pool is not None: - self._pool.shutdown(wait=False) - self._pool = None + with self._cv: + self._closed = True + self._heap.clear() + self._cv.notify_all() + + +_DEFAULT_WORKERS = 16 + +#: Threads started per fetcher, so the viewer can raise the limit live. +MAX_WORKERS = 64 + +#: The choices the viewer offers for tiles in flight. +WORKER_CHOICES = (8, 16, 32, 64) def _default_workers() -> int: - """Chunk-fetcher worker count. Env override for the cloud-latency case.""" + """Chunks fetched at once at launch; ``NDI_LIGHTSHEET_WORKERS`` overrides. + + 16 by default. A cloud tile is seconds of waiting on the network and + almost no CPU, so overlap pays -- until the line is full: on a + 100 Mb/s connection 32 tiles of a few MB each just share the same + bandwidth and every one arrives later. The viewer can change it live + (:meth:`_ChunkFetcher.setWorkers`). + """ try: - n = int(os.environ.get("NDI_LIGHTSHEET_WORKERS", "8")) + n = int(os.environ.get("NDI_LIGHTSHEET_WORKERS", str(_DEFAULT_WORKERS))) except ValueError: - return 8 + return _DEFAULT_WORKERS return max(1, n) +def configureDaskPool(workers: int) -> int: + """Size dask's thread pool for I/O-bound chunk reads; returns its size. + + napari computes slices through dask's threaded scheduler, whose pool + defaults to one thread per CPU -- right for computation, wrong here: + each chunk read mostly waits on the network, so the pool, not the + fetcher, caps how many tiles are in flight. This sets the pool to at + least ``workers`` threads (never fewer than the CPU count), process + wide, so napari's reads can keep every fetcher worker busy. + """ + from concurrent.futures import ThreadPoolExecutor + + import dask + + n = max(os.cpu_count() or 4, int(workers)) + dask.config.set(pool=ThreadPoolExecutor(n, thread_name_prefix="ndi-dask")) + return n + + +def _ndicLocation(props: dict, name: str) -> str: + """The ``ndic://`` location recorded for file ``name``, or "".""" + from ndi.cloud.filehandler import NDIC_SCHEME + + files = props.get("files") if isinstance(props, dict) else None + if not isinstance(files, dict): + return "" + file_info = files.get("file_info") + infos = file_info if isinstance(file_info, list) else [file_info] if file_info else [] + for info in infos: + if not isinstance(info, dict) or str(info.get("name", "")).lower() != name.lower(): + continue + locations = info.get("locations") + locs = locations if isinstance(locations, list) else [locations] if locations else [] + for loc in locs: + where = loc.get("location", "") if isinstance(loc, dict) else "" + if str(where).startswith(NDIC_SCHEME): + return str(where) + return "" + + def levelArrays( session: Any, pyramid_doc: Any, @@ -496,7 +1066,8 @@ def levelArrays( _ = channel # single-channel narrowing lands with the magicgui panels - fetcher = _ChunkFetcher(session, workers=workers if workers is not None else _default_workers()) + limit = workers if workers is not None else _default_workers() + fetcher = _ChunkFetcher(session, workers=limit, max_workers=max(limit, MAX_WORKERS)) # Upsample-fallback context: build once from the coarsest level's # document so every fine-level delayed task can look up covering @@ -509,6 +1080,12 @@ def levelArrays( fetcher._fallback_context = fallback_context # type: ignore[attr-defined] arrays: list[Any] = [] + # Per-level chunk geometry, finest first (same order as ARRAYS). + # The viewer reads this to work out which chunks the current view + # needs, so it can say "37 of 120 tiles" rather than only how many + # are in flight. Rides on the fetcher like the fallback context. + level_meta: list[dict] = [] + fetcher._level_meta = level_meta # type: ignore[attr-defined] for i, doc in enumerate(docs): p = doc.document_properties["lightsheetZarrLevel"] shape = tuple(int(v) for v in p["shape"]) @@ -520,6 +1097,16 @@ def levelArrays( stored = _storedChunkNames(doc) _assertStoredMatchesLevel(doc, p, stored) + level_meta.append( + { + "doc": doc, + "axes": str(p.get("axes_order", "")).lower(), + "chunks": chunks, + "chunk_grid": chunk_grid, + "stored": stored, + } + ) + n_blocks = 1 for g in chunk_grid: n_blocks *= g @@ -1179,19 +1766,24 @@ def layerSpec( # only a graph rewrite that says "block[c] instead of block[all]". import dask.array as da - # Default is single-level (only the coarsest, as a plain non- - # multiscale layer). Napari 0.5's multiscale slicer has been - # observed to never mark layer.loaded=True on a lazy cloud-backed - # 3D multiscale pyramid: the channel-list spinner spins forever - # and nothing draws. Single-level takes the multiscale slicer out - # of the loop entirely; the layer draws, then a magicgui panel - # lets the user swap between levels manually (see - # :func:`_attach_level_selector` in viewer.py). + # Default is multiscale: user A/B preferred this over single-level + # because the transition between levels is smoother -- single-level + # swaps layer.data under napari, which shows up as a short black + # frame on every level change, while a native multiscale layer + # just picks a finer level and refines. Both modes have been + # observed to hit the same napari slicer wedge on this dataset + # (vispy "QBasicTimer destroyed dispatcher" spam, no tile + # requests), so single-level is not actually safer -- it just + # looks janker when it does paint. # - # NDI_LIGHTSHEET_MULTISCALE=1 opts back into the multiscale path - # for anyone testing whether the napari-side bug has been fixed - # or for a data shape that does not hit it. - single_level = not _env_true("NDI_LIGHTSHEET_MULTISCALE") + # NDI_LIGHTSHEET_SINGLE_LEVEL=1 opts back into the earlier + # single-level + Resolution-picker mode for anyone who wants + # the manual swap flow (or is debugging the multiscale slicer + # bug on a different dataset shape). + # NDI_LIGHTSHEET_ASYNC=0 is the escape hatch when the slicer + # wedges: it blocks add_image until the first slice is on + # screen, bypassing napari's async slicer entirely. + single_level = _env_true("NDI_LIGHTSHEET_SINGLE_LEVEL") # Build per-channel arrays once; each is a list of one 3D lazy # dask array per level. The level dropdown swaps between the diff --git a/src/ndi/pyramid/ndi_matlab_python_bridge.yaml b/src/ndi/pyramid/ndi_matlab_python_bridge.yaml index 98c57a0e..438fa364 100644 --- a/src/ndi/pyramid/ndi_matlab_python_bridge.yaml +++ b/src/ndi/pyramid/ndi_matlab_python_bridge.yaml @@ -100,6 +100,18 @@ functions: One layer per channel. Kept here (rather than viewer.py) so a headless caller can build the ladders without a display. + - name: configureDaskPool + matlab_path: null + python_path: "ndi/pyramid/multiscale.py" + output_arguments: + - name: n + type_python: int + decision_log: >- + Python-only in this namespace. Widens dask's threaded-scheduler pool + (default: one thread per CPU) to the chunk fetcher's worker count, so + napari's I/O-bound chunk reads can keep every worker busy. MATLAB has + no dask; nothing to mirror. + - name: prefetchCoarsestLevel matlab_path: null python_path: "ndi/pyramid/multiscale.py" diff --git a/src/ndi/pyramid/upsample_fallback.py b/src/ndi/pyramid/upsample_fallback.py index d22824fd..cd8b1311 100644 --- a/src/ndi/pyramid/upsample_fallback.py +++ b/src/ndi/pyramid/upsample_fallback.py @@ -28,7 +28,7 @@ of interest. Bicubic is prettier but the point is "any pixel beats black" not "publication quality". -Off by default -- opt in with ``NDI_LIGHTSHEET_UPSAMPLE_FALLBACK=1``. +On by default; opt out with ``NDI_LIGHTSHEET_UPSAMPLE_FALLBACK=0``. When on, the reader path in :func:`multiscale._build_block_grid` routes each block through :func:`readChunkWithFallback` instead of :func:`multiscale._read_chunk_from_fetcher`. Reader also fires an @@ -42,18 +42,35 @@ from __future__ import annotations import os +import sys import threading from typing import Any def env_on() -> bool: - """True when the upsample fallback is opted-in via the env var.""" - return os.environ.get("NDI_LIGHTSHEET_UPSAMPLE_FALLBACK", "").strip().lower() in ( - "1", - "true", - "on", - "yes", - ) + """False unless explicitly enabled via NDI_LIGHTSHEET_UPSAMPLE_FALLBACK. + + Default flipped back to OFF after live A/B testing on the Maddie + lightsheet pyramid showed the fallback actively hiding fresh fine + data: once Path 2 returned an upsampled-coarse block for a chunk + on the first slice compute, napari treated that slice as "loaded" + and did not re-render even after our refresh hint fired and Path 1 + returned real fine data on the next compute (hit_fine climbed in + the stats while the on-screen pixels stayed coarse). Flipping the + knob to 0 made the view refresh correctly as fine chunks arrived. + + The historical reason to default it ON was multi-minute black + stretches during level swaps on a slow cloud connection; keep the + knob so cloud users can still opt in, but default OFF because + "coarse pixels forever that never refine" is worse UX than "black + briefly then correct fine pixels". + + ``NDI_LIGHTSHEET_UPSAMPLE_FALLBACK=1`` (or true/on/yes) opts in. + Any other value, including empty (env var unset), leaves the + fallback off. + """ + value = os.environ.get("NDI_LIGHTSHEET_UPSAMPLE_FALLBACK", "").strip().lower() + return value in ("1", "true", "on", "yes") def _fallback_debug() -> bool: @@ -280,12 +297,17 @@ def upsampleToBlock( _STATS_LOCK = threading.Lock() _STATS = { "hit_fine": 0, # chunkPathIfCached found it -> real fine returned + "fine_decode_failed": 0, # fine chunk on disk but decode raised (silent-fallback) "upsampled": 0, # fine missing, upsampled from coarse "zero_no_coarse_cover": 0, # no covering coarse chunk in-range "zero_coarse_missing": 0, # covering coarse chunk not on disk "zero_upsample_failed": 0, # upsampler returned None "zero_read_fail": 0, # decode of coarse chunk raised "prefetches_queued": 0, # prefetchAsync calls + "refresh_hints_null": 0, # reader called with no refresh hint installed + "refresh_hints_called": 0, # RefreshHint.__call__ invoked (from fetch completion) + "refresh_hints_scheduled": 0, # QTimer.singleShot was accepted + "refresh_hints_fired": 0, # RefreshHint._fire ran (on the Qt main thread) } @@ -299,13 +321,79 @@ def fallbackStatsSummary() -> str: they land (refresh hint not triggering a slice compute). """ with _STATS_LOCK: - parts = [f"{k}={v}" for k, v in _STATS.items()] + parts = [f"{k}={v}" for k, v in _STATS.items() if not k.startswith("_")] return "upsample-fallback " + " ".join(parts) def _bump(key: str, n: int = 1) -> None: with _STATS_LOCK: _STATS[key] = _STATS.get(key, 0) + n + _STATS["_dirty"] = _STATS.get("_dirty", 0) + n + + +# Expose module-level bump so RefreshHint can log its activity in +# the same stats stream the user already watches. +bump = _bump + + +def totalReaderActivity() -> int: + """How many times the fallback reader has returned a block. + + Sum of hit_fine + upsampled + every zero_* bucket. Used by the + viewer heartbeat to tell "napari actually asked for a slice" from + "napari has no local-disk fetches so the cloud-only _FetchCounter + stayed at zero even though the slicer was working all along." + """ + activity_keys = {"hit_fine", "upsampled", "fine_decode_failed"} + with _STATS_LOCK: + return sum(v for k, v in _STATS.items() if k in activity_keys or k.startswith("zero_")) + + +_HEARTBEAT_STARTED = False +_HEARTBEAT_LOCK = threading.Lock() + + +def _ensure_debug_heartbeat() -> None: + """Spin up one daemon thread that dumps fallback stats every ~5 s. + + Only runs when ``_fallback_debug()`` is on. Prints a stats line + only when something changed since the last tick, so a quiet + session stays quiet. Starts lazily on the first fallback call so + there is nothing to clean up at shutdown. + """ + global _HEARTBEAT_STARTED + if _HEARTBEAT_STARTED or not _fallback_debug(): + return + with _HEARTBEAT_LOCK: + if _HEARTBEAT_STARTED: + return + _HEARTBEAT_STARTED = True + + def _loop(): + import time + + last_signature = None + while True: + time.sleep(5.0) + with _STATS_LOCK: + dirty = _STATS.get("_dirty", 0) + if dirty == 0: + continue + _STATS["_dirty"] = 0 + snapshot = {k: v for k, v in _STATS.items() if not k.startswith("_")} + sig = tuple(sorted(snapshot.items())) + if sig == last_signature: + continue + last_signature = sig + parts = " ".join(f"{k}={v}" for k, v in snapshot.items()) + print( + f"[lightsheet] upsample-fallback live: {parts}", + file=sys.stderr, + flush=True, + ) + + t = threading.Thread(target=_loop, name="ndi-lightsheet-fallback-stats", daemon=True) + t.start() def readChunkWithFallback( @@ -344,6 +432,8 @@ def readChunkWithFallback( """ from ndi.pyramid.multiscale import _read_chunk_from_fetcher, _zero_block + _ensure_debug_heartbeat() + # Path 1: fine chunk on disk -> normal read. fine_path = fetcher.chunkPathIfCached(fine_doc, fine_filename) if fine_path is not None: @@ -353,10 +443,25 @@ def readChunkWithFallback( ) _bump("hit_fine") return result - except Exception: # noqa: BLE001 - a bad decode is fallback time - pass + except Exception as decode_exc: # noqa: BLE001 - a bad decode is fallback time + # The fine chunk is on disk but failed to decode: a + # truncated download, a codec mismatch, or a corrupt + # blob. Falling through to coarse-upsample keeps the + # view from going black, but if we don't LOG this, the + # user sees level-3 pixels forever while the Auto-level + # picker reports level 0 and no error ever appears -- + # that was the silent-blurry symptom before this change. + _bump("fine_decode_failed") + print( + f"[lightsheet] fine decode failed, upsampling from coarse: " + f"{fine_filename!r} ({type(decode_exc).__name__}: {decode_exc})", + file=sys.stderr, + flush=True, + ) # Path 2: fine missing -> upsample coarse if we can, and fetch fine. + if refresh_hint is None: + _bump("refresh_hints_null") fetcher.prefetchAsync(fine_doc, fine_filename, on_complete=refresh_hint) _bump("prefetches_queued") diff --git a/tests/conftest.py b/tests/conftest.py index a2425eaa..2065d458 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -223,3 +223,43 @@ def _isolate_did_file_cache(tmp_path_factory): else: os.environ[env_var] = previous did_common._cached_cache = None + + +# --------------------------------------------------------------------------- +# Isolate the signed-URL disk cache for the whole test session. +# +# Waltham-Data-Science/NDI-python#322: ``ndi.cloud.signed_url_disk_cache`` +# persists signed-URL-set responses under ``~/.ndi/signed-url-cache/`` so a +# scientist reopening the same lightsheet the next morning pays the +# ~85 min async job cost once. That is exactly the trap DID's file cache +# used to be: a live-cloud test that hits the production +# ``batch_signed_url.get_default()`` populates the user's real cache, and +# subsequent test runs against different accounts / different environments +# could either read stale entries or silently pollute the developer's home +# directory. +# +# Point the cache dir at a session-scoped tmp directory and unset the +# safety-buffer override so each test starts from the documented 30 min +# default. Same shape as the DID isolation above, same reasoning. +# --------------------------------------------------------------------------- + + +@_pytest.fixture(autouse=True, scope="session") +def _isolate_signed_url_disk_cache(tmp_path_factory): + cache_dir = tmp_path_factory.mktemp("signed-url-cache") + + previous_dir = os.environ.get("NDI_SIGNED_URL_CACHE_DIR") + previous_safety = os.environ.get("NDI_SIGNED_URL_CACHE_SAFETY_SECONDS") + os.environ["NDI_SIGNED_URL_CACHE_DIR"] = str(cache_dir) + os.environ.pop("NDI_SIGNED_URL_CACHE_SAFETY_SECONDS", None) + try: + yield cache_dir + finally: + if previous_dir is None: + os.environ.pop("NDI_SIGNED_URL_CACHE_DIR", None) + else: + os.environ["NDI_SIGNED_URL_CACHE_DIR"] = previous_dir + if previous_safety is None: + os.environ.pop("NDI_SIGNED_URL_CACHE_SAFETY_SECONDS", None) + else: + os.environ["NDI_SIGNED_URL_CACHE_SAFETY_SECONDS"] = previous_safety diff --git a/tests/fixtures/lightsheet_cloud_fixture.json b/tests/fixtures/lightsheet_cloud_fixture.json new file mode 100644 index 00000000..1d1bb812 --- /dev/null +++ b/tests/fixtures/lightsheet_cloud_fixture.json @@ -0,0 +1,8 @@ +{ + "cloud_dataset_id": "6accf3a0044c77d93a6ebdb7", + "user": "1", + "environment": "prod", + "created_at": "2026-10-12T14:50:35Z", + "workflow_run": "https://github.com/Waltham-Data-Science/NDI-python/actions/runs/38265299422", + "notes": "Small synthetic lightsheet blob uploaded by ndi.test.cloud.lightsheet_blob_cloud_roundtrip (MATLAB), 12 documents / 15 files / 1.92 MB. Regenerate with the Build lightsheet blob fixture workflow. tests/test_pyramid_loader_integration.py reads this JSON by default when NDI_LIGHTSHEET_TEST_FIXTURE_ID is not set." +} diff --git a/tests/test_cloud_api_feedback.py b/tests/test_cloud_api_feedback.py index f80730a0..755f38fd 100644 --- a/tests/test_cloud_api_feedback.py +++ b/tests/test_cloud_api_feedback.py @@ -67,6 +67,9 @@ def make_client(resp, token="test-token"): client = CloudClient.__new__(CloudClient) client.config = CloudConfig(api_url=API_URL, token=token) client._session = Transport(resp) + # A hand-built client never refreshes against env credentials, so a + # 401/403 surfaces as CloudAuthError rather than a re-login attempt. + client._can_reauth = False return client diff --git a/tests/test_cloud_api_signed_url_set_job.py b/tests/test_cloud_api_signed_url_set_job.py new file mode 100644 index 00000000..de703a75 --- /dev/null +++ b/tests/test_cloud_api_signed_url_set_job.py @@ -0,0 +1,376 @@ +"""Unit tests for the async signed-URL-set-job family (NDI-python#206). + +Covers the four new ``ndi.cloud.api.files`` entry points: + +* :func:`createSignedURLSetJob` -- POST /.../signed-url-set-jobs. +* :func:`getSignedURLSetJob` -- GET /signed-url-set-jobs/{jobId}. +* :func:`waitForSignedURLSetJob`-- exponential-backoff poll to a terminal + state or timeout. +* :func:`getSignedURLSetResult` -- download and parse the gzipped result + blob a ready job produced. + +The live round trip against a real cloud dataset lives elsewhere; this +file is the offline half, driven by mocked clients and a scripted result +URL, so no credentials are required. +""" + +from __future__ import annotations + +import gzip +import json +from unittest.mock import MagicMock, patch + +import pytest + +# --------------------------------------------------------------------------- +# createSignedURLSetJob +# --------------------------------------------------------------------------- + + +class TestCreateSignedURLSetJob: + def test_posts_to_cloud_id_route_by_default(self): + """id_namespace='cloud' (default) targets the by-_id route.""" + from ndi.cloud.api import files as files_api + + client = MagicMock() + client.post.return_value = {"jobId": "j1", "datasetId": "ds1", "documentId": "doc1"} + + result = files_api.createSignedURLSetJob("ds1", "doc1", client=client) + + assert result == {"jobId": "j1", "datasetId": "ds1", "documentId": "doc1"} + client.post.assert_called_once() + args, kwargs = client.post.call_args + assert args[0] == "/datasets/{datasetId}/documents/{documentId}/signed-url-set-jobs" + assert kwargs.get("datasetId") == "ds1" + assert kwargs.get("documentId") == "doc1" + # An empty JSON body, matching MATLAB, so a gateway that rejects + # zero-byte POSTs still accepts the call. + assert kwargs.get("json") == {} + + def test_ndi_namespace_targets_ndi_documents_route(self): + """id_namespace='ndi' is what NDI's own callers use.""" + from ndi.cloud.api import files as files_api + + client = MagicMock() + client.post.return_value = {"jobId": "j2"} + + files_api.createSignedURLSetJob("ds1", "doc1", id_namespace="ndi", client=client) + + args, _ = client.post.call_args + assert args[0] == "/datasets/{datasetId}/ndi-documents/{ndiDocumentId}/signed-url-set-jobs" + # The path param name changes with the route (documentId -> + # ndiDocumentId), mirroring MATLAB's endpointName switch. + _, kwargs = client.post.call_args + assert kwargs.get("ndiDocumentId") == "doc1" + + def test_file_series_folds_into_the_endpoint(self): + """Optional fileSeries becomes a query parameter on the URL.""" + from ndi.cloud.api import files as files_api + + client = MagicMock() + client.post.return_value = {"jobId": "j3"} + + files_api.createSignedURLSetJob( + "ds1", "doc1", file_series="level_a", id_namespace="ndi", client=client + ) + + args, kwargs = client.post.call_args + assert "fileSeries={fileSeries}" in args[0] + assert kwargs.get("fileSeries") == "level_a" + + +# --------------------------------------------------------------------------- +# getSignedURLSetJob +# --------------------------------------------------------------------------- + + +class TestGetSignedURLSetJob: + def test_calls_the_jobs_endpoint(self): + from ndi.cloud.api import files as files_api + + client = MagicMock() + client.get.return_value = {"jobId": "j1", "state": "running", "signedCount": 42} + + result = files_api.getSignedURLSetJob("j1", client=client) + + assert result["state"] == "running" + args, kwargs = client.get.call_args + assert args[0] == "/signed-url-set-jobs/{jobId}" + assert kwargs.get("jobId") == "j1" + + +# --------------------------------------------------------------------------- +# waitForSignedURLSetJob +# --------------------------------------------------------------------------- + + +class TestWaitForSignedURLSetJob: + def test_returns_ready_terminal_state(self): + """A job that goes queued -> running -> ready terminates on ready.""" + from ndi.cloud.api import files as files_api + + states = [ + {"state": "queued"}, + {"state": "running", "signedCount": 500}, + {"state": "ready", "resultUrl": "https://example/result", "fileCount": 1000}, + ] + + with ( + patch("ndi.cloud.api.files.getSignedURLSetJob", side_effect=states), + patch("ndi.cloud.api.files.time.sleep", lambda _s: None), + ): + result = files_api.waitForSignedURLSetJob( + "j1", + initial_interval=0.01, + max_interval=0.01, + timeout=10.0, + client=MagicMock(), + ) + + assert result["state"] == "ready" + assert result["resultUrl"] == "https://example/result" + + def test_returns_failed_terminal_state(self): + """A job that goes to failed returns the failure payload verbatim.""" + from ndi.cloud.api import files as files_api + + with patch( + "ndi.cloud.api.files.getSignedURLSetJob", + return_value={"state": "failed", "error": "boom"}, + ): + result = files_api.waitForSignedURLSetJob( + "j1", initial_interval=0.01, timeout=1.0, client=MagicMock() + ) + + assert result["state"] == "failed" + assert result["error"] == "boom" + + def test_timeout_returns_state_timeout(self): + """When the deadline elapses, state='timeout' and elapsed is set.""" + from ndi.cloud.api import files as files_api + + with ( + patch( + "ndi.cloud.api.files.getSignedURLSetJob", + return_value={"state": "running", "signedCount": 10}, + ), + patch("ndi.cloud.api.files.time.sleep", lambda _s: None), + ): + result = files_api.waitForSignedURLSetJob( + "j1", + initial_interval=1.0, + max_interval=1.0, + timeout=0.01, # basically immediate + client=MagicMock(), + ) + + assert result["state"] == "timeout" + assert "elapsed" in result + + def test_transient_raise_is_not_terminal(self): + """A gateway blip during a poll does not mark the job dead.""" + from ndi.cloud.api import files as files_api + + seq = [ + ConnectionError("blip"), + {"state": "running"}, + {"state": "ready", "resultUrl": "https://example/r"}, + ] + + def poll(*args, **kwargs): + v = seq.pop(0) + if isinstance(v, Exception): + raise v + return v + + with ( + patch("ndi.cloud.api.files.getSignedURLSetJob", side_effect=poll), + patch("ndi.cloud.api.files.time.sleep", lambda _s: None), + ): + result = files_api.waitForSignedURLSetJob( + "j1", + initial_interval=0.01, + max_interval=0.01, + timeout=10.0, + client=MagicMock(), + ) + + assert result["state"] == "ready" + + +# --------------------------------------------------------------------------- +# getSignedURLSetResult +# --------------------------------------------------------------------------- + + +def _make_response(body: bytes, *, status: int = 200): + resp = MagicMock() + resp.status_code = status + resp.content = body + resp.text = body.decode("utf-8", errors="replace") + return resp + + +class TestGetSignedURLSetResult: + def test_parses_gzipped_blob(self): + """A gzipped payload (typical S3 response) is inflated then parsed.""" + from ndi.cloud.api import files as files_api + + payload = { + "jobId": "j1", + "fileCount": 2, + "generatedAt": "2026-09-27T00:00:00Z", + "files": {"u1": "https://s3.example/u1", "u2": "https://s3.example/u2"}, + } + body_gz = gzip.compress(json.dumps(payload).encode("utf-8")) + + session = MagicMock() + session.get.return_value = _make_response(body_gz) + with patch("ndi.cloud.api.files._download_session", return_value=session): + result = files_api.getSignedURLSetResult("https://example/result.gz") + + assert result["files"] == payload["files"] + assert result["fileCount"] == 2 + assert result["generatedAt"] == "2026-09-27T00:00:00Z" + + def test_parses_uncompressed_blob(self): + """A body the transport already inflated is parsed as-is.""" + from ndi.cloud.api import files as files_api + + payload = {"files": {"u1": "https://s3/u1"}, "fileCount": 1} + body = json.dumps(payload).encode("utf-8") + + session = MagicMock() + session.get.return_value = _make_response(body) + with patch("ndi.cloud.api.files._download_session", return_value=session): + result = files_api.getSignedURLSetResult("https://example/result") + + assert result["files"] == payload["files"] + + def test_http_error_raises(self): + from ndi.cloud.api import files as files_api + + session = MagicMock() + session.get.return_value = _make_response(b"AccessDenied", status=403) + with patch("ndi.cloud.api.files._download_session", return_value=session): + with pytest.raises(RuntimeError, match="HTTP 403"): + files_api.getSignedURLSetResult("https://example/result") + + def test_files_field_wrong_type_raises(self): + """A payload whose 'files' isn't a dict is a caller-visible failure.""" + from ndi.cloud.api import files as files_api + + payload = {"files": "not-a-dict"} + session = MagicMock() + session.get.return_value = _make_response(json.dumps(payload).encode("utf-8")) + with patch("ndi.cloud.api.files._download_session", return_value=session): + with pytest.raises(RuntimeError, match="'files' arrived"): + files_api.getSignedURLSetResult("https://example/result") + + +# --------------------------------------------------------------------------- +# batch_signed_url _default_signer -- end-to-end wiring through the new path +# --------------------------------------------------------------------------- + + +class TestDefaultSignerHappyPath: + """A single happy pass through create -> wait -> read must produce the + same (True, {files: ...}) shape the paged signer used to. + """ + + def test_returns_the_files_map_when_the_job_is_ready(self): + from ndi.cloud.batch_signed_url import _default_signer + + files_map = {"u1": "https://s3/u1", "u2": "https://s3/u2"} + + with ( + patch( + "ndi.cloud.api.files.createSignedURLSetJob", + return_value={"jobId": "j1"}, + ) as m_create, + patch( + "ndi.cloud.api.files.waitForSignedURLSetJob", + return_value={"state": "ready", "resultUrl": "https://example/r"}, + ) as m_wait, + patch( + "ndi.cloud.api.files.getSignedURLSetResult", + return_value={"files": files_map, "fileCount": 2}, + ) as m_read, + ): + ok, answer = _default_signer( + "ds1", + "doc1", + file_series="level_a", + client=MagicMock(), + retry_delays=(), + ) + + assert ok is True + assert answer["files"] == files_map + m_create.assert_called_once() + m_wait.assert_called_once() + m_read.assert_called_once_with("https://example/r") + + # The create call is what carries the NDI namespace and the series + # scope; verify those explicitly so a future edit that drops the + # id_namespace='ndi' argument fails here. + _, create_kwargs = m_create.call_args + assert create_kwargs.get("id_namespace") == "ndi" + assert create_kwargs.get("file_series") == "level_a" + + def test_failed_state_reports_the_server_error(self): + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, _default_signer + + with ( + patch( + "ndi.cloud.api.files.createSignedURLSetJob", + return_value={"jobId": "j2"}, + ), + patch( + "ndi.cloud.api.files.waitForSignedURLSetJob", + return_value={"state": "failed", "error": "signer crashed"}, + ), + ): + with pytest.raises(BatchScopeUnreachable, match="signer crashed"): + _default_signer("ds1", "doc1", client=MagicMock(), retry_delays=()) + + def test_wait_timeout_is_reported_as_failure(self): + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, _default_signer + + with ( + patch( + "ndi.cloud.api.files.createSignedURLSetJob", + return_value={"jobId": "j3"}, + ), + patch( + "ndi.cloud.api.files.waitForSignedURLSetJob", + return_value={"state": "timeout", "elapsed": 900.0}, + ), + ): + with pytest.raises(BatchScopeUnreachable) as exc_info: + _default_signer("ds1", "doc1", client=MagicMock(), retry_delays=()) + + message = str(exc_info.value) + assert ( + "did not finish" in message or "timeout" in message.lower() + ), f"failure must name the timeout: {message!r}" + + def test_ready_without_result_url_is_reported_as_failure(self): + """A ready job that carries no resultUrl is a server bug we must not + follow into an assertion-free read. See NDI-matlab#1009 for the + same guard on the MATLAB side. + """ + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, _default_signer + + with ( + patch( + "ndi.cloud.api.files.createSignedURLSetJob", + return_value={"jobId": "j4"}, + ), + patch( + "ndi.cloud.api.files.waitForSignedURLSetJob", + return_value={"state": "ready"}, # no resultUrl + ), + ): + with pytest.raises(BatchScopeUnreachable, match="resultUrl"): + _default_signer("ds1", "doc1", client=MagicMock(), retry_delays=()) diff --git a/tests/test_cloud_batch_signed_url.py b/tests/test_cloud_batch_signed_url.py index 3e7d5347..4fd00810 100644 --- a/tests/test_cloud_batch_signed_url.py +++ b/tests/test_cloud_batch_signed_url.py @@ -323,6 +323,91 @@ def test_a_scope_is_only_retried_once(self): # The one uid that IS in the map still resolves. assert lookup.lookup("ds1", "doc1", "chunks", "u_2") == "https://s3/u_2" + def test_a_multi_wave_schedule_retries_until_a_hit_or_exhaustion(self): + """A schedule of several delays fires each in order until either + the map settles or the schedule is exhausted. + + On User-1-prod after a bulk upload the batch endpoint has been + observed lagging further than the original single 0.5 s retry + covered (Waltham-Data-Science/NDI-python#320), so the default + schedule is now a bounded exponential backoff rather than one + shot. Each wave still costs one scope refetch, not one per uid. + """ + + class _EventualSigner: + """Partial map for the first N answers, then the full map.""" + + def __init__(self, misses_before_hit: int): + self.misses_before_hit = misses_before_hit + self.calls = 0 + + def __call__(self, dataset_id, document_id, *, file_series="", client=None): + self.calls += 1 + if self.calls <= self.misses_before_hit: + return True, {"files": {"u_present": "https://s3/u_present"}} + return True, { + "files": { + "u_present": "https://s3/u_present", + "u_lagged": "https://s3/u_lagged", + } + } + + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + # Three waves scheduled; the map settles on the SECOND wave, so + # one initial fetch + two retries = three signer calls, then no + # more. + sleeps: list[float] = [] + signer = _EventualSigner(misses_before_hit=2) + lookup = BatchSignedUrlLookup( + signer=signer, + partial_map_retry_delays=(0.1, 0.2, 0.4), + sleep=sleeps.append, + ) + + assert lookup.lookup("ds1", "doc1", "chunks", "u_lagged") == "https://s3/u_lagged" + stats = lookup.stats() + assert stats.partial_map_retries == 2 + assert stats.signer_calls == 3 + assert stats.uid_misses == 0 + assert sleeps == [0.1, 0.2], f"expected two waves' worth of sleeps; got {sleeps!r}" + + def test_a_schedule_that_never_settles_bounds_retries_to_len(self): + """If every wave still returns a partial map, retries stop at + the schedule length -- not one per uid, not unbounded.""" + + class _AlwaysPartial: + def __init__(self): + self.calls = 0 + + def __call__(self, dataset_id, document_id, *, file_series="", client=None): + self.calls += 1 + return True, {"files": {"u_present": "https://s3/u_present"}} + + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + sleeps: list[float] = [] + signer = _AlwaysPartial() + lookup = BatchSignedUrlLookup( + signer=signer, + partial_map_retry_delays=(0.05, 0.1, 0.2), + sleep=sleeps.append, + ) + + assert lookup.lookup("ds1", "doc1", "chunks", "u_lagged") == "" + stats = lookup.stats() + assert stats.partial_map_retries == 3, "all three waves fire before giving up" + assert stats.signer_calls == 4, "one initial fetch + three retries" + assert stats.uid_misses == 1 + assert sleeps == [0.05, 0.1, 0.2] + + # A second uid on the same scope must not spend another wave. + signer_calls_before = signer.calls + assert lookup.lookup("ds1", "doc1", "chunks", "u_also_lagged") == "" + assert signer.calls == signer_calls_before, "schedule exhausted; no more retries" + stats = lookup.stats() + assert stats.partial_map_retries == 3 + def test_a_retry_that_returns_no_map_at_all_still_falls_back(self): """The retry can itself fail (a fetch that raises, a bad payload). The caller must still get an empty string, not a crash.""" @@ -368,6 +453,166 @@ def test_clear_drops_everything(self): assert lookup.stats().signer_calls == 1 # refetched after clear +class TestBatchLookupPrefetch: + """The prefetch path warms one scope without asking about a uid. + + The whole point is to hide the 20-80 s signed-URL-set wall time + behind an initial render the user is already watching -- a later + ``lookup()`` on any uid in the scope must then be a pure cache + hit and NOT run another signer call. + """ + + def test_prefetch_scope_populates_the_cache(self): + """After a prefetch, lookup() serves the map without an extra call.""" + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + files = {"u1": "https://s3/u1", "u2": "https://s3/u2"} + signer = _FakeSigner({("ds1", "doc1", "chunk.bin"): files}) + lookup = BatchSignedUrlLookup(signer=signer) + + assert lookup.prefetch_scope("ds1", "doc1", "chunk.bin") is True + assert len(signer.calls) == 1 + + # A subsequent lookup on any uid in the scope must be a pure cache hit. + assert lookup.lookup("ds1", "doc1", "chunk.bin", "u1") == "https://s3/u1" + assert lookup.lookup("ds1", "doc1", "chunk.bin", "u2") == "https://s3/u2" + assert len(signer.calls) == 1, ( + "the prefetched scope must serve subsequent lookups from cache; " + f"signer was called {len(signer.calls)} times" + ) + + def test_prefetch_scope_returns_true_when_already_cached(self): + """A second prefetch of a fresh scope is a no-op; no extra call.""" + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + signer = _FakeSigner({("ds1", "doc1", "chunk.bin"): {"u1": "https://s3/u1"}}) + lookup = BatchSignedUrlLookup(signer=signer) + + assert lookup.prefetch_scope("ds1", "doc1", "chunk.bin") is True + assert lookup.prefetch_scope("ds1", "doc1", "chunk.bin") is True + assert len(signer.calls) == 1, ( + "a second prefetch of a fresh scope must not fire another signer call; " + f"got {len(signer.calls)} calls" + ) + + def test_prefetch_scope_returns_false_on_empty_document_id(self): + """No document id means nothing to batch against.""" + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + signer = _FakeSigner({}) + lookup = BatchSignedUrlLookup(signer=signer) + + assert lookup.prefetch_scope("ds1", "", "chunk.bin") is False + assert signer.calls == [] + + def test_start_prefetch_iterates_scopes_in_the_background(self): + """The background thread warms every scope it is handed.""" + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + scopes_files = { + ("ds1", "doc_a", "chunk.bin"): {"u_a1": "https://s3/a1"}, + ("ds1", "doc_b", "chunk.bin"): {"u_b1": "https://s3/b1"}, + ("ds1", "doc_c", "chunk.bin"): {"u_c1": "https://s3/c1"}, + } + signer = _FakeSigner(scopes_files) + lookup = BatchSignedUrlLookup(signer=signer) + + thread = lookup.start_prefetch( + [ + ("ds1", "doc_a", "chunk.bin"), + ("ds1", "doc_b", "chunk.bin"), + ("ds1", "doc_c", "chunk.bin"), + ] + ) + thread.join(timeout=5.0) + assert not thread.is_alive(), "prefetch thread did not finish" + + # All three scopes must now be cached, so lookup() serves without + # another signer call. + signer_calls_before = len(signer.calls) + assert lookup.lookup("ds1", "doc_a", "chunk.bin", "u_a1") == "https://s3/a1" + assert lookup.lookup("ds1", "doc_b", "chunk.bin", "u_b1") == "https://s3/b1" + assert lookup.lookup("ds1", "doc_c", "chunk.bin", "u_c1") == "https://s3/c1" + assert len(signer.calls) == signer_calls_before, ( + "lookups after prefetch must be pure cache hits; " + f"signer went from {signer_calls_before} to {len(signer.calls)}" + ) + assert ( + signer_calls_before == 3 + ), f"expected one signer call per prefetched scope, got {signer_calls_before}" + + def test_start_prefetch_skips_empty_document_ids(self): + """A scope with an empty ndi_document_id is skipped, not fetched.""" + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + scopes_files = { + ("ds1", "doc_a", "chunk.bin"): {"u_a1": "https://s3/a1"}, + ("ds1", "doc_c", "chunk.bin"): {"u_c1": "https://s3/c1"}, + } + signer = _FakeSigner(scopes_files) + lookup = BatchSignedUrlLookup(signer=signer) + + thread = lookup.start_prefetch( + [ + ("ds1", "doc_a", "chunk.bin"), + ("ds1", "", "chunk.bin"), # skipped + ("ds1", "doc_c", "chunk.bin"), + ] + ) + thread.join(timeout=5.0) + assert not thread.is_alive() + + assert len(signer.calls) == 2, ( + "the empty-doc-id scope must be skipped; " + f"expected 2 signer calls, got {len(signer.calls)}" + ) + scopes_fetched = {tuple(c) for c in signer.calls} + assert scopes_fetched == { + ("ds1", "doc_a", "chunk.bin"), + ("ds1", "doc_c", "chunk.bin"), + } + + def test_start_prefetch_continues_after_a_failure(self): + """A BatchScopeUnreachable on one scope must not stop the rest.""" + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, BatchSignedUrlLookup + + good_files = {"u_b1": "https://s3/b1"} + good_files_c = {"u_c1": "https://s3/c1"} + + state = {"calls": 0} + + def signer(dataset_id, document_id, *, file_series="", client=None): + state["calls"] += 1 + if document_id == "doc_a": + raise BatchScopeUnreachable("simulated async-job failure for doc_a") + if document_id == "doc_b": + return True, {"files": dict(good_files)} + if document_id == "doc_c": + return True, {"files": dict(good_files_c)} + return False, {"message": "no such scope"} + + lookup = BatchSignedUrlLookup(signer=signer) + + thread = lookup.start_prefetch( + [ + ("ds1", "doc_a", "chunk.bin"), # raises + ("ds1", "doc_b", "chunk.bin"), + ("ds1", "doc_c", "chunk.bin"), + ] + ) + thread.join(timeout=5.0) + assert not thread.is_alive() + + # The remaining two scopes still landed in the cache. + signer_calls_before = state["calls"] + assert lookup.lookup("ds1", "doc_b", "chunk.bin", "u_b1") == "https://s3/b1" + assert lookup.lookup("ds1", "doc_c", "chunk.bin", "u_c1") == "https://s3/c1" + assert state["calls"] == signer_calls_before, ( + "lookups after a partial prefetch must still be cache hits; " + f"signer went from {signer_calls_before} to {state['calls']}" + ) + + # --------------------------------------------------------------------------- # fetch_cloud_file wiring: the batch cache is consulted, and its answer is # what gets streamed. On a batch miss, the per-uid getFileDetails is called. @@ -393,7 +638,11 @@ def test_batch_answer_replaces_getFileDetails(self, tmp_path): patch("ndi.cloud.api.files.getFile") as mock_get_file, ): - def fake_get_file(url, path, timeout=300): + def fake_get_file(url, path, timeout=300, **kwargs): + # **kwargs so fetch_cloud_file's opt-in error_out sink for + # the batch-cache 403 hook (NDI-python#322 follow-on) does + # not blow up the mock; the mock never fails, so error_out + # goes untouched. Path(path).write_bytes(b"member bytes") return True @@ -512,13 +761,20 @@ def test_series_member_uses_context_for_batch_scope(self, tmp_path): assert kwargs["ndi_document_id"] == "doc_ndi" assert kwargs["series_name"] == "stack" - def test_ordinary_file_still_carries_document_scope(self, tmp_path): - """Two documents each with one file must not share a scope. - - An ordinary file (no seriesName) with a documentId still keys its - batch scope on that documentId, so the endpoint returns just that - document's map -- exactly one uid, but the cache is still primed - for the next uid in the same document. + def test_a_single_file_fetch_bypasses_batch(self, tmp_path): + """An ordinary file (no seriesName) is a SINGLE-uid fetch, so it + must take the direct ``getFileDetails`` path -- NOT ask the batch + endpoint for a whole-document scope to answer one question. + + For a lightsheet-scale pyramid document that scope names 15k+ + files, so the batch endpoint spends 60-90 s signing a set the + caller has no use for, delaying the ONE URL that the download + actually needs by more than a minute. This is the same reasoning + that ``_fetch_manifest`` uses for the internal manifest fetch; + the DID handler path has to make the same choice or the pyramid + pathology reappears whenever DID reads a series member (DID + fetches the manifest via the handler with seriesName="" before + it fetches any members). See Waltham-Data-Science/NDI-python#320. """ from ndi.cloud.filehandler import download_file_from_cloud @@ -531,7 +787,9 @@ def test_ordinary_file_still_carries_document_scope(self, tmp_path): ) _, kwargs = mock_fetch.call_args - assert kwargs["ndi_document_id"] == "doc_ndi" + # documentId is discarded on the single-file path so + # fetch_cloud_file skips the batch scope entirely. + assert kwargs["ndi_document_id"] == "" assert kwargs["series_name"] == "" def test_no_context_disables_batch(self, tmp_path): @@ -608,3 +866,160 @@ def unique_cursor(*args, **kwargs): with pytest.raises(files_api.SignedURLSetMaxPagesReached) as exc: files_api.getSignedURLSetAll("ds1", "doc1", max_pages=3, client=MagicMock()) assert exc.value.merged["pages"] == 3 + + +# --------------------------------------------------------------------------- +# The default signer retries transient batch failures. +# --------------------------------------------------------------------------- + + +class TestDefaultSignerRetries: + """A ConnectionFailed-style raise from the batch API must not collapse + a whole scope to the O(N) per-member fallback on the first try. See + Waltham-Data-Science/NDI-python#322 and the parallel + VH-Lab/NDI-matlab#1010: a residential-network TLS blip used to hang + downloads for hours by tripping this cascade. + + NDI-python#206 rewired the default signer from the paged + ``getSignedURLSetAll`` walk to the async job path + (``createSignedURLSetJob`` -> ``waitForSignedURLSetJob`` -> + ``getSignedURLSetResult``). The retry contract is unchanged: any raise + from the three-step exchange is retried per ``retry_delays``. These + tests patch the first step (``createSignedURLSetJob``) to trip the + retry, which is enough to exercise the loop without also needing to + script the wait and result calls. + """ + + def test_a_transient_raise_then_success_returns_success(self): + from ndi.cloud.batch_signed_url import _default_signer + + calls = {"n": 0} + + def flaky_create(*args, **kwargs): + calls["n"] += 1 + if calls["n"] == 1: + raise ConnectionError("first try, transient") + return {"jobId": "job-1"} + + ready_status = {"state": "ready", "resultUrl": "https://example/result"} + result_payload = {"files": {"u1": "https://s3.example.com/u1"}, "fileCount": 1} + + with ( + patch("ndi.cloud.api.files.createSignedURLSetJob", side_effect=flaky_create), + patch("ndi.cloud.api.files.waitForSignedURLSetJob", return_value=ready_status), + patch("ndi.cloud.api.files.getSignedURLSetResult", return_value=result_payload), + ): + ok, answer = _default_signer( + "ds1", + "doc1", + client=MagicMock(), + retry_delays=(0.0, 0.0, 0.0), + sleep=lambda _s: None, + ) + + assert ok is True + assert answer["files"] == {"u1": "https://s3.example.com/u1"} + assert calls["n"] == 2, "should have retried exactly once before succeeding" + + def test_every_attempt_raising_raises_batch_scope_unreachable(self): + """After every retry attempt fails, raise instead of silent fallback. + + Falling back per-uid on an actually-broken async job path + would hide the bug behind slow-but-working chunk fetches. See + BatchScopeUnreachable's docstring. + """ + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, _default_signer + + calls = {"n": 0} + + def always_fails(*args, **kwargs): + calls["n"] += 1 + raise ConnectionError(f"try {calls['n']}") + + with patch("ndi.cloud.api.files.createSignedURLSetJob", side_effect=always_fails): + with pytest.raises(BatchScopeUnreachable) as exc_info: + _default_signer( + "ds1", + "doc1", + client=MagicMock(), + retry_delays=(0.0, 0.0), # 3 attempts total + sleep=lambda _s: None, + ) + + assert calls["n"] == 3, "should have made 3 attempts (initial + 2 retries)" + message = str(exc_info.value) + assert "ConnectionError" in message + assert "3 attempts" in message, f"should name the attempt count: {message!r}" + # The underlying transport error is chained for programmatic access. + assert isinstance(exc_info.value.__cause__, ConnectionError) + + def test_zero_retry_delays_is_a_single_attempt(self): + """Passing an empty retry_delays disables retry entirely. + + Useful anywhere a caller wants the pre-retry behavior (a test, a + fast-fail probe, or an environment where the signer already handles + its own retries). + """ + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, _default_signer + + calls = {"n": 0} + + def always_fails(*args, **kwargs): + calls["n"] += 1 + raise ConnectionError("nope") + + with patch("ndi.cloud.api.files.createSignedURLSetJob", side_effect=always_fails): + with pytest.raises(BatchScopeUnreachable): + _default_signer( + "ds1", + "doc1", + client=MagicMock(), + retry_delays=(), + sleep=lambda _s: None, + ) + + assert calls["n"] == 1, "empty retry_delays should mean one attempt, no retries" + + def test_retry_delays_are_slept_in_order(self): + """The backoff delays are consumed in order, once per failed attempt.""" + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, _default_signer + + slept: list[float] = [] + + def always_fails(*args, **kwargs): + raise ConnectionError("nope") + + with patch("ndi.cloud.api.files.createSignedURLSetJob", side_effect=always_fails): + with pytest.raises(BatchScopeUnreachable): + _default_signer( + "ds1", + "doc1", + client=MagicMock(), + retry_delays=(1.0, 4.0, 16.0), + sleep=slept.append, + ) + + assert slept == [ + 1.0, + 4.0, + 16.0, + ], f"expected the three backoff delays consumed in order, got {slept}" + + def test_fetch_scope_propagates_batch_scope_unreachable(self): + """The strict-mode raise must not be caught by _fetch_scope. + + _fetch_scope catches Exception from injected signers so tests can + script arbitrary failures (existing behavior). But when the DEFAULT + signer's async job path exhausts its retries, the resulting + BatchScopeUnreachable must propagate all the way to the caller so + the real error surfaces. + """ + from ndi.cloud.batch_signed_url import BatchScopeUnreachable, BatchSignedUrlLookup + + def unreachable_signer(*args, **kwargs): + raise BatchScopeUnreachable("simulated async-job failure") + + lookup = BatchSignedUrlLookup(signer=unreachable_signer) + + with pytest.raises(BatchScopeUnreachable, match="simulated async-job failure"): + lookup.lookup("ds1", "doc1", "chunks", "u_1") diff --git a/tests/test_cloud_client_retry.py b/tests/test_cloud_client_retry.py index f4a8d42f..64e8df6e 100644 --- a/tests/test_cloud_client_retry.py +++ b/tests/test_cloud_client_retry.py @@ -103,15 +103,21 @@ def slept(monkeypatch): return waits -def make_client(*script): +def make_client(*script, can_reauth: bool = True): """A client whose transport is the given script. ``__new__`` rather than the constructor: the real one builds a ``requests.Session`` that would have to be replaced anyway. + + ``can_reauth`` defaults to True so ordinary retry tests continue to + behave as if the client were obtained via ``from_env()``. Tests that + verify the "hand-built CloudClient(CloudConfig(token=...)) does not + refresh against env credentials" behavior pass ``can_reauth=False``. """ client = CloudClient.__new__(CloudClient) client.config = CloudConfig() client._session = Transport(*script) + client._can_reauth = can_reauth return client @@ -193,13 +199,29 @@ def test_a_non_transient_status_fails_on_the_first_attempt(self, status, slept): assert client._session.attempts == 1 assert slept == [] - def test_a_401_is_not_retried(self, slept): - """Credentials will not become valid by asking again.""" + def test_a_401_that_reauth_cannot_fix_still_surfaces(self, slept, monkeypatch): + """A 401 triggers reauthentication once; if reauth itself fails, the + original 401 surfaces without the client entering a retry storm. + + Bad or missing creds are the same class of failure they always + were -- credentials do not become valid by asking again. The + reauth branch that :meth:`_request` gained to survive + token-expired-mid-run is a one-shot per request; when it cannot + produce a fresh token, the original 4xx is what the caller sees. + """ client = make_client(response(401, text="nope")) + def failing_reauth(): + raise CloudAuthError("no creds available for reauth") + + monkeypatch.setattr(client, "_reauthenticate", failing_reauth) + with pytest.raises(CloudAuthError): client.get("/datasets") + # Reauth failed -- the original request was made once and the + # 401 surfaces from _handle_response. Reauth itself does not go + # through this session. assert client._session.attempts == 1 def test_a_404_is_not_retried(self, slept): @@ -351,5 +373,132 @@ def test_the_delay_is_jittered(self): assert len(draws) > 1, "delays are identical; the backoff has no jitter" +# ====================================================================== +# Token-expired-mid-run: reauth once and retry +# ====================================================================== +class TestReauthOnTokenExpiry: + """A long-running operation can outlive its bearer token's TTL. + + Observed today on a signed-URL-set-job poll loop: the server was + signing steadily, the client polled every ~30 s, and at ~9 min the + token expired. Every subsequent poll returned 401/403 with a "Token + is invalid or expired" body, but the client kept using the same + stale token until the outer timeout fired. A reauth-and-retry + branch in :meth:`_request` turns that quietly-fatal case into one + invisible-to-the-caller refresh followed by success. + + A second 401/403 after reauth is a real auth failure (bad creds, + revoked key) and must not loop -- one refresh per request is the + invariant these tests pin. + """ + + def test_a_401_triggers_reauth_and_retries_once(self, slept, monkeypatch): + """Happy path: token expired, reauth succeeds, retry succeeds.""" + client = make_client(response(401, text="expired"), OK) + + refreshed = {"n": 0} + + def fresh_token(): + refreshed["n"] += 1 + client.config.token = f"new-token-{refreshed['n']}" + + monkeypatch.setattr(client, "_reauthenticate", fresh_token) + + result = client.get("/datasets") + + assert result.status_code == 200 + assert client._session.attempts == 2, "expected one retry after reauth" + assert refreshed["n"] == 1, "reauth should fire exactly once" + + def test_a_403_also_triggers_reauth(self, slept, monkeypatch): + """The server returned 403 with 'Token is invalid or expired' in + our real observation; the client must treat 403 the same way as + 401 for reauth purposes.""" + client = make_client(response(403, text="expired"), OK) + monkeypatch.setattr( + client, "_reauthenticate", lambda: setattr(client.config, "token", "new") + ) + + result = client.get("/datasets") + assert result.status_code == 200 + assert client._session.attempts == 2 + + def test_a_second_401_after_reauth_does_not_loop(self, slept, monkeypatch): + """Reauth is one-shot per request. A 401 that persists past a + successful reauth is a real auth problem and belongs at the + surface, not in a retry storm.""" + client = make_client(response(401, text="expired"), response(401, text="still expired")) + + calls = {"n": 0} + + def refresh(): + calls["n"] += 1 + client.config.token = f"new-{calls['n']}" + + monkeypatch.setattr(client, "_reauthenticate", refresh) + + with pytest.raises(CloudAuthError): + client.get("/datasets") + + assert calls["n"] == 1, "reauth must fire exactly once, not once per 401" + assert client._session.attempts == 2 + + def test_reauth_updates_the_authorization_header(self, slept, monkeypatch): + """The retry uses the NEW token, not the stale one. Without + this, we'd refresh in place and then send the old header + anyway -- and the whole exercise would be for nothing. + + Captures the header at request time (rather than reading the + session's post-facto record, which holds a single mutating dict + for both calls). + """ + client = make_client(response(401, text="expired"), OK) + + captured_authorizations: list[str] = [] + real_request = client._session.request + + def capturing_request(method, url, **kwargs): + captured_authorizations.append(kwargs.get("headers", {}).get("Authorization")) + return real_request(method, url, **kwargs) + + client._session.request = capturing_request + + def refresh(): + client.config.token = "the-fresh-token" + + monkeypatch.setattr(client, "_reauthenticate", refresh) + client.config.token = "the-stale-token" + + client.get("/datasets") + + assert captured_authorizations[0] == "Bearer the-stale-token" + assert captured_authorizations[1] == "Bearer the-fresh-token" + + def test_a_hand_built_client_does_not_reauth_against_env(self, slept, monkeypatch): + """``CloudClient(CloudConfig(token=...))`` uses the given token as + given: a 401 surfaces as ``CloudAuthError`` rather than silently + being replaced by whatever ``authenticate()`` would return from + env. Without this, a test that hard-codes a bad token to verify + auth failure "passes" because CI happens to have valid + ``NDI_CLOUD_USERNAME``/``NDI_CLOUD_PASSWORD`` in env, and the + client swaps in a fresh valid token behind the caller's back. + """ + client = make_client(response(401, text="expired"), can_reauth=False) + + reauth_calls = {"n": 0} + + def should_not_be_called(): + reauth_calls["n"] += 1 + client.config.token = "would-be-fresh" + + monkeypatch.setattr(client, "_reauthenticate", should_not_be_called) + + with pytest.raises(CloudAuthError): + client.get("/datasets") + + assert reauth_calls["n"] == 0, "hand-built client must not refresh from env" + assert client._session.attempts == 1, "no retry when reauth is disabled" + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/tests/test_cloud_file_series_round_trip.py b/tests/test_cloud_file_series_round_trip.py index 95b6e095..099d24f4 100644 --- a/tests/test_cloud_file_series_round_trip.py +++ b/tests/test_cloud_file_series_round_trip.py @@ -348,21 +348,26 @@ def test_members_survive_a_download_from_the_cloud( f"partial_map_retries={stats.partial_map_retries}, " f"failure_reason={stats.last_failure_reason!r}" ) - # One presign call for the scope, plus at most one retry if the - # first answer was a partial map (Waltham-Data-Science/ - # NDI-python#309 -- some environments lag briefly after the bulk - # upload). Anything beyond that would be per-uid fallback, which - # the uid_misses assertion above already rules out; this is just - # for a clearer failure message. + # One presign call for the scope, plus up to len(retry_delays) + # partial-map retries if the first answer(s) missed + # (Waltham-Data-Science/NDI-python#309 and #320 -- User 1 prod + # can lag long enough after ``waitForAllBulkUploads`` that the + # first retry is not enough, so the default schedule is now + # 1 s / 3 s / 9 s). Anything beyond that would be per-uid + # fallback, which the uid_misses assertion above already rules + # out; this is just for a clearer failure message. + from ndi.cloud.batch_signed_url import DEFAULT_PARTIAL_MAP_RETRY_DELAYS + assert stats.signer_calls == 1 + stats.partial_map_retries, ( f"the members of one series should cost ONE presign call, " - f"plus at most one retry if the first map was partial. " + f"plus at most one retry per partial-map wave. " f"Got signer_calls={stats.signer_calls}, " f"partial_map_retries={stats.partial_map_retries}." ) - assert stats.partial_map_retries <= 1, ( - f"the retry is bounded to once per scope; " - f"got partial_map_retries={stats.partial_map_retries}." + assert stats.partial_map_retries <= len(DEFAULT_PARTIAL_MAP_RETRY_DELAYS), ( + f"the retry is bounded by the delay schedule; " + f"got partial_map_retries={stats.partial_map_retries}, " + f"schedule length={len(DEFAULT_PARTIAL_MAP_RETRY_DELAYS)}." ) def test_members_survive_a_download_from_the_cloud_with_sync_files_false( diff --git a/tests/test_cloud_filehandler.py b/tests/test_cloud_filehandler.py index 4fe1ef24..7be62f44 100644 --- a/tests/test_cloud_filehandler.py +++ b/tests/test_cloud_filehandler.py @@ -284,6 +284,23 @@ def fake_get_file(url, path, timeout=300): class TestGetOrCreateCloudClient: """Tests for get_or_create_cloud_client.""" + @pytest.fixture(autouse=True) + def reset_ambient_client(self): + """The ambient CloudClient is cached process-wide, so any earlier + test in this session that did a real cloud fetch (e.g. the live + Cloud API suites) leaves it populated -- and then + ``test_env_vars_present`` sees the cached client returned instead + of a fresh ``from_env`` call, and fails "Called 0 times". Reset + both slots before AND after each test in this class so the tests + exercise the create path they mean to.""" + import ndi.cloud.filehandler as fh + + fh._ambient_cloud_client = None + fh._ambient_client_key = None + yield + fh._ambient_cloud_client = None + fh._ambient_client_key = None + def test_missing_env_vars(self): from ndi.cloud.exceptions import CloudAuthError from ndi.cloud.filehandler import get_or_create_cloud_client diff --git a/tests/test_cloud_live.py b/tests/test_cloud_live.py index c98089ff..c8ec5597 100644 --- a/tests/test_cloud_live.py +++ b/tests/test_cloud_live.py @@ -87,10 +87,20 @@ def cloud_config(): @pytest.fixture(scope="module") def client(cloud_config): - """Return an authenticated CloudClient.""" + """Return an authenticated CloudClient. + + The fixture's config was obtained via ``login()`` from env credentials, + so a mid-run 401/403 (typical after the bearer's TTL expires on a + long-running poll) can be resolved by re-authenticating against the + same env. Callers who hand-build ``CloudClient(CloudConfig(token=...))`` + with a specific token get the default (no reauth) so that a hard-coded + token is used exactly as given. + """ from ndi.cloud.client import CloudClient - return CloudClient(cloud_config) + client = CloudClient(cloud_config) + client._can_reauth = True + return client @pytest.fixture(scope="module") diff --git a/tests/test_cloud_signed_url_disk_cache.py b/tests/test_cloud_signed_url_disk_cache.py new file mode 100644 index 00000000..17267b86 --- /dev/null +++ b/tests/test_cloud_signed_url_disk_cache.py @@ -0,0 +1,442 @@ +"""Tests for the persistent (on-disk) signed-URL-set cache. + +Mirrors the MATLAB SignedUrlDiskCacheTest (VH-Lab/NDI-matlab commit +27661a3 on the same branch). Covers save/load round-trip, past-expiry, +30 min safety buffer, no-expiry uncacheable, forget, cacheDir env +override, atomic-write behavior under alternating writers, and series +name escaping. + +Nothing here hits the network. Each test isolates its own cache dir via +``NDI_SIGNED_URL_CACHE_DIR`` so a leaked file cannot leak into the +user's real ``~/.ndi/signed-url-cache``, nor across tests. +""" + +from __future__ import annotations + +import gzip +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def isolated_cache_dir(tmp_path, monkeypatch): + """One tempdir per test, wired through NDI_SIGNED_URL_CACHE_DIR.""" + cache = tmp_path / "signed-url-cache" + monkeypatch.setenv("NDI_SIGNED_URL_CACHE_DIR", str(cache)) + # Ensure the safety-seconds override is not carried over from a + # previous test / the ambient shell. + monkeypatch.delenv("NDI_SIGNED_URL_CACHE_SAFETY_SECONDS", raising=False) + return cache + + +def future_iso(seconds_ahead: float) -> str: + """ISO-8601 UTC timestamp N seconds from now, server-shaped.""" + dt = datetime.now(timezone.utc) + timedelta(seconds=seconds_ahead) + return dt.strftime("%Y-%m-%dT%H:%M:%SZ") + + +def sample_payload(**overrides): + """Signer-shaped payload the cache accepts.""" + payload = { + "files": { + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6": "https://s3/one", + "4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6": "https://s3/two?sig=abc%2F123", + }, + "filesExpireAt": future_iso(23 * 3600), + "expiresAt": future_iso(23 * 3600), + "generatedAt": future_iso(-30), + "fileCount": 2, + } + payload.update(overrides) + return payload + + +# --------------------------------------------------------------------------- +# Round-trip +# --------------------------------------------------------------------------- + + +class TestSaveLoadRoundTrip: + """What goes in comes back out on the fields callers actually read.""" + + def test_round_trip_preserves_files_and_expiry(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + payload = sample_payload() + cache.save("ds1", "doc1", "chunk.bin", payload) + + got = cache.load("ds1", "doc1", "chunk.bin") + + assert got is not None, "a just-written scope should load" + assert got["files"] == payload["files"] + assert ( + got["files"]["4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6"] == "https://s3/two?sig=abc%2F123" + ), "URL with url-escaped bytes must survive JSON encoding" + assert got["filesExpireAt"] == payload["filesExpireAt"] + assert got["fileCount"] == 2 + + def test_written_file_is_valid_gzipped_json(self, isolated_cache_dir): + """The bytes on disk are the documented shape. + + NDI-matlab and NDI-python have to write the same file so a + cache dir written by one can be read by the other. If the + shape drifts, that portability is silently gone. + """ + from ndi.cloud import signed_url_disk_cache as cache + + cache.save("ds1", "doc1", "", sample_payload()) + + path = isolated_cache_dir / "ds1" / "doc1.json.gz" + assert path.is_file(), "whole-doc scope lands at /.json.gz" + + with gzip.open(path, "rt", encoding="utf-8") as fh: + data = json.load(fh) + assert data["schemaVersion"] == 1 + assert data["datasetId"] == "ds1" + assert data["documentId"] == "doc1" + assert data["seriesName"] == "" + assert isinstance(data["files"], dict) + assert ( + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6" in data["files"] + ), "digit-prefixed uids must survive JSON as literal keys" + + +# --------------------------------------------------------------------------- +# Expiry / safety buffer +# --------------------------------------------------------------------------- + + +class TestExpiryAndSafetyBuffer: + """Never hand a caller a URL that would 403 before they can use it.""" + + def test_expired_payload_is_a_miss(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + cache.save( + "ds1", + "doc1", + "", + sample_payload(filesExpireAt=future_iso(-3600), expiresAt=""), + ) + assert cache.load("ds1", "doc1", "") is None + + def test_safety_buffer_guards_against_almost_expired(self, isolated_cache_dir, monkeypatch): + """20 min ≤ default 30 min buffer is a miss.""" + from ndi.cloud import signed_url_disk_cache as cache + + cache.save( + "ds1", + "doc1", + "", + sample_payload(filesExpireAt=future_iso(20 * 60), expiresAt=""), + ) + + assert cache.load("ds1", "doc1", "") is None, "20 min < 30 min safety buffer must be a miss" + + # Tighten the buffer and the SAME payload becomes visible -- + # proving the check is the buffer doing the work, not the + # write refusing to persist. + monkeypatch.setenv("NDI_SIGNED_URL_CACHE_SAFETY_SECONDS", "60") + assert cache.load("ds1", "doc1", "") is not None + + def test_payload_without_expiry_is_uncacheable(self, isolated_cache_dir): + """save() refuses; load() sees nothing on disk.""" + from ndi.cloud import signed_url_disk_cache as cache + + cache.save( + "ds1", + "doc1", + "", + sample_payload(filesExpireAt="", expiresAt=""), + ) + assert cache.load("ds1", "doc1", "") is None + # And nothing wrote itself into the tree either. + ds_dir = isolated_cache_dir / "ds1" + assert not ds_dir.exists() or not any(ds_dir.iterdir()) + + def test_load_falls_back_to_expires_at(self, isolated_cache_dir): + """The paging path's older field also age-checks the cache.""" + from ndi.cloud import signed_url_disk_cache as cache + + cache.save( + "ds1", + "doc1", + "", + sample_payload(filesExpireAt="", expiresAt=future_iso(3600)), + ) + assert cache.load("ds1", "doc1", "") is not None + + +# --------------------------------------------------------------------------- +# Forget / invalidation +# --------------------------------------------------------------------------- + + +class TestForget: + """The S3-403 hook: drop a scope whose cached URLs are dead.""" + + def test_forget_removes_file(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + cache.save("ds1", "doc1", "chunk.bin", sample_payload()) + assert cache.load("ds1", "doc1", "chunk.bin") is not None + + cache.forget("ds1", "doc1", "chunk.bin") + + assert cache.load("ds1", "doc1", "chunk.bin") is None + assert not (isolated_cache_dir / "ds1" / "doc1_chunk.bin.json.gz").exists() + + def test_forget_on_unknown_scope_is_silent(self, isolated_cache_dir): + """S3-403 recovery must not need to check-first.""" + from ndi.cloud import signed_url_disk_cache as cache + + # No pytest.warns / raises: forget is best-effort and idempotent. + cache.forget("nosuch-ds", "nosuch-doc", "nosuch-series") + + +# --------------------------------------------------------------------------- +# cacheDir env override +# --------------------------------------------------------------------------- + + +class TestCacheDir: + """The env var override is documented on-disk contract for NDI-python.""" + + def test_cache_dir_respects_env_override(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + assert Path(cache.cache_dir()) == Path(isolated_cache_dir) + + def test_cache_dir_default_when_env_unset(self, tmp_path, monkeypatch): + from ndi.cloud import signed_url_disk_cache as cache + + # Pretend HOME is empty ground so we can see the default shape + # without touching the user's real ~/.ndi/. + home = tmp_path / "home" + home.mkdir() + monkeypatch.setenv("HOME", str(home)) + monkeypatch.delenv("NDI_SIGNED_URL_CACHE_DIR", raising=False) + + result = Path(cache.cache_dir()) + assert result == home / ".ndi" / "signed-url-cache" + + +# --------------------------------------------------------------------------- +# Concurrency +# --------------------------------------------------------------------------- + + +class TestAtomicWrite: + """Two writers hitting the same scope: last write wins, no corruption.""" + + def test_alternating_writes_leave_a_valid_file(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + payload_a = sample_payload(files={"aa_bb": "https://s3/first"}) + payload_b = sample_payload(files={"cc_dd": "https://s3/second"}) + + for _ in range(10): + cache.save("ds1", "doc1", "chunks", payload_a) + cache.save("ds1", "doc1", "chunks", payload_b) + + got = cache.load("ds1", "doc1", "chunks") + assert got is not None, "after alternating writes, file must be valid" + assert isinstance(got["files"], dict) + # Whichever landed last is fine; what matters is no half-write. + assert len(got["files"]) == 1 + + def test_temp_files_do_not_linger_next_to_the_cache_file(self, isolated_cache_dir): + """A successful save leaves ONE file: no .tmp scraps around.""" + from ndi.cloud import signed_url_disk_cache as cache + + for _ in range(5): + cache.save("ds1", "doc1", "chunks", sample_payload()) + + ds_dir = isolated_cache_dir / "ds1" + names = sorted(p.name for p in ds_dir.iterdir()) + # Under a race with a second process the two writers could each + # briefly hold a .tmp file, but a fully successful run should + # never leak one. + assert names == ["doc1_chunks.json.gz"], f"expected exactly one cache file, got {names!r}" + + +# --------------------------------------------------------------------------- +# Series name escaping +# --------------------------------------------------------------------------- + + +class TestSeriesNameEscaping: + """Real series names can hold '/', spaces, unicode -- the FS can't. + + The percent-encoding + SHA-1-hash-above-96-chars rule is the on-disk + contract MATLAB mirrors byte-for-byte. + """ + + def test_pathy_series_name_saves_and_loads(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + pathy = "level 3/chunk.bin" + cache.save("ds1", "doc1", pathy, sample_payload()) + assert cache.load("ds1", "doc1", pathy) is not None + + def test_two_series_names_land_in_two_files(self, isolated_cache_dir): + """Distinct series must NOT collide on disk.""" + from ndi.cloud import signed_url_disk_cache as cache + + payload = sample_payload() + cache.save("ds1", "doc1", "level 3/chunk.bin", payload) + cache.save("ds1", "doc1", "level 3_chunk.bin", payload) + + ds_dir = isolated_cache_dir / "ds1" + n_gz = sum(1 for p in ds_dir.iterdir() if p.name.endswith(".json.gz")) + assert n_gz == 2, "two distinct series names must produce two distinct cache files" + + def test_very_long_series_name_hashes(self, isolated_cache_dir): + """A 500-char series name lands as ``hash-``, never overflows.""" + from ndi.cloud import signed_url_disk_cache as cache + + long_name = "x" * 500 + cache.save("ds1", "doc1", long_name, sample_payload()) + + ds_dir = isolated_cache_dir / "ds1" + files = list(ds_dir.iterdir()) + assert len(files) == 1 + name = files[0].name + # Anything short-enough with 'hash-<40 hex>' is fine. + assert name.startswith("doc1_hash-") + assert name.endswith(".json.gz") + assert len(name) < 100 # not 500+ + + +# --------------------------------------------------------------------------- +# Path convention (on-disk contract with NDI-matlab) +# --------------------------------------------------------------------------- + + +class TestOnDiskLayout: + """The layout MATLAB writes / reads: whole-doc vs. scoped file names.""" + + def test_whole_doc_scope_lands_at_doc_json_gz(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + cache.save("ds1", "doc1", "", sample_payload()) + assert (isolated_cache_dir / "ds1" / "doc1.json.gz").is_file() + + def test_scoped_lands_at_doc_underscore_series(self, isolated_cache_dir): + from ndi.cloud import signed_url_disk_cache as cache + + cache.save("ds1", "doc1", "chunk.bin", sample_payload()) + assert (isolated_cache_dir / "ds1" / "doc1_chunk.bin.json.gz").is_file() + + def test_corrupt_file_is_treated_as_miss(self, isolated_cache_dir): + """A truncated / non-gzip file must not crash load().""" + from ndi.cloud import signed_url_disk_cache as cache + + ds_dir = isolated_cache_dir / "ds1" + ds_dir.mkdir(parents=True) + (ds_dir / "doc1.json.gz").write_bytes(b"not a gzip") + + assert cache.load("ds1", "doc1", "") is None + + +# --------------------------------------------------------------------------- +# Integration with BatchSignedUrlLookup +# --------------------------------------------------------------------------- + + +class TestBatchLookupDiskCacheWiring: + """Second call across a fresh in-memory cache must skip the signer. + + This is the whole reopen story: run 1 signs, disk saves; run 2 + reads disk and never touches the signer -- whether that signer + would have been the paging walk or the async job. + """ + + def _counting_signer(self, files_map, expiry_iso): + """Signer that counts calls and hands back a valid, cacheable answer.""" + + state = {"count": 0} + + def signer(dataset_id, document_id, *, file_series="", client=None): + state["count"] += 1 + return True, { + "files": dict(files_map), + "filesExpireAt": expiry_iso, + "expiresAt": expiry_iso, + "generatedAt": expiry_iso, + "fileCount": len(files_map), + } + + return signer, state + + def test_reopen_reads_disk_and_skips_signer(self, isolated_cache_dir): + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + files_map = { + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6": "https://s3/one", + "4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6": "https://s3/two", + } + signer, state = self._counting_signer(files_map, future_iso(23 * 3600)) + + # Run 1: cold. One signer call answers both uids. + lookup1 = BatchSignedUrlLookup(signer=signer, disk_cache=True) + u1a = lookup1.lookup( + "ds-e2e", + "doc-e2e", + "chunk.bin", + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6", + ) + u1b = lookup1.lookup( + "ds-e2e", + "doc-e2e", + "chunk.bin", + "4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6", + ) + assert u1a == "https://s3/one" + assert u1b == "https://s3/two" + assert state["count"] == 1 + + # And the disk file lands. + disk_file = isolated_cache_dir / "ds-e2e" / "doc-e2e_chunk.bin.json.gz" + assert disk_file.is_file() + + # Run 2: fresh in-memory cache; the disk should carry both uids. + lookup2 = BatchSignedUrlLookup(signer=signer, disk_cache=True) + u2a = lookup2.lookup( + "ds-e2e", + "doc-e2e", + "chunk.bin", + "4192a3c0dd1b4e00_3fe8a1b2c3d4e5f6", + ) + u2b = lookup2.lookup( + "ds-e2e", + "doc-e2e", + "chunk.bin", + "4192a3c0dd1b4e00_4fe8a1b2c3d4e5f6", + ) + assert u2a == "https://s3/one" + assert u2b == "https://s3/two" + assert state["count"] == 1, ( + "run 2 must have hit the disk cache -- signer must not have " "been called again" + ) + + def test_disk_cache_off_by_default_signs_every_cold_run(self, isolated_cache_dir): + """The default (disk_cache=False) preserves pre-cache behaviour.""" + from ndi.cloud.batch_signed_url import BatchSignedUrlLookup + + signer, state = self._counting_signer({"aa_bb": "https://s3/x"}, future_iso(23 * 3600)) + + for _ in range(3): + lookup = BatchSignedUrlLookup(signer=signer) # disk_cache=False + lookup.lookup("ds-off", "doc-off", "", "aa_bb") + + assert state["count"] == 3 + # And nothing lands on disk. + assert not (isolated_cache_dir / "ds-off").exists() diff --git a/tests/test_gui_feedback.py b/tests/test_gui_feedback.py new file mode 100644 index 00000000..cce77185 --- /dev/null +++ b/tests/test_gui_feedback.py @@ -0,0 +1,261 @@ +"""A user can report a problem from inside a viewer in one step. + +The report carries what the viewer knows (versions, state, recent log) +without carrying credentials -- viewer logs do contain signed S3 URLs +and can contain bearer tokens -- and fits the server's size limits. +""" + +import io +import json +import os +import sys + +import pytest + +from ndi.gui import feedback + +# ------------------------------------------------------------ scrub + + +class TestScrub: + def test_signed_url_credentials_are_removed(self): + url = ( + "https://bucket.s3.amazonaws.com/chunk.bin_7?X-Amz-Algorithm=AWS4-HMAC-SHA256" + "&X-Amz-Credential=ASIAXYZ%2F20261012&X-Amz-Security-Token=IQoJb3JpZ2lu" + "&X-Amz-Signature=deadbeef1234&X-Amz-Expires=3600" + ) + out = feedback.scrub(url) + for secret in ("ASIAXYZ", "IQoJb3JpZ2lu", "deadbeef1234"): + assert secret not in out + # The non-secret parts stay, so the log is still readable. + assert "chunk.bin_7" in out and "X-Amz-Expires=3600" in out + + def test_bearer_tokens_and_jwts_are_removed(self): + jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NSJ9.c2lnbmF0dXJl" + out = feedback.scrub(f"Authorization: Bearer abc.def-123 and later {jwt}") + assert "abc.def-123" not in out and jwt not in out + + def test_secret_named_values_are_removed(self): + out = feedback.scrub('password=hunter2 "api_key": "k-999" NDI_CLOUD_TOKEN=xyz') + for secret in ("hunter2", "k-999", "xyz"): + assert secret not in out + + def test_ordinary_log_lines_survive(self): + line = "12:00:01.123 [lightsheet] tiles: 56 loaded / 112 requested (in flight: 8)" + assert feedback.scrub(line) == line + + +# ------------------------------------------------------------ log buffer + + +class TestLogBuffer: + def test_lines_are_kept_and_redraws_keep_their_last_state(self): + buf = feedback.LogBuffer(max_lines=10) + buf.feed("first line\n[lightsheet] 10%\r[lightsheet] 50%\r[lightsheet] 100%\nthird") + assert buf.text().splitlines() == ["first line", "[lightsheet] 100%", "third"] + + def test_only_the_newest_lines_are_kept(self): + buf = feedback.LogBuffer(max_lines=3) + buf.feed("".join(f"line {i}\n" for i in range(10))) + assert buf.text().splitlines() == ["line 7", "line 8", "line 9"] + + def test_the_tee_still_writes_through(self): + real = io.StringIO() + buf = feedback.LogBuffer() + tee = feedback._Tee(real, buf) + print("hello", file=tee) + assert real.getvalue() == "hello\n" + assert buf.text() == "hello" + + +# ------------------------------------------------------------ report + + +class TestBuildReport: + def test_payload_fields_match_the_api(self): + rep = feedback.buildReport( + "lightsheetZarr-viewer", + "performance", + "slow zoom", + datasetId="ds1", + documentId="doc1", + state={"view": {"ready": 56, "needed": 112}}, + log="a\nb", + ) + assert rep["app"] == "lightsheetZarr-viewer" + assert rep["category"] == "performance" + assert rep["description"] == "slow zoom" + assert rep["datasetId"] == "ds1" and rep["documentId"] == "doc1" + assert rep["state"] == {"view": {"ready": 56, "needed": 112}} + assert rep["log"] == "a\nb" + assert rep["appVersion"].startswith("NDI-python") + assert "python" in rep["environment"] + # Every key is one submitFeedback accepts. + import inspect + + from ndi.cloud.api.feedback import submitFeedback + + params = inspect.signature(submitFeedback).parameters + assert set(rep) <= set(params) + + def test_unset_optional_fields_are_left_out(self): + rep = feedback.buildReport("app", "bug", "d") + for key in ("datasetId", "documentId", "state", "log", "traceback"): + assert key not in rep + + def test_secrets_in_state_env_and_log_are_scrubbed(self, monkeypatch): + monkeypatch.setenv("NDI_CLOUD_PASSWORD", "hunter2") + monkeypatch.setenv("NDI_LIGHTSHEET_DEBUG", "1") + rep = feedback.buildReport( + "app", + "bug", + "d", + state={"url": "https://x/y?X-Amz-Signature=abc123", "token": "t0k"}, + log="Bearer sekrit", + ) + blob = json.dumps(rep) + for secret in ("hunter2", "abc123", "t0k", "sekrit"): + assert secret not in blob + assert rep["environment"]["flags"]["NDI_LIGHTSHEET_DEBUG"] == "1" + + def test_a_huge_log_is_cut_to_the_server_limit_keeping_the_end(self): + log = "".join(f"line {i}\n" for i in range(200_000)) + rep = feedback.buildReport("app", "bug", "d", log=log) + assert len(rep["log"]) <= feedback.MAX_LOG + assert rep["log"].rstrip().endswith("line 199999") + assert "earlier characters cut" in rep["log"] + + def test_an_oversized_state_is_replaced_by_a_note(self): + rep = feedback.buildReport("app", "bug", "d", state={"big": "x" * 300_000}) + assert "_note" in rep["state"] + assert len(json.dumps(rep["state"])) < feedback.MAX_STATE_JSON + 1000 + + +class TestSendAndSave: + def test_send_returns_the_stored_id(self, monkeypatch): + seen = {} + + def fake_submit(**kwargs): + seen.update(kwargs) + return (True, "6710aa", 201, "https://api/v1/feedback") + + import ndi.cloud.api.feedback as api + + monkeypatch.setattr(api, "submitFeedback", fake_submit) + rep = feedback.buildReport("app", "bug", "d") + assert feedback.sendReport(rep) == "6710aa" + assert seen["app"] == "app" + + def test_save_writes_the_report_with_its_schema_version(self, tmp_path): + path = tmp_path / "r.json" + feedback.saveReport(feedback.buildReport("app", "other", "d"), str(path)) + body = json.loads(path.read_text()) + assert body["schemaVersion"] == 1 and body["description"] == "d" + + @pytest.mark.parametrize( + "status, phrase", + [(429, "Too many"), (413, "too large"), (404, "does not accept"), (500, "HTTP 500")], + ) + def test_errors_read_as_something_a_user_can_act_on(self, status, phrase): + from ndi.cloud.exceptions import CloudAPIError + + assert phrase in feedback.describeSendError(CloudAPIError("x", status_code=status)) + + def test_not_signed_in_says_so(self): + from ndi.cloud.exceptions import CloudAuthError + + assert "Not signed in" in feedback.describeSendError(CloudAuthError("no")) + + +# ------------------------------------------------------------ dialog + + +@pytest.fixture +def qapp(): + os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") + try: + from qtpy.QtWidgets import QApplication + except Exception: + pytest.skip("no Qt binding") + return QApplication.instance() or QApplication([]) + + +def _wait(qapp, cond, timeout=5.0): + import time + + end = time.monotonic() + timeout + while not cond() and time.monotonic() < end: + qapp.processEvents() + time.sleep(0.02) + return cond() + + +class TestDialog: + def test_send_is_off_until_the_user_says_something(self, qapp): + dlg = feedback.FeedbackDialog(app="a", collect=dict, send=lambda r: "id") + assert not dlg.sendButton.isEnabled() + dlg.description.setPlainText("it froze") + assert dlg.sendButton.isEnabled() + + def test_sending_reports_the_id_and_carries_the_collected_state(self, qapp): + sent = [] + dlg = feedback.FeedbackDialog( + app="lightsheetZarr-viewer", + collect=lambda: {"state": {"view": {"ready": 1, "needed": 2}}, "documentId": "p1"}, + send=lambda rep: sent.append(rep) or "6710aa", + ) + dlg.category.setCurrentIndex(1) # Slow + dlg.description.setPlainText("zoom is slow") + dlg.sendButton.click() + assert _wait(qapp, lambda: dlg.feedbackID == "6710aa") + assert "6710aa" in dlg.status.text() + assert not dlg.sendButton.isEnabled(), "one report per dialog" + (rep,) = sent + assert rep["category"] == "performance" + assert rep["state"]["view"] == {"ready": 1, "needed": 2} + assert rep["documentId"] == "p1" + + def test_a_failed_send_says_what_to_do_and_can_be_retried(self, qapp): + from ndi.cloud.exceptions import CloudAuthError + + def boom(rep): + raise CloudAuthError("no token") + + dlg = feedback.FeedbackDialog(app="a", collect=dict, send=boom) + dlg.description.setPlainText("x") + dlg.sendButton.click() + assert _wait(qapp, lambda: "Not signed in" in dlg.status.text()) + assert dlg.sendButton.isEnabled() + + def test_the_preview_shows_exactly_the_report(self, qapp): + dlg = feedback.FeedbackDialog( + app="a", collect=lambda: {"log": "Bearer sekrit"}, send=lambda r: "id" + ) + dlg.description.setPlainText("x") + dlg.toggle.setChecked(True) + shown = json.loads(dlg.preview.toPlainText()) + assert shown["app"] == "a" and "sekrit" not in shown["log"] + + def test_a_broken_collector_still_produces_a_report(self, qapp): + def broken(): + raise RuntimeError("viewer is wedged") + + dlg = feedback.FeedbackDialog(app="a", collect=broken, send=lambda r: "id") + dlg.description.setPlainText("x") + assert "viewer is wedged" in json.dumps(dlg.report()) + + +def test_installing_the_log_buffer_tees_stderr(monkeypatch): + monkeypatch.setattr(feedback, "_installed", None) + real = io.StringIO() + monkeypatch.setattr(sys, "stderr", real) + buf = feedback.installLogBuffer() + print("[lightsheet] hello", file=sys.stderr) + assert "[lightsheet] hello" in buf.text() + assert real.getvalue() == "[lightsheet] hello\n" + assert feedback.installLogBuffer() is buf + import logging + + for h in list(logging.getLogger().handlers): + if isinstance(h, feedback._BufferHandler): + logging.getLogger().removeHandler(h) diff --git a/tests/test_gui_genepyramid_controls.py b/tests/test_gui_genepyramid_controls.py index 6f1e5c4b..b048f9b1 100644 --- a/tests/test_gui_genepyramid_controls.py +++ b/tests/test_gui_genepyramid_controls.py @@ -522,3 +522,60 @@ def test_a_failing_appearance_panel_still_leaves_a_tour(monkeypatch, capsys): _args, kwargs = tour.calls[0] assert kwargs.get("appearance") is None assert "gene appearance" in capsys.readouterr().err + + +def _stubPanels(monkeypatch): + monkeypatch.setattr(controls, "_importQtWidgets", lambda: None) + for name in ( + "addDisplayPanel", + "addCellTypePanel", + "addGenePanel", + "addRotationPanel", + "addGeneAppearancePanel", + "addGeneTourPanel", + ): + monkeypatch.setattr(controls, name, _Recorder()) + + +def test_a_cloud_session_gets_the_report_button_on_the_cloud_panel(monkeypatch): + """Reporting a problem needs a sign-in, and the cloud panel is where + the reader signs in -- so that is where the button goes.""" + _stubPanels(monkeypatch) + cloud, report = _Recorder(), _Recorder() + monkeypatch.setattr(controls, "addCloudPanel", cloud) + monkeypatch.setattr(controls, "addReportPanel", report) + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + collect = dict + + made = controls.addAllPanels( + object(), object(), object(), object(), True, report_collect=collect + ) + assert cloud.calls[0][1] == {"report_collect": collect} + assert not report.calls + assert "report" not in made + + +def test_a_local_session_still_gets_a_way_to_report(monkeypatch): + """No token, no cloud panel -- but a local reader can still hit a + problem, and Save to file / Copy work without signing in.""" + _stubPanels(monkeypatch) + cloud, report = _Recorder(), _Recorder() + monkeypatch.setattr(controls, "addCloudPanel", cloud) + monkeypatch.setattr(controls, "addReportPanel", report) + monkeypatch.delenv("NDI_CLOUD_TOKEN", raising=False) + + made = controls.addAllPanels(object(), object(), object(), object(), True, report_collect=dict) + assert not cloud.calls + assert report.calls and made["report"] is not None + + +def test_without_a_collector_nothing_changes(monkeypatch): + _stubPanels(monkeypatch) + cloud, report = _Recorder(), _Recorder() + monkeypatch.setattr(controls, "addCloudPanel", cloud) + monkeypatch.setattr(controls, "addReportPanel", report) + monkeypatch.setenv("NDI_CLOUD_TOKEN", "anything") + + controls.addAllPanels(object(), object(), object(), object(), True) + assert cloud.calls[0][1] == {} + assert not report.calls diff --git a/tests/test_gui_genepyramid_progress.py b/tests/test_gui_genepyramid_progress.py index e19a8022..c7b72720 100644 --- a/tests/test_gui_genepyramid_progress.py +++ b/tests/test_gui_genepyramid_progress.py @@ -236,6 +236,11 @@ def test_no_window_and_no_terminal_is_off_not_a_crash(self): class TestClosingIsAlwaysSafe(unittest.TestCase): + def setUp(self): + # Closing latches _window_closed; put it back so later tests can + # still build a window. + self.addCleanup(setattr, progress, "_window_closed", progress._window_closed) + def test_closing_when_there_never_was_a_window(self): self.addCleanup(setattr, progress, "_window", progress._window) progress._window = None diff --git a/tests/test_gui_lightsheetZarr_viewtiles.py b/tests/test_gui_lightsheetZarr_viewtiles.py new file mode 100644 index 00000000..27e71ba3 --- /dev/null +++ b/tests/test_gui_lightsheetZarr_viewtiles.py @@ -0,0 +1,228 @@ +"""The viewer says how many tiles the current view needs, not only how +many are in flight; and the launch window, once closed, stays closed. + +Before this, zooming into a new region showed "12 in flight" with no way +to tell whether that was most of the view or a sliver of it. The chunk +grid and the stored-chunk list on the level documents make the total +knowable: napari draws one level over one region at one slice. +""" + +import io +import types + +import numpy as np +import pytest + +from ndi.gui.app.lightsheetZarr import progress, viewtiles +from ndi.gui.app.lightsheetZarr.viewer import _FetchCounter + + +def _meta(chunks=(1, 1, 4, 10, 10), grid=(1, 2, 3, 5, 5), stored=None, axes="tczyx"): + return { + "doc": types.SimpleNamespace(id="lvl"), + "axes": axes, + "chunks": chunks, + "chunk_grid": grid, + "stored": stored, + } + + +# ------------------------------------------------------------ chunksInBox + + +class TestChunksInBox: + def test_the_box_covers_the_chunks_it_touches(self): + m = _meta(chunks=(10, 10), grid=(5, 5), axes="yx") + names = viewtiles.chunksInBox(m, [5, 15], [12, 29]) + # rows 0..1, cols 1..2 -> linear 1-based (r*5 + c) + 1 + assert sorted(names) == sorted(["chunk.bin_2", "chunk.bin_3", "chunk.bin_7", "chunk.bin_8"]) + + def test_sparse_chunks_need_no_fetch_and_are_not_counted(self): + m = _meta(chunks=(10, 10), grid=(5, 5), axes="yx", stored={"chunk.bin_2"}) + assert viewtiles.chunksInBox(m, [0, 0], [19, 29]) == ["chunk.bin_2"] + + def test_a_box_past_the_edge_is_clipped(self): + m = _meta(chunks=(10, 10), grid=(2, 2), axes="yx") + assert len(viewtiles.chunksInBox(m, [-5, -5], [999, 999])) == 4 + + def test_a_huge_box_is_not_tallied(self, monkeypatch): + monkeypatch.setattr(viewtiles, "MAX_POSITIONS", 3) + m = _meta(chunks=(10, 10), grid=(2, 2), axes="yx") + assert viewtiles.chunksInBox(m, [0, 0], [19, 19]) is None + + def test_linear_index_matches_the_reader(self): + from ndi.pyramid.multiscale import _linear_chunk_index + + m = _meta() + names = viewtiles.chunksInBox(m, [0, 1, 8, 30, 40], [0, 1, 8, 30, 40]) + assert names == [f"chunk.bin_{_linear_chunk_index((0, 1, 2, 3, 4), m['chunk_grid'])}"] + + +# ------------------------------------------------------------ layerBox + + +def _fakeViewer(point, displayed=(1, 2), ndim=3): + dims = types.SimpleNamespace(point=point, displayed=displayed, ndim=ndim) + return types.SimpleNamespace(dims=dims) + + +def _fakeLayer(level, corners, n_levels=2, factors=None, visible=True): + factors = factors or [(1, 1, 1), (2, 2, 2)] + return types.SimpleNamespace( + multiscale=True, + data=[None] * n_levels, + data_level=level, + corner_pixels=np.array(corners), + ndim=3, + visible=visible, + downsample_factors=np.array(factors, dtype=float), + world_to_data=lambda p: list(p), + ) + + +class TestLayerBox: + def test_displayed_axes_come_from_corner_pixels_and_z_from_the_slice(self): + metas = [_meta(), _meta()] + layer = _fakeLayer(1, [[0, 3, 4], [0, 25, 31]]) + viewer = _fakeViewer(point=(10.0, 0.0, 0.0)) + level, lo, hi = viewtiles.layerBox(layer, viewer, metas, channel=1) + assert level == 1 + # t, c, z, y, x on the grid; z is world 10 / factor 2 = 5. + assert lo == [0, 1, 5, 3, 4] + assert hi == [0, 1, 5, 25, 31] + + def test_a_tzyx_layer_maps_onto_a_tczyx_grid(self): + layer = _fakeLayer(0, [[0, 0, 2, 3], [0, 0, 8, 9]], n_levels=1, factors=[(1, 1, 1, 1)]) + layer.ndim = 4 + viewer = _fakeViewer(point=(0.0, 6.0, 0.0, 0.0), displayed=(2, 3), ndim=4) + _level, lo, hi = viewtiles.layerBox(layer, viewer, [_meta()], channel=2) + assert lo == [0, 2, 6, 2, 3] + assert hi == [0, 2, 6, 8, 9] + + def test_a_single_level_layer_is_not_tallied(self): + layer = _fakeLayer(0, [[0, 0, 0], [0, 1, 1]]) + layer.multiscale = False + assert viewtiles.layerBox(layer, _fakeViewer((0, 0, 0)), [_meta()]) is None + + def test_a_ladder_that_does_not_match_the_docs_is_not_tallied(self): + layer = _fakeLayer(0, [[0, 0, 0], [0, 1, 1]], n_levels=3) + assert viewtiles.layerBox(layer, _fakeViewer((0, 0, 0)), [_meta(), _meta()]) is None + + def test_napari_without_the_attributes_is_unknown_not_a_crash(self): + layer = types.SimpleNamespace(multiscale=True, data=[None]) + assert viewtiles.layerBox(layer, _fakeViewer((0, 0, 0)), [_meta()]) is None + assert "could not read napari layer state" in viewtiles.lastReason + + def test_a_level_mismatch_says_so(self): + layer = _fakeLayer(0, [[0, 0, 0], [0, 1, 1]], n_levels=3) + viewtiles.layerBox(layer, _fakeViewer((0, 0, 0)), [_meta(), _meta()]) + assert viewtiles.lastReason == ( + "layer has 3 level(s) but the pyramid has 2 level document(s)" + ) + + +class TestViewTally: + def test_ready_counts_what_is_on_disk_and_shared_chunks_count_once(self): + # Channel chunk spans both channels -> two layers, same chunks. + m = _meta(chunks=(1, 2, 4, 10, 10), grid=(1, 1, 3, 5, 5)) + layers = [_fakeLayer(0, [[0, 0, 0], [0, 19, 9]], n_levels=1) for _ in range(2)] + viewer = _fakeViewer(point=(0.0, 0.0, 0.0)) + + class Fetcher: + def chunkPathIfCached(self, doc, name): + return "/x" if name == "chunk.bin_1" else None + + assert viewtiles.viewTally(viewer, layers, [m], Fetcher()) == (1, 2) + + def test_hidden_layers_need_nothing(self): + m = _meta(chunks=(10, 10, 10), grid=(1, 2, 2), axes="zyx") + layer = _fakeLayer(0, [[0, 0, 0], [0, 19, 19]], n_levels=1, visible=False) + + class Fetcher: + def chunkPathIfCached(self, doc, name): + return None + + assert viewtiles.viewTally(_fakeViewer((0, 0, 0)), [layer], [m], Fetcher()) == (0, 0) + + +# ------------------------------------------------------------ counter + + +class _Reporter: + def __init__(self): + self.pushed = [] + + def push(self, message, *, overlay=""): + self.pushed.append((message, overlay)) + + +class TestCounterReportsTheView: + def _counter(self): + out = io.StringIO() + rep = _Reporter() + c = _FetchCounter(out=out, idle_interval=3600, status_reporter=rep) + return c, out, rep + + def test_status_and_overlay_say_how_much_of_the_view_is_left(self): + c, _out, rep = self._counter() + try: + c("start", "u1", 0, None) + c.set_view_tally(37, 120) + message, overlay = rep.pushed[-1] + assert "view 37/120 tiles ready" in message + assert overlay == "Loading view: 37 / 120 tiles (1 in flight)" + finally: + c.stop() + + def test_a_complete_view_hides_the_overlay_and_logs_how_long_it_took(self): + c, out, rep = self._counter() + try: + c.set_view_tally(0, 4) + c.set_view_tally(4, 4) + assert rep.pushed[-1][1] == "" + assert "view complete: 4 tile(s)" in out.getvalue() + finally: + c.stop() + + def test_an_untallied_view_falls_back_to_in_flight(self): + c, _out, rep = self._counter() + try: + c("start", "u1", 0, None) + c.set_view_tally(None, None) + assert rep.pushed[-1][1] == "Loading tiles ... (1 in flight)" + finally: + c.stop() + + +# ------------------------------------------------------------ launch window + + +class TestTheLaunchWindowStaysClosed: + def test_a_note_after_closing_does_not_build_a_new_window(self, monkeypatch): + """closeLaunchWindow then note("viewer running ...") used to go + through _mode() -> _launchWindow(), find no window, and open a + fresh "starting ..." window that nothing ever closed.""" + monkeypatch.setattr(progress, "_window", None) + monkeypatch.setattr(progress, "_window_failed", False) + monkeypatch.setattr(progress, "_window_closed", False) + monkeypatch.delenv("NDI_LIGHTSHEET_PROGRESS", raising=False) + monkeypatch.delenv("NDI_LIGHTSHEET_QUIET", raising=False) + + class Widget: + def __getattr__(self, _name): + return lambda *a, **k: None + + monkeypatch.setattr( + progress, "_window", {"widget": Widget(), "app": Widget(), "step": None, "bar": None} + ) + progress.closeLaunchWindow() + progress.note("viewer running") + + assert progress._launchWindow() is None + # It short-circuited on the close latch; it never got as far as + # trying (and failing) to build a window. + assert progress._window_failed is False + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/tests/test_ndi_common_matlab_sync.py b/tests/test_ndi_common_matlab_sync.py index 32a5590d..4682dbef 100644 --- a/tests/test_ndi_common_matlab_sync.py +++ b/tests/test_ndi_common_matlab_sync.py @@ -132,18 +132,7 @@ def _require_matlab_repo() -> Path: # must match its copy on the branch byte-for-byte, and the entry must go once # the branch merges (test_branch_pending_files_match_their_branch). Section 8 # of docs/developer_notes/ndi_matlab_python_bridge.yaml. -_LIGHTSHEET_BRANCH = "claude/lightsheet-zarr-ndi-viewer-djp5vk" -MATLAB_BRANCH_PENDING: dict[str, str] = { - # The lightsheet OME-Zarr document types, landed here with the napari - # viewer (src/ndi/gui/app/lightsheetZarr) and authored as a pair with - # the MATLAB writer on this branch. - "database_documents/data/lightsheetZarrLevel.json": _LIGHTSHEET_BRANCH, - "database_documents/data/lightsheetZarrLevel.md": _LIGHTSHEET_BRANCH, - "database_documents/data/lightsheetZarrPyramid.json": _LIGHTSHEET_BRANCH, - "database_documents/data/lightsheetZarrPyramid.md": _LIGHTSHEET_BRANCH, - "schema_documents/data/lightsheetZarrLevel_schema.json": _LIGHTSHEET_BRANCH, - "schema_documents/data/lightsheetZarrPyramid_schema.json": _LIGHTSHEET_BRANCH, -} +MATLAB_BRANCH_PENDING: dict[str, str] = {} # Files present on both sides that intentionally do not match byte-for-byte. diff --git a/tests/test_pyramid_fetch_direct.py b/tests/test_pyramid_fetch_direct.py new file mode 100644 index 00000000..1b087f82 --- /dev/null +++ b/tests/test_pyramid_fetch_direct.py @@ -0,0 +1,297 @@ +"""Session-free chunk resolution in ndi.pyramid.multiscale._ChunkFetcher. + +A pyramid chunk is a file series member. Resolving one needs no database: +the document is in memory, its manifest names each member's uid, and DID's +cached_path_for_uid finds a member on disk with no SQL. So fetcher workers +are plain threads, and a worker opens a session of its own only when a chunk +must go the long way. These tests pin that down: + +* a member on disk resolves from a worker thread without any session being + reopened; +* a member not on disk is fetched through the cloud handler with DID's + context (seriesName, member uid) and lands where cached_path_for_uid + finds it; +* an absent (sparse) slot resolves to None, as open_doc's NoSuchMember; +* NDI_LIGHTSHEET_DIRECT=0 goes back through the session; +* configureDaskPool widens dask's pool to the worker count; +* fetch_cloud_file records per-stage timing for the debug log. +""" + +from __future__ import annotations + +import copy +import os +import threading + +import pytest + +from ndi.pyramid import multiscale +from ndi.pyramid.multiscale import _ChunkFetcher + +SERIES = "chunkdata.bin" +COUNT = 4 + + +def _content(i: int) -> bytes: + return bytes([(i * 37 + k) % 256 for k in range(64)]) + + +@pytest.fixture +def series_session(tmp_path): + """A local dataset holding one demoNDISeries document with COUNT members.""" + from did.document import Document as DIDDocument + + from ndi.dataset import ndi_dataset_dir + from ndi.document import ndi_document + + members = tmp_path / "members" + members.mkdir() + paths = [] + for i in range(COUNT): + p = members / f"chunk_{i + 1}.bin" + p.write_bytes(_content(i)) + paths.append(str(p)) + + local = ndi_dataset_dir("direct_ds", str(tmp_path / "ds")) + seed = ndi_document("demoNDISeries", **{"base.name": "direct_series", "demoNDISeries.value": 1}) + did_doc = DIDDocument(seed.document_properties) + did_doc.add_file_series(SERIES, paths) + doc = ndi_document(did_doc.document_properties) + local.database_add(doc) + stored = local.database_search( + __import__("ndi.query", fromlist=["ndi_query"]).ndi_query("base.id") == doc.id + )[0] + return local, stored + + +def _no_reopen(fetcher): + """Make any session reopen fail loudly: the direct path must not need one.""" + + def boom(): + raise AssertionError("a worker reopened a session") + + fetcher._reopen = boom + + +def _resolve_on_worker(fetcher, doc, filename): + out = {} + + def run(): + try: + out["path"] = fetcher.chunkPath(doc, filename) + except BaseException as exc: # noqa: BLE001 + out["exc"] = exc + + t = threading.Thread(target=run) + t.start() + t.join(30) + assert not t.is_alive(), "chunkPath did not return" + if "exc" in out: + raise out["exc"] + return out["path"] + + +def test_member_on_disk_resolves_without_a_session(series_session): + session, doc = series_session + fetcher = _ChunkFetcher(session, workers=4) + _no_reopen(fetcher) + try: + for i in range(COUNT): + path = _resolve_on_worker(fetcher, doc, f"{SERIES}_{i + 1}") + assert path is not None + with open(path, "rb") as f: + assert f.read() == _content(i), f"member {i + 1} resolved to the wrong file" + assert fetcher._fallbacks == 0 + finally: + fetcher.close() + + +def test_member_not_on_disk_is_fetched_through_the_cloud_handler( + series_session, monkeypatch, tmp_path +): + from did.file import cached_path_for_uid, read_series_manifest + + import ndi.cloud.filehandler as fh + + session, doc = series_session + fetcher = _ChunkFetcher(session, workers=2) + _no_reopen(fetcher) + + # The manifest's uid list, and the member we will make remote. + props = copy.deepcopy(doc.document_properties) + manifest_uid = None + for info in props["files"]["file_info"]: + if info["name"] == SERIES: + locs = info["locations"] + locs = locs if isinstance(locs, list) else [locs] + manifest_uid = locs[0]["uid"] + # Record a cloud copy of the manifest, as a downloaded dataset does. + locs.append( + { + "location": f"ndic://dsX/{manifest_uid}", + "location_type": "ndicloud", + "uid": manifest_uid, + "ingest": 0, + "delete_original": 0, + } + ) + info["locations"] = locs + assert manifest_uid + manifest_path = cached_path_for_uid(manifest_uid, additional_roots=[fetcher._file_dir]) + uids = read_series_manifest(manifest_path)["uids"] + member_uid = uids[1] + on_disk = cached_path_for_uid(member_uid, additional_roots=[fetcher._file_dir]) + os.remove(on_disk) + + calls = [] + + def fake_download(dest, source, context=None): + calls.append((source, dict(context or {}))) + with open(dest, "wb") as f: + f.write(b"fetched-" + _content(1)) + + monkeypatch.setattr(fh, "download_file_from_cloud", fake_download) + + # Keep the fetched file out of the user's global DID cache. + import did.common + + def no_cache(): + raise OSError("no cache in tests") + + monkeypatch.setattr(did.common, "get_cache", no_cache) + + remote_doc = type(doc)(props) + try: + path = _resolve_on_worker(fetcher, remote_doc, f"{SERIES}_2") + with open(path, "rb") as f: + assert f.read() == b"fetched-" + _content(1) + assert len(calls) == 1 + source, context = calls[0] + assert source == f"ndic://dsX/{manifest_uid}" + assert context["uid"] == member_uid + assert context["seriesName"] == SERIES + assert context["documentId"] == doc.id + assert context["mode"] == "open" + # Where DID will find it next time, with no database. + assert cached_path_for_uid(member_uid, additional_roots=[fetcher._file_dir]) == path + assert fetcher._fallbacks == 0 + finally: + fetcher.close() + + +def test_sparse_slot_is_none(series_session): + session, doc = series_session + fetcher = _ChunkFetcher(session, workers=1) + _no_reopen(fetcher) + try: + assert _resolve_on_worker(fetcher, doc, f"{SERIES}_{COUNT + 3}") is None + finally: + fetcher.close() + + +def test_direct_path_can_be_turned_off(series_session, monkeypatch): + monkeypatch.setenv("NDI_LIGHTSHEET_DIRECT", "0") + session, doc = series_session + fetcher = _ChunkFetcher(session, workers=1) + try: + # Owner thread: goes through the session's database_openbinarydoc. + path = fetcher.chunkPath(doc, f"{SERIES}_3") + with open(path, "rb") as f: + assert f.read() == _content(2) + assert fetcher._direct is False + finally: + fetcher.close() + + +def test_default_workers_is_wide(monkeypatch): + monkeypatch.delenv("NDI_LIGHTSHEET_WORKERS", raising=False) + assert multiscale._default_workers() == 16 + monkeypatch.setenv("NDI_LIGHTSHEET_WORKERS", "5") + assert multiscale._default_workers() == 5 + + +def test_configure_dask_pool_matches_workers(): + import dask + + previous = dask.config.get("pool", None) + try: + n = multiscale.configureDaskPool(48) + assert n >= 48 + assert dask.config.get("pool")._max_workers == n + finally: + dask.config.set(pool=previous) + + +def test_fetch_cloud_file_records_stage_timing(monkeypatch, tmp_path): + import ndi.cloud.api.files as files_api + import ndi.cloud.filehandler as fh + + monkeypatch.setattr( + files_api, "getFileDetails", lambda ds, uid, client=None: {"downloadUrl": "https://x"} + ) + + def fake_get(url, dest, **kw): + with open(dest, "wb") as f: + f.write(b"x" * 1000) + return True + + monkeypatch.setattr(files_api, "getFile", fake_get) + assert fh.fetch_cloud_file("ndic://ds/uid1", tmp_path / "out", client=object()) + t = fh.lastFetchTiming() + assert t is not None + assert t["bytes"] == 1000 + assert t["ok"] is True + assert t["batch"] is False + assert t["sign_s"] >= 0 and t["transfer_s"] >= 0 + + +def test_worker_limit_caps_concurrency_and_changes_live(): + """setWorkers caps how many chunks fetch at once, and can be changed live.""" + import time + + class _Session: # no path, no database: nothing to reopen + pass + + fetcher = _ChunkFetcher(_Session(), workers=2, max_workers=4) + gate = threading.Event() + lock = threading.Lock() + state = {"now": 0, "peak": 0} + + def slow_resolve(doc, filename): + with lock: + state["now"] += 1 + state["peak"] = max(state["peak"], state["now"]) + gate.wait(10) + with lock: + state["now"] -= 1 + return f"/tmp/{filename}" + + fetcher._resolve = slow_resolve + done = [] + try: + for i in range(8): + fetcher.prefetchAsync(f"doc{i}", f"c_{i}", on_complete=done.append) + time.sleep(0.3) + assert state["peak"] == 2, f"limit 2 but {state['peak']} ran at once" + + assert fetcher.setWorkers(99) == 4 # clamped to the threads started + time.sleep(0.3) + assert state["peak"] == 4 + + assert fetcher.setWorkers(0) == 1 + assert fetcher.workerLimit == 1 + gate.set() + deadline = time.time() + 10 + while len(done) < 8 and time.time() < deadline: + time.sleep(0.05) + assert len(done) == 8 + finally: + gate.set() + fetcher.close() + + +def test_level_arrays_start_max_threads_with_default_limit(monkeypatch): + monkeypatch.delenv("NDI_LIGHTSHEET_WORKERS", raising=False) + assert multiscale._default_workers() == 16 + assert set(multiscale.WORKER_CHOICES) >= {8, 16, 32, 64} + assert multiscale.MAX_WORKERS >= max(multiscale.WORKER_CHOICES) diff --git a/tests/test_pyramid_fetch_priority.py b/tests/test_pyramid_fetch_priority.py new file mode 100644 index 00000000..b83c6f1d --- /dev/null +++ b/tests/test_pyramid_fetch_priority.py @@ -0,0 +1,194 @@ +"""The chunk fetcher serves the view the user is looking at first. + +napari slices on one thread, so a slice for a level the user zoomed +through held up the slice for where they stopped until every one of its +tiles had arrived, and the fetch queue was first-in-first-out on top of +that. Now: newest requests first, chunks in the current view ahead of +everything, and a blocking read for a chunk that has left the view is +released (StaleChunkRead) while its fetch carries on in the background. +""" + +import threading +import time +import types + +import pytest + +from ndi.pyramid.multiscale import StaleChunkRead, _ChunkFetcher + +DOC = types.SimpleNamespace(id="lvl0") +COARSE = types.SimpleNamespace(id="lvl3") + + +class _Session: + """Reopenable fake session. Fetches block until released, and the + order in which they start is recorded.""" + + started: list = [] + gate = threading.Event() + + def __init__(self, path): + self.path = path + + def database_openbinarydoc(self, doc, filename): + _Session.started.append(filename) + assert _Session.gate.wait(10) + return types.SimpleNamespace(fullpathfilename=__file__) + + def database_closebinarydoc(self, fh): + pass + + +@pytest.fixture +def fetcher(): + _Session.started = [] + _Session.gate = threading.Event() + f = _ChunkFetcher(_Session("/x"), workers=1) + f.warm() + yield f + _Session.gate.set() + f.close() + + +def _occupy(f): + """Tie up the single worker so later requests queue behind it.""" + f.prefetchAsync(DOC, "busy") + for _ in range(100): + if _Session.started: + return + time.sleep(0.01) + + +def _drain(f, n): + _Session.gate.set() + for _ in range(300): + if len(_Session.started) >= n: + return + time.sleep(0.01) + + +def test_newest_request_goes_first(fetcher): + _occupy(fetcher) + for name in ("a", "b", "c"): + fetcher.prefetchAsync(DOC, name) + _drain(fetcher, 4) + assert _Session.started == ["busy", "c", "b", "a"] + + +def test_the_current_view_jumps_the_queue_and_is_queued_unasked(fetcher): + _occupy(fetcher) + fetcher.prefetchAsync(DOC, "old") + fetcher.setView({("lvl0", "v1"): DOC, ("lvl0", "v2"): DOC}) + fetcher.prefetchAsync(DOC, "newer-but-not-in-view") + _drain(fetcher, 5) + assert set(_Session.started[1:3]) == {"v1", "v2"} + assert _Session.started[3:] == ["newer-but-not-in-view", "old"] + + +def test_a_chunk_that_leaves_the_view_drops_back(fetcher): + _occupy(fetcher) + fetcher.setView({("lvl0", "was-in-view"): DOC}) + fetcher.prefetchAsync(DOC, "other") + fetcher.setView({("lvl0", "now-in-view"): DOC}) + _drain(fetcher, 4) + assert _Session.started[1] == "now-in-view" + + +def test_a_waiting_read_is_released_when_its_chunk_leaves_the_view(fetcher): + _occupy(fetcher) + out = {} + + def read(): + try: + out["path"] = fetcher.chunkPath(DOC, "zoomed-past") + except StaleChunkRead as exc: + out["stale"] = exc + + t = threading.Thread(target=read) + t.start() + time.sleep(0.15) + fetcher.setView({("lvl0", "elsewhere"): DOC}) + t.join(2) + assert not t.is_alive(), "the read must not wait for a tile nobody is looking at" + assert "stale" in out + assert fetcher.staleReleased() == 1 + + # ...and the fetch still lands, slowly, for when they zoom back. + _drain(fetcher, 3) + for _ in range(200): + if fetcher.chunkPathIfCached(DOC, "zoomed-past"): + break + time.sleep(0.01) + assert fetcher.chunkPathIfCached(DOC, "zoomed-past") is not None + + +def test_a_read_in_view_is_never_released(fetcher): + _occupy(fetcher) + fetcher.setView({("lvl0", "wanted"): DOC}) + result = {} + t = threading.Thread(target=lambda: result.update(p=fetcher.chunkPath(DOC, "wanted"))) + t.start() + time.sleep(0.2) + fetcher.setView({("lvl0", "wanted"): DOC, ("lvl0", "more"): DOC}) + _drain(fetcher, 2) + t.join(2) + assert result.get("p") == __file__ + + +def test_a_read_that_started_after_the_view_was_set_is_not_judged_by_it(fetcher): + """The poll lags napari: a read for the NEW view can start before + the poll has seen the new view. It must not be released on the old + one's say-so.""" + _occupy(fetcher) + fetcher.setView({("lvl0", "old-view"): DOC}) + result = {} + t = threading.Thread(target=lambda: result.update(p=fetcher.chunkPath(DOC, "new-view"))) + t.start() + time.sleep(0.3) + assert t.is_alive() + _drain(fetcher, 2) + t.join(2) + assert result.get("p") == __file__ + + +def test_the_coarsest_level_is_never_released(fetcher): + _occupy(fetcher) + fetcher.setView({}, never_stale_doc_ids=["lvl3"]) + result = {} + t = threading.Thread(target=lambda: result.update(p=fetcher.chunkPath(COARSE, "c"))) + t.start() + time.sleep(0.15) + fetcher.setView({("lvl0", "x"): DOC}) + time.sleep(0.3) + assert t.is_alive() + _drain(fetcher, 3) + t.join(2) + assert result.get("p") == __file__ + + +def test_an_unknown_view_releases_nothing(fetcher): + _occupy(fetcher) + fetcher.setView(None) + t = threading.Thread(target=lambda: fetcher.chunkPath(DOC, "q")) + t.start() + time.sleep(0.3) + assert t.is_alive() + _drain(fetcher, 2) + t.join(2) + + +def test_one_chunk_is_fetched_once_however_many_ask(fetcher): + _occupy(fetcher) + done = [] + threads = [ + threading.Thread(target=lambda: done.append(fetcher.chunkPath(DOC, "shared"))) + for _ in range(3) + ] + for t in threads: + t.start() + fetcher.prefetchAsync(DOC, "shared") + _drain(fetcher, 2) + for t in threads: + t.join(2) + assert _Session.started.count("shared") == 1 + assert done == [__file__] * 3 diff --git a/tests/test_pyramid_loader.py b/tests/test_pyramid_loader.py index cd31be55..c4a68abb 100644 --- a/tests/test_pyramid_loader.py +++ b/tests/test_pyramid_loader.py @@ -197,9 +197,16 @@ def test_registering_without_a_fallback_context_is_a_noop(self): class TestStats(unittest.TestCase): - def test_before_build_the_snapshot_is_empty(self): + def test_before_build_no_fetcher_line(self): + """No fetcher yet means no fetcher line. The fallback stats line + is a module-level counter (env-gated, not fetcher-gated), so it + may still be present -- what this pins is that ``stats()`` does + not fabricate a fetcher summary before build.""" + import os + loader = ImagePyramidLoader(_FakeSession(), _FakeDoc()) - self.assertEqual(loader.stats(), {}) + with mock.patch.dict(os.environ, {"NDI_LIGHTSHEET_UPSAMPLE_FALLBACK": "0"}, clear=False): + self.assertEqual(loader.stats(), {}) def test_after_build_the_fetcher_line_is_present(self): fetcher = _FakeFetcher() @@ -210,6 +217,15 @@ def test_after_build_the_fetcher_line_is_present(self): self.assertEqual(snap.get("fetcher"), "cache=42, cloud=10") def test_fallback_line_only_appears_when_the_env_is_on(self): + """The fallback env defaults OFF after live A/B testing on the + Maddie lightsheet showed the fallback hiding newly-arrived + fine data. The stats line tracks whichever state ``env_on()`` + reports. + + With the env unset the default is OFF, and the stats do NOT + include 'fallback'. With ``=1`` the fallback is opted in and + the stats section is present. + """ import os fetcher = _FakeFetcher() diff --git a/tests/test_pyramid_loader_integration.py b/tests/test_pyramid_loader_integration.py new file mode 100644 index 00000000..f130c5a2 --- /dev/null +++ b/tests/test_pyramid_loader_integration.py @@ -0,0 +1,269 @@ +"""End-to-end loader test against a real NDI cloud fixture. + +The permanent fixture is built by the MATLAB roundtrip +(``ndi.test.cloud.lightsheet_blob_cloud_roundtrip``) and uploaded via +the ``Build lightsheet blob fixture`` GitHub Actions workflow. Its +identifier lives in ``tests/fixtures/lightsheet_cloud_fixture.json`` +so a regeneration is one edit in one place; the workflow overwrites +that JSON when re-run. + +Skipped automatically when the cloud credentials are not set (same +gate as ``test_cloud_live.py``), so a laptop without secrets keeps +running the rest of the unit tests as usual. In CI these tests run +under the same environment matrix as ``test-cloud-api.yml``. + +What the tests exercise: + +* Downloading the fixture with ``sync_files=False`` -- the loader + path we care about is on-demand fetch through the NDI cloud API, + not local disk. +* Opening the pyramid via :class:`ImagePyramidLoader`, walking every + level, ``.compute()``-ing each so the fetcher is actually driven. +* Verifying the finest level's shape / dtype / channel count against + what the MATLAB fixture writes. +* Reading ``loader.stats()`` at the end so the profile numbers + (cache hits, cloud fetches, mean fetch time) appear in the test + log for the "home vs. office vs. CI" comparison. + +Session-scoped download: opening every test would redownload; a +module fixture pulls once and every test in the file shares the +result. +""" + +from __future__ import annotations + +import json +import os +import time +from pathlib import Path + +import pytest + +# --------------------------------------------------------------------------- +# Skip gate + fixture registry. + + +_HAS_CREDS = bool(os.environ.get("NDI_CLOUD_USERNAME") and os.environ.get("NDI_CLOUD_PASSWORD")) +pytestmark = [ + pytest.mark.cloud, + pytest.mark.skipif(not _HAS_CREDS, reason="NDI cloud credentials not set"), +] + +FIXTURE_JSON = Path(__file__).parent / "fixtures" / "lightsheet_cloud_fixture.json" + + +def _fixture_id() -> str: + """Pick the dataset id the test should point at. + + Precedence: ``NDI_LIGHTSHEET_TEST_FIXTURE_ID`` (env override, for + a one-shot pointing at a freshly rebuilt fixture) -> the + ``cloud_dataset_id`` in ``lightsheet_cloud_fixture.json``. + """ + env = os.environ.get("NDI_LIGHTSHEET_TEST_FIXTURE_ID", "").strip() + if env: + return env + with open(FIXTURE_JSON) as f: + return str(json.load(f)["cloud_dataset_id"]) + + +# --------------------------------------------------------------------------- +# Shared download. + + +@pytest.fixture(scope="module") +def downloaded_dataset(tmp_path_factory) -> Path: + """Pull the fixture once for every test in this module. + + ``sync_files=False`` mirrors what the viewer opens on a normal + launch: documents land locally, chunk files are fetched from the + cloud on demand. That is the read path we want to exercise -- + with sync_files on, every chunk arrives during download and the + loader's cloud-fetch path never runs. + """ + from ndi.cloud.exceptions import CloudNotFoundError + from ndi.cloud.orchestration import downloadDataset + + target = tmp_path_factory.mktemp("lightsheet_fixture_download") + dataset_id = _fixture_id() + try: + dataset = downloadDataset( + dataset_id, + str(target), + sync_files=False, + verbose=False, + ) + except CloudNotFoundError as exc: + # The fixture lives on one account + env (see + # lightsheet_cloud_fixture.json; currently user 1, prod). The + # cloud-CI matrix runs every account x env combination, so + # three of the four cells cannot see the fixture and would + # crash with HTTP 404. Skip cleanly there rather than fail + # -- one green cell per fixture is the intended baseline. + # Set NDI_LIGHTSHEET_TEST_FIXTURE_ID to a dataset that IS + # visible on the account you are running under to point the + # tests at a different fixture. + pytest.skip( + f"fixture {dataset_id} not accessible from this account " + f"(NDI_CLOUD_USERNAME={os.environ.get('NDI_CLOUD_USERNAME', '?')!r}, " + f"CLOUD_API_ENVIRONMENT={os.environ.get('CLOUD_API_ENVIRONMENT', '?')!r}): " + f"{exc}" + ) + # downloadDataset returns an ndi.ndi_dataset backed by + # target/. The tests want the on-disk path so + # ImagePyramidLoader can open a session on it. + dataset_path = getattr(dataset, "_path", None) or (target / dataset_id) + return Path(dataset_path) + + +@pytest.fixture(scope="module") +def pyramid_doc(downloaded_dataset): + """The first ``lightsheetZarrPyramid`` on the downloaded dataset.""" + from ndi.dataset._dataset import ndi_dataset_dir + from ndi.query import ndi_query + + dataset = ndi_dataset_dir(str(downloaded_dataset)) + q = ndi_query("").isa("lightsheetZarrPyramid") + docs = list(dataset.database_search(q)) + assert docs, f"no lightsheetZarrPyramid documents in {downloaded_dataset}" + # The dataset itself owns database_search (it follows links into + # every session), so return both -- the loader wants the session + # or dataset that resolves further searches, plus the doc. + return dataset, docs[0] + + +# --------------------------------------------------------------------------- +# Tests. + + +class TestLoaderOpensTheFixture: + def test_the_loader_enumerates_the_levels_finest_first(self, pyramid_doc): + from ndi.pyramid.loader import ImagePyramidLoader + + session, doc = pyramid_doc + loader = ImagePyramidLoader(session, doc, reduction="mean") + try: + # numLevels does an implicit build via docs; a fixture + # that lands with no levels means the MATLAB ingest ran + # halfway. + n = loader.numLevels + assert n >= 2, f"pyramid should have at least 2 levels, got {n}" + # Levels are sorted finest-first, so index 0 has the + # smallest voxel size on the last docs (they're 0-based + # ints). + docs = loader.docs + levels = [int(d.document_properties["lightsheetZarrLevel"]["level"]) for d in docs] + assert levels == sorted( + levels + ), f"levelDocs should return finest-first, got levels {levels}" + finally: + loader.close() + + def test_the_loader_returns_one_spec_per_channel(self, pyramid_doc): + from ndi.pyramid.loader import ImagePyramidLoader + + session, doc = pyramid_doc + loader = ImagePyramidLoader(session, doc, reduction="mean") + try: + specs = loader.specs + # Fixture is 2 channels (makeBlobFixture defaults). + assert len(specs) == 2, f"expected 2 layer specs, got {len(specs)}" + for spec in specs: + assert "data" in spec + # Multiscale default: spec["data"] is a list of 3D arrays + # (channel axis stripped), one per level finest-first. + # Single-level mode (NDI_LIGHTSHEET_SINGLE_LEVEL=1) would + # give a single 3D array instead; accept either. + data = spec["data"] + if isinstance(data, list): + assert data, "expected at least one level in multiscale ladder" + first = data[0] + else: + first = data + data_shape = getattr(first, "shape", ()) + assert len(data_shape) == 3, f"expected 3D per-channel data, got shape {data_shape}" + finally: + loader.close() + + +class TestLoaderReadsRealBytes: + def test_the_coarsest_level_computes_and_is_not_all_zero(self, pyramid_doc): + # A pyramid the fetcher can open but never actually pulls + # from would compute to all-zero (fill_value) blocks -- the + # classic "silent fetch failure" bug that hid the DID series + # switch. Fixture has a ring + Gaussian + spike so any + # actually-fetched slice is non-zero. + import numpy as np + + from ndi.pyramid.loader import ImagePyramidLoader + + session, doc = pyramid_doc + loader = ImagePyramidLoader(session, doc, reduction="mean") + try: + specs = loader.specs + # Multiscale default: data is a finest-first list; coarsest is + # the last entry. Single-level mode gives a single array. + data = specs[0]["data"] + coarsest = data[-1] if isinstance(data, list) else data + arr = coarsest.compute() + assert arr.dtype == np.uint16, f"expected uint16, got {arr.dtype}" + assert arr.any(), "coarsest-level channel-0 slice is entirely fill_value" + finally: + loader.close() + + def test_every_level_can_be_computed(self, pyramid_doc): + # Walks the full ladder. On the small fixture (~5 levels max, + # 300**3 voxels finest) this is small enough to finish in a + # single-digit-seconds test on a reasonable link. If it grows + # to minutes, the fetcher or the network is the problem and + # the profile numbers below will name which. + from ndi.pyramid.loader import ImagePyramidLoader + + session, doc = pyramid_doc + loader = ImagePyramidLoader(session, doc, reduction="mean") + per_level: list[dict] = [] + try: + channel0_ladder = specs_channel_ladder(loader, 0) + for i, arr in enumerate(channel0_ladder): + t0 = time.perf_counter() + out = arr.compute() + dt = time.perf_counter() - t0 + per_level.append({"level": i, "shape": tuple(out.shape), "seconds": round(dt, 2)}) + finally: + loader.close() + print(f"\n[integration] per-level compute times: {per_level}") + assert per_level, "expected at least one level to compute" + + def test_stats_show_at_least_one_cloud_fetch(self, pyramid_doc): + # If every read hits the cache instead of the cloud, the + # fixture is not exercising the code we care about. On a + # freshly-downloaded fixture with sync_files=False, every + # chunk needs a cloud round trip. + from ndi.pyramid.loader import ImagePyramidLoader + + session, doc = pyramid_doc + loader = ImagePyramidLoader(session, doc, reduction="mean") + try: + for arr in specs_channel_ladder(loader, 0): + arr.compute() + snap = loader.stats() + print(f"\n[integration] loader.stats(): {snap}") + # stats["fetcher"] is a summary line like + # "resolves: N cache-hits (mean X.Xms), M cloud-fetches + # (mean X.Xs), K failures". We just check the summary is + # non-empty; the actual counts vary by pyramid shape. + assert snap.get("fetcher"), "loader.stats() has no fetcher summary" + finally: + loader.close() + + +def specs_channel_ladder(loader, channel: int) -> list: + """Every level's array for one channel (finest first). + + ``layerSpec`` stashes per-channel per-level arrays on each spec's + ``_ndi_level_arrays`` list (that is the ladder the level selector + swaps between). Take the ladder off spec[channel] rather than + calling into internals. + """ + specs = loader.specs + assert 0 <= channel < len(specs), f"channel {channel} out of range" + return specs[channel].get("_ndi_level_arrays", []) diff --git a/tests/test_pyramid_upsample_fallback.py b/tests/test_pyramid_upsample_fallback.py index 1360b6e1..62d40313 100644 --- a/tests/test_pyramid_upsample_fallback.py +++ b/tests/test_pyramid_upsample_fallback.py @@ -195,20 +195,29 @@ def test_it_respects_row_major_with_the_expected_stride(self): class TestEnvGate(unittest.TestCase): def test_absent_env_is_off(self): + """Default re-flipped OFF after live A/B testing: with the + fallback on, napari treated the first slice as 'loaded' with + upsampled-coarse data and did not re-render when fine chunks + arrived. Black briefly then correct beats blurry forever.""" with mock.patch.dict(os.environ, {}, clear=False): os.environ.pop("NDI_LIGHTSHEET_UPSAMPLE_FALLBACK", None) self.assertFalse(uf.env_on()) def test_truthy_env_is_on(self): + """The opt-in knob: cloud users on slow links who prefer a + blurry-but-present first paint to a black one can still get it.""" for value in ("1", "true", "on", "yes", "TRUE"): with mock.patch.dict( os.environ, {"NDI_LIGHTSHEET_UPSAMPLE_FALLBACK": value}, clear=False ): self.assertTrue(uf.env_on(), value) - def test_zero_is_off(self): - with mock.patch.dict(os.environ, {"NDI_LIGHTSHEET_UPSAMPLE_FALLBACK": "0"}, clear=False): - self.assertFalse(uf.env_on()) + def test_falsy_env_is_off(self): + for value in ("0", "false", "off", "no", "FALSE"): + with mock.patch.dict( + os.environ, {"NDI_LIGHTSHEET_UPSAMPLE_FALLBACK": value}, clear=False + ): + self.assertFalse(uf.env_on(), value) if __name__ == "__main__":