ViSR sprint: server-side improvements

README.md

@@ -35,8 +35,9 @@ setpoints in the run's start document with `setpoints=true`.
 
 Unless `width`/`height` or a full `xmin`/`xmax`/`ymin`/`ymax` range are given,
 the bins follow the scan's setpoint grid from its spec (one cell per setpoint,
-fixed from the first poll), provided the spec is a grid of at least 3×3 and the
-recorded positions lie on it. Positions that are off the grid by a constant
+fixed from the first poll), provided the spec lays its points out on a grid
+(any size, down to a line or a single point) and the recorded positions lie on
+it. Positions that are off the grid by a constant
 amount per axis (as fly scans currently record them) are first shifted onto it
 by the median of each point's position minus its setpoint, and the shift is
 returned as `position_offset`. Otherwise numpy's default 10×10 over the data.

src/dls_tiled/visr.py

@@ -227,10 +227,16 @@ def _xyz_order(axes: list, motors: object) -> list[int]:
     return sorted(range(len(axes)), key=lambda i: _AXIS_LETTERS.index(letters[i]))
 
 
-def _bin_edges(centres: numpy.ndarray) -> numpy.ndarray:
-    """Edges halfway between consecutive centres, extended half a step at each end."""
+def _bin_edges(centres: numpy.ndarray, single_width: float = 1.0) -> numpy.ndarray:
+    """Edges halfway between consecutive centres, extended half a step at each end.
+
+    A single centre has no neighbour to take a step from, so its cell is
+    ``single_width`` wide.
+    """
     if len(centres) == 1:
-        return numpy.array([centres[0] - 0.5, centres[0] + 0.5])
+        return numpy.array(
+            [centres[0] - single_width / 2, centres[0] + single_width / 2]
+        )
     mids = (centres[:-1] + centres[1:]) / 2
     first = centres[0] - (centres[1] - centres[0]) / 2
     last = centres[-1] + (centres[-1] - centres[-2]) / 2
@@ -241,9 +247,6 @@ def _bin_edges(centres: numpy.ndarray) -> numpy.ndarray:
 # poll (200 x 200 is about 2.3 MB of JSON), so the caller falls back to the
 # coarse default binning.
 MAX_GRID_CELLS = int(os.getenv("DLS_TILED_VISR_MAX_GRID_CELLS", "40000"))
-# The ViSR plot needs at least three points on each axis, so a scan narrower than
-# that keeps the previous default binning.
-MIN_CELLS_PER_AXIS = 3
 
 
 class ScanGrid(NamedTuple):
@@ -295,13 +298,23 @@ def _grid_from_spec(spec_json: str, motors_json: str) -> ScanGrid | None:
     y_setpoints = numpy.asarray(by_axis[axes[letters.index("y")]], dtype=float)
     xs = numpy.unique(numpy.round(x_setpoints, 9))
     ys = numpy.unique(numpy.round(y_setpoints, 9))
-    if min(len(xs), len(ys)) < MIN_CELLS_PER_AXIS:
-        return None  # a line, or too few cells for the plot's axes
+    if not len(xs) or not len(ys):
+        return None  # a spec with no points
     if len(xs) * len(ys) > min(n_total, MAX_GRID_CELLS):
         return None  # more cells than points: not a grid
+    # An axis with one setpoint (a line scan, or a single point) has no step of
+    # its own, so its cell takes the other axis's step, or 1 if neither has one.
+    step_x = float(numpy.min(numpy.diff(xs))) if len(xs) > 1 else None
+    step_y = float(numpy.min(numpy.diff(ys))) if len(ys) > 1 else None
+    single = step_x or step_y or 1.0
     named = None not in (_axis_name(a) for a in axes)
     return ScanGrid(
-        _bin_edges(xs), _bin_edges(ys), n_total, x_setpoints, y_setpoints, named
+        _bin_edges(xs, single),
+        _bin_edges(ys, single),
+        n_total,
+        x_setpoints,
+        y_setpoints,
+        named,
     )
 
 
@@ -876,6 +889,30 @@ def current_lengths() -> dict[str, int]:
         readbacks = readbacks[:, mask]
         data = {channel: d[mask] for channel, d in data.items()}
 
+    # A point with no position on x or y (NaN: a run that only moved x has no
+    # y readings, say) can't be placed in any bin, and numpy's default binning
+    # can't take a range from it. Drop such points; if that leaves none of a
+    # run's points, there is nothing to show, so say why.
+    positioned = numpy.isfinite(readbacks[x_dim_index]) & numpy.isfinite(
+        readbacks[y_dim_index]
+    )
+    if not positioned.all():
+        if not positioned.any():
+            missing = [
+                axis
+                for axis, dim in (("x", x_dim_index), ("y", y_dim_index))
+                if not numpy.isfinite(readbacks[dim]).any()
+            ]
+            raise HTTPException(
+                status_code=HTTP_422_UNPROCESSABLE_CONTENT,
+                detail=(
+                    f"Run '{uid}' has no recorded "
+                    f"{' or '.join(missing) or 'x and y'} positions to bin by"
+                ),
+            )
+        readbacks = readbacks[:, positioned]
+        data = {channel: d[positioned] for channel, d in data.items()}
+
     assert isinstance(red_total, (H5Dataset, numpy.ndarray))
     logger.debug(
         "readbacks shape %s, red_total shape %s (sliced=%s)",

tests/test_visr.py

@@ -6,7 +6,7 @@
 import numpy
 import pytest
 from fastapi import HTTPException
-from scanspec.specs import Line, Spiral
+from scanspec.specs import Ellipse, Line, Spiral
 from tiled.adapters.utils import DataNotReadyError
 from tiled.catalog import in_memory
 from tiled.client import Context, from_context
@@ -332,10 +332,44 @@ def test_grid_is_none_when_it_has_more_cells_than_the_cap(monkeypatch):
     visr._grid_from_spec.cache_clear()
 
 
-def test_grid_is_none_for_a_scan_narrower_than_three_setpoints():
-    # a line, and a 2-wide grid: too few points on an axis for the UI's plot axes
-    assert _grid_or_none(Line("y", 0, 2, 3) * Line("x", 3, 3, 1)) is None
-    assert _grid_or_none(Line("y", 0, 2, 3) * Line("x", 0, 1, 2)) is None
+def test_grid_of_a_two_by_two_scan():
+    x_edges, y_edges, n_total, *_ = _grid(Line("y", 0, 2, 2) * ~Line("x", 0, 2, 2))
+
+    assert x_edges.tolist() == [-1.0, 1.0, 3.0]
+    assert y_edges.tolist() == [-1.0, 1.0, 3.0]
+    assert n_total == 4
+
+
+def test_grid_of_a_line_takes_its_cell_height_from_the_scan_step():
+    # one setpoint in y: no step of its own, so its cell is as tall as x's step
+    x_edges, y_edges, *_ = _grid(Line("y", 5, 5, 1) * Line("x", 0, 2, 3))
+
+    assert x_edges.tolist() == [-0.5, 0.5, 1.5, 2.5]
+    assert y_edges.tolist() == [4.5, 5.5]
+
+
+def test_grid_of_a_single_point():
+    x_edges, y_edges, n_total, *_ = _grid(Line("y", 1, 1, 1) * Line("x", 2, 2, 1))
+
+    assert x_edges.tolist() == [1.5, 2.5]
+    assert y_edges.tolist() == [0.5, 1.5]
+    assert n_total == 1
+
+
+def test_grid_is_none_for_a_spec_with_no_points():
+    # an ellipse narrower than its step holds no setpoints at all
+    spec = Ellipse(
+        x_axis="x",
+        x_centre=0.3,
+        x_diameter=0.01,
+        x_step=1,
+        y_axis="y",
+        y_centre=0.3,
+        y_diameter=0.01,
+        y_step=1,
+    )
+
+    assert _grid_or_none(spec) is None
 
 
 def write_grid_scan(client, uid, n_points):
@@ -392,6 +426,94 @@ def test_binned_without_a_spec_still_reports_no_total(client):
     assert body["n_total"] is None
 
 
+def write_small_scan(client, uid, spec, x, y, x_shift=0.0, y_shift=0.0):
+    """A run with this spec and these recorded positions, values 0, 1, 2..."""
+    write_points(
+        client,
+        uid,
+        numpy.asarray(x, dtype=float) + x_shift,
+        numpy.asarray(y, dtype=float) + y_shift,
+        metadata={"start": {"spec": spec.serialize()}},
+    )
+
+
+def test_binned_shows_a_two_by_two_scan_on_its_grid(client):
+    write_small_scan(
+        client,
+        "two",
+        Line("y", 0, 2, 2) * ~Line("x", 0, 2, 2),
+        [0, 2, 2, 0],
+        [0, 0, 2, 2],
+    )
+
+    body = client.context.http_client.get("/api/v1/binned/two").json()
+
+    # snaked: the second row runs back, so its first point is on the right
+    assert body["RedTotal"] == [[0.0, 1.0], [3.0, 2.0]]
+    assert body["x_limits"] == [-1.0, 1.0, 3.0]
+
+
+def test_binned_shows_a_single_point_as_one_cell(client):
+    write_small_scan(client, "one", Line("y", 1, 1, 1) * Line("x", 2, 2, 1), [2], [1])
+
+    body = client.context.http_client.get("/api/v1/binned/one").json()
+
+    assert body["RedTotal"] == [[0.0]]
+    assert (body["x_limits"], body["y_limits"]) == ([1.5, 2.5], [0.5, 1.5])
+
+
+def test_binned_shows_a_line_scan_as_one_row(client):
+    write_small_scan(
+        client, "line", Line("y", 5, 5, 1) * Line("x", 0, 2, 3), [0, 1, 2], [5, 5, 5]
+    )
+
+    body = client.context.http_client.get("/api/v1/binned/line").json()
+
+    assert body["RedTotal"] == [[0.0, 1.0, 2.0]]
+    assert body["y_limits"] == [4.5, 5.5]
+
+
+def test_binned_keeps_the_default_binning_for_an_offset_line_scan(client):
+    # a single row never wraps, so an offset can't be told from a frame gained
+    # or lost at the start: no offset is taken, and the default binning stays
+    write_small_scan(
+        client,
+        "flyline",
+        Line("y", 5, 5, 1) * Line("x", 0, 2, 3),
+        [0, 1, 2],
+        [5, 5, 5],
+        x_shift=1.0,
+        y_shift=10.4,
+    )
+
+    body = client.context.http_client.get("/api/v1/binned/flyline").json()
+
+    assert body["position_offset"] is None
+    assert len(body["RedTotal"]) == 10
+
+
+def test_binned_leaves_out_points_without_a_position(client):
+    # no spec, so the default binning, whose range a NaN used to break
+    write_points(client, "gap", [0.0, 1.0, numpy.nan, 1.0], [0.0, 0.0, 1.0, 1.0])
+
+    body = client.context.http_client.get(
+        "/api/v1/binned/gap", params={"width": 2, "height": 2}
+    ).json()
+
+    assert body["RedTotal"] == [[0.0, 1.0], [None, 3.0]]
+    assert body["n_points"] == 3
+
+
+def test_binned_answers_422_for_a_run_with_no_y_positions(client):
+    # an x-only run: y is filled with NaN, which used to fail with a 500
+    write_points(client, "xonly", [0.0, 1.1, 2.2], [numpy.nan] * 3)
+
+    response = client.context.http_client.get("/api/v1/binned/xonly")
+
+    assert response.status_code == 422
+    assert "no recorded y positions" in response.json()["detail"]
+
+
 # --- sharing work between polls ------------------------------------------------------