diff --git a/pyproject.toml b/pyproject.toml index 99ee6d9..2d19d6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ gui = [ "tiled[client]", "bluesky-tiled-plugins", "matplotlib", + "pyqtgraph", "qtpy", "pyside6", "pexpect", diff --git a/src/hextools/gui/run_viewer.py b/src/hextools/gui/run_viewer.py new file mode 100644 index 0000000..7a698ce --- /dev/null +++ b/src/hextools/gui/run_viewer.py @@ -0,0 +1,662 @@ +"""Review one Bluesky run's data, read from Tiled, in its own window. + +The feature set follows DatView (https://github.com/algotom/datview): zoom, line +profiles, contrast, histogram, percentile density, image statistics and slicing +through a stack, plus 1D plots of scalar data and the run's metadata. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Any + +import numpy as np +import pyqtgraph as pg +from qtpy.QtCore import Qt +from qtpy.QtWidgets import ( + QCheckBox, + QComboBox, + QHBoxLayout, + QHeaderView, + QLabel, + QPushButton, + QSlider, + QSpinBox, + QSplitter, + QStackedWidget, + QTableWidget, + QTableWidgetItem, + QTabWidget, + QTreeWidget, + QTreeWidgetItem, + QVBoxLayout, + QWidget, +) + +from hextools.gui._threading import create_worker + +#: Bookkeeping columns of an event stream that are not data. +_SKIPPED_KEYS = {"seq_num"} +_INDEX = "Index" + + +@dataclass(frozen=True) +class DatasetInfo: + """One dataset of a run's event stream, as listed in the viewer.""" + + stream: str + key: str + #: Full shape for arrays; ``None`` for table columns (one value per event). + shape: tuple[int, ...] | None + dtype: str + + @property + def is_image(self) -> bool: + """Whether the dataset has at least two axes longer than one.""" + return self.shape is not None and len(_long_axes(self.shape)) >= 2 + + +def fill_metadata_tree(parent: QTreeWidgetItem, value: Any) -> None: + """Add ``value`` (nested dicts and lists) under ``parent``, one row per key.""" + items = value.items() if isinstance(value, dict) else enumerate(value) + for key, child in items: + item = QTreeWidgetItem(parent, [str(key)]) + if isinstance(child, (dict, list)) and child: + fill_metadata_tree(item, child) + else: + item.setText( + 1, child if isinstance(child, str) else json.dumps(child, default=str) + ) + + +def run_streams(run: Any) -> dict[str, Any]: + """Return the run's event streams by name, for either Tiled run layout.""" + container = run + # Older runs keep their streams under a "streams" namespace. + if "streams" in run and "BlueskyEventStream" not in { + s.name for s in run["streams"].specs + }: + container = run["streams"] + return {name: container[name] for name in container} + + +def list_datasets(stream_name: str, stream: Any) -> list[DatasetInfo]: + """List a stream's datasets in one request; ``stream[key]`` re-lists each time.""" + data_keys = stream.metadata.get("data_keys", {}) + datasets = [] + for key, item in stream.get_contents().items(): + attributes = item["attributes"] + structure = attributes["structure"] + if attributes["structure_family"] == "table": + for column in structure["columns"]: + if column in _SKIPPED_KEYS or column.startswith("ts_"): + continue + dtype = data_keys.get(column, {}).get("dtype", "") + datasets.append(DatasetInfo(stream_name, column, None, dtype)) + else: + data_type = structure.get("data_type") or {} + dtype = f"{data_type.get('kind', '')}{data_type.get('itemsize', '')}" + datasets.append( + DatasetInfo(stream_name, key, tuple(structure["shape"]), dtype) + ) + return datasets + + +def load_run(client: Any, uid: str) -> tuple[Any, dict[str, Any], list[DatasetInfo]]: + """Fetch run ``uid`` from ``client`` with its streams and their datasets.""" + run = client[uid] + streams = run_streams(run) + datasets = [ + info for name, stream in streams.items() for info in list_datasets(name, stream) + ] + return run, streams, datasets + + +def _long_axes(shape: tuple[int, ...]) -> list[int]: + return [axis for axis, length in enumerate(shape) if length > 1] + + +def slice_axes(shape: tuple[int, ...]) -> list[int]: + """Axes a stack can be sliced along: the last three longer than one, if three.""" + long_axes = _long_axes(shape) + return long_axes[-3:] if len(long_axes) >= 3 else [] + + +def frame_key( + shape: tuple[int, ...], slice_axis: int | None = None, index: int = 0 +) -> tuple[int | slice, ...]: + """Index that reads one 2D image out of an array of ``shape``. + + The image is ``index`` along ``slice_axis`` (one of :func:`slice_axes`), or the + whole array when only two axes are longer than one. Any other axis is read at 0. + """ + long_axes = _long_axes(shape) + if len(long_axes) < 2: + raise ValueError(f"Shape {shape} has fewer than two axes longer than one") + if slice_axis is None: + shown = long_axes[-2:] + else: + if slice_axis not in slice_axes(shape): + raise ValueError(f"Cannot slice shape {shape} along axis {slice_axis}") + shown = [axis for axis in long_axes[-3:] if axis != slice_axis] + key: list[int | slice] = [] + for axis in range(len(shape)): + if axis == slice_axis: + key.append(index) + elif axis in shown: + key.append(slice(None)) + else: + key.append(0) + return tuple(key) + + +def _finite(data: np.ndarray) -> np.ndarray: + values = np.asarray(data, dtype=float).ravel() + return values[np.isfinite(values)] + + +def image_statistics(data: np.ndarray) -> dict[str, float]: + """Minimum, maximum, mean, median, standard deviation and outer percentiles.""" + values = _finite(data) + if values.size == 0: + return {} + p1, p5, p95, p99 = np.percentile(values, [1, 5, 95, 99]) + return { + "Minimum": float(values.min()), + "Maximum": float(values.max()), + "Mean": float(values.mean()), + "Median": float(np.median(values)), + "Std. deviation": float(values.std()), + "1st percentile": float(p1), + "5th percentile": float(p5), + "95th percentile": float(p95), + "99th percentile": float(p99), + } + + +def percentile_density( + data: np.ndarray, bins: int = 100 +) -> tuple[np.ndarray, np.ndarray]: + """Density of values in each percentile bin, normalised to sum to one. + + Returns the bin-centre percentiles and densities, dropping bins of zero width. + """ + values = _finite(data) + if values.size == 0: + return np.empty(0), np.empty(0) + percentiles = np.linspace(0, 100, bins + 1) + edges = np.percentile(values, percentiles) + counts, _ = np.histogram(values, bins=edges) + widths = np.diff(edges) + keep = widths > 0 + density = counts[keep] / widths[keep] + if density.sum() > 0: + density = density / density.sum() + centres = percentiles[:-1] + 50 / bins + return centres[keep], density + + +def auto_levels( + data: np.ndarray, low: float = 1, high: float = 99 +) -> tuple[float, float]: + """Display levels that clip the lowest and highest percent of values.""" + values = _finite(data) + if values.size == 0: + return 0.0, 1.0 + lo, hi = (float(v) for v in np.percentile(values, [low, high])) + return (lo, hi) if hi > lo else (lo, lo + 1.0) + + +def _numeric(values: np.ndarray) -> np.ndarray | None: + """``values`` as floats, or ``None`` if they are not numbers (e.g. "Open").""" + try: + return np.asarray(values, dtype=float) + except (TypeError, ValueError): + return None + + +# Worker functions return what they were asked for alongside the result, so a +# result that arrives after the user moved on can be recognised and dropped. + + +def _fetch_array(stream: Any, info: DatasetInfo) -> tuple[DatasetInfo, Any]: + return info, stream[info.key] + + +def _read_frame( + array: Any, key: tuple[int | slice, ...] +) -> tuple[Any, tuple[int | slice, ...], np.ndarray]: + return array, key, np.asarray(array[key]) + + +def _read_columns( + stream: Any, stream_name: str, keys: list[str] +) -> dict[tuple[str, str], np.ndarray]: + return {(stream_name, key): np.asarray(stream[key].read()).ravel() for key in keys} + + +class QtRunViewer(QWidget): + """A window reviewing one run read from Tiled. + + Datasets with two or more long axes are shown as images, sliced along a chosen + axis when there are three; the rest are plotted against the event index or + another dataset of the same stream. + """ + + def __init__(self, client: Any, uid: str, parent: QWidget | None = None): + super().__init__(parent) + self.setWindowFlag(Qt.WindowType.Window) + self.setAttribute(Qt.WidgetAttribute.WA_DeleteOnClose) + self.setWindowTitle(f"Run {uid}") + self.resize(1300, 850) + self._uid = uid + self._streams: dict[str, Any] = {} + self._datasets: list[DatasetInfo] = [] + self._dataset: DatasetInfo | None = None + self._array: Any = None + self._image: np.ndarray | None = None + self._columns: dict[tuple[str, str], np.ndarray] = {} + self._frame_worker: Any = None + self._frame_pending = False + + self._status = QLabel("Loading run…") + + self._dataset_tree = QTreeWidget() + self._dataset_tree.setHeaderLabels(["Dataset", "Shape", "Type"]) + self._dataset_tree.header().setSectionResizeMode( + 0, QHeaderView.ResizeMode.ResizeToContents + ) + self._dataset_tree.currentItemChanged.connect(self._on_dataset_selected) + self._metadata_tree = QTreeWidget() + self._metadata_tree.setHeaderLabels(["Key", "Value"]) + self._metadata_tree.header().setSectionResizeMode( + 0, QHeaderView.ResizeMode.ResizeToContents + ) + side_tabs = QTabWidget() + side_tabs.addTab(self._dataset_tree, "Datasets") + side_tabs.addTab(self._metadata_tree, "Metadata") + + self._views = QStackedWidget() + self._views.addWidget(self._build_image_page()) + self._views.addWidget(self._build_plot_page()) + + self._analysis = QTabWidget() + self._profile_plot = pg.PlotWidget() + self._profile_curve = self._profile_plot.plot(pen="y") + self._histogram_plot = pg.PlotWidget() + self._histogram_plot.setLabel("bottom", "Value") + self._histogram_plot.setLabel("left", "Count") + self._histogram_curve = self._histogram_plot.plot( + stepMode="center", fillLevel=0, brush=(80, 140, 220, 120) + ) + self._density_plot = pg.PlotWidget() + self._density_plot.setLabel("bottom", "Percentile") + self._density_plot.setLabel("left", "Density") + self._density_curve = self._density_plot.plot(pen="c") + self._stats_table = QTableWidget(0, 2) + self._stats_table.setHorizontalHeaderLabels(["Statistic", "Value"]) + self._stats_table.verticalHeader().setVisible(False) + self._stats_table.horizontalHeader().setStretchLastSection(True) + self._analysis.addTab(self._profile_plot, "Line profile") + self._analysis.addTab(self._histogram_plot, "Histogram") + self._analysis.addTab(self._density_plot, "Percentile density") + self._analysis.addTab(self._stats_table, "Statistics") + + centre = QSplitter(Qt.Orientation.Vertical) + centre.addWidget(self._views) + centre.addWidget(self._analysis) + centre.setStretchFactor(0, 3) + centre.setStretchFactor(1, 1) + + splitter = QSplitter(Qt.Orientation.Horizontal) + splitter.addWidget(side_tabs) + splitter.addWidget(centre) + splitter.setStretchFactor(0, 1) + splitter.setStretchFactor(1, 3) + splitter.setSizes([320, 980]) + + layout = QVBoxLayout(self) + layout.addWidget(splitter, stretch=1) + layout.addWidget(self._status) + + self._start(load_run, client, uid, returned=self._on_run_loaded) + + def _build_image_page(self) -> QWidget: + self._slice_axis = QComboBox() + self._slice_axis.setSizeAdjustPolicy( + QComboBox.SizeAdjustPolicy.AdjustToContents + ) + self._slice_axis.currentIndexChanged.connect(self._on_slice_axis_changed) + self._slider = QSlider(Qt.Orientation.Horizontal) + self._index = QSpinBox() + self._slider.valueChanged.connect(self._index.setValue) + self._index.valueChanged.connect(self._slider.setValue) + self._index.valueChanged.connect(self._request_frame) + self._profile_direction = QComboBox() + self._profile_direction.addItems(["Horizontal", "Vertical"]) + self._profile_direction.currentIndexChanged.connect(self._on_profile_direction) + self._zoom_box = QCheckBox("Zoom box") + self._zoom_box.toggled.connect(self._on_zoom_box) + reset_zoom = QPushButton("Reset zoom") + reset_zoom.clicked.connect(lambda: self._image_view.autoRange()) + auto_contrast = QPushButton("Auto contrast") + auto_contrast.clicked.connect(self._auto_contrast) + + controls = QHBoxLayout() + controls.addWidget(QLabel("Slice axis:")) + controls.addWidget(self._slice_axis) + controls.addWidget(self._slider, stretch=1) + controls.addWidget(self._index) + controls.addWidget(QLabel("Profile:")) + controls.addWidget(self._profile_direction) + controls.addWidget(self._zoom_box) + controls.addWidget(reset_zoom) + controls.addWidget(auto_contrast) + + plot = pg.PlotWidget() + self._image_view = plot.getViewBox() + self._image_view.setAspectLocked(True) + self._image_view.invertY(True) + self._image_item = pg.ImageItem(axisOrder="row-major") + plot.addItem(self._image_item) + self._profile_line = pg.InfiniteLine(angle=0, movable=True, pen="y") + self._profile_line.sigPositionChanged.connect(self._update_profile) + plot.addItem(self._profile_line) + self._contrast = pg.HistogramLUTWidget(image=self._image_item) + + image_row = QHBoxLayout() + image_row.addWidget(plot, stretch=1) + image_row.addWidget(self._contrast) + + page = QWidget() + layout = QVBoxLayout(page) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(controls) + layout.addLayout(image_row, stretch=1) + return page + + def _build_plot_page(self) -> QWidget: + self._x_key = QComboBox() + self._x_key.currentIndexChanged.connect(self._on_x_key_changed) + controls = QHBoxLayout() + controls.addWidget(QLabel("X axis:")) + controls.addWidget(self._x_key) + controls.addStretch() + self._line_plot = pg.PlotWidget() + self._line_plot.showGrid(x=True, y=True, alpha=0.3) + self._line_curve = self._line_plot.plot(pen="y", symbol="o", symbolSize=4) + + page = QWidget() + layout = QVBoxLayout(page) + layout.setContentsMargins(0, 0, 0, 0) + layout.addLayout(controls) + layout.addWidget(self._line_plot, stretch=1) + return page + + def _start( + self, + func: Callable, + *args: Any, + returned: Callable[[Any], None], + finished: Callable[[], None] | None = None, + ) -> Any: + """Run ``func`` off the GUI thread; ``returned`` gets its result. + + Pass bound methods: a lambda would run in the worker's thread. + """ + connect: dict[str, Callable | Sequence[Callable]] = { + "returned": returned, + "errored": self._on_error, + } + if finished is not None: + connect["finished"] = finished + return create_worker(func, *args, _connect=connect) + + def _on_error(self, error: Exception) -> None: + self._status.setText(f"Error: {type(error).__name__}: {error}") + + def _on_run_loaded( + self, result: tuple[Any, dict[str, Any], list[DatasetInfo]] + ) -> None: + run, self._streams, self._datasets = result + metadata = dict(run.metadata) + start = metadata.get("start", {}) + self.setWindowTitle( + f"Scan {start.get('scan_id', '?')} — {start.get('plan_name', '')}" + f" — {self._uid}" + ) + fill_metadata_tree(self._metadata_tree.invisibleRootItem(), metadata) + stream_items: dict[str, QTreeWidgetItem] = {} + for info in self._datasets: + parent = stream_items.get(info.stream) + if parent is None: + parent = QTreeWidgetItem(self._dataset_tree, [info.stream]) + parent.setExpanded(True) + stream_items[info.stream] = parent + shape = "per event" if info.shape is None else str(info.shape) + item = QTreeWidgetItem(parent, [info.key, shape, info.dtype]) + item.setData(0, Qt.ItemDataRole.UserRole, info) + self._status.setText( + f"{len(self._datasets)} dataset(s) in {len(self._streams)} stream(s). " + "Click one to view it." + ) + + def _on_dataset_selected( + self, item: QTreeWidgetItem | None, _previous: QTreeWidgetItem | None = None + ) -> None: + info = None if item is None else item.data(0, Qt.ItemDataRole.UserRole) + if not isinstance(info, DatasetInfo): + return + self._dataset = info + self._array = None + self._status.setText(f"Loading {info.stream}/{info.key}…") + if info.is_image: + stream = self._streams[info.stream] + self._start(_fetch_array, stream, info, returned=self._on_array_ready) + else: + self._show_plot_page(info) + + # Images + + def _on_array_ready(self, result: tuple[DatasetInfo, Any]) -> None: + info, array = result + if info != self._dataset or info.shape is None: + return + self._array = array + self._image = None + self._views.setCurrentIndex(0) + axes = slice_axes(info.shape) + self._slice_axis.blockSignals(True) + self._slice_axis.clear() + for axis in axes: + self._slice_axis.addItem(f"{axis} (length {info.shape[axis]})", axis) + self._slice_axis.blockSignals(False) + self._slice_axis.setEnabled(bool(axes)) + self._on_slice_axis_changed() + + def _on_slice_axis_changed(self, _index: int = 0) -> None: + axis = self._slice_axis.currentData() + shape = None if self._dataset is None else self._dataset.shape + length = 1 if axis is None or shape is None else shape[axis] + self._image = None + for widget in (self._slider, self._index): + widget.blockSignals(True) + widget.setRange(0, length - 1) + widget.setValue(0) + widget.blockSignals(False) + widget.setEnabled(length > 1) + self._request_frame() + + def _current_frame_key(self) -> tuple[int | slice, ...] | None: + if self._array is None or self._dataset is None or self._dataset.shape is None: + return None + return frame_key( + self._dataset.shape, self._slice_axis.currentData(), self._index.value() + ) + + def _request_frame(self, _value: int = 0) -> None: + """Read the selected image; while a read runs, only the latest request waits.""" + key = self._current_frame_key() + if key is None: + return + if self._frame_worker is not None: + self._frame_pending = True + return + self._frame_worker = self._start( + _read_frame, + self._array, + key, + returned=self._show_image, + finished=self._on_frame_finished, + ) + + def _on_frame_finished(self) -> None: + self._frame_worker = None + if self._frame_pending: + self._frame_pending = False + self._request_frame() + + def _show_image( + self, result: tuple[Any, tuple[int | slice, ...], np.ndarray] + ) -> None: + array, key, image = result + info = self._dataset + # Matching the array alone let a frame the slider had moved past through. + if array is not self._array or key != self._current_frame_key() or info is None: + return + first = self._image is None + self._image = image + self._image_item.setImage(image, autoLevels=False) + if first: + self._image_item.setLevels(auto_levels(image)) + self._image_view.autoRange() + vertical = self._profile_direction.currentIndex() == 1 + self._profile_line.setValue(image.shape[1 if vertical else 0] // 2) + self._update_profile() + self._update_analysis(image) + self._status.setText( + f"{info.stream}/{info.key}: image {image.shape}, {image.dtype}" + ) + + def _auto_contrast(self) -> None: + if self._image is not None: + self._image_item.setLevels(auto_levels(self._image)) + + def _on_zoom_box(self, checked: bool) -> None: + mode = pg.ViewBox.RectMode if checked else pg.ViewBox.PanMode + self._image_view.setMouseMode(mode) + + def _on_profile_direction(self, index: int) -> None: + vertical = index == 1 + self._profile_line.setAngle(90 if vertical else 0) + if self._image is not None: + self._profile_line.setValue(self._image.shape[1 if vertical else 0] // 2) + self._update_profile() + + def _update_profile(self) -> None: + if self._image is None: + return + vertical = self._profile_direction.currentIndex() == 1 + limit = self._image.shape[1 if vertical else 0] - 1 + position = int(np.clip(round(self._profile_line.value()), 0, limit)) + profile = self._image[:, position] if vertical else self._image[position, :] + self._profile_curve.setData(np.asarray(profile, dtype=float)) + self._profile_plot.setTitle(f"{'Column' if vertical else 'Row'} {position}") + + def _update_analysis(self, data: np.ndarray) -> None: + values = _finite(data) + if values.size: + counts, edges = np.histogram(values, bins=256) + self._histogram_curve.setData(edges, counts) + else: + self._histogram_curve.setData([], []) + self._density_curve.setData(*percentile_density(data)) + stats = {"Shape": str(data.shape), "Type": str(data.dtype)} + stats.update( + {name: f"{value:.6g}" for name, value in image_statistics(data).items()} + ) + self._stats_table.setRowCount(len(stats)) + for row, (name, value) in enumerate(stats.items()): + self._stats_table.setItem(row, 0, QTableWidgetItem(name)) + self._stats_table.setItem(row, 1, QTableWidgetItem(value)) + + def _clear_analysis(self) -> None: + self._histogram_curve.setData([], []) + self._density_curve.setData([], []) + self._stats_table.setRowCount(0) + + # 1D data + + def _show_plot_page(self, info: DatasetInfo) -> None: + self._views.setCurrentIndex(1) + self._analysis.setCurrentIndex(3) + self._x_key.blockSignals(True) + self._x_key.clear() + self._x_key.addItem(_INDEX) + for other in self._datasets: + if other.stream == info.stream and other != info and not other.is_image: + self._x_key.addItem(other.key) + self._x_key.blockSignals(False) + self._line_plot.setLabel("left", info.key) + self._plot_1d() + + def _on_x_key_changed(self, _index: int = 0) -> None: + self._plot_1d() + + def _plot_1d(self) -> None: + info = self._dataset + if info is None or info.is_image: + return + keys = [info.key] + x_key = self._x_key.currentText() + if x_key and x_key != _INDEX: + keys.append(x_key) + missing = [key for key in keys if (info.stream, key) not in self._columns] + if missing: + stream = self._streams[info.stream] + self._start( + _read_columns, + stream, + info.stream, + missing, + returned=self._on_columns_read, + ) + return + y = _numeric(self._columns[(info.stream, info.key)]) + if y is None: + self._line_curve.clear() + self._clear_analysis() + self._status.setText( + f"{info.stream}/{info.key} holds values that are not numbers; " + "nothing to plot." + ) + return + x = None + if x_key != _INDEX: + x = _numeric(self._columns[(info.stream, x_key)]) + problem = None + if x is None: + problem = "holds values that are not numbers" + elif x.shape != y.shape: + problem = f"has {x.size} values and {info.key} has {y.size}" + x = None + if problem: + self._status.setText(f"{x_key} {problem}; plotting against index.") + self._line_plot.setLabel("bottom", _INDEX if x is None else x_key) + if x is not None and x_key == "time" and x.size: + x = x - x[0] + self._line_plot.setLabel("bottom", "Time since first event", units="s") + if x is None: + self._line_curve.setData(y) + else: + self._line_curve.setData(x, y) + self._line_plot.autoRange() + self._update_analysis(y) + if x is not None or x_key == _INDEX: + self._status.setText(f"{info.stream}/{info.key}: {y.size} value(s)") + + def _on_columns_read(self, columns: dict[tuple[str, str], np.ndarray]) -> None: + self._columns.update(columns) + self._plot_1d() diff --git a/src/hextools/gui/scan_browser.py b/src/hextools/gui/scan_browser.py index 46e58a7..dba1ed6 100644 --- a/src/hextools/gui/scan_browser.py +++ b/src/hextools/gui/scan_browser.py @@ -2,7 +2,6 @@ from __future__ import annotations -import json from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, time, timedelta @@ -17,6 +16,7 @@ QHBoxLayout, QHeaderView, QLabel, + QLineEdit, QMenu, QMessageBox, QPushButton, @@ -24,7 +24,6 @@ QTableWidget, QTableWidgetItem, QTreeWidget, - QTreeWidgetItem, QVBoxLayout, QWidget, ) @@ -32,6 +31,7 @@ from tiled.structures.core import Spec from hextools.gui._threading import create_worker +from hextools.gui.run_viewer import QtRunViewer, fill_metadata_tree #: Called with the uid of the run it was invoked on. ScanCallback = Callable[[str], Any] @@ -49,19 +49,33 @@ class RunSummary: def search_runs( - client: Any, spec: Spec, since: float, until: float, data_session: str | None + client: Any, + spec: Spec, + since: float, + until: float, + data_session: str | None, + detector: str | None = None, ) -> list[RunSummary]: - """Find runs with ``spec`` started in ``[since, until)``, newest first.""" + """Find runs with ``spec`` started in ``[since, until)``, newest first. + + ``detector`` keeps only runs with a detector whose name contains it, ignoring + case, so "kinetix" finds runs with any of the Kinetix detectors. + """ results = client.search(SpecQuery(spec.name)) results = results.search(Key("start.time") >= since).search( Key("start.time") < until ) if data_session: results = results.search(Key("start.data_session") == data_session) + wanted = detector.strip().lower() if detector else "" runs = [] for uid, run in results.items(): metadata = dict(run.metadata) start = metadata.get("start", {}) + if wanted and not any( + wanted in str(name).lower() for name in start.get("detectors") or [] + ): + continue runs.append( RunSummary( uid=uid, @@ -80,7 +94,8 @@ class QtScanBrowser(QWidget): Selecting a run shows its metadata, plus a button for each callback attached (with :meth:`add_callback`) to one of the run's specs. The same callbacks are - offered in the run's right-click menu. + offered in the run's right-click menu. Any run opens in a :class:`QtRunViewer` + from its button or menu, or by double-clicking it. """ _COLUMNS = ["Time", "Scan ID", "UID"] @@ -105,6 +120,12 @@ def __init__( for date_edit in (self._since, self._until): date_edit.setCalendarPopup(True) date_edit.setDisplayFormat("yyyy-MM-dd") + self._detector = QLineEdit() + self._detector.setPlaceholderText("any") + self._detector.setToolTip( + "Only runs with a detector whose name contains this, e.g. kinetix" + ) + self._detector.returnPressed.connect(self.search) self._search_button = QPushButton("Search") self._search_button.clicked.connect(self.search) self._status = QLabel() @@ -116,6 +137,8 @@ def __init__( controls.addWidget(self._since) controls.addWidget(QLabel("To:")) controls.addWidget(self._until) + controls.addWidget(QLabel("Detector:")) + controls.addWidget(self._detector) controls.addWidget(self._search_button) controls.addWidget(self._status, stretch=1) @@ -130,6 +153,7 @@ def __init__( header.setSectionResizeMode(1, QHeaderView.ResizeMode.ResizeToContents) header.setStretchLastSection(True) self._table.itemSelectionChanged.connect(self._on_selection_changed) + self._table.cellDoubleClicked.connect(self._on_double_click) self._table.setContextMenuPolicy(Qt.ContextMenuPolicy.CustomContextMenu) self._table.customContextMenuRequested.connect(self._on_context_menu) @@ -183,6 +207,7 @@ def search(self) -> None: since, until, data_session, + self._detector.text(), _connect={ "returned": self._on_results, "errored": self._on_search_error, @@ -230,14 +255,15 @@ def _on_selection_changed(self) -> None: run = self._selected_run() if run is None: return - for label, callback in self._callbacks_for(run): + actions = [("Open in viewer", self.open_viewer), *self._callbacks_for(run)] + for label, callback in actions: button = QPushButton(label) button.clicked.connect( lambda _=False, cb=callback, uid=run.uid: self._invoke(cb, uid) ) self._callback_bar.addWidget(button) self._callback_bar.addStretch() - _fill_tree(self._metadata.invisibleRootItem(), run.metadata) + fill_metadata_tree(self._metadata.invisibleRootItem(), run.metadata) for i in range(self._metadata.topLevelItemCount()): self._metadata.topLevelItem(i).setExpanded(True) @@ -248,29 +274,25 @@ def _on_context_menu(self, pos: QPoint) -> None: self._table.selectRow(row) run = self._runs[row] menu = QMenu(self) - callbacks = self._callbacks_for(run) - for label, callback in callbacks: + actions = [("Open in viewer", self.open_viewer), *self._callbacks_for(run)] + for label, callback in actions: menu.addAction( label, lambda cb=callback, uid=run.uid: self._invoke(cb, uid) ) - if not callbacks: - menu.addAction("No actions for this run").setEnabled(False) menu.exec(self._table.viewport().mapToGlobal(pos)) + def _on_double_click(self, row: int, _column: int) -> None: + self._invoke(self.open_viewer, self._runs[row].uid) + + def open_viewer(self, uid: str) -> QtRunViewer: + """Open run ``uid`` in a new viewer window, reading it from the catalog.""" + # Parented so Qt keeps it alive; it is still its own window. + viewer = QtRunViewer(self._client, uid, parent=self) + viewer.show() + return viewer + def _invoke(self, callback: ScanCallback, uid: str) -> None: try: callback(uid) except Exception as ex: QMessageBox.critical(self, "Action failed", f"{type(ex).__name__}: {ex}") - - -def _fill_tree(parent: QTreeWidgetItem, value: Any) -> None: - items = value.items() if isinstance(value, dict) else enumerate(value) - for key, child in items: - item = QTreeWidgetItem(parent, [str(key)]) - if isinstance(child, (dict, list)) and child: - _fill_tree(item, child) - else: - item.setText( - 1, child if isinstance(child, str) else json.dumps(child, default=str) - ) diff --git a/tests/gui/test_run_viewer.py b/tests/gui/test_run_viewer.py new file mode 100644 index 0000000..230ffd5 --- /dev/null +++ b/tests/gui/test_run_viewer.py @@ -0,0 +1,371 @@ +import os +import threading + +import numpy as np +import pytest + +os.environ.setdefault("QT_QPA_PLATFORM", "offscreen") +pytest.importorskip("qtpy") +pytest.importorskip("pyqtgraph") + +from bluesky import RunEngine # noqa: E402 +from qtpy.QtCore import QElapsedTimer # noqa: E402 +from qtpy.QtWidgets import QApplication, QTreeWidgetItem # noqa: E402 +from tiled.structures.core import Spec # noqa: E402 + +from hextools.gui.run_viewer import ( # noqa: E402 + DatasetInfo, + QtRunViewer, + auto_levels, + frame_key, + image_statistics, + list_datasets, + percentile_density, + run_streams, + slice_axes, +) +from hextools.gui.scan_browser import QtScanBrowser, search_runs # noqa: E402 + + +class FakeArray: + """Tiled array client: indexing reads a slice, ``read()`` reads it all.""" + + def __init__(self, data): + self.data = data + self.keys_read = [] + + def __getitem__(self, key): + self.keys_read.append(key) + return self.data[key] + + def read(self): + return self.data + + +class FakeSpec: + def __init__(self, name): + self.name = name + + +class FakeStream: + """An event stream: one table of per-event columns plus external arrays.""" + + specs = (FakeSpec("BlueskyEventStream"),) + + def __init__(self, columns, arrays): + self.metadata = {"data_keys": {key: {"dtype": "number"} for key in columns}} + self._columns = { + "seq_num": np.arange(1, 4), + "ts_angle": np.zeros(3), + **columns, + } + self.arrays = {key: FakeArray(value) for key, value in arrays.items()} + + def get_contents(self): + contents = { + "internal": { + "attributes": { + "structure_family": "table", + "structure": {"columns": list(self._columns)}, + } + } + } + for key, array in self.arrays.items(): + contents[key] = { + "attributes": { + "structure_family": "array", + "structure": { + "shape": list(array.data.shape), + "data_type": { + "kind": array.data.dtype.kind, + "itemsize": array.data.dtype.itemsize, + }, + }, + } + } + return contents + + def __getitem__(self, key): + if key in self.arrays: + return self.arrays[key] + return FakeArray(self._columns[key]) + + +class FakeRun(dict): + def __init__(self, streams, start, nested=False): + super().__init__({"streams": FakeNamespace(streams)} if nested else streams) + self.metadata = {"start": start, "stop": {"exit_status": "success"}} + self.specs = [FakeSpec("BlueskyRun")] + + +class FakeNamespace(dict): + specs = () + + +class FakeResults: + def __init__(self, runs): + self._runs = runs + + def search(self, _query): + return self + + def items(self): + return list(self._runs.items()) + + +class FakeClient: + def __init__(self, runs): + self._runs = runs + + def __getitem__(self, uid): + return self._runs[uid] + + def search(self, _query): + return FakeResults(self._runs) + + +# Projections are (events, frames per event, rows, columns) as Kinetix writes them. +PROJECTIONS = np.arange(3 * 1 * 4 * 5, dtype="uint16").reshape(3, 1, 4, 5) +ANGLES = np.array([0.0, 90.0, 180.0]) + + +def _tomo_run(nested=False, detectors=("kinetix-det1",)): + stream = FakeStream( + columns={"time": np.array([10.0, 11.0, 12.0]), "angle": ANGLES}, + arrays={"kinetix-det1_image": PROJECTIONS}, + ) + start = {"uid": "abc", "scan_id": 7, "time": 100.0, "detectors": list(detectors)} + return FakeRun({"primary": stream}, start, nested=nested) + + +def _wait_for(condition, timeout_ms=5000): + app = QApplication.instance() or QApplication([]) + timer = QElapsedTimer() + timer.start() + while not condition(): + app.processEvents() + assert timer.elapsed() < timeout_ms, "timed out waiting for the viewer" + + +def _select(viewer, key): + root = viewer._dataset_tree.invisibleRootItem() + for i in range(root.childCount()): + stream_item = root.child(i) + for j in range(stream_item.childCount()): + item: QTreeWidgetItem = stream_item.child(j) + if item.text(0) == key: + viewer._dataset_tree.setCurrentItem(item) + return + raise AssertionError(f"{key} not listed") + + +def test_frame_key_reads_one_image_and_skips_unit_axes(): + shape = (3, 1, 4, 5) + assert slice_axes(shape) == [0, 2, 3] + assert frame_key(shape, 0, 2) == (2, 0, slice(None), slice(None)) + # Slicing along a row gives a sinogram: events by columns. + assert frame_key(shape, 2, 1) == (slice(None), 0, 1, slice(None)) + assert PROJECTIONS[frame_key(shape, 2, 1)].shape == (3, 5) + assert slice_axes((4, 5)) == [] + assert frame_key((1, 4, 5)) == (0, slice(None), slice(None)) + + +@pytest.mark.parametrize( + ("shape", "axis"), [((5,), None), ((1, 5), None), ((3, 4, 5), 1 + 5)] +) +def test_frame_key_refuses_what_is_not_an_image(shape, axis): + with pytest.raises(ValueError): + frame_key(shape, axis) + + +def test_image_statistics_and_levels(): + data = np.array([[0.0, 1.0], [2.0, np.nan]]) + stats = image_statistics(data) + assert stats["Minimum"] == 0.0 + assert stats["Maximum"] == 2.0 + assert stats["Mean"] == pytest.approx(1.0) + assert stats["Median"] == 1.0 + assert image_statistics(np.full(3, np.nan)) == {} + assert auto_levels(np.full((2, 2), 5.0)) == (5.0, 6.0) + + +def test_percentile_density_is_normalised_and_drops_empty_bins(): + percentiles, density = percentile_density(np.arange(1000.0)) + assert density.sum() == pytest.approx(1.0) + assert percentiles[0] == pytest.approx(0.5) + assert np.allclose(density, density[0], rtol=0.05) + assert percentile_density(np.zeros(10))[0].size == 0 + + +def test_list_datasets_reads_columns_and_arrays_in_one_listing(): + stream = _tomo_run()["primary"] + datasets = list_datasets("primary", stream) + assert datasets == [ + DatasetInfo("primary", "time", None, "number"), + DatasetInfo("primary", "angle", None, "number"), + DatasetInfo("primary", "kinetix-det1_image", (3, 1, 4, 5), "u2"), + ] + assert [d.is_image for d in datasets] == [False, False, True] + + +@pytest.mark.parametrize("nested", [False, True]) +def test_run_streams_handles_both_tiled_layouts(nested): + assert list(run_streams(_tomo_run(nested=nested))) == ["primary"] + + +def test_search_runs_filters_by_detector_name(): + client = FakeClient( + { + "a": _tomo_run(detectors=("kinetix-det1",)), + "b": _tomo_run(detectors=("phantom",)), + "c": _tomo_run(detectors=()), + } + ) + spec = Spec("TomoFlyscan") + assert [r.uid for r in search_runs(client, spec, 0, 1, None, "Kinetix")] == ["a"] + assert len(search_runs(client, spec, 0, 1, None, "")) == 3 + + +def test_viewer_slices_a_stack_read_from_tiled(): + QApplication.instance() or QApplication([]) + run = _tomo_run() + viewer = QtRunViewer(FakeClient({"abc": run}), "abc") + _wait_for(lambda: viewer._datasets) + assert "Scan 7" in viewer.windowTitle() + + _select(viewer, "kinetix-det1_image") + _wait_for(lambda: viewer._image is not None) + np.testing.assert_array_equal(viewer._image, PROJECTIONS[0, 0]) + assert viewer._stats_table.rowCount() > 2 + + viewer._index.setValue(2) + _wait_for(lambda: viewer._image is not None and viewer._image[0, 0] == 40) + array = run["primary"].arrays["kinetix-det1_image"] + # Only single frames were requested, never the whole stack. + assert array.keys_read == [ + (0, 0, slice(None), slice(None)), + (2, 0, slice(None), slice(None)), + ] + + viewer._profile_direction.setCurrentIndex(1) + xs, ys = viewer._profile_curve.getData() + assert len(ys) == PROJECTIONS.shape[2] + viewer.close() + + +def test_viewer_plots_a_column_against_another(): + QApplication.instance() or QApplication([]) + viewer = QtRunViewer(FakeClient({"abc": _tomo_run()}), "abc") + _wait_for(lambda: viewer._datasets) + + _select(viewer, "angle") + _wait_for(lambda: viewer._line_curve.getData()[1] is not None) + np.testing.assert_array_equal(viewer._line_curve.getData()[1], ANGLES) + + viewer._x_key.setCurrentText("time") + _wait_for(lambda: viewer._line_curve.getData()[0][-1] == 2.0) + viewer.close() + + +def test_scan_browser_opens_a_run_in_the_viewer(): + QApplication.instance() or QApplication([]) + client = FakeClient({"abc": _tomo_run()}) + browser = QtScanBrowser(client, [Spec("TomoFlyscan")], RunEngine({})) + viewer = browser.open_viewer("abc") + assert viewer.parent() is browser + assert viewer.isWindow() + _wait_for(lambda: viewer._datasets) + viewer.close() + + +class GatedArray(FakeArray): + """An array whose reads wait while the gate is closed, as a slow Tiled would.""" + + def __init__(self, data): + super().__init__(data) + self.gate = threading.Event() + self.gate.set() + + def __getitem__(self, key): + self.gate.wait(5) + return super().__getitem__(key) + + +def _open_gated_image(monkeypatch): + QApplication.instance() or QApplication([]) + run = _tomo_run() + array = GatedArray(PROJECTIONS) + run["primary"].arrays["kinetix-det1_image"] = array + viewer = QtRunViewer(FakeClient({"abc": run}), "abc") + _wait_for(lambda: viewer._datasets) + _select(viewer, "kinetix-det1_image") + _wait_for(lambda: viewer._image is not None and viewer._frame_worker is None) + shown = [] + set_image = viewer._image_item.setImage + + def spy(image, **kwargs): + shown.append(np.array(image)) + set_image(image, **kwargs) + + monkeypatch.setattr(viewer._image_item, "setImage", spy) + return viewer, array, shown + + +# PR 96 review: a frame read finishing after the controls moved on was shown, +# so the image and statistics disagreed with the slider. +def test_viewer_drops_a_frame_the_controls_moved_past(monkeypatch): + viewer, array, shown = _open_gated_image(monkeypatch) + array.gate.clear() + viewer._index.setValue(1) + _wait_for(lambda: viewer._frame_worker is not None) + viewer._index.setValue(2) + array.gate.set() + _wait_for(lambda: viewer._frame_worker is None and not viewer._frame_pending) + _wait_for(lambda: len(shown) >= 1) + assert [frame[0, 0] for frame in shown] == [PROJECTIONS[2, 0, 0, 0]] + np.testing.assert_array_equal(viewer._image, PROJECTIONS[2, 0]) + viewer.close() + + +# PR 96 review: after a slice-axis change, a stale projection used up the +# first-frame reset, leaving the sinogram with the projection's contrast. +def test_viewer_resets_contrast_for_the_new_axis_not_a_stale_frame(monkeypatch): + viewer, array, _shown = _open_gated_image(monkeypatch) + array.gate.clear() + viewer._index.setValue(1) + _wait_for(lambda: viewer._frame_worker is not None) + viewer._slice_axis.setCurrentIndex(1) # slice along rows: a sinogram + array.gate.set() + _wait_for(lambda: viewer._frame_worker is None and not viewer._frame_pending) + sinogram = PROJECTIONS[:, 0, 0, :] + np.testing.assert_array_equal(viewer._image, sinogram) + assert tuple(viewer._image_item.getLevels()) == pytest.approx(auto_levels(sinogram)) + viewer.close() + + +# PR 96 review: a text column (e.g. a shutter's "Open"/"Closed") raised in the +# plotting callback and left the previous plot under the new label. +@pytest.mark.parametrize("axis", ["y", "x"]) +def test_viewer_refuses_to_plot_text_values(axis): + QApplication.instance() or QApplication([]) + run = _tomo_run() + stream = run["primary"] + stream._columns["shutter"] = np.array(["Open", "Closed", "Open"]) + stream.metadata["data_keys"]["shutter"] = {"dtype": "string"} + viewer = QtRunViewer(FakeClient({"abc": run}), "abc") + _wait_for(lambda: viewer._datasets) + _select(viewer, "angle") + _wait_for(lambda: viewer._line_curve.getData()[1] is not None) + + if axis == "y": + _select(viewer, "shutter") + else: + viewer._x_key.setCurrentText("shutter") + _wait_for(lambda: "not numbers" in viewer._status.text()) + if axis == "y": + assert viewer._line_curve.getData()[1] is None + else: + # Plotted against the index instead. + np.testing.assert_array_equal(viewer._line_curve.getData()[0], [0, 1, 2]) + np.testing.assert_array_equal(viewer._line_curve.getData()[1], ANGLES) + viewer.close()