diff --git a/.env.example b/.env.example index dea99356..8b31c3f7 100644 --- a/.env.example +++ b/.env.example @@ -114,6 +114,14 @@ WHISPER_API_TOKEN= WHISPER_CA_CERT=/app/certs/whisper-live.pem # WHISPER_CONNECT_TIMEOUT=10 +# Assisted segmentation (GPU worker). Server-side only; leave empty to hide the Assist tool. +# The worker's base URL, without a path: its public https:// address behind a reverse proxy, +# or its service address on a Docker network shared with web (docs/setup.md section 8). +# ASSIST_WORKER_TOKEN is the worker's WORKER_API_TOKEN. +ASSIST_WORKER_URL= +ASSIST_WORKER_TOKEN= +# ASSIST_CONNECT_TIMEOUT=10 + # App server (entrypoint.sh) # AUTO_MIGRATE=1 # set 0 to skip 'migrate' on container start # RUN_DEV_SERVER=0 # set 1 for Django dev server with auto-reload diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0fdef5e8..bd00266d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,10 +129,11 @@ Where the bundle fits in the system as a whole: [docs/architecture.md](docs/arch ### Third-party CDNs -CDNs are allowed. Today exactly one template uses one: -`templates/common/user_activity_stats.html` loads two pinned Chart.js scripts from -jsDelivr, both with SRI. `templates/base.html` loads **no** CDN scripts at all. Three -rules apply to anything new: +CDNs are allowed. Today two templates use one, both jsDelivr with SRI: +`templates/common/user_activity_stats.html` loads two pinned Chart.js scripts, and +`templates/laparoscopy/patient_detail_content.html` loads driver.js for the Tutorial (its +stylesheet is vendored under `static/vendor/`, because the CSP's `style-src` is `'self'`). +`templates/base.html` loads **no** CDN scripts at all. Three rules apply to anything new: - **Webfonts stay self-hosted.** A font CDN sees every page view of every visitor, which is a GDPR question a JavaScript library does not raise. IBM Plex and Font diff --git a/annotations/tests_image_segmentation.py b/annotations/tests_image_segmentation.py index 7384a696..c75994ee 100644 --- a/annotations/tests_image_segmentation.py +++ b/annotations/tests_image_segmentation.py @@ -22,7 +22,7 @@ from django.apps import apps from django.contrib.auth.models import User from django.core.exceptions import ValidationError -from django.test import SimpleTestCase, TestCase +from django.test import Client, SimpleTestCase, TestCase from django.urls import reverse from annotations.adapters import image_segmentation as codec @@ -183,6 +183,11 @@ def setUp(self): self.user = User.objects.create_user(username=f"u{uuid.uuid4().hex[:8]}", password="pw") # noqa: S106 ProjectAccess.objects.create(user=self.user, project=self.project, role="annotator") self.client.force_login(self.user) + # Labels are a project admin's to add, rename and retire. + admin = User.objects.create_user(username=f"a{uuid.uuid4().hex[:8]}", password="pw") # noqa: S106 + ProjectAccess.objects.create(user=admin, project=self.project, role="admin") + self.admin_client = Client() + self.admin_client.force_login(admin) kw = {"patient_id": self.patient.patient_id} self.save_url = reverse("laparoscopy:api_save_image_segmentation", kwargs=kw) self.state_url = reverse("laparoscopy:api_image_segmentation_state", kwargs=kw) @@ -194,12 +199,18 @@ def setUp(self): # -- helpers ------------------------------------------------------------ def _label(self, name): - response = self.client.post( + response = self.admin_client.post( self.labels_url, json.dumps({"name": name}), content_type="application/json" ) self.assertEqual(response.status_code, 201, response.content) return response.json()["label"]["code"] + def _label_url(self, code): + return reverse( + "laparoscopy:api_image_segmentation_label", + kwargs={"patient_id": self.patient.patient_id, "code": code}, + ) + def _state(self): response = self.client.get(self.state_url) self.assertEqual(response.status_code, 200) @@ -233,7 +244,7 @@ def test_labels_get_stable_codes_and_distinct_colours(self): self.assertNotEqual(labels[0]["color"], labels[1]["color"]) def test_a_duplicate_name_is_refused_case_insensitively(self): - response = self.client.post( + response = self.admin_client.post( self.labels_url, json.dumps({"name": "liver"}), content_type="application/json" ) self.assertEqual(response.status_code, 400) @@ -243,7 +254,7 @@ def test_a_rename_changes_the_name_and_never_the_code_or_value(self): "laparoscopy:api_image_segmentation_label", kwargs={"patient_id": self.patient.patient_id, "code": self.liver}, ) - response = self.client.patch( + response = self.admin_client.patch( url, json.dumps({"name": "Hepar", "color": "#112233"}), content_type="application/json" ) label = response.json()["label"] @@ -254,11 +265,47 @@ def test_a_retired_label_is_kept_and_its_value_is_not_reused(self): "laparoscopy:api_image_segmentation_label", kwargs={"patient_id": self.patient.patient_id, "code": self.tumour}, ) - self.assertFalse(self.client.delete(url).json()["label"]["active"]) + self.assertFalse(self.admin_client.delete(url).json()["label"]["active"]) self.assertEqual(self._label("Spleen"), "l3", "retiring must not free a value") labels = {label["code"]: label for label in self._state()["labels"]} self.assertFalse(labels[self.tumour]["active"]) + def test_only_a_project_admin_adds_renames_retires_or_restores_a_label(self): + url = self._label_url(self.liver) + refused = [ + self.client.post(self.labels_url, json.dumps({"name": "Spleen"}), content_type="application/json"), + self.client.patch(url, json.dumps({"name": "Hepar"}), content_type="application/json"), + self.client.patch(url, json.dumps({"active": False}), content_type="application/json"), + self.client.patch(url, json.dumps({"name": "Hepar", "color": "#112233"}), content_type="application/json"), + self.client.delete(url), + ] + self.assertEqual([response.status_code for response in refused], [403] * 5) + labels = self.admin_client.get(self.labels_url).json()["labels"] + self.assertEqual([(label["name"], label["active"]) for label in labels], [("Liver", True), ("Tumour", True)]) + mine = {label["code"]: label["color"] for label in self._state()["labels"]} + self.assertNotEqual(mine[self.liver], "#112233", "a refused request writes nothing, colour included") + + def test_a_colour_is_the_users_own(self): + shared = self.admin_client.get(self.labels_url).json()["labels"][0]["color"] + response = self.client.patch( + self._label_url(self.liver), json.dumps({"color": "#ABCDEF"}), content_type="application/json" + ) + self.assertEqual(response.status_code, 200, response.content) + self.assertEqual(response.json()["label"]["color"], "#abcdef") + mine = {label["code"]: label["color"] for label in self._state()["labels"]} + self.assertEqual(mine[self.liver], "#abcdef") + theirs = {label["code"]: label["color"] for label in self.admin_client.get(self.state_url).json()["labels"]} + self.assertEqual(theirs[self.liver], shared, "nobody else sees it") + + # A viewer may pick colours too; a blank one goes back to the label's own. + ProjectAccess.objects.filter(user=self.user).update(role="viewer") + self.client.patch(self._label_url(self.liver), json.dumps({"color": ""}), content_type="application/json") + self.assertEqual({label["code"]: label["color"] for label in self._state()["labels"]}[self.liver], shared) + bad = self.client.patch(self._label_url(self.liver), json.dumps({"color": "red"}), content_type="application/json") + self.assertEqual(bad.status_code, 400) + missing = self.client.patch(self._label_url("l99"), json.dumps({"color": "#000000"}), content_type="application/json") + self.assertEqual(missing.status_code, 404) + # -- saving and reading ------------------------------------------------- def test_a_frame_round_trips_through_save_state_and_read(self): diff --git a/annotations/tests_quadrants.py b/annotations/tests_quadrants.py index 372b573b..33c83f67 100644 --- a/annotations/tests_quadrants.py +++ b/annotations/tests_quadrants.py @@ -9,7 +9,7 @@ from django.apps import apps from django.contrib.auth.models import User -from django.test import TestCase +from django.test import Client, TestCase from django.urls import reverse from annotations.models import AnnotationSet, LabelSchema @@ -51,6 +51,11 @@ def setUp(self): self.user = User.objects.create_user(username=f"u{uuid.uuid4().hex[:8]}", password="pw") # noqa: S106 ProjectAccess.objects.create(user=self.user, project=self.project, role="annotator") self.client.force_login(self.user) + # Quadrants are a project admin's to add, rename and retire. + admin = User.objects.create_user(username=f"a{uuid.uuid4().hex[:8]}", password="pw") # noqa: S106 + ProjectAccess.objects.create(user=admin, project=self.project, role="admin") + self.admin_client = Client() + self.admin_client.force_login(admin) kw = {"patient_id": self.patient.patient_id} self.url = reverse("laparoscopy:api_quadrants", kwargs=kw) self.labels_url = reverse("laparoscopy:api_quadrant_labels", kwargs=kw) @@ -59,7 +64,7 @@ def setUp(self): self.lower = self._label("Lower right") def _label(self, name): - response = self.client.post( + response = self.admin_client.post( self.labels_url, json.dumps({"name": name}), content_type="application/json" ) self.assertEqual(response.status_code, 201, response.content) @@ -72,7 +77,7 @@ def _put(self, markers, *, expected=None, file_id=None): return self.client.put(self.url, json.dumps(body), content_type="application/json") def test_the_vocabulary_is_separate_from_the_segmentation_labels(self): - self.client.post( + self.admin_client.post( self.seg_labels_url, json.dumps({"name": "Liver"}), content_type="application/json" ) quadrants = self.client.get(self.url).json()["labels"] @@ -131,11 +136,33 @@ def test_a_retired_quadrant_keeps_its_markers(self): "laparoscopy:api_quadrant_label", kwargs={"patient_id": self.patient.patient_id, "code": self.upper}, ) - self.assertFalse(self.client.delete(url).json()["label"]["active"]) + self.assertFalse(self.admin_client.delete(url).json()["label"]["active"]) state = self.client.get(self.client.get(self.url).wsgi_request.path).json() self.assertEqual(state["markers"], [{"timeMs": 0, "code": self.upper}]) self.assertEqual(self._put(state["markers"]).status_code, 200) + def test_quadrants_are_an_admins_but_their_colours_are_each_users(self): + url = reverse( + "laparoscopy:api_quadrant_label", + kwargs={"patient_id": self.patient.patient_id, "code": self.upper}, + ) + refused = [ + self.client.post(self.labels_url, json.dumps({"name": "Middle"}), content_type="application/json"), + self.client.patch(url, json.dumps({"name": "Top"}), content_type="application/json"), + self.client.delete(url), + ] + self.assertEqual([response.status_code for response in refused], [403] * 3) + + shared = self.admin_client.get(self.url).json()["labels"][0]["color"] + response = self.client.patch(url, json.dumps({"color": "#123456"}), content_type="application/json") + self.assertEqual(response.status_code, 200, response.content) + mine = {label["code"]: label for label in self.client.get(self.url).json()["labels"]} + self.assertEqual((mine[self.upper]["name"], mine[self.upper]["color"]), ("Upper left", "#123456")) + self.assertEqual(self.admin_client.get(self.url).json()["labels"][0]["color"], shared) + # Saving markers sends the state back: still in this user's colours. + saved = self._put([{"timeMs": 0, "code": self.upper}]).json() + self.assertEqual(saved["labels"][0]["color"], "#123456") + def test_a_project_without_the_method_cannot_save(self): self.project.annotation_methods.clear() self.assertEqual(self._put([]).status_code, 403) diff --git a/annotations/views_image_segmentation.py b/annotations/views_image_segmentation.py index 0ae915f6..7b80a433 100644 --- a/annotations/views_image_segmentation.py +++ b/annotations/views_image_segmentation.py @@ -12,7 +12,8 @@ import json from django.contrib.auth.decorators import login_required -from django.core.exceptions import ValidationError +from django.core.exceptions import PermissionDenied, ValidationError +from django.db import transaction from django.http import JsonResponse from django.views.decorators.http import require_http_methods, require_POST @@ -20,14 +21,15 @@ from annotations.constants import CoordinateSystem from annotations.services import AnnotationConflict, AnnotationNotAllowed from annotations.services import image_segmentation as service -from annotations.validators.image_segmentation import validate_time_ms +from annotations.validators.image_segmentation import validate_color, validate_time_ms from annotations.views import ( _BadRequest, _checked_file, _first_message, _patient_model, ) -from common.permissions import get_patient_for +from common.models import UserPreference +from common.permissions import get_patient_for, user_is_patient_admin def _json_body(request): @@ -58,13 +60,88 @@ def _project(patient): return project +# --- Labels: admins manage them, every user picks their own colours --------------------- +# +# Adding, renaming, retiring and restoring a label change the project's vocabulary, so they +# are a project admin's. A colour is a viewing preference: it is stored per user in +# ``common.UserPreference.label_colors`` and swapped into every label list that user is +# sent, so the label's own colour is only the default. + + +def _color_key(project, vocabulary, code): + return f"{project.pk}:{vocabulary}:{code}" + + +def _personal_colors(user, project, vocabulary, labels): + """``labels`` as ``user`` sees them: their own colour wherever they picked one.""" + preference = UserPreference.objects.filter(user=user).first() + mine = preference.label_colors if preference else {} + if not mine or project is None: + return labels + return [ + {**label, "color": mine.get(_color_key(project, vocabulary, label["code"]), label["color"])} + for label in labels + ] + + +def _set_personal_color(user, project, vocabulary, code, color): + """Store ``user``'s colour for one label; a blank colour goes back to the label's own.""" + color = validate_color(color) + key = _color_key(project, vocabulary, code) + with transaction.atomic(): + preference, _ = UserPreference.objects.select_for_update().get_or_create(user=user) + if color: + preference.label_colors[key] = color + else: + preference.label_colors.pop(key, None) + preference.save(update_fields=["label_colors", "updated_at"]) + + +def label_detail_response(request, patient_id, code, labels, missing): + """PATCH (rename/restore: admin; colour: anyone who can read) or DELETE (retire: admin). + + ``labels`` is the service that owns the vocabulary: ``create_label``, ``update_label``, + ``list_labels`` and ``VOCABULARY``. + """ + patient = _patient(request, patient_id, "admin" if request.method == "DELETE" else "read") + try: + project = _project(patient) + if request.method == "DELETE": + changes, color = {"active": False}, None + else: + body = _json_body(request) + changes = {key: body[key] for key in ("name", "active") if key in body} + color = body.get("color") + if changes and not user_is_patient_admin(request.user, patient): + raise PermissionDenied("Only a project admin can rename, retire or restore labels.") + if changes: + label = labels.update_label(project, code, **changes) + else: + label = next((item for item in labels.list_labels(project) if item["code"] == code), None) + if label is None: + raise LookupError(code) + if color is not None: + _set_personal_color(request.user, project, labels.VOCABULARY, code, color) + except _BadRequest as exc: + return JsonResponse({"error": str(exc)}, status=exc.status) + except LookupError: + return JsonResponse({"error": missing}, status=404) + except ValidationError as exc: + return JsonResponse({"error": _first_message(exc)}, status=400) + [label] = _personal_colors(request.user, project, labels.VOCABULARY, [label]) + return JsonResponse({"label": label}) + + @login_required def image_segmentation_state_api(request, patient_id): """The vocabulary, the revision a save must quote, and which frames hold masks.""" patient = _patient(request, patient_id, "read") state = service.image_segmentation_state(patient) updated_at = state["updatedAt"] - return JsonResponse({**state, "updatedAt": updated_at.isoformat() if updated_at else None}) + labels = _personal_colors(request.user, patient.project, service.VOCABULARY, state["labels"]) + return JsonResponse( + {**state, "labels": labels, "updatedAt": updated_at.isoformat() if updated_at else None} + ) @login_required @@ -173,12 +250,15 @@ def save_image_segmentation_api(request, patient_id): @login_required @require_http_methods(["GET", "POST"]) def image_segmentation_labels_api(request, patient_id): - """List (read) or create (write) the project's labels.""" - patient = _patient(request, patient_id, "read" if request.method == "GET" else "write") + """List (read) or create (admin) the project's labels.""" + patient = _patient(request, patient_id, "read" if request.method == "GET" else "admin") try: project = _project(patient) if request.method == "GET": - return JsonResponse({"labels": service.list_labels(project)}) + labels = service.list_labels(project) + return JsonResponse( + {"labels": _personal_colors(request.user, project, service.VOCABULARY, labels)} + ) body = _json_body(request) label = service.create_label( project, name=body.get("name"), color=body.get("color", "") @@ -193,26 +273,9 @@ def image_segmentation_labels_api(request, patient_id): @login_required @require_http_methods(["PATCH", "DELETE"]) def image_segmentation_label_detail_api(request, patient_id, code): - """Rename/recolour/restore (PATCH) or retire (DELETE) one label. + """Rename/restore (PATCH, admin), recolour for yourself (PATCH) or retire (DELETE, admin). DELETE retires rather than deletes: the label's pixels stay in every revision that has them, and a deleted definition would leave planes that name nothing. """ - patient = _patient(request, patient_id, "write") - try: - project = _project(patient) - if request.method == "DELETE": - changes = {"active": False} - else: - body = _json_body(request) - changes = { - key: body[key] for key in ("name", "color", "active") if key in body - } - label = service.update_label(project, code, **changes) - except _BadRequest as exc: - return JsonResponse({"error": str(exc)}, status=exc.status) - except LookupError: - return JsonResponse({"error": "No such label."}, status=404) - except ValidationError as exc: - return JsonResponse({"error": _first_message(exc)}, status=400) - return JsonResponse({"label": label}) + return label_detail_response(request, patient_id, code, service, "No such label.") diff --git a/annotations/views_quadrants.py b/annotations/views_quadrants.py index a7b90a42..20816712 100644 --- a/annotations/views_quadrants.py +++ b/annotations/views_quadrants.py @@ -16,10 +16,18 @@ _integer_or_none, _json_body, _patient, + _personal_colors, _project, + label_detail_response, ) +def _state(request, patient): + state = service.quadrant_state(patient) + labels = _personal_colors(request.user, patient.project, service.VOCABULARY, state["labels"]) + return JsonResponse({**state, "labels": labels}) + + @login_required @require_http_methods(["GET", "PUT"]) def quadrants_api(request, patient_id): @@ -29,7 +37,7 @@ def quadrants_api(request, patient_id): """ patient = _patient(request, patient_id, "read" if request.method == "GET" else "write") if request.method == "GET": - return JsonResponse(service.quadrant_state(patient)) + return _state(request, patient) try: body = _json_body(request) expected = _integer_or_none(body, "expectedRevision") @@ -51,14 +59,14 @@ def quadrants_api(request, patient_id): return JsonResponse({"error": str(exc)}, status=403) except ValidationError as exc: return JsonResponse({"error": _first_message(exc)}, status=400) - return JsonResponse(service.quadrant_state(patient)) + return _state(request, patient) @login_required @require_http_methods(["POST"]) def quadrant_labels_api(request, patient_id): - """Create a quadrant (write). The list comes with the state.""" - patient = _patient(request, patient_id, "write") + """Create a quadrant (admin). The list comes with the state.""" + patient = _patient(request, patient_id, "admin") try: body = _json_body(request) label = service.create_label( @@ -74,20 +82,5 @@ def quadrant_labels_api(request, patient_id): @login_required @require_http_methods(["PATCH", "DELETE"]) def quadrant_label_detail_api(request, patient_id, code): - """Rename/recolour/restore (PATCH) or retire (DELETE) one quadrant.""" - patient = _patient(request, patient_id, "write") - try: - project = _project(patient) - if request.method == "DELETE": - changes = {"active": False} - else: - body = _json_body(request) - changes = {k: body[k] for k in ("name", "color", "active") if k in body} - label = service.update_label(project, code, **changes) - except _BadRequest as exc: - return JsonResponse({"error": str(exc)}, status=exc.status) - except LookupError: - return JsonResponse({"error": "No such quadrant."}, status=404) - except ValidationError as exc: - return JsonResponse({"error": _first_message(exc)}, status=400) - return JsonResponse({"label": label}) + """Rename/restore (PATCH, admin), recolour for yourself (PATCH) or retire (DELETE, admin).""" + return label_detail_response(request, patient_id, code, service, "No such quadrant.") diff --git a/common/assist.py b/common/assist.py new file mode 100644 index 00000000..b57dca1d --- /dev/null +++ b/common/assist.py @@ -0,0 +1,150 @@ +"""Assisted segmentation: the DjangoWorker GPU worker seen from Yggdrasil. + +The browser never talks to the worker. ``common.consumers.AssistSessionConsumer`` holds the +browser's WebSocket for the page visit; everything the worker is told goes through here. +""" + +import base64 +import binascii +import json +import math +import secrets +import urllib.request + +from django.conf import settings + +from common.object_storage import get_object_storage + +METHOD_SLUG = "assisted_segmentation" +MAX_POINTS = 64 +MAX_WINDOW = 50 +# The worker downloads the video once, when its session opens; this only has to outlast +# that wait (seconds, or minutes behind other sessions' loads). +LINK_TTL_SECONDS = 600 + + +class PromptError(ValueError): + """A prompt from the browser that the worker must not see.""" + + +def is_configured(): + return bool(settings.ASSIST_WORKER_URL and settings.ASSIST_WORKER_TOKEN) + + +def new_session_id(file_id): + # New per relay connection: a page reload's late DELETE must not evict the new session. + return f"lap-{file_id}-{secrets.token_hex(4)}" + + +def video_link(key): + """A signed, short-lived download link for one stored video. + + The worker gets this, never storage credentials: it can read this one file, for + ``LINK_TTL_SECONDS``, and nothing else in the bucket. + """ + return get_object_storage().presign_get(key, expires_seconds=LINK_TTL_SECONDS) + + +def _base_url(): + url = settings.ASSIST_WORKER_URL.rstrip("/") + # Only ever http(s): urllib would happily open file:// or a custom scheme. + if not url.startswith(("http://", "https://")): + raise ValueError("ASSIST_WORKER_URL must be an http(s) URL") + return url + + +def upstream_ws_url(session_id): + scheme, rest = _base_url().split("://", 1) + return f"{'wss' if scheme == 'https' else 'ws'}://{rest}/ws/session/{session_id}/" + + +def auth_headers(): + return {"Authorization": f"Bearer {settings.ASSIST_WORKER_TOKEN}"} + + +def worker_request(method, path, body=None): + """One call to the worker's HTTP API. Blocking: from async code, run it in a thread.""" + request = urllib.request.Request( # noqa: S310 -- scheme checked in _base_url() + _base_url() + path, + data=None if body is None else json.dumps(body).encode(), + method=method, + headers={**auth_headers(), "Content-Type": "application/json"}, + ) + with urllib.request.urlopen(request, timeout=settings.ASSIST_CONNECT_TIMEOUT) as response: # noqa: S310 + return response.status + + +def probe_of(file_obj): + """Width, height, fps and frame count of a video with a recorded probe, else ``None``.""" + probe = (file_obj.metadata or {}).get("probe") or {} + try: + width, height = int(probe["width"]), int(probe["height"]) + fps, frame_count = float(probe["fps"]), int(probe["frame_count"]) + except (KeyError, TypeError, ValueError): + return None + if min(width, height, frame_count) <= 0 or not fps > 0: + return None + return {"width": width, "height": height, "fps": fps, "frame_count": frame_count} + + +def _is_number(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) + + +def prompt_body(session_id, message, probe): + """The worker's ``/api/session/prompt/`` body for one browser prompt. + + Checked against the video's own probe: a frame of this video, points inside the frame, + a mask of exactly this frame's size. ``window`` is how many frames to track forward after + the prompted one (0: that frame only). + """ + frame_index = message.get("frameIndex") + if type(frame_index) is not int or not 0 <= frame_index < probe["frame_count"]: + raise PromptError("That is not a frame of this video.") + window = message.get("window", 0) + if type(window) is not int or not 0 <= window <= MAX_WINDOW: + raise PromptError(f"Propagate over 0 to {MAX_WINDOW} frames.") + points, labels = message.get("points"), message.get("labels") + if not isinstance(points, list) or not 1 <= len(points) <= MAX_POINTS: + raise PromptError(f"Send between 1 and {MAX_POINTS} points.") + if ( + not isinstance(labels, list) + or len(labels) != len(points) + or any(type(label) is not int or label not in (0, 1) for label in labels) + ): + raise PromptError("Each point needs an include (1) or exclude (0) label.") + for point in points: + if ( + not isinstance(point, list) + or len(point) != 2 + or not all(_is_number(value) for value in point) + or not (0 <= point[0] < probe["width"] and 0 <= point[1] < probe["height"]) + ): + raise PromptError("A point lies outside the frame.") + region = { + "region_id": "1", + "points": [[float(x), float(y)] for x, y in points], + "point_labels": labels, + } + mask = message.get("mask") + if mask is not None: + try: + raw = base64.b64decode(mask, validate=True) + except (TypeError, ValueError, binascii.Error): + raise PromptError("The mask is not base64.") + if len(raw) != math.ceil(probe["width"] * probe["height"] / 8): + raise PromptError("The mask is not this frame's size.") + region.update( + mask_b64=mask, + mask_encoding="bitpack_u1_v1", + mask_shape=[probe["height"], probe["width"]], + ) + return { + "video_id": session_id, + "frame_index": frame_index, + # The worker tracks int(window_seconds * fps) frames; the half frame keeps float + # error from rounding that down to window - 1. + "window_seconds": (window + 0.5) / probe["fps"] if window else 0, + "normalized": False, + "regions": [region], + } diff --git a/common/consumers.py b/common/consumers.py index db671fd5..3e4035e1 100644 --- a/common/consumers.py +++ b/common/consumers.py @@ -11,8 +11,9 @@ from django.apps import apps from django.conf import settings +from common import assist from common.domains import DOMAINS -from common.permissions import user_can_write_patient_annotations +from common.permissions import project_allows_annotation, user_can_write_patient_annotations SUPPORTED_LANGUAGES = frozenset({"it", "en", "es", "fr", "de"}) @@ -170,3 +171,181 @@ async def _relay_upstream(self): pass finally: await self.close(code=4502) + + +# What the browser may be told about a worker error: never the worker's own text. +_WORKER_ERRORS = { + "session_expired": "The assistant session ended. Reload the page to start it again.", + "load_failed": "The assistant could not load this video.", + "inference_failed": "The assistant could not segment this frame.", +} + + +@database_sync_to_async +def _assist_target(user, domain, patient_id, file_id): + """``(close_code, None)`` or ``(None, (video_link, probe))`` for the video to assist on. + + The video is found from the patient, never taken from the browser: ``fileId`` must name + *this* patient's subsampled derivative, the file Annotation Mode's frames come from. + The worker is sent a signed link to it (``assist.video_link``), never a storage key it + would need credentials for. + """ + if not user or not user.is_authenticated: + return 4401, None + if domain not in DOMAINS: + return 4404, None + Patient = apps.get_model(domain, "Patient") + patient = Patient.objects.select_related("project").filter(patient_id=patient_id).first() + if patient is None: + return 4404, None + if not user_can_write_patient_annotations(user, patient): + return 4403, None + if not project_allows_annotation(patient, assist.METHOD_SLUG): + return 4403, None + video = patient.files.filter( + pk=file_id, file_type="video_processed", subtype="subsampled" + ).first() + probe = assist.probe_of(video) if video else None + if probe is None: + return 4404, None + return None, (assist.video_link(video.file_path), probe) + + +class AssistSessionConsumer(AsyncWebsocketConsumer): + """One page visit's assisted-segmentation session, relayed to the GPU worker. + + The browser names the patient (URL) and the video (``?fileId=``); everything the worker + is told -- the video link, the session id, the token -- is decided here. The worker + starts precomputing on connect; leaving the page closes this socket, and that closes the + worker session at once instead of after its grace period. + """ + + upstream = None + upstream_task = None + session_id = None + probe = None + ready = False + + async def connect(self): + user = self.scope.get("user") + kwargs = self.scope["url_route"]["kwargs"] + query = parse_qs(self.scope.get("query_string", b"").decode()) + try: + file_id = int(query.get("fileId", [""])[0]) + except ValueError: + await self._refuse(4400, "fileId is not an integer") + return + if not assist.is_configured(): + await self._refuse(4503, "ASSIST_WORKER_URL or ASSIST_WORKER_TOKEN is unset") + return + code, target = await _assist_target(user, kwargs["domain"], kwargs["patient_id"], file_id) + if code: + await self._refuse( + code, + f"{user} may not assist on {kwargs['domain']} patient {kwargs['patient_id']} file {file_id}", + ) + return + video_link, self.probe = target + self.session_id = assist.new_session_id(file_id) + try: + # Subscribe before /ready: the worker's session_ready is pub/sub, and is lost if + # nobody is listening when precompute finishes. + self.upstream = await websockets.connect( + assist.upstream_ws_url(self.session_id), + additional_headers=assist.auth_headers(), + open_timeout=settings.ASSIST_CONNECT_TIMEOUT, + max_size=8 * 1024 * 1024, + ) + await asyncio.to_thread( + assist.worker_request, + "POST", + "/api/session/ready/", + {"video_id": self.session_id, "video_source": video_link}, + ) + except Exception as exc: + logger.warning("Could not open an assist session: %s", exc) + # The socket may be open and /ready may have reached the worker: release both. + await self.disconnect(4502) + await self.close(code=4502) + return + await self.accept() + await self.send(text_data=json.dumps({"type": "preparing"})) + self.upstream_task = asyncio.create_task(self._relay_upstream()) + + async def _refuse(self, code, reason): + logger.warning("Refusing assist session (%s): %s", code, reason) + await self.close(code=code) + + async def _error(self, message): + await self.send(text_data=json.dumps({"type": "error", "message": message})) + + async def receive(self, text_data=None, bytes_data=None): + try: + message = json.loads(text_data or "") + except json.JSONDecodeError: + message = None + if not isinstance(message, dict) or message.get("type") != "prompt": + await self._error("Unknown message.") + return + if not self.ready: + await self._error("The assistant is still preparing this video.") + return + try: + body = assist.prompt_body(self.session_id, message, self.probe) + await asyncio.to_thread(assist.worker_request, "POST", "/api/session/prompt/", body) + except assist.PromptError as exc: + await self._error(str(exc)) + except Exception as exc: + logger.warning("Assist prompt failed: %s", exc) + await self._error("The assistant could not be reached.") + + async def disconnect(self, close_code): + current = asyncio.current_task() + if self.upstream_task and self.upstream_task is not current: + self.upstream_task.cancel() + # Whatever the relay died of, the cleanup below must still run. + with suppress(asyncio.CancelledError, Exception): + await self.upstream_task + if self.upstream is not None: + with suppress(Exception): + await self.upstream.close() + self.upstream = None + if self.session_id: + # Free the worker's RAM now, not after its grace period. + with suppress(Exception): + await asyncio.to_thread( + assist.worker_request, "DELETE", f"/api/session/{self.session_id}/" + ) + self.session_id = None + + async def _relay_upstream(self): + try: + async for raw in self.upstream: + try: + message = json.loads(raw) + except (TypeError, json.JSONDecodeError): + continue + kind = message.get("type") + if kind == "session_ready": + self.ready = True + await self.send(text_data=json.dumps({"type": "ready"})) + elif kind == "frame_result": + await self.send( + text_data=json.dumps( + { + "type": "mask", + "frameIndex": message.get("frame_index"), + "mask": message.get("mask_b64"), + "shape": message.get("mask_shape"), + } + ) + ) + elif kind == "inference_done": + await self.send(text_data=json.dumps({"type": "done"})) + elif kind == "error": + await self._error(_WORKER_ERRORS.get(message.get("message"), "The assistant failed.")) + except websockets.ConnectionClosed: + pass + # Reached only when the worker ended the stream. When the browser leaves, the task is + # cancelled instead, and closing the already-closed socket would raise in disconnect(). + await self.close(code=4502) diff --git a/common/migrations/0062_userpreference_label_colors.py b/common/migrations/0062_userpreference_label_colors.py new file mode 100644 index 00000000..c1fdc297 --- /dev/null +++ b/common/migrations/0062_userpreference_label_colors.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.17 on 2026-10-08 08:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('common', '0061_image_segmentation_method'), + ] + + operations = [ + migrations.AddField( + model_name='userpreference', + name='label_colors', + field=models.JSONField(blank=True, default=dict), + ), + ] diff --git a/common/migrations/0063_assisted_segmentation_method.py b/common/migrations/0063_assisted_segmentation_method.py new file mode 100644 index 00000000..6c8d1a45 --- /dev/null +++ b/common/migrations/0063_assisted_segmentation_method.py @@ -0,0 +1,37 @@ +"""Register the ``assisted_segmentation`` annotation method and enable it on laparoscopy projects. + +The Assist tool is gated on it (and on ``ASSIST_WORKER_URL``/``ASSIST_WORKER_TOKEN``); a fresh +project gets it from ``setup_laparoscopy_modalities``. +""" + +from django.db import migrations + +SLUG = "assisted_segmentation" + + +def forwards(apps, schema_editor): + AnnotationMethod = apps.get_model("common", "AnnotationMethod") + Project = apps.get_model("common", "Project") + method, _created = AnnotationMethod.objects.get_or_create( + slug=SLUG, + defaults={ + "name": "Assisted Segmentation", + "description": "Masks from clicked points, painted by a model on the GPU worker.", + "icon": "fas fa-wand-magic-sparkles", + "domain": "laparoscopy", + "is_active": True, + }, + ) + for project in Project.objects.filter(domain="laparoscopy"): + project.annotation_methods.add(method) + + +class Migration(migrations.Migration): + + dependencies = [ + ("common", "0062_userpreference_label_colors"), + ] + + operations = [ + migrations.RunPython(forwards, migrations.RunPython.noop), + ] diff --git a/common/models.py b/common/models.py index 063ea1e4..b6e9ab21 100644 --- a/common/models.py +++ b/common/models.py @@ -929,6 +929,9 @@ class UserPreference(models.Model): user = models.OneToOneField(User, on_delete=models.CASCADE, related_name='ygg_preference') report_language = models.CharField(max_length=5, choices=LANGUAGE_CHOICES, default='it') + # The colour this user sees a project's label in, overriding the label's own: + # {"::": "#rrggbb"}. Display only -- nobody else sees it. + label_colors = models.JSONField(default=dict, blank=True) updated_at = models.DateTimeField(auto_now=True) def __str__(self): diff --git a/common/routing.py b/common/routing.py index ad36d22b..5efa24be 100644 --- a/common/routing.py +++ b/common/routing.py @@ -1,6 +1,6 @@ from django.urls import path -from common.consumers import LiveTranscriptionConsumer +from common.consumers import AssistSessionConsumer, LiveTranscriptionConsumer websocket_urlpatterns = [ @@ -9,4 +9,9 @@ LiveTranscriptionConsumer.as_asgi(), name="live_transcription", ), + path( + "ws/assist///", + AssistSessionConsumer.as_asgi(), + name="assist_session", + ), ] diff --git a/common/tests_assist.py b/common/tests_assist.py new file mode 100644 index 00000000..04b66db8 --- /dev/null +++ b/common/tests_assist.py @@ -0,0 +1,317 @@ +import asyncio +import base64 +import json +from unittest import mock + +from channels.db import database_sync_to_async +from channels.routing import URLRouter +from channels.testing import WebsocketCommunicator +from django.apps import apps +from django.contrib.auth.models import AnonymousUser, User +from django.test import SimpleTestCase, TransactionTestCase, override_settings + +from common import assist +from common.consumers import AssistSessionConsumer +from common.models import AnnotationMethod, FileRegistry, Project, ProjectAccess +from common.object_storage import get_object_storage +from common.routing import websocket_urlpatterns + +PROBE = {"width": 16, "height": 8, "fps": 25.0, "frame_count": 100} + + +class PromptBodyTests(SimpleTestCase): + def body(self, **message): + base = {"frameIndex": 3, "points": [[2, 1]], "labels": [1]} + return assist.prompt_body("lap-1-0a1b2c3d", {**base, **message}, PROBE) + + def test_a_click_becomes_one_region_on_that_frame_only(self): + self.assertEqual( + self.body(), + { + "video_id": "lap-1-0a1b2c3d", + "frame_index": 3, + "window_seconds": 0, + "normalized": False, + "regions": [{"region_id": "1", "points": [[2.0, 1.0]], "point_labels": [1]}], + }, + ) + + def test_a_window_tracks_exactly_that_many_frames_forward(self): + body = self.body(window=5) + # The worker tracks int(window_seconds * fps) frames after the prompted one. + self.assertEqual(int(body["window_seconds"] * PROBE["fps"]), 5) + self.assertEqual(self.body(window=0)["window_seconds"], 0) + thirds = assist.prompt_body("lap-1-0a1b2c3d", {"frameIndex": 3, "points": [[2, 1]], "labels": [1], "window": 5}, {**PROBE, "fps": 3.0}) + self.assertEqual(int(thirds["window_seconds"] * 3.0), 5, "no float rounding down to 4") + + def test_the_current_mask_rides_along_for_refinement(self): + mask = base64.b64encode(bytes(16)).decode() # 16 x 8 pixels = 16 bytes + region = self.body(mask=mask)["regions"][0] + self.assertEqual( + (region["mask_b64"], region["mask_encoding"], region["mask_shape"]), + (mask, "bitpack_u1_v1", [8, 16]), + ) + + def test_what_the_worker_must_not_see_is_refused(self): + for message in ( + {"frameIndex": 100}, {"frameIndex": -1}, {"frameIndex": "3"}, {"frameIndex": True}, + {"points": []}, {"points": [[0, 0]] * 65, "labels": [1] * 65}, + {"points": [[16, 0]]}, {"points": [[0, 8]]}, {"points": [[-1, 0]]}, {"points": [[1]]}, + {"points": [["1", 1]]}, {"labels": [2]}, {"labels": [1, 0]}, {"labels": [True]}, + {"mask": "not base64!"}, {"mask": base64.b64encode(bytes(15)).decode()}, + {"window": -1}, {"window": 51}, {"window": "5"}, {"window": True}, + ): + with self.subTest(message=message), self.assertRaises(assist.PromptError): + self.body(**message) + + +class WorkerAddressTests(SimpleTestCase): + def test_each_session_id_is_new_and_names_the_file(self): + first, second = assist.new_session_id(7), assist.new_session_id(7) + self.assertRegex(first, r"^lap-7-[0-9a-f]{8}$") + self.assertNotEqual(first, second) + + @override_settings(ASSIST_WORKER_URL="https://worker.example:8443/", ASSIST_WORKER_TOKEN="s3cret") + def test_the_socket_follows_the_http_scheme_and_carries_the_token(self): + self.assertEqual( + assist.upstream_ws_url("lap-7-0a1b2c3d"), + "wss://worker.example:8443/ws/session/lap-7-0a1b2c3d/", + ) + self.assertEqual(assist.auth_headers(), {"Authorization": "Bearer s3cret"}) + + @override_settings(ASSIST_WORKER_URL="file:///etc/passwd", ASSIST_WORKER_TOKEN="t") + def test_only_an_http_worker_url_is_ever_opened(self): + with self.assertRaises(ValueError): + assist.worker_request("GET", "/") + with self.assertRaises(ValueError): + assist.upstream_ws_url("lap-1-0a1b2c3d") + + def test_a_probe_must_be_complete(self): + class File: + def __init__(self, metadata): + self.metadata = metadata + + good = {"probe": {"width": 16, "height": 8, "fps": 25.0, "frame_count": 100}} + self.assertEqual(assist.probe_of(File(good)), PROBE) + for metadata in (None, {}, {"probe": {"width": 16}}, {"probe": {**good["probe"], "width": 0}}): + with self.subTest(metadata=metadata): + self.assertIsNone(assist.probe_of(File(metadata))) + + +KEY = "laparoscopy/processed/video/job_7/subsampled.mp4" + + +class FakeWorkerSocket: + def __init__(self): + self.messages = asyncio.Queue() + self.closed = False + + async def push(self, message): + await self.messages.put(json.dumps(message)) + + async def close(self): + self.closed = True + await self.messages.put(None) + + def __aiter__(self): + return self + + async def __anext__(self): + message = await self.messages.get() + if message is None: + raise StopAsyncIteration + return message + + +async def _until(predicate): + for _ in range(100): + if predicate(): + return + await asyncio.sleep(0.01) + raise AssertionError("timed out") + + +@override_settings(ASSIST_WORKER_URL="http://assist-worker.test:8000", ASSIST_WORKER_TOKEN="s3cret") +class AssistSessionConsumerTests(TransactionTestCase): + reset_sequences = True + + def setUp(self): + Patient = apps.get_model("laparoscopy", "Patient") + Folder = apps.get_model("laparoscopy", "Folder") + self.project = Project.objects.create(slug="lap-assist", name="Lap assist", domain="laparoscopy") + self.method, _ = AnnotationMethod.objects.get_or_create( + slug=assist.METHOD_SLUG, defaults={"name": "Assisted Segmentation", "domain": "laparoscopy"} + ) + self.project.annotation_methods.add(self.method) + self.user = User.objects.create_user(username="annotator", password="x") + ProjectAccess.objects.create(user=self.user, project=self.project, role="annotator") + folder = Folder.objects.create(name="F", project=self.project) + self.patient = Patient.objects.create(name="P", folder=folder, project=self.project) + self.video = FileRegistry.objects.create( + domain="laparoscopy", file_type="video_processed", subtype="subsampled", + laparoscopy_patient=self.patient, file_path=KEY, file_size=1, file_hash="a" * 64, + metadata={"probe": {"width": 16, "height": 8, "fps": 25.0, "frame_count": 100}}, + ) + self.upstreams = [] + self.calls = [] + + async def connect(url, **kwargs): + socket = FakeWorkerSocket() + socket.url, socket.kwargs = url, kwargs + self.upstreams.append(socket) + return socket + + for patcher in ( + mock.patch("common.consumers.websockets.connect", new=connect), + mock.patch("common.assist.worker_request", side_effect=lambda *call: self.calls.append(call) or 202), + ): + patcher.start() + self.addCleanup(patcher.stop) + + def communicator(self, user=None, file_id=None): + communicator = WebsocketCommunicator( + URLRouter(websocket_urlpatterns), + f"/ws/assist/laparoscopy/{self.patient.patient_id}/?fileId={file_id or self.video.pk}", + ) + communicator.scope["user"] = user or self.user + return communicator + + async def connected(self): + communicator = self.communicator() + ok, _ = await communicator.connect() + self.assertTrue(ok) + self.assertEqual(await communicator.receive_json_from(), {"type": "preparing"}) + return communicator + + async def refused(self, communicator): + ok, code = await communicator.connect() + self.assertFalse(ok) + return code + + async def test_refuses_anonymous(self): + self.assertEqual(await self.refused(self.communicator(AnonymousUser())), 4401) + + async def test_refuses_a_viewer(self): + await database_sync_to_async(ProjectAccess.objects.filter(user=self.user).update)(role="viewer") + self.assertEqual(await self.refused(self.communicator()), 4403) + + async def test_refuses_when_the_project_has_the_method_off(self): + await database_sync_to_async(self.project.annotation_methods.remove)(self.method) + self.assertEqual(await self.refused(self.communicator()), 4403) + + async def test_refuses_when_no_worker_is_configured(self): + with self.settings(ASSIST_WORKER_TOKEN=""): + self.assertEqual(await self.refused(self.communicator()), 4503) + + async def test_refuses_a_file_that_is_not_this_patients_subsampled_video(self): + raw = await database_sync_to_async(FileRegistry.objects.create)( + domain="laparoscopy", file_type="video_raw", laparoscopy_patient=self.patient, + file_path="laparoscopy/raw/case.mp4", file_size=1, file_hash="b" * 64, + ) + self.assertEqual(await self.refused(self.communicator(file_id=raw.pk)), 4404) + self.assertEqual(await self.refused(self.communicator(file_id=999999)), 4404) + self.assertEqual(self.calls, [], "the worker heard nothing") + + async def test_opens_a_worker_session_on_the_patients_own_video_and_closes_it_on_leave(self): + communicator = await self.connected() + method, path, body = self.calls[0] + self.assertEqual((method, path), ("POST", "/api/session/ready/")) + # A short-lived signed link to this one video: the worker holds no storage credentials. + self.assertEqual( + body["video_source"], + get_object_storage().presign_get(KEY, expires_seconds=assist.LINK_TTL_SECONDS), + ) + self.assertNotEqual(body["video_source"], KEY) + session_id = body["video_id"] + self.assertRegex(session_id, rf"^lap-{self.video.pk}-[0-9a-f]{{8}}$") + upstream = self.upstreams[0] + self.assertEqual(upstream.url, f"ws://assist-worker.test:8000/ws/session/{session_id}/") + self.assertEqual(upstream.kwargs["additional_headers"], {"Authorization": "Bearer s3cret"}) + + await communicator.disconnect() + await _until(lambda: self.calls[-1][:2] == ("DELETE", f"/api/session/{session_id}/")) + self.assertTrue(upstream.closed) + + async def test_leaving_frees_the_worker_session_even_when_the_socket_is_already_gone(self): + # Uvicorn refuses a second close once the browser has left ("Unexpected ASGI message + # 'websocket.close'"); that must not stop the worker session from being freed. + communicator = await self.connected() + session_id = self.calls[0][2]["video_id"] + refuse = RuntimeError("Unexpected ASGI message 'websocket.close'") + with mock.patch.object(AssistSessionConsumer, "close", side_effect=refuse): + try: + await communicator.disconnect() + except RuntimeError: + pass + await _until(lambda: self.calls[-1][:2] == ("DELETE", f"/api/session/{session_id}/")) + self.assertTrue(self.upstreams[0].closed) + + async def test_each_connection_gets_its_own_worker_session(self): + first = await self.connected() + await first.disconnect() + second = await self.connected() + ready = [call[2]["video_id"] for call in self.calls if call[1] == "/api/session/ready/"] + self.assertEqual(len(set(ready)), 2) + await second.disconnect() + + async def test_a_prompt_before_the_worker_is_ready_is_refused(self): + communicator = await self.connected() + await communicator.send_json_to({"type": "prompt", "frameIndex": 4, "points": [[1, 1]], "labels": [1]}) + self.assertEqual( + await communicator.receive_json_from(), + {"type": "error", "message": "The assistant is still preparing this video."}, + ) + self.assertFalse(any(call[1] == "/api/session/prompt/" for call in self.calls)) + await communicator.disconnect() + + async def test_relays_a_prompt_and_its_mask(self): + communicator = await self.connected() + upstream = self.upstreams[0] + await upstream.push({"type": "session_ready"}) + self.assertEqual(await communicator.receive_json_from(), {"type": "ready"}) + + await communicator.send_json_to({"type": "prompt", "frameIndex": 4, "points": [[1, 1]], "labels": [1]}) + await _until(lambda: self.calls[-1][1] == "/api/session/prompt/") + sent = self.calls[-1][2] + self.assertEqual((sent["frame_index"], sent["window_seconds"]), (4, 0)) + + await upstream.push({"type": "frame_result", "frame_index": 4, "mask_b64": "AA==", "mask_shape": [8, 16], "region_id": "1"}) + self.assertEqual( + await communicator.receive_json_from(), + {"type": "mask", "frameIndex": 4, "mask": "AA==", "shape": [8, 16]}, + ) + await upstream.push({"type": "inference_done", "job_id": "j1"}) + self.assertEqual(await communicator.receive_json_from(), {"type": "done"}) + await communicator.disconnect() + + async def test_a_bad_prompt_is_explained_and_never_forwarded(self): + communicator = await self.connected() + await self.upstreams[0].push({"type": "session_ready"}) + await communicator.receive_json_from() + await communicator.send_json_to({"type": "prompt", "frameIndex": 4, "points": [[99, 1]], "labels": [1]}) + self.assertEqual( + await communicator.receive_json_from(), + {"type": "error", "message": "A point lies outside the frame."}, + ) + self.assertFalse(any(call[1] == "/api/session/prompt/" for call in self.calls)) + await communicator.disconnect() + + async def test_worker_errors_are_translated_never_passed_through(self): + communicator = await self.connected() + upstream = self.upstreams[0] + await upstream.push({"type": "error", "message": "inference_failed"}) + self.assertEqual( + await communicator.receive_json_from(), + {"type": "error", "message": "The assistant could not segment this frame."}, + ) + await upstream.push({"type": "error", "message": "Traceback: /app/secret/path"}) + self.assertEqual( + await communicator.receive_json_from(), + {"type": "error", "message": "The assistant failed."}, + ) + await communicator.disconnect() + + async def test_a_dead_worker_closes_the_browser_socket(self): + communicator = await self.connected() + await self.upstreams[0].close() + self.assertEqual(await communicator.receive_output(), {"type": "websocket.close", "code": 4502}) diff --git a/docs/setup.md b/docs/setup.md index 37b51a29..9ff4a63a 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -142,3 +142,35 @@ current certificate contains the IP address but not that DNS name. The public reverse proxy must pass WebSocket upgrade requests for `/ws/live-transcription/` to the web container. The standard nginx-proxy setup used by `docker-compose.yml` handles WebSocket upgrades automatically. + +## 8. Configure assisted segmentation (optional) + +The **Assist** tool in laparoscopy Annotation Mode turns clicked points into masks using a +model on the DjangoWorker GPU worker. The browser never reaches the worker. Its WebSocket +(`/ws/assist/…`) ends at Yggdrasil, which opens the worker session itself. Leave both +values empty to hide the tool. + +```dotenv +# Base URL, no path. Yggdrasil calls /api/session/... and opens /ws/session/... +ASSIST_WORKER_URL=https://gpu-worker.example.org +# The worker's WORKER_API_TOKEN +ASSIST_WORKER_TOKEN=replace-with-the-worker-token +# ASSIST_CONNECT_TIMEOUT=10 +``` + +- **Where the worker is.** Behind a reverse proxy (which must route `/api/session` and + `/ws/session` to it), use its public `https://` URL; the socket then uses `wss://`. + If it shares a Docker network with `web`, use its service address there, e.g. + `http://assist-worker:8000`. Only `http(s)` URLs are accepted, and certificates are + checked against the system CAs. +- **The worker gets a link, not storage credentials.** For each session Yggdrasil signs a + 10-minute download link to that one video, against its own `OBJECT_STORAGE_ENDPOINT_URL`. + So that endpoint must be reachable from the worker, and the worker's `VIDEO_SOURCE_ORIGINS` + must list its origin (`scheme://host[:port]`, e.g. `https://s3.example.org`). The worker + refuses links to any other origin and does not follow redirects. One worker can serve + several deployments by listing each one's storage origin. +- **The project.** The tool also needs the `assisted_segmentation` annotation method on the + project. Migration `0063` enables it on every laparoscopy project, and + `setup_laparoscopy_modalities` does the same for a new one. +- After changing `.env`, recreate `web` (`docker compose up -d web`); a restart does not + re-read `env_file`. The page shows the tool as *Unavailable* if the worker cannot be reached. diff --git a/frontend/entries/image-segment.js b/frontend/entries/image-segment.js index 5dceaeca..9b1869f4 100644 --- a/frontend/entries/image-segment.js +++ b/frontend/entries/image-segment.js @@ -112,6 +112,8 @@ function prepare() { ToolGroupManager, segmentation, HistoryMemo: coreUtilities.HistoryMemo, + worldToImageCoords: coreUtilities.worldToImageCoords, + imageToWorldCoords: coreUtilities.imageToWorldCoords, Pan: PanTool, Zoom: ZoomTool, toolConfiguration: configuration, diff --git a/frontend/imaging/segment/assist.js b/frontend/imaging/segment/assist.js new file mode 100644 index 00000000..5ac83497 --- /dev/null +++ b/frontend/imaging/segment/assist.js @@ -0,0 +1,192 @@ +/** + * Assisted segmentation, the parts that are not Cornerstone: the worker's mask format, the + * prompt points, and the relay socket (`common.consumers.AssistSessionConsumer`). + * + * The socket opens with the page so the worker precomputes while the annotator is still + * watching the video; closing the page closes the worker session. + */ + +/** A 0/1 plane as the worker's `bitpack_u1_v1`: row-major, LSB-first in each byte, base64. */ +export function packMask(plane) { + const bytes = new Uint8Array(Math.ceil(plane.length / 8)); + for (let index = 0; index < plane.length; index += 1) { + if (plane[index]) { + bytes[index >> 3] |= 1 << (index & 7); + } + } + let binary = ''; + for (const byte of bytes) { + binary += String.fromCharCode(byte); + } + return btoa(binary); +} + +/** + * The inverse, for a mask the worker says is `shape` = [height, width]. Refuses anything that + * is not exactly this frame: a transposed mask has the same byte count and would paint + * plausible nonsense. + */ +export function unpackMask(b64, shape, width, height) { + const binary = atob(b64); + if (shape?.[0] !== height || shape?.[1] !== width || binary.length !== Math.ceil((width * height) / 8)) { + throw new Error('The assistant answered with a mask for another frame size.'); + } + const plane = new Uint8Array(width * height); + for (let index = 0; index < plane.length; index += 1) { + plane[index] = (binary.charCodeAt(index >> 3) >> (index & 7)) & 1; + } + return plane; +} + +export const hasPixels = (plane) => plane.some((value) => value !== 0); + +/** + * The points of the (frame, label) being worked on that the worker has not answered yet. + * Changing frame or label starts over; an answered point is forgotten, so it never steers a + * later prompt (the mask it produced does, as that prompt's starting shape). + */ +export function createPromptPoints() { + let key = null; + let points = []; + function pointsFor(frameIndex, code) { + const next = `${frameIndex}:${code}`; + if (next !== key) { + key = next; + points = []; + } + return points; + } + return { + for: pointsFor, + add(frameIndex, code, x, y, label) { + pointsFor(frameIndex, code).push({ x, y, label }); + }, + forget(frameIndex, code, answered) { + const current = pointsFor(frameIndex, code); + const keep = current.filter((point) => !answered.includes(point)); + current.splice(0, current.length, ...keep); + }, + }; +} + +/** The label a returned mask belongs on, or null when the annotator has moved on. */ +export function maskTarget(sent, message, current) { + if (!sent || message.frameIndex !== sent.frameIndex) { + return null; + } + if (current.frameIndex !== sent.frameIndex || current.code !== sent.code) { + return null; + } + return sent.code; +} + +export function assistSocketUrl({ protocol, host }, { projectNamespace, patientId, fileId }) { + const scheme = protocol === 'https:' ? 'wss' : 'ws'; + return `${scheme}://${host}/ws/assist/${projectNamespace}/${patientId}/?fileId=${fileId}`; +} + +/** + * The relay socket. One prompt in flight at a time. A prompt is *built* only when the channel + * is free -- `request(build)` keeps the newest builder, and `build()` returns + * `{payload, context}` or null -- so it carries the points still unanswered and the mask as it + * is after the previous answer, not a snapshot from the moment of the click. + */ +export function createAssistChannel({ + url, + WebSocketImpl = globalThis.WebSocket, + onStatus = () => {}, + onMask = () => {}, + onError = () => {}, +}) { + const socket = new WebSocketImpl(url); + let status = 'connecting'; + let inFlight = null; + let waiting = null; + + function setStatus(next) { + status = next; + onStatus(next); + } + + function flush() { + if (inFlight || !waiting || status !== 'ready') { + return; + } + const next = waiting(); + waiting = null; + if (!next) { + return; + } + inFlight = next; + socket.send(JSON.stringify({ type: 'prompt', ...inFlight.payload })); + onStatus('busy'); + } + + socket.addEventListener('message', (event) => { + let message; + try { + message = JSON.parse(event.data); + } catch { + return; + } + if (message.type === 'preparing') { + setStatus('preparing'); + } else if (message.type === 'ready') { + setStatus('ready'); + flush(); + } else if (message.type === 'mask') { + onMask(message, inFlight?.context ?? null); + } else if (message.type === 'done' || message.type === 'error') { + if (message.type === 'error') { + onError(message.message); + } + inFlight = null; + if (status === 'ready') { + onStatus('ready'); + } + flush(); + } + }); + socket.addEventListener('close', () => { + inFlight = null; + waiting = null; + setStatus('closed'); + }); + + return { + get status() { + return status; + }, + request(build) { + waiting = build; + flush(); + }, + close() { + socket.close(); + }, + }; +} + +/** The key a frame's mask is stored under, as `frameSource` computes it. */ +export function frameTimeMs(frameIndex, fps) { + return Math.round((frameIndex * 1000) / fps); +} + +/** + * Whether tracking may fill this frame for `code`: only when the stored manifest has no mask + * of that label there. Frames already annotated are kept as they are. + */ +export function labelIsEmptyOn(manifestFrames, timeMs, code) { + return !manifestFrames?.[String(timeMs)]?.labels?.[code]; +} + +/** + * A tracked frame to save: the planes stored for it plus the tracked label's. A save replaces + * a frame wholesale, so the other labels must ride along; a painted plane is never replaced. + */ +export function mergeTrackedPlane(storedPlanes, code, plane) { + if (storedPlanes[code] && hasPixels(storedPlanes[code])) { + return { ...storedPlanes }; + } + return { ...storedPlanes, [code]: plane }; +} diff --git a/frontend/imaging/segment/bootstrap.js b/frontend/imaging/segment/bootstrap.js index 7de004eb..377ac315 100644 --- a/frontend/imaging/segment/bootstrap.js +++ b/frontend/imaging/segment/bootstrap.js @@ -10,9 +10,22 @@ import { createApi, createQuadrantsApi, endpointFor, quadrantsEndpointFor } from './api.js'; import { mountQuadrants } from './quadrants.js'; import { createFrameSource } from './frameSource.js'; -import { renderLabelPanel, describeSave, labelAfterRemoval, paintableLabels } from './labelPanel.js'; +import { renderLabelPanel, describeSave, labelAfterRemoval } from './labelPanel.js'; import { createSaver, SaveStatus } from './saver.js'; -import { SEGMENT_TOOLS } from './tools.js'; +import { ASSIST_TOOL, SEGMENT_TOOLS } from './tools.js'; +import { + assistSocketUrl, + createAssistChannel, + createPromptPoints, + frameTimeMs, + hasPixels, + labelIsEmptyOn, + maskTarget, + mergeTrackedPlane, + packMask, + unpackMask, +} from './assist.js'; +import { decodePlane, encodePlane } from './planes.js'; const DATA_ELEMENT_ID = 'imageSegmentData'; @@ -98,7 +111,6 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } const messages = $(doc, 'segment-message'); const timeline = $(doc, 'video-timeline-range'); const timelineTime = $(doc, 'video-timeline-time'); - const timelineMarks = $(doc, 'video-timeline-marks'); const timelineControls = [...doc.querySelectorAll('[data-video-seek]')]; const startButton = $(doc, 'video-start'); const stopButton = $(doc, 'video-stop'); @@ -142,6 +154,162 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } let navigating = false; const hidden = new Set(); + // Assisted segmentation --------------------------------------------------------- + // The relay opens with the page, so the worker precomputes while the annotator is + // still watching the video; leaving the page closes it (and the worker session). + const assistButton = doc.querySelector(`[data-segment-tool="${ASSIST_TOOL}"]`); + const assistStatus = $(doc, 'assist-status'); + const assistModes = [...doc.querySelectorAll('[data-assist-mode]')]; + const assistWindow = $(doc, 'assist-window'); + // Masks the worker tracked onto later frames, waiting for the autosaver: timeMs -> {code, plane}. + const tracked = new Map(); + const assistOverlay = $(doc, 'segment-assist-points'); + const promptPoints = createPromptPoints(); + const ASSIST_STATUS = { + connecting: 'Connecting…', + preparing: 'Preparing video…', + ready: 'Ready', + busy: 'Working…', + closed: 'Unavailable', + }; + const SVG_NS = 'http://www.w3.org/2000/svg'; + let includeNext = true; + const assist = + data.canAssist && assistButton + ? createAssistChannel({ + url: assistSocketUrl(doc.defaultView.location, data), + onStatus: describeAssist, + onMask: applyAssistMask, + onError: (text) => say(text, true), + }) + : null; + + function describeAssist(status) { + assistButton.disabled = status !== 'ready' && status !== 'busy'; + if (assistStatus) { + assistStatus.textContent = ASSIST_STATUS[status] ?? ''; + assistStatus.dataset.state = status; + } + if (status === 'closed' && toolId === ASSIST_TOOL) { + say('The assistant is unavailable', true); + doc.querySelector('[data-segment-tool="brush"]')?.click(); + } + } + + function applyAssistMask(message, sent) { + // A prompt's answers count only while the annotator is still where they clicked. + if (!session || !sent || session.frame.frameIndex !== sent.frameIndex || selected !== sent.code) { + return; + } + if (message.frameIndex > sent.frameIndex) { + trackAssistMask(message, sent); + return; + } + const code = maskTarget(sent, message, { frameIndex: session.frame.frameIndex, code: selected }); + if (!code) { + return; + } + try { + const { width, height } = session.frame; + session.applyMask(code, unpackMask(message.mask, message.shape, width, height)); + // The answered points are done: the mask they made steers the next prompt. + promptPoints.forget(sent.frameIndex, sent.code, sent.points); + renderAssistPoints(); + refreshUndoRedoSoon(); + } catch (error) { + say(error.message, true); + } + } + + function renderAssistPoints() { + if (!assistOverlay) { + return; + } + assistOverlay.replaceChildren(); + if (toolId !== ASSIST_TOOL || !session || !selected) { + return; + } + for (const point of promptPoints.for(session.frame.frameIndex, selected)) { + const [cx, cy] = session.canvasPointOf([point.x, point.y]); + const dot = doc.createElementNS(SVG_NS, 'circle'); + dot.setAttribute('cx', String(cx)); + dot.setAttribute('cy', String(cy)); + dot.setAttribute('r', '5'); + dot.setAttribute('class', point.label ? 'is-add' : 'is-remove'); + assistOverlay.appendChild(dot); + } + } + + /** + * A mask tracked onto a later frame: queued on the autosaver (the one writer, so no + * revision conflicts with brush strokes), and only where the label is still empty there. + */ + function trackAssistMask(message, sent) { + if (!saver) { + return; + } + const timeMs = frameTimeMs(message.frameIndex, data.fps); + if (!labelIsEmptyOn(state?.sources?.[String(data.fileId)]?.frames, timeMs, sent.code)) { + return; + } + let plane; + try { + plane = unpackMask(message.mask, message.shape, data.width, data.height); + } catch (error) { + say(error.message, true); + return; + } + if (!hasPixels(plane)) { + return; // tracking lost the object on this frame + } + tracked.set(String(timeMs), { code: sent.code, plane }); + saver.markDirty(timeMs); + } + + /** A tracked frame as the save endpoint wants it: what is stored there, plus the tracked plane. */ + async function trackedFrame(timeMs, { code, plane }) { + const stored = {}; + if (String(timeMs) in (state.sources?.[String(data.fileId)]?.frames ?? {})) { + const saved = await api.frame(data.fileId, timeMs); + for (const [savedCode, encoded] of Object.entries(saved.planes)) { + stored[savedCode] = await decodePlane(encoded, saved.width, saved.height); + } + } + const planes = {}; + for (const [planeCode, merged] of Object.entries(mergeTrackedPlane(stored, code, plane))) { + planes[planeCode] = await encodePlane(merged); + } + return { timeMs, width: data.width, height: data.height, planes }; + } + + function windowFrames() { + const value = Number(assistWindow?.value ?? 0); + return Number.isInteger(value) ? Math.max(0, Math.min(50, value)) : 0; + } + + /** Built when the channel is free: the points still unanswered, the mask as it is now. */ + function buildAssistPrompt() { + if (!session || !selected) { + return null; + } + const { frameIndex } = session.frame; + const points = [...promptPoints.for(frameIndex, selected)]; + if (!points.length) { + return null; + } + const plane = session.labels.readPlane(selected); + return { + payload: { + frameIndex, + window: windowFrames(), + points: points.map((point) => [point.x, point.y]), + labels: points.map((point) => point.label), + ...(hasPixels(plane) ? { mask: packMask(plane) } : {}), + }, + context: { frameIndex, code: selected, points }, + }; + } + function formatTime(seconds) { const wholeSeconds = Math.max(0, Math.floor(seconds || 0)); return `${Math.floor(wholeSeconds / 60)}:${String(wholeSeconds % 60).padStart(2, '0')}`; @@ -167,32 +335,6 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } quadrants?.showNow(); } - /** - * Tick the timeline wherever a frame already has a mask, so "where have I worked" is - * visible without stepping through 187 frames to find out. - */ - function renderMarks(sources) { - if (!timelineMarks) { - return; - } - const frames = sources?.[String(data.fileId)]?.frames ?? {}; - const duration = data.frameCount / data.fps; - timelineMarks.replaceChildren(); - let count = 0; - for (const timeMs of Object.keys(frames)) { - const seconds = Number(timeMs) / 1000; - if (!Number.isFinite(seconds) || seconds < 0 || seconds > duration) { - continue; - } - const tick = doc.createElement('span'); - tick.className = 'video-timeline__mark'; - tick.style.left = `${(seconds / duration) * 100}%`; - timelineMarks.appendChild(tick); - count += 1; - } - timelineMarks.title = count === 1 ? '1 annotated frame' : `${count} annotated frames`; - } - function setTimelineBusy(isBusy) { if (timeline) { timeline.disabled = isBusy; @@ -227,28 +369,12 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } } } - function refreshPicker() { - const picker = $(doc, 'segment-label-select'); - if (!picker || !state) { - return; - } - picker.replaceChildren( - ...paintableLabels(state.labels).map((label) => { - const option = doc.createElement('option'); - option.value = label.code; - option.textContent = label.name; - option.selected = label.code === selected; - return option; - }) - ); - picker.disabled = !picker.options.length; - } function refreshPanel() { if (!labelContainer || !state) { return; } - refreshPicker(); + renderAssistPoints(); renderLabelPanel({ container: labelContainer, labels: state.labels, @@ -256,7 +382,7 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } selected, hidden, opacity: (code) => session?.opacityOf(code) ?? 0.5, - canEdit: Boolean(data.canModify), + canManage: Boolean(data.canManageLabels), // Visibility and opacity act on the open frame; with no session there is none. showDisplay: Boolean(session), }, @@ -353,23 +479,39 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } } async function send(keys, expectedRevision) { - // Only the open frame can be dirty (one session = one frame), so the batch is it. - if (!session || !keys.includes(String(session.frame.timeMs))) { + // The open frame (one session = one frame) and any frames Assist tracked onto. + const frames = []; + const open = session ? String(session.frame.timeMs) : null; + if (open && keys.includes(open)) { + frames.push(await session.collect()); + } + const sentTracked = []; + for (const key of keys) { + const entry = tracked.get(key); + if (entry && key !== open) { + frames.push(await trackedFrame(Number(key), entry)); + sentTracked.push([key, entry]); + } + } + if (!frames.length) { return { revision: expectedRevision }; } - const frame = await session.collect(); const result = await api.save( buildSaveBody({ fileId: data.fileId, revision: expectedRevision, fps: data.fps, - frames: [frame], + frames, }) ); // A frame may be revisited before Annotation Mode closes, so keep the manifest used // by openSession current rather than treating this freshly saved frame as empty. state.sources = result.sources ?? state.sources; - renderMarks(state.sources); + for (const [key, entry] of sentTracked) { + if (tracked.get(key) === entry) { + tracked.delete(key); + } + } return { revision: result.revision }; } @@ -465,10 +607,10 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } const current = frameSource.nearestFrame(playbackTime); const frame = await frameSource.grab(current.frameIndex); state = await api.state(); - renderMarks(state.sources); // Painting needs a label, and "no labels yet" under the image was easy to miss, - // so a project's first annotation session starts with one. - if (data.canModify && !state.labels.some((label) => label.active)) { + // so a project's first annotation session starts with one -- when the user is + // allowed to add labels at all (project admins). + if (data.canManageLabels && !state.labels.some((label) => label.active)) { const { label } = await api.createLabel(FIRST_LABEL_NAME); state.labels.push(label); } @@ -606,14 +748,11 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } } }); updateTimeline(); - // The marks are useful before Annotation Mode is opened too; a failed read just leaves - // the track plain. // The labels are managed from the Labels card at any time, so the vocabulary is read on // load rather than when Annotation Mode opens. void api.state().then( (current) => { state = current; - renderMarks(current.sources); refreshPanel(); }, () => {} @@ -637,6 +776,7 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } }), fileId: data.fileId, canModify: Boolean(data.canModify), + canManage: Boolean(data.canManageLabels), getDurationMs: () => timelineDuration() * 1000, getTimeMs: () => (session ? session.frame.timeMs : video.currentTime * 1000), ask, @@ -660,12 +800,7 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } }); // Toolbar -------------------------------------------------------------------- - $(doc, 'segment-label-select')?.addEventListener('change', (event) => { - selected = event.target.value || null; - session?.selectLabel(selected); - refreshPanel(); - }); - for (const tool of SEGMENT_TOOLS) { + for (const tool of [...SEGMENT_TOOLS, { id: ASSIST_TOOL }]) { const button = doc.querySelector(`[data-segment-tool="${tool.id}"]`); button?.addEventListener('click', () => { toolId = tool.id; @@ -675,6 +810,7 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } } const sizeRow = $(doc, 'segment-size-row'); sizeRow?.classList.toggle('d-none', !tool.hasSize); + renderAssistPoints(); }); } $(doc, 'segment-brush-size')?.addEventListener('input', (event) => { @@ -692,6 +828,23 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } session?.redo(); refreshUndoRedo(); }); + // Ctrl+Z undoes, Ctrl+Y or Ctrl+Shift+Z redoes (Cmd on macOS). Through the buttons, so a + // disabled one does nothing; a text field keeps its own undo. + doc.addEventListener('keydown', (event) => { + if (!session || !(event.ctrlKey || event.metaKey) || event.altKey) { + return; + } + if (event.target?.closest?.('textarea, select, [contenteditable="true"], input:not([type="range"])')) { + return; + } + const key = event.key.toLowerCase(); + const redo = key === 'y' || (key === 'z' && event.shiftKey); + if (key !== 'z' && !redo) { + return; + } + event.preventDefault(); + $(doc, redo ? 'segment-redo' : 'segment-undo')?.click(); + }); // Leaving the page with unsaved work: the browser's own prompt, and a best-effort save. doc.defaultView?.addEventListener('beforeunload', (event) => { @@ -701,5 +854,29 @@ export async function bootstrapSegmentation({ doc, openSession, fetchImpl, ask } } }); + viewportElement.addEventListener('click', (event) => { + if (!assist || toolId !== ASSIST_TOOL || !session || !selected || event.button !== 0) { + return; + } + const { frameIndex, width, height } = session.frame; + const [x, y] = session.imagePointAt(event.clientX, event.clientY); + if (!(x >= 0 && y >= 0 && x < width && y < height)) { + return; + } + promptPoints.add(frameIndex, selected, x, y, includeNext ? 1 : 0); + renderAssistPoints(); + assist.request(buildAssistPrompt); + }); + // coreEnums.Events.CAMERA_MODIFIED: zoom and pan move the points with the frame. + viewportElement.addEventListener('CORNERSTONE_CAMERA_MODIFIED', () => renderAssistPoints()); + for (const button of assistModes) { + button.addEventListener('click', () => { + includeNext = button.dataset.assistMode === 'add'; + for (const other of assistModes) { + other.setAttribute('aria-checked', String(other === button)); + } + }); + } + return { enter, leave, get session() { return session; } }; } diff --git a/frontend/imaging/segment/labelPanel.js b/frontend/imaging/segment/labelPanel.js index b571bff2..561311eb 100644 --- a/frontend/imaging/segment/labelPanel.js +++ b/frontend/imaging/segment/labelPanel.js @@ -66,7 +66,8 @@ function element(doc, tag, props = {}, children = []) { * @param {HTMLElement} options.container emptied and refilled. * @param {Array<{code, name, color, active}>} options.labels * @param {{selected: string|null, hidden: Set, opacity: (code: string) => number, - * canEdit: boolean}} options.view + * canManage: boolean}} options.view `canManage`: add, rename and retire (project admins). + * Anyone may change a colour: it is their own, stored per user by the server. * @param {object} options.handlers `select(code)`, `toggleVisible(code)`, `opacity(code, alpha)`, * `color(code, hex)`, `rename(code)`, `retire(code)`, `add()`. */ @@ -81,7 +82,9 @@ export function renderLabelPanel({ container, labels, view, handlers }) { container.appendChild( element(doc, 'p', { class: 'segment-labels__empty', - text: view.canEdit ? `No ${noun}s yet. Add one to get started.` : `No ${noun}s.`, + text: view.canManage + ? `No ${noun}s yet. Add one to get started.` + : `No ${noun}s yet. Ask a project admin to add one.`, }) ); } @@ -101,9 +104,8 @@ export function renderLabelPanel({ container, labels, view, handlers }) { type: 'color', class: 'segment-label__swatch', value: label.color || '#3498db', - title: view.canEdit ? 'Change colour' : 'Colour', + title: 'Change colour (only you see it)', 'aria-label': `Colour of ${label.name}`, - ...(view.canEdit ? {} : { disabled: 'disabled' }), oninput: (event) => handlers.color(label.code, event.target.value), }), element(doc, 'button', { @@ -157,7 +159,7 @@ export function renderLabelPanel({ container, labels, view, handlers }) { ]) ] : []), - ...(view.canEdit + ...(view.canManage ? [ element( doc, @@ -191,7 +193,7 @@ export function renderLabelPanel({ container, labels, view, handlers }) { container.appendChild(row); } - if (view.canEdit) { + if (view.canManage) { container.appendChild( element( doc, diff --git a/frontend/imaging/segment/quadrants.js b/frontend/imaging/segment/quadrants.js index 503d6633..4802e5aa 100644 --- a/frontend/imaging/segment/quadrants.js +++ b/frontend/imaging/segment/quadrants.js @@ -125,7 +125,8 @@ function element(doc, tag, props = {}) { * @param {Document} options.doc * @param {object} options.api `createQuadrantsApi(...)`. * @param {number} options.fileId the video the markers are filed against. - * @param {boolean} options.canModify + * @param {boolean} options.canModify place and move markers. + * @param {boolean} [options.canManage] add, rename and retire quadrants (project admins). * @param {() => number} options.getDurationMs * @param {() => number} options.getTimeMs the playhead, in ms. * @param {Function} [options.ask] themed text prompt. @@ -139,6 +140,7 @@ export async function mountQuadrants({ api, fileId, canModify, + canManage = false, getDurationMs, getTimeMs, ask, @@ -209,7 +211,7 @@ export async function mountQuadrants({ selected: active, hidden: new Set(), opacity: () => 1, - canEdit: canModify, + canManage, noun: 'quadrant', showDisplay: false, selectTitle: 'Use this quadrant for "Mark here"', @@ -348,7 +350,9 @@ export async function mountQuadrants({ mark.disabled = !active; mark.querySelector('span').textContent = active ? `Mark “${find(active)?.name}” from here` - : 'Add a quadrant in Labels'; + : canManage + ? 'Add a quadrant in Labels' + : 'No quadrants yet — ask a project admin'; } renderBar(); renderLabels(); @@ -563,7 +567,13 @@ export async function mountQuadrants({ if (canModify) { items.push({ heading: 'Start a quadrant here' }); if (!activeLabels().length) { - items.push({ text: 'No quadrants yet — add one in Labels', disabled: true, run: () => {} }); + items.push({ + text: canManage + ? 'No quadrants yet — add one in Labels' + : 'No quadrants yet — ask a project admin', + disabled: true, + run: () => {}, + }); } for (const label of activeLabels()) { items.push({ diff --git a/frontend/imaging/segment/session.js b/frontend/imaging/segment/session.js index 792b5660..d54dce4e 100644 --- a/frontend/imaging/segment/session.js +++ b/frontend/imaging/segment/session.js @@ -22,7 +22,7 @@ import { decodePlane, encodePlane } from './planes.js'; import { createLabelmapStack } from './labelmapStack.js'; -import { ALL_TOOL_NAMES, toolNameFor } from './tools.js'; +import { ALL_TOOL_NAMES, ASSIST_TOOL, toolNameFor } from './tools.js'; import { frameImageId } from './frameImageLoader.js'; /** @@ -201,6 +201,10 @@ export async function openSession({ for (const name of ALL_TOOL_NAMES) { toolGroup.setToolPassive(name); } + // Assist paints nothing itself: its clicks are prompt points for the worker. + if (toolId === ASSIST_TOOL) { + return; + } toolGroup.setToolActive(toolNameFor(toolId), { bindings: [{ mouseButton: toolsEnums.MouseBindings.Primary }], }); @@ -258,6 +262,28 @@ export async function openSession({ opacityOf: (code) => opacity.get(code) ?? DEFAULT_OPACITY, setColor: (code, color) => stack.setColor(code, color), + /** The frame pixel under a page point (an event's clientX/clientY), at any zoom or pan. */ + imagePointAt(clientX, clientY) { + const box = element.getBoundingClientRect(); + const world = viewport.canvasToWorld([clientX - box.left, clientY - box.top]); + return cornerstone.worldToImageCoords(imageId, world); + }, + /** Where a frame pixel is drawn, in the element's CSS pixels. */ + canvasPointOf([x, y]) { + return viewport.worldToCanvas(cornerstone.imageToWorldCoords(imageId, [x, y])); + }, + /** + * Replace a label's mask with the assistant's, as one undoable edit. `writePlane` + * fires the data-modified event, so the saver saves it like a brush stroke. + */ + applyMask(code, plane) { + const before = stack.readPlane(code); + stack.writePlane(code, plane); + HistoryMemo.DefaultHistoryMemo.push({ + restoreMemo: (undo) => stack.writePlane(code, undo ? before : plane), + }); + }, + undo() { HistoryMemo.DefaultHistoryMemo.undo(); onEdit(frame.timeMs); diff --git a/frontend/imaging/segment/tools.js b/frontend/imaging/segment/tools.js index 08e1bb7d..ac6b83d3 100644 --- a/frontend/imaging/segment/tools.js +++ b/frontend/imaging/segment/tools.js @@ -96,6 +96,9 @@ export const SEGMENT_TOOLS = Object.freeze([ { id: 'polygon', label: 'Polygon', fill: 'PolygonFill', erase: null }, ]); +/** The Assist tool: not a Cornerstone tool, its clicks are prompt points (`bootstrap.js`). */ +export const ASSIST_TOOL = 'assist'; + /** Every registered name, for the tool group to add and to switch off. */ export const ALL_TOOL_NAMES = Object.freeze( SEGMENT_TOOLS.flatMap((tool) => [tool.fill, tool.erase]).filter(Boolean) diff --git a/frontend/tests/segmentAssist.test.js b/frontend/tests/segmentAssist.test.js new file mode 100644 index 00000000..3ce29926 --- /dev/null +++ b/frontend/tests/segmentAssist.test.js @@ -0,0 +1,180 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; + +import { + assistSocketUrl, + createAssistChannel, + createPromptPoints, + frameTimeMs, + hasPixels, + labelIsEmptyOn, + maskTarget, + mergeTrackedPlane, + packMask, + unpackMask, +} from '../imaging/segment/assist.js'; + +test('packMask is the worker bitpack_u1_v1: row-major, LSB-first, base64', () => { + // numpy: np.packbits([1,0,0,0,0,0,0,0, 0,1], bitorder="little") == [1, 2] -> "AQI=" + assert.equal(packMask(Uint8Array.from([1, 0, 0, 0, 0, 0, 0, 0, 0, 1])), 'AQI='); +}); + +test('unpackMask inverts packMask on a frame that is not a multiple of 8', () => { + const plane = Uint8Array.from({ length: 37 * 5 }, (_, index) => ((index * 7) % 3 === 0 ? 1 : 0)); + assert.deepEqual(unpackMask(packMask(plane), [5, 37], 37, 5), plane); +}); + +test('unpackMask refuses a mask for another frame size or orientation', () => { + const mask = packMask(new Uint8Array(8 * 6)); + assert.throws(() => unpackMask(mask, [8, 6], 8, 6), /frame/, 'transposed: same byte count'); + assert.throws(() => unpackMask(packMask(new Uint8Array(64)), [8, 9], 9, 8), /frame/); +}); + +test('hasPixels', () => { + assert.equal(hasPixels(new Uint8Array(4)), false); + assert.equal(hasPixels(Uint8Array.from([0, 0, 1, 0])), true); +}); + +test('prompt points start over when the frame or the label changes', () => { + const points = createPromptPoints(); + points.add(4, 'l1', 10, 20, 1); + points.add(4, 'l1', 30, 40, 0); + assert.deepEqual(points.for(4, 'l1'), [{ x: 10, y: 20, label: 1 }, { x: 30, y: 40, label: 0 }]); + assert.deepEqual(points.for(5, 'l1'), [], 'another frame'); + assert.deepEqual(points.for(4, 'l1'), [], 'going back does not bring them back'); + points.add(4, 'l1', 1, 1, 1); + assert.deepEqual(points.for(4, 'l2'), [], 'another label'); +}); + +test('answered points are forgotten; points clicked meanwhile stay for the next prompt', () => { + const points = createPromptPoints(); + points.add(4, 'l1', 10, 20, 1); + const sent = [...points.for(4, 'l1')]; + points.add(4, 'l1', 30, 40, 0); + points.forget(4, 'l1', sent); + assert.deepEqual(points.for(4, 'l1'), [{ x: 30, y: 40, label: 0 }]); +}); + +test('a mask is written only where its prompt was sent from', () => { + const sent = { frameIndex: 4, code: 'l1' }; + const here = { frameIndex: 4, code: 'l1' }; + assert.equal(maskTarget(sent, { frameIndex: 4 }, here), 'l1'); + assert.equal(maskTarget(sent, { frameIndex: 5 }, here), null, 'a frame the prompt was not for'); + assert.equal(maskTarget(sent, { frameIndex: 4 }, { frameIndex: 5, code: 'l1' }), null, 'moved to another frame'); + assert.equal(maskTarget(sent, { frameIndex: 4 }, { frameIndex: 4, code: 'l2' }), null, 'selected another label'); + assert.equal(maskTarget(null, { frameIndex: 4 }, here), null, 'nothing in flight'); +}); + +test('assistSocketUrl follows the page scheme', () => { + const data = { projectNamespace: 'laparoscopy', patientId: 3, fileId: 9 }; + assert.equal(assistSocketUrl({ protocol: 'https:', host: 'h' }, data), 'wss://h/ws/assist/laparoscopy/3/?fileId=9'); + assert.equal(assistSocketUrl({ protocol: 'http:', host: 'localhost:8000' }, data), 'ws://localhost:8000/ws/assist/laparoscopy/3/?fileId=9'); +}); + +class FakeSocket { + constructor(url) { + this.url = url; + this.sent = []; + this.listeners = {}; + FakeSocket.last = this; + } + addEventListener(type, listener) { + (this.listeners[type] ??= []).push(listener); + } + send(data) { + this.sent.push(JSON.parse(data)); + } + close() { + this.emit('close', {}); + } + emit(type, event) { + for (const listener of this.listeners[type] ?? []) listener(event); + } + receive(message) { + this.emit('message', { data: JSON.stringify(message) }); + } +} + +function channel(overrides = {}) { + const seen = { statuses: [], masks: [], errors: [] }; + const assist = createAssistChannel({ + url: 'ws://x', + WebSocketImpl: FakeSocket, + onStatus: (status) => seen.statuses.push(status), + onMask: (message, context) => seen.masks.push([message.frameIndex, context]), + onError: (text) => seen.errors.push(text), + ...overrides, + }); + return { assist, socket: FakeSocket.last, seen }; +} + +test('a prompt is built when the channel is free, from what is still unanswered then', () => { + const { assist, socket, seen } = channel(); + let pending = [1]; + const build = () => (pending.length ? { payload: { points: [...pending] }, context: [...pending] } : null); + assist.request(build); + assert.equal(socket.sent.length, 0, 'nothing before the worker is ready'); + socket.receive({ type: 'preparing' }); + socket.receive({ type: 'ready' }); + assert.deepEqual(socket.sent, [{ type: 'prompt', points: [1] }]); + pending.push(2); + assist.request(build); + pending.push(3); + assist.request(build); + assert.equal(socket.sent.length, 1, 'one prompt in flight'); + socket.receive({ type: 'mask', frameIndex: 0 }); + pending = pending.filter((point) => point !== 1); // the answered point is forgotten + socket.receive({ type: 'done' }); + assert.deepEqual(socket.sent, [{ type: 'prompt', points: [1] }, { type: 'prompt', points: [2, 3] }]); + assert.deepEqual(seen.masks, [[0, [1]]]); + assert.deepEqual(seen.statuses.slice(0, 3), ['preparing', 'ready', 'busy']); +}); + +test('nothing is sent when there is nothing left to ask', () => { + const { assist, socket } = channel(); + socket.receive({ type: 'ready' }); + assist.request(() => null); + assert.equal(socket.sent.length, 0); +}); + +test('an error ends the prompt in flight and is reported', () => { + const { assist, socket, seen } = channel(); + socket.receive({ type: 'ready' }); + assist.request(() => ({ payload: { frameIndex: 1 }, context: 'a' })); + assist.request(() => ({ payload: { frameIndex: 2 }, context: 'b' })); + socket.receive({ type: 'error', message: 'The assistant could not segment this frame.' }); + assert.deepEqual(seen.errors, ['The assistant could not segment this frame.']); + assert.deepEqual(socket.sent.at(-1), { type: 'prompt', frameIndex: 2 }); +}); + +test('a closed socket says so and sends nothing more', () => { + const { assist, socket, seen } = channel(); + socket.receive({ type: 'ready' }); + socket.close(); + assert.equal(assist.status, 'closed'); + assert.equal(seen.statuses.at(-1), 'closed'); + assist.request(() => ({ payload: { frameIndex: 1 }, context: 'a' })); + assert.equal(socket.sent.length, 0); +}); + +test('a tracked frame is stored under the same time key as frameSource uses', () => { + assert.equal(frameTimeMs(7, 1), 7000); + assert.equal(frameTimeMs(1, 3), 333, 'round(n * 1000 / fps)'); +}); + +test('tracking fills only frames where the label is still empty', () => { + const frames = { 3000: { labels: { l1: { bbox: [0, 0, 1, 1] } } }, 4000: { labels: { l2: {} } } }; + assert.equal(labelIsEmptyOn(frames, 3000, 'l1'), false, 'already annotated: kept'); + assert.equal(labelIsEmptyOn(frames, 4000, 'l1'), true, 'another label only'); + assert.equal(labelIsEmptyOn(frames, 5000, 'l1'), true, 'nothing stored'); + assert.equal(labelIsEmptyOn(undefined, 5000, 'l1'), true); +}); + +test('a tracked plane joins the stored planes and never replaces a painted one', () => { + const other = Uint8Array.from([1, 0, 0, 0]); + const tracked = Uint8Array.from([0, 1, 1, 0]); + assert.deepEqual(mergeTrackedPlane({ l2: other }, 'l1', tracked), { l2: other, l1: tracked }); + const painted = Uint8Array.from([0, 0, 0, 1]); + assert.deepEqual(mergeTrackedPlane({ l1: painted }, 'l1', tracked), { l1: painted }); + assert.deepEqual(mergeTrackedPlane({ l1: new Uint8Array(4) }, 'l1', tracked), { l1: tracked }, 'an empty stored plane is replaced'); +}); diff --git a/laparoscopy/management/commands/setup_laparoscopy_modalities.py b/laparoscopy/management/commands/setup_laparoscopy_modalities.py index 4d7aeab7..c4404915 100644 --- a/laparoscopy/management/commands/setup_laparoscopy_modalities.py +++ b/laparoscopy/management/commands/setup_laparoscopy_modalities.py @@ -30,6 +30,17 @@ def handle(self, *args, **options): ) if project_created: project.annotation_methods.add(method) + assist, _ = AnnotationMethod.objects.get_or_create( + slug='assisted_segmentation', + defaults={ + 'name': 'Assisted Segmentation', + 'description': 'Masks from clicked points, painted by a model on the GPU worker.', + 'icon': 'fas fa-wand-magic-sparkles', + 'domain': 'laparoscopy', + }, + ) + if project_created: + project.annotation_methods.add(assist) if project_created: self.stdout.write(self.style.SUCCESS(f'Created project: {project.name}')) diff --git a/laparoscopy/tests.py b/laparoscopy/tests.py index ad4cbc1a..b95490c8 100644 --- a/laparoscopy/tests.py +++ b/laparoscopy/tests.py @@ -1,5 +1,5 @@ from django.contrib.auth.models import User -from django.test import TestCase +from django.test import TestCase, override_settings from django.urls import reverse from common.models import FileRegistry, Project, ProjectAccess @@ -81,6 +81,24 @@ def test_a_patient_without_video_says_so(self): self.assertContains(response, "No video uploaded for this patient.") self.assertNotContains(response, " .btn { border-radius: 0; margin-left: -1px; } +/* The -1px overlap paints each button's right border under its neighbour: lift the + highlighted one above it, as Bootstrap does. */ +.btn-group > .btn { position: relative; } +.btn-group > .btn:hover, .btn-group > .btn:focus-visible, .btn-group > .btn.active { z-index: 1; } .btn-group > .btn:first-child { border-top-left-radius: var(--ygg-radius-md); border-bottom-left-radius: var(--ygg-radius-md); margin-left: 0; } .btn-group > .btn:last-child { border-top-right-radius: var(--ygg-radius-md); border-bottom-right-radius: var(--ygg-radius-md); } .btn-lg { padding: 0.65rem 1.25rem; font-size: 1rem; } diff --git a/static/css/patient_detail.css b/static/css/patient_detail.css index 7fbd17e4..3ab5c73f 100644 --- a/static/css/patient_detail.css +++ b/static/css/patient_detail.css @@ -1982,22 +1982,6 @@ height: 1.9rem; } -.laparoscopy-patient-detail .video-timeline__marks { - position: absolute; - inset: 0 7px; /* the thumb's half-width, so ticks line up with it */ - pointer-events: none; -} - -.laparoscopy-patient-detail .video-timeline__mark { - position: absolute; - top: 50%; - width: 3px; - height: 12px; - margin: -6px 0 0 -1.5px; - border-radius: 2px; - background: var(--ygg-accent2); -} - .laparoscopy-patient-detail .video-timeline__range { position: absolute; inset: 0; @@ -2145,18 +2129,6 @@ grid-column: 1 / -1; } -.laparoscopy-patient-detail .segment-toolbar__label { - display: inline-flex; - align-items: center; - gap: 0.4rem; - font-size: 0.85rem; -} - -.laparoscopy-patient-detail .segment-toolbar__label select { - width: auto; - max-width: 12rem; -} - .laparoscopy-patient-detail .segment-label__swatch { width: 1.25rem; height: 1.25rem; @@ -2597,3 +2569,129 @@ is no longer a place to scroll from: keep it shorter than the screen. */ #scan-viewer { height: min(85vw, 400px); } } + +/* Laparoscopy Tutorial (static/js/laparoscopy_tutorial.js): driver.js's popover on the + theme tokens, so it follows light/dark. The popover is appended to , hence the + popoverClass scope rather than .laparoscopy-patient-detail. */ +.driver-popover.ygg-tour { + --driver-popover-font-family: var(--ygg-font-body); + background: var(--ygg-surface); + color: var(--ygg-text); + border: 1px solid var(--ygg-border); + border-radius: var(--ygg-radius-md); + box-shadow: var(--ygg-shadow-pop); + max-width: 340px; +} +.ygg-tour .driver-popover-title { font-size: 16px; } +.ygg-tour .driver-popover-progress-text, +.ygg-tour .driver-popover-close-btn { color: var(--ygg-text-muted); } +.ygg-tour .driver-popover-close-btn:hover, +.ygg-tour .driver-popover-close-btn:focus { color: var(--ygg-text); } +.ygg-tour .driver-popover-footer-btn { + background: var(--ygg-surface-alt); + color: var(--ygg-text); + border-color: var(--ygg-border-strong); + border-radius: var(--ygg-radius-sm); + padding: 4px 10px; + text-shadow: none; +} +.ygg-tour .driver-popover-footer-btn:hover, +.ygg-tour .driver-popover-footer-btn:focus { background: var(--ygg-surface-3); } +.ygg-tour .driver-popover-next-btn { + background: var(--ygg-primary); + border-color: var(--ygg-primary); + color: var(--ygg-on-primary); +} +.ygg-tour .driver-popover-next-btn:hover, +.ygg-tour .driver-popover-next-btn:focus { background: var(--ygg-primary-hover); } +/* Only the arrow's visible side takes the surface colour; the others stay transparent. */ +.ygg-tour .driver-popover-arrow-side-left { border-left-color: var(--ygg-surface); } +.ygg-tour .driver-popover-arrow-side-right { border-right-color: var(--ygg-surface); } +.ygg-tour .driver-popover-arrow-side-top { border-top-color: var(--ygg-surface); } +.ygg-tour .driver-popover-arrow-side-bottom { border-bottom-color: var(--ygg-surface); } + +/* Assist (assisted segmentation): the prompt points drawn over the frame. */ +.laparoscopy-patient-detail .segment-viewport-wrap { + position: relative; +} +.laparoscopy-patient-detail .segment-assist-points { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + pointer-events: none; + overflow: visible; +} +.laparoscopy-patient-detail .segment-assist-points circle { + stroke: #fff; + stroke-width: 2; +} +.laparoscopy-patient-detail .segment-assist-points .is-add { fill: var(--ygg-accent2); } +.laparoscopy-patient-detail .segment-assist-points .is-remove { fill: var(--ygg-danger); } +.laparoscopy-patient-detail .assist-card__head { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + margin-bottom: 0.55rem; +} +.laparoscopy-patient-detail .assist-card__head .segment-labels-card__title { margin: 0; } +/* The relay's state (bootstrap.js describeAssist): a dot plus one word. */ +.laparoscopy-patient-detail .assist-card__status { + display: inline-flex; + align-items: center; + gap: 5px; + font-size: 0.75rem; + color: var(--ygg-text-muted); +} +.laparoscopy-patient-detail .assist-card__status::before { + content: ""; + width: 7px; + height: 7px; + border-radius: 50%; + background: var(--ygg-warn); +} +.laparoscopy-patient-detail .assist-card__status[data-state="ready"]::before { background: var(--ygg-ok); } +.laparoscopy-patient-detail .assist-card__status[data-state="closed"]::before { background: var(--ygg-danger); } +.laparoscopy-patient-detail .assist-card__row { + display: flex; + align-items: center; + gap: 8px; +} +/* Positive (green) and negative (red) prompt points, matching the dots on the frame. */ +.laparoscopy-patient-detail .assist-mode { + display: flex; + flex: 1; +} +.laparoscopy-patient-detail .assist-mode__btn { + --assist-tone: var(--ygg-accent2); + flex: 1; + border: 1px solid var(--assist-tone); + background: transparent; + color: var(--assist-tone); + padding: 3px 8px; + font-size: 13px; + font-weight: 600; +} +.laparoscopy-patient-detail .assist-mode__btn.is-add { border-radius: var(--ygg-radius-sm) 0 0 var(--ygg-radius-sm); } +.laparoscopy-patient-detail .assist-mode__btn.is-remove { + --assist-tone: var(--ygg-danger); + border-left: 0; + border-radius: 0 var(--ygg-radius-sm) var(--ygg-radius-sm) 0; +} +.laparoscopy-patient-detail .assist-mode__btn[aria-checked="true"] { + background: var(--assist-tone); + color: var(--ygg-surface); +} +.laparoscopy-patient-detail .assist-window { + display: flex; + align-items: center; + gap: 5px; + margin: 0; + font-size: 13px; + color: var(--ygg-text-muted); + white-space: nowrap; +} +.laparoscopy-patient-detail .assist-window input { + width: 3.5rem; +} diff --git a/static/js/laparoscopy_tutorial.js b/static/js/laparoscopy_tutorial.js new file mode 100644 index 00000000..2740b607 --- /dev/null +++ b/static/js/laparoscopy_tutorial.js @@ -0,0 +1,191 @@ +/** + * Laparoscopy patient page — the Tutorial button's guided tour (driver.js, loaded by the + * template from jsDelivr). + * + * One tour: the quadrant timeline and labels, then it opens Annotation Mode itself (the + * "Next" on that step clicks the page's own toggle) and walks the drawing tools. When the + * tour ends it closes Annotation Mode again if it was the one that opened it. Opening it + * is a real entry: on a project with no labels yet the page creates the first one. + */ +(function () { + 'use strict'; + + var button = document.getElementById('tutorial-btn'); + var toggle = document.getElementById('annotation-toggle-btn'); + if (!button || !toggle) { return; } + + var openedByTour = false; + + // The page disables the toggle while Annotation Mode opens or closes. + function whenIdle(callback) { + (function check() { + if (toggle.disabled) { window.setTimeout(check, 100); } else { callback(); } + })(); + } + + function isOpen() { + return toggle.dataset.active === 'true'; + } + + function openAnnotationMode(element, step, opts) { + if (isOpen()) { opts.driver.moveNext(); return; } + if (toggle.disabled) { return; } + openedByTour = true; + toggle.click(); + var next = document.querySelector('.ygg-tour .driver-popover-next-btn'); + if (next) { next.textContent = 'Opening…'; } + whenIdle(function () { + if (!opts.driver.isActive()) { return; } + if (isOpen()) { opts.driver.moveNext(); } else { opts.driver.destroy(); } + }); + } + + var STEPS = [ + { + element: '#video-timeline', + popover: { + title: 'Playback', + description: 'Play and pause, drag the slider to scrub, or jump −10 / −1 / +1 / +10 ' + + 'seconds.' + } + }, + { + element: '#quadrant-timeline', + popover: { + title: 'Quadrant timeline', + description: 'Which abdominal quadrant the camera is in as the procedure goes on. A quadrant ' + + 'starts at its marker and runs until the next one.' + } + }, + { + element: '[data-quadrant-mark]', + popover: { + title: 'Mark a quadrant', + description: 'Pick a quadrant in the list, move the playhead to where the camera enters it, ' + + 'then press Mark.' + } + }, + { + element: '#quadrant-bar', + popover: { + title: 'Fine-tune markers', + description: 'Drag a needle (or focus it and use ← / →) to move a marker. ' + + 'Right-click the bar or a needle to jump to, change or remove one.' + } + }, + { + element: '#labels-card', + popover: { + title: 'Labels', + description: 'Segmentation holds the regions you paint, Quadrants the ones the ' + + 'timeline uses. Click a swatch to pick your own colour (only you see it); project ' + + 'admins add, rename and remove labels.' + } + }, + { + element: '.side-panel', + popover: { + title: 'Files, Settings, Activity', + description: 'Files: the uploaded video, its processed versions and extra uploads. ' + + 'Settings: this scan\'s tags. Activity: what happened to it and when.' + } + }, + { + element: '#annotation-toggle-btn', + popover: { + title: 'Annotation Mode', + description: 'Freezes the frame you are paused on so you can paint masks on it. ' + + 'Next opens it for you.', + onNextClick: openAnnotationMode + } + }, + { + element: '#segment-viewport', + popover: { + title: 'Paint on the frame', + description: 'Left mouse button paints. Right-drag or Ctrl + scroll zooms, middle-drag pans. ' + + 'The buttons at the top right of the toolbar zoom, reset the view and undo / redo ' + + '(Ctrl+Z / Ctrl+Y).' + } + }, + { + element: '#segment-toolbar [aria-label="Segmentation tools"]', + popover: { + title: 'Drawing tools', + description: 'Brush paints the selected label, Eraser removes it. ' + + 'Polygon: click to place points, click the first one (or press Enter) to fill; ' + + 'Backspace removes a point, Esc cancels.' + } + }, + { + element: '#segment-size-row', + popover: { + title: 'Brush size', + description: 'Sets the brush and eraser width. You paint with the label selected in the ' + + 'Labels card.' + } + }, + { + element: '#assist-card', + popover: { + title: 'Assist', + description: 'Choose Segment by clicking, then click the object on the frame: the ' + + 'model paints it into the selected label. Add (green) / Remove (red) ' + + 'set what the next click does; each click refines the mask, and undo works as usual. ' + + 'The result is also tracked onto the next N frames, wherever the label is still empty.' + } + }, + { + element: '#video-timeline', + popover: { + title: 'Other frames', + description: '← / → step one frame; the slider and −10 … +10 jump further.' + } + }, + { + element: '#annotation-toggle-btn', + popover: { + title: 'Saving and leaving', + description: 'Your work saves automatically (the status is under the frame). This button ' + + 'takes you back to the video.' + } + } + ]; + + // In Annotation Mode the page steps frames on ←/→ keydown; driver.js navigates on keyup. + // Swallow the keydown while the tour is open so one key press does not do both. + function swallowArrows(event) { + if (event.key === 'ArrowLeft' || event.key === 'ArrowRight') { + event.stopPropagation(); + } + } + + button.addEventListener('click', function () { + if (!window.driver || !window.driver.js) { + if (typeof window.appNotify === 'function') { + window.appNotify('warning', 'The tutorial could not load. Check your connection and reload the page.'); + } + return; + } + openedByTour = false; + window.addEventListener('keydown', swallowArrows, true); + window.driver.js.driver({ + steps: STEPS, + skipMissingElement: true, // a read-only user has no Mark button + showProgress: true, + progressText: '{{current}} of {{total}}', + nextBtnText: 'Next', + prevBtnText: 'Back', + doneBtnText: 'Done', + popoverClass: 'ygg-tour', + stagePadding: 6, + stageRadius: 8, + onDestroyed: function () { + window.removeEventListener('keydown', swallowArrows, true); + if (openedByTour) { + whenIdle(function () { if (isOpen()) { toggle.click(); } }); + } + } + }).drive(); + }); +})(); diff --git a/static/vendor/cornerstone/ff220b1a/app/chunk-O2OOUNWU.js b/static/vendor/cornerstone/112f296c/app/chunk-3QTYWGYT.js similarity index 100% rename from static/vendor/cornerstone/ff220b1a/app/chunk-O2OOUNWU.js rename to static/vendor/cornerstone/112f296c/app/chunk-3QTYWGYT.js diff --git a/static/vendor/cornerstone/ff220b1a/app/chunk-TTTJJYWU.js b/static/vendor/cornerstone/112f296c/app/chunk-4D2KK7NG.js similarity index 100% rename from static/vendor/cornerstone/ff220b1a/app/chunk-TTTJJYWU.js rename to static/vendor/cornerstone/112f296c/app/chunk-4D2KK7NG.js diff --git a/static/vendor/cornerstone/ff220b1a/app/chunk-EUNK5TYG.js b/static/vendor/cornerstone/112f296c/app/chunk-6K6V66OC.js similarity index 99% rename from static/vendor/cornerstone/ff220b1a/app/chunk-EUNK5TYG.js rename to static/vendor/cornerstone/112f296c/app/chunk-6K6V66OC.js index 28c8bf33..872a957f 100644 --- a/static/vendor/cornerstone/ff220b1a/app/chunk-EUNK5TYG.js +++ b/static/vendor/cornerstone/112f296c/app/chunk-6K6V66OC.js @@ -1,4 +1,4 @@ -import{a as G,b as te}from"/static/vendor/cornerstone/ff220b1a/app/chunk-HPLBK6MT.js";import{e as F}from"/static/vendor/cornerstone/ff220b1a/app/chunk-IVUMTZUA.js";import{b as z,c as ee,d as k}from"/static/vendor/cornerstone/ff220b1a/app/chunk-O2OOUNWU.js";function v({scl_slope:e,scl_inter:n}={}){let t=Number.isFinite(e)&&e!==0,o=Number.isFinite(n);return{rescaleSlope:t?e:1,rescaleIntercept:o?n:0}}function qt(e){return v(e??{})}function Jt(e,{rescaleSlope:n,rescaleIntercept:t}){return e*n+t}function ze(e){let{rescaleSlope:n,rescaleIntercept:t}=v(e??{});return!(n!==1&&t!==0)}function ke(e){return!ze(e)}function Kt(e){return ke(e)?{rescaleSlope:1,rescaleIntercept:0}:v(e??{})}function Qt(e,{rescaleSlope:n,rescaleIntercept:t}){return(e-t)/n}var _=4,x=Object.freeze({AXIAL:"axial",SAGITTAL:"sagittal",CORONAL:"coronal",RENDER:"render"}),Ge=Object.freeze([x.AXIAL,x.SAGITTAL,x.CORONAL]),L=Object.freeze({ORTHOGRAPHIC:"orthographic",VOLUME_3D:"volume3d"}),V=Object.freeze({AXIAL:"axial",SAGITTAL:"sagittal",CORONAL:"coronal"});function en(e){switch(e){case x.AXIAL:return{type:L.ORTHOGRAPHIC,orientation:V.AXIAL};case x.SAGITTAL:return{type:L.ORTHOGRAPHIC,orientation:V.SAGITTAL};case x.CORONAL:return{type:L.ORTHOGRAPHIC,orientation:V.CORONAL};case x.RENDER:return{type:L.VOLUME_3D,orientation:null};default:throw new Error(`Unknown orientation '${e}'.`)}}function M(e){return Ge.includes(e)}var ne=Object.freeze([Object.freeze({window:0,orientation:x.AXIAL,lazy:!1}),Object.freeze({window:1,orientation:x.SAGITTAL,lazy:!1}),Object.freeze({window:2,orientation:x.CORONAL,lazy:!1}),Object.freeze({window:3,orientation:x.RENDER,lazy:!1})]),oe=Object.freeze([Object.freeze({window:0,orientation:x.AXIAL,lazy:!1})]),re=Object.freeze(Array.from({length:4},(e,n)=>Object.freeze({window:n,orientation:x.AXIAL,lazy:!1})));function tn(e=[]){return new Set(e.filter(t=>!t.lazy&&M(t.orientation)).map(t=>t.orientation)).size>=2}function ie(e){return $(e),`ygg-grid-${e}`}var nn="nifti",Be="ygg-volume";function on(e){if(typeof e!="string"||e.length===0)throw new Error("A volume id needs the loader URL it is derived from.");return`${Be}:${e}`}function rn(e){return e===x.RENDER?"ygg-grid-3d":"ygg-grid-2d"}function $(e){if(!Number.isInteger(e)||e<0||e>=4)throw new Error(`Window index must be an integer in 0..3, got ${JSON.stringify(e)}.`)}function an({ViewportType:e,OrientationAxis:n}){let t=[];for(let[o,r]of Object.entries(L))e?.[o]!==r&&t.push(`ViewportType.${o}: expected '${r}', got '${e?.[o]}'`);for(let[o,r]of Object.entries(V))n?.[o]!==r&&t.push(`OrientationAxis.${o}: expected '${r}', got '${n?.[o]}'`);if(t.length)throw new Error(`Cornerstone enum values have changed under frontend/imaging/grid/layout.js: +import{a as G,b as te}from"/static/vendor/cornerstone/112f296c/app/chunk-VX5OPZ5X.js";import{e as F}from"/static/vendor/cornerstone/112f296c/app/chunk-6RMYE36T.js";import{b as z,c as ee,d as k}from"/static/vendor/cornerstone/112f296c/app/chunk-3QTYWGYT.js";function v({scl_slope:e,scl_inter:n}={}){let t=Number.isFinite(e)&&e!==0,o=Number.isFinite(n);return{rescaleSlope:t?e:1,rescaleIntercept:o?n:0}}function qt(e){return v(e??{})}function Jt(e,{rescaleSlope:n,rescaleIntercept:t}){return e*n+t}function ze(e){let{rescaleSlope:n,rescaleIntercept:t}=v(e??{});return!(n!==1&&t!==0)}function ke(e){return!ze(e)}function Kt(e){return ke(e)?{rescaleSlope:1,rescaleIntercept:0}:v(e??{})}function Qt(e,{rescaleSlope:n,rescaleIntercept:t}){return(e-t)/n}var _=4,x=Object.freeze({AXIAL:"axial",SAGITTAL:"sagittal",CORONAL:"coronal",RENDER:"render"}),Ge=Object.freeze([x.AXIAL,x.SAGITTAL,x.CORONAL]),L=Object.freeze({ORTHOGRAPHIC:"orthographic",VOLUME_3D:"volume3d"}),V=Object.freeze({AXIAL:"axial",SAGITTAL:"sagittal",CORONAL:"coronal"});function en(e){switch(e){case x.AXIAL:return{type:L.ORTHOGRAPHIC,orientation:V.AXIAL};case x.SAGITTAL:return{type:L.ORTHOGRAPHIC,orientation:V.SAGITTAL};case x.CORONAL:return{type:L.ORTHOGRAPHIC,orientation:V.CORONAL};case x.RENDER:return{type:L.VOLUME_3D,orientation:null};default:throw new Error(`Unknown orientation '${e}'.`)}}function M(e){return Ge.includes(e)}var ne=Object.freeze([Object.freeze({window:0,orientation:x.AXIAL,lazy:!1}),Object.freeze({window:1,orientation:x.SAGITTAL,lazy:!1}),Object.freeze({window:2,orientation:x.CORONAL,lazy:!1}),Object.freeze({window:3,orientation:x.RENDER,lazy:!1})]),oe=Object.freeze([Object.freeze({window:0,orientation:x.AXIAL,lazy:!1})]),re=Object.freeze(Array.from({length:4},(e,n)=>Object.freeze({window:n,orientation:x.AXIAL,lazy:!1})));function tn(e=[]){return new Set(e.filter(t=>!t.lazy&&M(t.orientation)).map(t=>t.orientation)).size>=2}function ie(e){return $(e),`ygg-grid-${e}`}var nn="nifti",Be="ygg-volume";function on(e){if(typeof e!="string"||e.length===0)throw new Error("A volume id needs the loader URL it is derived from.");return`${Be}:${e}`}function rn(e){return e===x.RENDER?"ygg-grid-3d":"ygg-grid-2d"}function $(e){if(!Number.isInteger(e)||e<0||e>=4)throw new Error(`Window index must be an integer in 0..3, got ${JSON.stringify(e)}.`)}function an({ViewportType:e,OrientationAxis:n}){let t=[];for(let[o,r]of Object.entries(L))e?.[o]!==r&&t.push(`ViewportType.${o}: expected '${r}', got '${e?.[o]}'`);for(let[o,r]of Object.entries(V))n?.[o]!==r&&t.push(`OrientationAxis.${o}: expected '${r}', got '${n?.[o]}'`);if(t.length)throw new Error(`Cornerstone enum values have changed under frontend/imaging/grid/layout.js: `+t.join(` `)+` Update the inlined constants and the tests that pin them.`)}var Ue=1e5;function sn(){return{minimal:{enabled:!0,lineLengthInPx:Ue},mobile:{enabled:!1}}}function un(e=[]){let n=Array.from({length:4},(t,o)=>({index:o,orientation:x.AXIAL,modality:null,fileId:null,volumeId:null,loading:!1,error:null,loadGeneration:0,freeScroll:!1,lazy:!1,orientationWarning:null}));for(let t of e)$(t.window),n[t.window].orientation=t.orientation,n[t.window].lazy=!!t.lazy;return{windows:n}}function S(e,n){return $(n),e.windows[n]}function dn(e,n,{modality:t=null,fileId:o=null,volumeId:r=null}={}){let i=S(e,n);return i.loadGeneration+=1,i.loading=!0,i.error=null,i.orientationWarning=null,i.modality=t,i.fileId=o,i.volumeId=r,i.loadGeneration}function ae(e,n,t){return S(e,n).loadGeneration===t}function fn(e,n,t,{orientationWarning:o=null}={}){if(!ae(e,n,t))return!1;let r=S(e,n);return r.loading=!1,r.error=null,r.orientationWarning=o,!0}function mn(e,n,t,o){if(!ae(e,n,t))return!1;let r=S(e,n);return r.loading=!1,r.error=o,!0}function pn(e,n,t){if(!Object.values(x).includes(t))throw new Error(`Unknown orientation '${t}'.`);S(e,n).orientation=t}function gn(e,n,t){S(e,n).freeScroll=!!t}function hn(e,n){let t=S(e,n);return t.freeScroll||!M(t.orientation)?[]:e.windows.filter(o=>o.index!==n&&!o.freeScroll&&M(o.orientation)&&o.volumeId!==null).map(o=>o.index)}function yn(e,n){let t=S(e,n);return t.freeScroll||t.loading||t.volumeId===null||!M(t.orientation)?[]:We(e,t.orientation).filter(o=>o!==n&&!S(e,o).loading)}function xn(e){let n=e.windows.filter(o=>o.volumeId!==null);if(n.length<2)return[];let[t]=n;return n.some(o=>o.volumeId!==t.volumeId)?[]:n.map(o=>o.index)}function We(e,n){return e.windows.filter(t=>t.orientation===n&&!t.freeScroll&&t.volumeId!==null).map(t=>t.index)}function wn(e){return[...new Set(e.windows.map(n=>n.volumeId).filter(Boolean))]}function Xe(){return{done:!1,framesLoaded:0,framesProcessed:0,totalNumFrames:0,failures:[]}}function He(e,n){let t=Number(n?.totalNumFrames)||e.totalNumFrames,o=Number(n?.framesProcessed)||0,r=n?.success===!1?[...e.failures,{imageId:n.imageId,error:String(n.error??"unknown")}]:e.failures;return{totalNumFrames:t,framesProcessed:Math.max(e.framesProcessed,o),framesLoaded:Math.max(e.framesLoaded,Number(n?.framesLoaded)||0),failures:r,done:t>0&&o>=t}}function Ye(e){if(!e.done)return{ok:!1,message:`The volume did not finish loading: ${e.framesProcessed} of ${e.totalNumFrames||"an unknown number of"} frames were processed.`};if(e.failures.length){let[n]=e.failures;return{ok:!1,message:`${e.failures.length} of ${e.totalNumFrames} frames failed to load (first: ${n.imageId??"unknown image"} -- ${n.error}). A volume with missing frames must not be measured: the gaps read as data.`}}return{ok:!0,message:null}}function se(e,{timeoutMs:n=3e5}={}){return new Promise((t,o)=>{let r=Xe(),i=!1,a=setTimeout(()=>{i||(i=!0,o(new Error(`The volume did not finish loading within ${Math.round(n/1e3)}s (${r.framesProcessed} of ${r.totalNumFrames} frames). Refusing to measure a partially loaded volume.`)))},n),l=()=>{i=!0,clearTimeout(a);let s=Ye(r);s.ok?t(r):o(new Error(s.message))};if(e.load(s=>{i||(r=He(r,s),r.done&&l())}),!i&&e.loadStatus?.loaded){let s=e.imageIds?.length??0;r={...r,done:!0,totalNumFrames:s,framesProcessed:s},l()}})}function le(e){let n=e?.voxelManager;if(!n)throw new Error("The volume exposes no voxel manager; it has not finished loading.");let t=n.getCompleteScalarDataArray?n.getCompleteScalarDataArray():n.getScalarData?.();if(!t)throw new Error("The volume exposes no scalar data.");let o=qe(e);if(t.length===0)throw new Error("The volume cached no voxels at all. `getCompleteScalarDataArray` returns an empty array when no slice has data yet, so this is a read before the load completed rather than an empty study.");if(o&&t.length!==o)throw new Error(`The volume cached ${t.length} voxels but its dimensions imply ${o}. Measuring a short array would silently treat the missing tail as zeros.`);return t}function qe(e){let n=e?.dimensions;return!Array.isArray(n)&&!ArrayBuffer.isView(n)?0:Number(n[0])*Number(n[1])*Number(n[2])}async function En(e){let n=globalThis.nifti;if(!n)throw new Error("The vendored nifti-reader is not loaded; orientation cannot be checked.");let t=await fetch(e,{credentials:"same-origin",headers:{Range:"bytes=0-1023"}});if(!t.ok&&t.status!==206)throw new Error(`HTTP ${t.status} fetching the volume header.`);let o=await t.arrayBuffer(),r=n.isCompressed(o)?n.decompress(o):o;return n.readHeader(r)}var ce=Object.freeze([Object.freeze(["L","R"]),Object.freeze(["P","A"]),Object.freeze(["I","S"])]),ue=1e-6;function de(e){return typeof e=="number"&&Number.isFinite(e)}function me(e){let[n,t,o]=[e[0][0],e[0][1],e[0][2]],[r,i,a]=[e[1][0],e[1][1],e[1][2]],[l,s,c]=[e[2][0],e[2][1],e[2][2]];return n*(i*c-a*s)-t*(r*c-a*l)+o*(r*s-i*l)}function B(e){if(!e||e.length<4)return!1;for(let t=0;t<3;t+=1){let o=e[t];if(!o||o.length<4)return!1;for(let r=0;r<4;r+=1)if(!de(o[r]))return!1}let n=me(e);return de(n)&&Math.abs(n)>=1e-9}function Je(e){if(!B(e))return null;let n=[];for(let t=0;t<3;t+=1){let o=[e[0][t],e[1][t],e[2][t]],r=0;for(let i=1;i<3;i+=1)Math.abs(o[i])>Math.abs(o[r])&&(r=i);if(o[r]===0)return null;n.push(o[r]>0?ce[r][1]:ce[r][0])}return n.join("")}function Ke(e){let n=[e[0][3],e[1][3],e[2][3]],t=[0,1,2].map(r=>Math.sqrt(e[0][r]**2+e[1][r]**2+e[2][r]**2)),o=[];for(let r=0;r<3;r+=1)for(let i=0;i<3;i+=1)o.push(e[r][i]/t[i]);return{origin:n,orientation:o,spacing:t}}function Qe(e){let{origin:n,orientation:t,spacing:o}=Ke(e);return{origin:[-n[0],-n[1],n[2]],direction:[-t[0],-t[3],t[6],-t[1],-t[4],t[7],-t[2],-t[5],t[8]],spacing:o}}function Ze(e,n){let[t,o,r]=n;return[0,1,2].map(i=>e[i][0]*t+e[i][1]*o+e[i][2]*r+e[i][3])}function An(e,n){let[t,o,r]=Ze(e,n);return[-t,-o,r]}function et(e){let n=[0,1,2].map(o=>e.slice(o*3,o*3+3)),t=0;for(let o=0;o<3;o+=1)for(let r=o;r<3;r+=1){let i=n[o][0]*n[r][0]+n[o][1]*n[r][1]+n[o][2]*n[r][2];t=Math.max(t,Math.abs(i-(o===r?1:0)))}return t}function tt(e){let n=Number(e?.qform_code)|0,t=Number(e?.sform_code)|0;return n>=1||t>=1}function nt(e){let n=[1,2,3].map(t=>{let o=Number(e?.[t]);return Number.isFinite(o)&&o>0?o:1});return[[n[0],0,0,0],[0,n[1],0,0],[0,0,n[2],0],[0,0,0,1]]}function pe(e){let n=tt(e),t=[],o=e?.affine??null;if(n||(t.push("qform_code and sform_code are both 0: the orientation is inferred from pixel dimensions and may be mirrored (finding F2)."),B(o)||(o=nt(e?.pixDims))),!B(o))return t.push("The affine is degenerate or unreadable."),{affine:o??null,declared:n,hasMetadata:!1,axcodes:null,determinant:NaN,handedness:0,origin:[NaN,NaN,NaN],direction:new Array(9).fill(NaN),spacing:[NaN,NaN,NaN],dimensions:fe(e),orthonormalityDefect:NaN,issues:t};let r=Je(o);r===null&&t.push("The affine could not be mapped to anatomical axes.");let{origin:i,direction:a,spacing:l}=Qe(o),s=me(o),c=et(a);return c>ue&&t.push(`Direction cosines are not orthonormal to ${ue} (worst deviation ${c.toExponential(3)}).`),{affine:o,declared:n,hasMetadata:n&&r!==null,axcodes:r,determinant:s,handedness:Math.sign(s),origin:i,direction:a,spacing:l,dimensions:fe(e),orthonormalityDefect:c,issues:t}}function fe(e){let n=e?.dims;return!Array.isArray(n)&&!ArrayBuffer.isView(n)?[0,0,0]:[1,2,3].map(t=>Number(n[t])|0)}var ot="patient_lps_mm",ge="Crosshairs";function rt(e){let n=pe(e),{rescaleSlope:t,rescaleIntercept:o}=v(e);return{shape:n.dimensions,spacing_mm:n.spacing,affine:n.affine,orientation:n.axcodes,qform_code:Number(e?.qform_code)|0,sform_code:Number(e?.sform_code)|0,spatial_codes_absent:!n.declared,scl_slope:t,scl_inter:o,recorded_by:"volume-grid"}}function he({fileId:e,bundleKey:n,annotations:t,header:o,expectedRevision:r}){if(!Number.isInteger(e)||e<=0)throw new Error(`fileId must be a positive integer, got ${JSON.stringify(e)}.`);return ee(t,r),{fileId:e,...n&&n!=="primary"?{fileKey:n}:{},expectedRevision:r,coordinateSystem:ot,volumeDescriptor:rt(o),annotations:t}}var it=Object.freeze({resetView:"resetCBCTView",save:"cbctSaveMeasurements",clear:"cbctClearMeasurements",expand3D:"cbctExpand3D",annotationMode:"cbctAnnotationMode",annotationTools:"cbctAnnotationTools",status:"cbctRenderStatus"}),at="is-3d-expanded",st="[data-ygg-tool]";function lt(e){let n={};for(let[t,o]of Object.entries(it))n[t]=e.getElementById(o);return n.tools=Array.from(e.querySelectorAll(st)),n}function j(e,n){for(let t of e){let o=t.dataset.yggTool===n;t.setAttribute("aria-pressed",String(o)),t.classList?.toggle("is-active",o)}}var W="Measurements saved.",ct="Remove every measurement drawn on this study? They come back if you reload without saving.",X="Measurements removed. Save to make it permanent.";function ut(e,n){let t=globalThis.appNotify;typeof t=="function"?t(e,n):console.info(`[ygg-grid] ${e}: ${n}`)}function dt(e){return e?.annotationMode?.getAttribute?.("aria-checked")==="true"}function U({plan:e,enabled:n}){let t=!!n;if(e.annotationMode){e.annotationMode.setAttribute("aria-checked",String(t));let o=e.annotationMode.querySelector?.("[data-mode-state]");o&&(o.textContent=t?"on":"off")}e.annotationTools&&(e.annotationTools.hidden=!t)}var xe="ygg:annotation-mode:";function ft(){try{return globalThis.localStorage??null}catch{return null}}function mt(e,n){if(!e||!n)return null;try{let t=n.getItem(xe+e);return t==="on"?!0:t==="off"?!1:null}catch{return null}}function pt(e,n,t){if(!(!e||!n))try{n.setItem(xe+e,t?"on":"off")}catch{}}function we({grid:e,doc:n=globalThis.document,onSave:t,onClear:o,onAnnotationMode:r,annotationsOn:i=!1,rememberAs:a=null,storage:l=ft(),notify:s=ut}){let c=lt(n),m=[],h=d=>{c.status&&(c.status.textContent=d||"")},g=(d,p)=>async u=>{try{await p(u)}catch(f){h(`${d} failed: ${f.message}`)}};c.resetView&&(c.resetView.addEventListener("click",g("Reset view",()=>{e.resetCameras(),h("")})),m.push("resetView"));for(let d of c.tools){let p=d.dataset.yggTool;d.addEventListener("click",g(p,()=>{e.setPrimaryTool(p),j(c.tools,p),h("")}))}if(c.tools.length&&m.push("tools"),c.expand3D&&(c.expand3D.addEventListener("click",g("Expand 3D",()=>{let p=c.expand3D.ownerDocument.querySelector(".viewer-grid")?.classList.toggle(at)??!1;c.expand3D.setAttribute("aria-pressed",String(p)),e.resize?.()})),m.push("expand3D")),c.annotationMode){let p=mt(a,l)??i===!0;U({plan:c,enabled:p}),r?.(p),c.annotationMode.addEventListener("click",g("Annotation mode",()=>{let u=!dt(c);U({plan:c,enabled:u}),pt(a,l,u),r?.(u),u||j(c.tools,e.navigationTool??ge),h("")})),m.push("annotationMode")}return c.clear&&o&&(c.clear.addEventListener("click",async()=>{let d=c.clear.ownerDocument?.defaultView??globalThis;if(!(d.confirm&&!d.confirm(ct)))try{let p=await o();s(p?.level??"success",p?.message??X)}catch(p){s("danger",`Could not clear the measurements: ${p.message}`)}}),m.push("clear")),c.save&&t&&(c.save.addEventListener("click",async()=>{c.save.setAttribute("aria-busy","true");try{let d=await t();s(d?.level??"success",d?.message??W)}catch(d){s("danger",`The save failed: ${d.message}`)}finally{c.save.removeAttribute("aria-busy")}}),m.push("save")),{bound:m,setStatus:h,plan:c,setAnnotationMode:d=>{U({plan:c,enabled:d}),r?.(d)}}}function be(e){let n=e?.ownerDocument;if(!n)return{update:()=>{},done:()=>{}};let t=n.createElement("div");t.className="ygg-loading",t.setAttribute("role","status"),t.setAttribute("aria-live","polite");let o=n.createElement("div");o.className="ygg-loading__spinner",t.appendChild(o);let r=n.createElement("div");return r.className="ygg-loading__text",r.textContent="Loading volume\u2026",t.appendChild(r),e.appendChild(t),{update(i,a){if(a)r.textContent=a;else if(Number.isFinite(i)){let l=Math.min(100,Math.max(0,Math.round(i*100)));r.textContent=`Loading volume\u2026 ${l}%`}},done(){t.remove()}}}var R=Object.freeze({root:"ygg-orient",button:"ygg-orient__btn",link:"ygg-orient__link"}),gt=Object.freeze([Object.freeze({orientation:x.AXIAL,letter:"A",title:"Axial view"}),Object.freeze({orientation:x.SAGITTAL,letter:"S",title:"Sagittal view"}),Object.freeze({orientation:x.CORONAL,letter:"C",title:"Coronal view"})]),ye=Object.freeze({linked:Object.freeze({icon:"fa-link",title:"Scrolls with the other windows on this plane. Click to scroll this one on its own."}),free:Object.freeze({icon:"fa-link-slash",title:"Scrolls on its own. Click to re-sync it with the other windows on this plane."})});function Ee(e,n){if(!e)return;let t=n?ye.free:ye.linked;e.setAttribute("aria-pressed",String(!!n)),e.classList?.toggle("is-active",!!n),e.title=t.title,e.setAttribute("aria-label",t.title),e.icon&&(e.icon.className=`fas ${t.icon}`)}function ht(e,n){for(let t of e){let o=t.dataset.yggOrientation===n;t.setAttribute("aria-pressed",String(o)),t.classList?.toggle("is-active",o)}}function yt(e,{doc:n=e?.ownerDocument,freeScroll:t=!1}={}){if(!e||!n)return null;e.querySelector?.(`.${R.root}`)?.remove?.();let o=n.createElement("div");o.className=R.root,o.setAttribute("role","group"),o.setAttribute("aria-label","View plane");let r=gt.map(a=>{let l=n.createElement("button");return l.type="button",l.className=R.button,l.dataset.yggOrientation=a.orientation,l.title=a.title,l.setAttribute("aria-label",a.title),l.textContent=a.letter,o.appendChild(l),l}),i=null;if(t){i=n.createElement("button"),i.type="button",i.className=`${R.button} ${R.link}`,i.dataset.yggFreeScroll="";let a=n.createElement("i");a.setAttribute("aria-hidden","true"),i.appendChild(a),i.icon=a,Ee(i,!1),o.appendChild(i)}for(let a of["mousedown","pointerdown"])o.addEventListener(a,l=>l.stopPropagation());return o.addEventListener("contextmenu",a=>{a.stopPropagation(),a.preventDefault()}),e.appendChild(o),{root:o,buttons:r,freeScrollButton:i}}function Oe({grid:e,elements:n=[],doc:t=globalThis.document,setStatus:o=()=>{},windowOrientation:r,windowFreeScroll:i}){let a=new Map,l=typeof i=="function"&&n.length>1,s=c=>{let m=a.get(c);m&&(ht(m.buttons,r(c)),l&&Ee(m.freeScrollButton,i(c)))};for(let[c,m]of n.entries()){let h=yt(m,{doc:t,freeScroll:l});if(h){a.set(c,h),s(c);for(let g of h.buttons){let E=g.dataset.yggOrientation;g.addEventListener("click",async()=>{try{await e.setWindowOrientation(c,E),o("")}catch(d){o(`View plane failed: ${d.message}`)}s(c)})}h.freeScrollButton?.addEventListener("click",()=>{try{e.setFreeScroll(c,!i(c)),o("")}catch(g){o(`Free scroll failed: ${g.message}`)}s(c)})}}return{controls:a,refresh:s}}var xt=".modality-chip[data-modality]",D="drag-over",Ae="is-armed";var Se="application/json",Te="text/plain";function C(e){return!e||typeof e.closest!="function"?null:e.closest(".viewer-window[data-window-index]")}function Ie({doc:e,elements:n,onDrop:t}){let o=Array.from(e.querySelectorAll(xt)),r=[],i=(a,l,s,c=!1)=>{a.addEventListener(l,s,c),r.push(()=>a.removeEventListener(l,s,c))};for(let a of o)i(a,"dragstart",l=>{let s=a.dataset.modality;l.dataTransfer.setData(Te,s),l.dataTransfer.setData(Se,JSON.stringify({modality:s})),l.dataTransfer.effectAllowed="copy",a.classList.add("is-dragging")}),i(a,"dragend",()=>a.classList.remove("is-dragging"));wt({doc:e,chips:o,elements:n,onDrop:t,on:i});for(let a of n)a&&(i(a,"dragover",l=>{let s=C(l.target)||a;l.preventDefault(),l.dataTransfer.dropEffect="copy",s.classList.add(D)},!0),i(a,"dragleave",l=>{let s=C(l.target)||a;l.relatedTarget&&s.contains(l.relatedTarget)||s.classList.remove(D)},!0),i(a,"drop",l=>{let s=C(l.target)||a;l.preventDefault(),s.classList.remove(D);let c=bt(l.dataTransfer),m=Number(s.dataset.windowIndex);!c||!Number.isInteger(m)||t(m,c)},!0));return()=>r.forEach(a=>a())}function wt({doc:e,chips:n,elements:t,onDrop:o,on:r}){let i=t.filter(Boolean),a=null,l=null,s=null,c=d=>d.pointerType==="touch"||d.pointerType==="pen",m=(d,p)=>{let u=C(e.elementFromPoint?.(d,p)??null);return u&&i.includes(u)?u:null},h=d=>{s!==d&&(s?.classList.remove(D),s=d,s?.classList.add(D))},g=()=>{a?.classList.remove(Ae),a=null},E=(d,p)=>{let u=Number(d.dataset.windowIndex);p&&Number.isInteger(u)&&o(u,p)};for(let d of n){r(d,"pointerdown",u=>{c(u)&&(l={chip:d,pointerId:u.pointerId,x:u.clientX,y:u.clientY,dragging:!1},d.setPointerCapture?.(u.pointerId))}),r(d,"pointermove",u=>{if(!l||l.chip!==d||u.pointerId!==l.pointerId)return;let f=Math.hypot(u.clientX-l.x,u.clientY-l.y);!l.dragging&&f>=8&&(l.dragging=!0,g(),d.classList.add("is-dragging")),l.dragging&&(u.preventDefault?.(),h(m(u.clientX,u.clientY)))});let p=(u,f)=>{if(!l||l.chip!==d||u.pointerId!==l.pointerId)return;let{dragging:w}=l;if(l=null,d.classList.remove("is-dragging"),h(null),f)return;if(w){let y=m(u.clientX,u.clientY);y&&E(y,d.dataset.modality);return}let b=a===d;g(),b||(a=d,d.classList.add(Ae))};r(d,"pointerup",u=>p(u,!1)),r(d,"pointercancel",u=>p(u,!0))}for(let d of i)r(d,"pointerup",p=>{if(!a||!c(p))return;let u=C(p.target)||d,f=a.dataset.modality;g(),E(u,f)},!0)}function bt(e){try{let t=JSON.parse(e.getData(Se));if(t?.modality)return String(t.modality)}catch{}let n=e.getData(Te);return n?String(n):null}var Et=" // Find maximum/minimum intensity along the ray.",Ot=[" vec4 previousSelectedValue = selectedValue;"," vec4 currentValue = getTextureValue(posIS);"," selectedValue = OP(selectedValue, currentValue);"," if (previousSelectedValue != selectedValue) {"," selectedPosVC = posVC;"," selectedPosIS = posIS;"," }"].join(` diff --git a/static/vendor/cornerstone/ff220b1a/app/chunk-WHYFTSVR.js b/static/vendor/cornerstone/112f296c/app/chunk-6NRANYVL.js similarity index 99% rename from static/vendor/cornerstone/ff220b1a/app/chunk-WHYFTSVR.js rename to static/vendor/cornerstone/112f296c/app/chunk-6NRANYVL.js index d26f843b..24633b68 100644 --- a/static/vendor/cornerstone/ff220b1a/app/chunk-WHYFTSVR.js +++ b/static/vendor/cornerstone/112f296c/app/chunk-6NRANYVL.js @@ -1,4 +1,4 @@ -import{a as x,b as y,c as w,d as E}from"/static/vendor/cornerstone/ff220b1a/app/chunk-IVUMTZUA.js";var v="yggweb",b=255;function D({fileId:t,filename:e,namespace:a="api",origin:o}={}){y(e);let n=E(w(x({fileId:t,filename:e,namespace:a}),{origin:o}));return`${v}:${n}`}function P(t){if(typeof t!="string")throw new Error("imageId must be a string.");let e=`${v}:`;if(!t.startsWith(e))throw new Error(`${JSON.stringify(t)} is not a ${e} imageId. This loader is registered for that scheme only.`);let a=t.slice(e.length);if(!a)throw new Error("A yggweb: imageId must carry a URL.");return a}function L({imageId:t,width:e,height:a,pixelData:o,numberOfComponents:n,voxelManagerFactory:r}){if(!Number.isInteger(e)||!Number.isInteger(a)||e<=0||a<=0)throw new Error(`A web image needs positive integer dimensions, got ${e}x${a}.`);if(n!==1&&n!==3)throw new Error(`numberOfComponents must be 1 (greyscale) or 3 (RGB), got ${n}. Four would leave vtk.js rendering the alpha channel as a fourth band.`);let i=e*a*n;if(o?.length!==i)throw new Error(`pixelData is ${o?.length} values; ${e}x${a} at ${n} components needs ${i}. A short buffer renders as a partly black image rather than as an error.`);let s=n>1,g=r({scalarData:o,width:e,height:a,numberOfComponents:n});return{imageId:t,rows:a,columns:e,height:a,width:e,color:s,rgba:!1,numberOfComponents:n,photometricInterpretation:s?"RGB":"MONOCHROME2",minPixelValue:0,maxPixelValue:b,slope:1,intercept:0,windowCenter:(b+1)/2,windowWidth:b+1,voiLUTFunction:"LINEAR",invert:!1,rowPixelSpacing:null,columnPixelSpacing:null,sizeInBytes:o.byteLength,voxelManager:g,getPixelData:()=>g.getScalarData(),getCanvas:()=>{}}}function M(t,e){if(t.length!==e*4)throw new Error(`Expected ${e*4} RGBA values, got ${t.length}.`);let a=!0;for(let n=0;nglobalThis.fetch(...r)),n=a??A;return function(i){let s=new AbortController,g=P(i);return{promise:(async()=>{let l=await o(g,{credentials:"same-origin",signal:s.signal});if(!l.ok)throw new Error(`HTTP ${l.status} loading ${g}`);let{width:d,height:p,rgba:m}=await n(await l.blob()),{pixelData:f,numberOfComponents:u}=M(m,d*p);return L({imageId:i,width:d,height:p,pixelData:f,numberOfComponents:u,voxelManagerFactory:t})})(),cancelFn:()=>s.abort()}}}async function A(t){let e=await globalThis.createImageBitmap(t);try{let o=new globalThis.OffscreenCanvas(e.width,e.height).getContext("2d",{willReadFrequently:!0});o.drawImage(e,0,0);let{data:n}=o.getImageData(0,0,e.width,e.height);return{width:e.width,height:e.height,rgba:n}}finally{e.close?.()}}var U=Object.freeze(["imagePlaneModule","imagePixelModule","generalSeriesModule","calibratedPixelSpacing"]),C="User";function I(t){let e={rows:t.height,columns:t.width,rowCosines:[1,0,0],columnCosines:[0,1,0],imagePositionPatient:[0,0,0],imageOrientationPatient:[1,0,0,0,1,0]},a=t.pixelSpacingMm;return a?{...e,rowPixelSpacing:a.y_mm,columnPixelSpacing:a.x_mm,pixelSpacing:[a.y_mm,a.x_mm]}:e}function $(t){let e=t.numberOfComponents>1;return{samplesPerPixel:e?3:1,photometricInterpretation:e?"RGB":"MONOCHROME2",rows:t.height,columns:t.width,bitsAllocated:8,bitsStored:8,highBit:7,pixelRepresentation:0,planarConfiguration:e?0:void 0,windowCenter:128,windowWidth:256}}function k(t){return{modality:"OT",seriesInstanceUID:void 0,studyInstanceUID:void 0,seriesNumber:t.index??void 0,seriesDescription:t.description??""}}function N(t){let e=t.pixelSpacingMm;if(e)return{rowPixelSpacing:e.y_mm,columnPixelSpacing:e.x_mm,type:C}}var S=new Set,O=Object.freeze({get(t){for(let e of S){let a=e.get(t);if(a)return a}}});function q(t){return S.add(t),t}function B(t=O){return function(a,o){if(typeof o!="string")return;let n=t.get(o);if(n)switch(a){case"imagePlaneModule":return I(n);case"imagePixelModule":return $(n);case"generalSeriesModule":return k(n);case"calibratedPixelSpacing":return N(n);default:return}}}var R="ygg-photo-dialog";function h(t,{title:e,icon:a,body:o,confirmLabel:n,cancelLabel:r}){let i=t.createElement("div");i.className=`modal ${R}`,i.setAttribute("tabindex","-1"),i.setAttribute("role","dialog"),i.setAttribute("aria-modal","true"),i.innerHTML=` +import{a as x,b as y,c as w,d as E}from"/static/vendor/cornerstone/112f296c/app/chunk-6RMYE36T.js";var v="yggweb",b=255;function D({fileId:t,filename:e,namespace:a="api",origin:o}={}){y(e);let n=E(w(x({fileId:t,filename:e,namespace:a}),{origin:o}));return`${v}:${n}`}function P(t){if(typeof t!="string")throw new Error("imageId must be a string.");let e=`${v}:`;if(!t.startsWith(e))throw new Error(`${JSON.stringify(t)} is not a ${e} imageId. This loader is registered for that scheme only.`);let a=t.slice(e.length);if(!a)throw new Error("A yggweb: imageId must carry a URL.");return a}function L({imageId:t,width:e,height:a,pixelData:o,numberOfComponents:n,voxelManagerFactory:r}){if(!Number.isInteger(e)||!Number.isInteger(a)||e<=0||a<=0)throw new Error(`A web image needs positive integer dimensions, got ${e}x${a}.`);if(n!==1&&n!==3)throw new Error(`numberOfComponents must be 1 (greyscale) or 3 (RGB), got ${n}. Four would leave vtk.js rendering the alpha channel as a fourth band.`);let i=e*a*n;if(o?.length!==i)throw new Error(`pixelData is ${o?.length} values; ${e}x${a} at ${n} components needs ${i}. A short buffer renders as a partly black image rather than as an error.`);let s=n>1,g=r({scalarData:o,width:e,height:a,numberOfComponents:n});return{imageId:t,rows:a,columns:e,height:a,width:e,color:s,rgba:!1,numberOfComponents:n,photometricInterpretation:s?"RGB":"MONOCHROME2",minPixelValue:0,maxPixelValue:b,slope:1,intercept:0,windowCenter:(b+1)/2,windowWidth:b+1,voiLUTFunction:"LINEAR",invert:!1,rowPixelSpacing:null,columnPixelSpacing:null,sizeInBytes:o.byteLength,voxelManager:g,getPixelData:()=>g.getScalarData(),getCanvas:()=>{}}}function M(t,e){if(t.length!==e*4)throw new Error(`Expected ${e*4} RGBA values, got ${t.length}.`);let a=!0;for(let n=0;nglobalThis.fetch(...r)),n=a??A;return function(i){let s=new AbortController,g=P(i);return{promise:(async()=>{let l=await o(g,{credentials:"same-origin",signal:s.signal});if(!l.ok)throw new Error(`HTTP ${l.status} loading ${g}`);let{width:d,height:p,rgba:m}=await n(await l.blob()),{pixelData:f,numberOfComponents:u}=M(m,d*p);return L({imageId:i,width:d,height:p,pixelData:f,numberOfComponents:u,voxelManagerFactory:t})})(),cancelFn:()=>s.abort()}}}async function A(t){let e=await globalThis.createImageBitmap(t);try{let o=new globalThis.OffscreenCanvas(e.width,e.height).getContext("2d",{willReadFrequently:!0});o.drawImage(e,0,0);let{data:n}=o.getImageData(0,0,e.width,e.height);return{width:e.width,height:e.height,rgba:n}}finally{e.close?.()}}var U=Object.freeze(["imagePlaneModule","imagePixelModule","generalSeriesModule","calibratedPixelSpacing"]),C="User";function I(t){let e={rows:t.height,columns:t.width,rowCosines:[1,0,0],columnCosines:[0,1,0],imagePositionPatient:[0,0,0],imageOrientationPatient:[1,0,0,0,1,0]},a=t.pixelSpacingMm;return a?{...e,rowPixelSpacing:a.y_mm,columnPixelSpacing:a.x_mm,pixelSpacing:[a.y_mm,a.x_mm]}:e}function $(t){let e=t.numberOfComponents>1;return{samplesPerPixel:e?3:1,photometricInterpretation:e?"RGB":"MONOCHROME2",rows:t.height,columns:t.width,bitsAllocated:8,bitsStored:8,highBit:7,pixelRepresentation:0,planarConfiguration:e?0:void 0,windowCenter:128,windowWidth:256}}function k(t){return{modality:"OT",seriesInstanceUID:void 0,studyInstanceUID:void 0,seriesNumber:t.index??void 0,seriesDescription:t.description??""}}function N(t){let e=t.pixelSpacingMm;if(e)return{rowPixelSpacing:e.y_mm,columnPixelSpacing:e.x_mm,type:C}}var S=new Set,O=Object.freeze({get(t){for(let e of S){let a=e.get(t);if(a)return a}}});function q(t){return S.add(t),t}function B(t=O){return function(a,o){if(typeof o!="string")return;let n=t.get(o);if(n)switch(a){case"imagePlaneModule":return I(n);case"imagePixelModule":return $(n);case"generalSeriesModule":return k(n);case"calibratedPixelSpacing":return N(n);default:return}}}var R="ygg-photo-dialog";function h(t,{title:e,icon:a,body:o,confirmLabel:n,cancelLabel:r}){let i=t.createElement("div");i.className=`modal ${R}`,i.setAttribute("tabindex","-1"),i.setAttribute("role","dialog"),i.setAttribute("aria-modal","true"),i.innerHTML=` + {% if image_segment_data.canAssist %} +
+
+

Assist

+ Preparing… +
+ +
+
+ + +
+ +
+
+ {% endif %} {% endif %}
@@ -195,4 +227,15 @@

Labels

{% if image_segment_data and 'image_segmentation' in allowed_annotations %} {{ image_segment_data|json_script:"imageSegmentData" }} {% cornerstone_entry 'image-segment' %} +{% comment %} + The Tutorial button (static/js/laparoscopy_tutorial.js) runs a driver.js tour that opens + Annotation Mode itself, so it exists only alongside it. The script comes pinned + SRI from + jsDelivr (CONTRIBUTING: Third-party CDNs); the stylesheet is vendored because the CSP's + style-src is 'self' only. +{% endcomment %} + + + {% endif %} diff --git a/yggdrasil/settings.py b/yggdrasil/settings.py index 602f4a93..9696c3fe 100644 --- a/yggdrasil/settings.py +++ b/yggdrasil/settings.py @@ -105,6 +105,12 @@ def _read_app_version() -> str: ) WHISPER_CONNECT_TIMEOUT = config("WHISPER_CONNECT_TIMEOUT", default=10, cast=int) +# Assisted segmentation: the DjangoWorker GPU worker, reached only from the server +# (common.consumers.AssistSessionConsumer; docs/setup.md section 8). Empty hides the Assist tool. +ASSIST_WORKER_URL = config("ASSIST_WORKER_URL", default="") +ASSIST_WORKER_TOKEN = config("ASSIST_WORKER_TOKEN", default="") +ASSIST_CONNECT_TIMEOUT = config("ASSIST_CONNECT_TIMEOUT", default=10, cast=int) + MIDDLEWARE = [ # Outermost, so it sees the final response body (see its docstring). "yggdrasil.middleware.AsyncStreamingMiddleware",