feat(cameras): ghost overlay of a saved reference frame to realign the rig

frontend/src/components/control/BrowserCameraPreview.test.tsx

@@ -39,3 +39,13 @@ test("a stream at another size keeps the recorded aspect ratio", () => {
   expect(video.style.width).toBe("100%");
   expect(video.style.objectFit).toBe("cover");
 });
+test("the reference ghost sits over the video with its rotation and opacity", () => {
+  act(() => root.render(<BrowserCameraPreview deviceId="wrist" rotation={90} ghostSrc="blob:ref" ghostOpacity={0.3} />));
+  const video = container.querySelector("video")!, ghost = container.querySelector("img")!;
+  expect(ghost.getAttribute("src")).toBe("blob:ref");
+  expect(ghost.style.opacity).toBe("0.3");
+  expect(ghost.style.transform).toBe(video.style.transform);
+  expect(ghost.style.width).toBe(video.style.width);
+  act(() => root.render(<BrowserCameraPreview deviceId="wrist" />));
+  expect(container.querySelector("img")).toBeNull();
+});

frontend/src/components/control/BrowserCameraPreview.tsx

@@ -11,6 +11,9 @@ interface Props {
   height?: number;
   className?: string;
   fallback?: string;
+  /** Saved reference frame, drawn over the live video as a translucent ghost. */
+  ghostSrc?: string;
+  ghostOpacity?: number;
 }
 
 /**
@@ -26,6 +29,8 @@ export default function BrowserCameraPreview({
   height = 480,
   className = "",
   fallback,
+  ghostSrc,
+  ghostOpacity = 0.5,
 }: Props) {
   const rotationError = cameraRotationError({ rotation });
   const { videoRef, hasError } = useCameraStream(deviceId, paused || !!rotationError, { width, height });
@@ -39,13 +44,24 @@ export default function BrowserCameraPreview({
       style={{ aspectRatio: geometry.aspectRatio }}
     >
       {showVideo ? (
+        <>
         <video
           ref={videoRef}
           autoPlay
           muted
           playsInline
           style={geometry.videoStyle}
         />
+        {ghostSrc && (
+          <img
+            src={ghostSrc}
+            alt=""
+            aria-hidden
+            className="pointer-events-none"
+            style={{ ...geometry.videoStyle, opacity: ghostOpacity }}
+          />
+        )}
+        </>
       ) : (
         <div className="absolute inset-0 flex flex-col items-center justify-center">
           <VideoOff className="w-5 h-5 text-gray-500 mb-1" />

frontend/src/components/landing/RecordingModal.tsx

@@ -282,10 +282,13 @@ const RecordingModal: React.FC<RecordingModalProps> = ({
                   onCamerasChange={setCameras}
                   releaseStreamsRef={releaseStreamsRef}
                   readOnly
+                  robotName={robot?.name}
                 />
                 <p className="text-xs text-gray-500">
                   These are the cameras set up for this robot. To add or change a
-                  camera, configure it on the Calibration page.
+                  camera, configure it on the Calibration page. Save a reference
+                  frame to see it as a ghost over the live preview and realign
+                  the rig next session.
                 </p>
               </div>
             )}

frontend/src/components/recording/CameraConfiguration.test.tsx

@@ -6,6 +6,8 @@ import CameraConfiguration from "./CameraConfiguration";
 import { CameraConfig, normalizeCameraConfig, serializeCameras } from "@/lib/cameraConfig";
 vi.mock("@/hooks/useAvailableCameras", () => ({ useAvailableCameras: () => ({ cameras: [], isLoading: false, refresh: vi.fn() }) }));
 vi.mock("@/hooks/use-toast", () => ({ useToast: () => ({ toast: vi.fn() }) }));
+const reference = { url: null as string | null, error: null, save: vi.fn(), clear: vi.fn() };
+vi.mock("@/hooks/useCameraReference", () => ({ useCameraReference: () => reference }));
 vi.mock("@/hooks/useCameraStream", () => ({ useCameraStream: () => ({ videoRef: React.createRef<HTMLVideoElement>(), hasError: false }) }));
 let root: Root;
 let container: HTMLDivElement;
@@ -43,3 +45,17 @@ test("an invalid stored rotation stays visible until the user selects a valid va
   act(() => { select.value = "180"; select.dispatchEvent(new Event("change", { bubbles: true })); });
   expect(change).toHaveBeenCalledWith([{ ...camera, rotation: 180 }]);
 });
+test("reference controls appear only for a named robot in read-only mode", () => {
+  act(() => root.render(<CameraConfiguration cameras={[camera]} onCamerasChange={vi.fn()} readOnly />));
+  expect(container.textContent).not.toContain("Set reference");
+  act(() => root.render(<CameraConfiguration cameras={[camera]} onCamerasChange={vi.fn()} readOnly robotName="arm" />));
+  expect(container.textContent).toContain("Set reference");
+  expect(container.querySelector('input[type="range"]')).toBeNull();
+  reference.url = "blob:ref";
+  act(() => root.render(<CameraConfiguration cameras={[camera]} onCamerasChange={vi.fn()} readOnly robotName="arm" />));
+  expect(container.textContent).toContain("Replace reference");
+  expect(container.querySelector("img")!.getAttribute("src")).toBe("blob:ref");
+  const slider = container.querySelector<HTMLInputElement>('input[type="range"]')!;
+  expect(slider).not.toBeNull();
+  reference.url = null;
+});

frontend/src/components/recording/CameraConfiguration.tsx

@@ -1,4 +1,4 @@
-import React, { useState, useEffect, useCallback } from "react";
+import React, { useState, useEffect, useCallback, useRef } from "react";
 import { Button } from "@/components/ui/button";
 import { Label } from "@/components/ui/label";
 import {
@@ -18,8 +18,10 @@ import {
 } from "@/components/ui/collapsible";
 import { useToast } from "@/hooks/use-toast";
 import { useAvailableCameras } from "@/hooks/useAvailableCameras";
+import { useCameraReference } from "@/hooks/useCameraReference";
 import BrowserCameraPreview from "@/components/control/BrowserCameraPreview";
 import CameraRotationSelect from "@/components/control/CameraRotationSelect";
+import CameraReferenceControls from "@/components/recording/CameraReferenceControls";
 import {
   CameraConfig,
   cameraRotation,
@@ -54,13 +56,17 @@ interface CameraConfigurationProps {
   // cameras as read-only live previews. Used in the recording dialog, where
   // cameras are managed on the Calibration page rather than added ad hoc.
   readOnly?: boolean;
+  // Robot the cameras belong to. In read-only mode this enables the reference
+  // frame controls, which are stored per robot and camera.
+  robotName?: string;
 }
 
 const CameraConfiguration: React.FC<CameraConfigurationProps> = ({
   cameras,
   onCamerasChange,
   releaseStreamsRef,
   readOnly = false,
+  robotName,
 }) => {
   const { toast } = useToast();
 
@@ -307,6 +313,7 @@ const CameraConfiguration: React.FC<CameraConfigurationProps> = ({
                 deviceId={previewDeviceId(camera, availableCameras)}
                 paused={streamsPaused}
                 readOnly={readOnly}
+                robotName={robotName}
                 onRemove={() => removeCamera(camera.id)}
                 onUpdate={(updates) => updateCamera(camera.id, updates)}
               />
@@ -334,6 +341,7 @@ interface CameraPreviewProps {
   deviceId: string;
   paused: boolean;
   readOnly: boolean;
+  robotName?: string;
   onRemove: () => void;
   onUpdate: (updates: Partial<CameraConfig>) => void;
 }
@@ -343,22 +351,30 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
   deviceId,
   paused,
   readOnly,
+  robotName,
   onRemove,
   onUpdate,
 }) => {
+  const previewRef = useRef<HTMLDivElement>(null);
+  const [ghostOpacity, setGhostOpacity] = useState(0.5);
+  const reference = useCameraReference(readOnly ? robotName : undefined, camera.name);
   const rotationError = cameraRotationError(camera);
   const rotation = rotationError ? null : cameraRotation(camera.rotation);
   const output = rotation === null ? null : rotatedDimensions(camera.width, camera.height, rotation);
   return (
     <div className="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
+      <div ref={previewRef}>
       <BrowserCameraPreview
         deviceId={deviceId}
         paused={paused}
+        ghostSrc={reference.url ?? undefined}
+        ghostOpacity={ghostOpacity}
         rotation={camera.rotation}
         width={camera.width}
         height={camera.height}
         fallback={paused ? "Preview paused" : deviceId ? undefined : "No browser match"}
       />
+      </div>
 
       {/* Camera Info */}
       <div className="p-3 space-y-2">
@@ -376,6 +392,20 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
           )}
         </div>
 
+        {readOnly && robotName && (
+          <CameraReferenceControls
+            hasReference={!!reference.url}
+            opacity={ghostOpacity}
+            onOpacityChange={setGhostOpacity}
+            onSave={() => {
+              const video = previewRef.current?.querySelector("video");
+              if (video) void reference.save(video);
+            }}
+            onClear={() => void reference.clear()}
+            disabled={paused || !!rotationError}
+            error={reference.error}
+          />
+        )}
         {rotationError && <p role="alert" className="text-xs text-red-400">{rotationError} Select a valid rotation.</p>}
         {readOnly ? (
           <p className="text-xs text-gray-400">

frontend/src/components/recording/CameraReferenceControls.tsx

@@ -0,0 +1,67 @@
+import React from "react";
+import { Button } from "@/components/ui/button";
+
+interface Props {
+  hasReference: boolean;
+  opacity: number;
+  onOpacityChange: (opacity: number) => void;
+  onSave: () => void;
+  onClear: () => void;
+  disabled: boolean;
+  error: string | null;
+}
+
+/** Save the current frame as the reference, then slide the ghost overlay over the live preview. */
+const CameraReferenceControls: React.FC<Props> = ({
+  hasReference,
+  opacity,
+  onOpacityChange,
+  onSave,
+  onClear,
+  disabled,
+  error,
+}) => (
+  <div className="space-y-1.5">
+    <div className="flex items-center gap-2">
+      <Button
+        type="button"
+        size="sm"
+        variant="outline"
+        onClick={onSave}
+        disabled={disabled}
+        className="h-7 px-2 text-xs bg-gray-800 border-gray-700 text-white hover:bg-gray-700"
+      >
+        {hasReference ? "Replace reference" : "Set reference"}
+      </Button>
+      {hasReference && (
+        <Button
+          type="button"
+          size="sm"
+          variant="ghost"
+          onClick={onClear}
+          className="h-7 px-2 text-xs text-gray-400 hover:text-white"
+        >
+          Clear
+        </Button>
+      )}
+    </div>
+    {hasReference && (
+      <label className="flex items-center gap-2 text-xs text-gray-400">
+        Ghost
+        <input
+          type="range"
+          min={0}
+          max={1}
+          step={0.05}
+          value={opacity}
+          onChange={(e) => onOpacityChange(Number(e.target.value))}
+          className="flex-1"
+          aria-label="Reference overlay opacity"
+        />
+      </label>
+    )}
+    {error && <p role="alert" className="text-xs text-red-400">{error}</p>}
+  </div>
+);
+
+export default CameraReferenceControls;

frontend/src/hooks/useCameraReference.ts

@@ -0,0 +1,68 @@
+import { useCallback, useEffect, useState } from "react";
+import { useApi } from "@/contexts/ApiContext";
+import {
+  captureVideoFrame,
+  deleteCameraReference,
+  getCameraReference,
+  saveCameraReference,
+} from "@/lib/cameraReferenceApi";
+
+/**
+ * The saved reference frame for one robot camera, as an object URL ready to use
+ * as an `<img src>`. Pass `robot = undefined` to disable it.
+ */
+export function useCameraReference(robot: string | undefined, camera: string) {
+  const { baseUrl, fetchWithHeaders } = useApi();
+  const [url, setUrl] = useState<string | null>(null);
+  const [error, setError] = useState<string | null>(null);
+
+  const show = useCallback((blob: Blob | null) => {
+    setUrl((prev) => {
+      if (prev) URL.revokeObjectURL(prev);
+      return blob ? URL.createObjectURL(blob) : null;
+    });
+  }, []);
+
+  useEffect(() => {
+    if (!robot) return;
+    const controller = new AbortController();
+    getCameraReference(baseUrl, fetchWithHeaders, robot, camera, controller.signal)
+      .then(show)
+      .catch((e) => {
+        if (!controller.signal.aborted) setError(String(e.message ?? e));
+      });
+    return () => {
+      controller.abort();
+      show(null);
+    };
+  }, [robot, camera, baseUrl, fetchWithHeaders, show]);
+
+  const save = useCallback(
+    async (video: HTMLVideoElement) => {
+      if (!robot) return;
+      setError(null);
+      try {
+        const png = await captureVideoFrame(video);
+        if (!png) throw new Error("No frame to capture yet");
+        await saveCameraReference(baseUrl, fetchWithHeaders, robot, camera, png);
+        show(png);
+      } catch (e) {
+        setError(e instanceof Error ? e.message : String(e));
+      }
+    },
+    [robot, camera, baseUrl, fetchWithHeaders, show],
+  );
+
+  const clear = useCallback(async () => {
+    if (!robot) return;
+    setError(null);
+    try {
+      await deleteCameraReference(baseUrl, fetchWithHeaders, robot, camera);
+      show(null);
+    } catch (e) {
+      setError(e instanceof Error ? e.message : String(e));
+    }
+  }, [robot, camera, baseUrl, fetchWithHeaders, show]);
+
+  return { url, error, save, clear };
+}

frontend/src/lib/cameraReferenceApi.test.ts

@@ -0,0 +1,25 @@
+import { expect, test, vi } from "vitest";
+
+import { getCameraReference, saveCameraReference } from "./cameraReferenceApi";
+
+const BASE = "http://localhost:8000";
+
+test("a missing reference reads as null rather than an error", async () => {
+  const fetcher = vi.fn().mockResolvedValue(new Response("{}", { status: 404 }));
+  await expect(getCameraReference(BASE, fetcher, "my arm", "front")).resolves.toBeNull();
+  expect(fetcher.mock.calls[0][0]).toBe(`${BASE}/robots/my%20arm/camera-reference/front`);
+});
+
+test("saving puts the PNG bytes and surfaces a rejection", async () => {
+  const png = new Blob(["png"], { type: "image/png" });
+  const fetcher = vi.fn().mockResolvedValueOnce(new Response("{}", { status: 200 }));
+  await saveCameraReference(BASE, fetcher, "arm", "front", png);
+  expect(fetcher).toHaveBeenCalledWith(`${BASE}/robots/arm/camera-reference/front`, {
+    method: "PUT",
+    headers: { "Content-Type": "image/png" },
+    body: png,
+  });
+
+  fetcher.mockResolvedValueOnce(new Response("{}", { status: 400 }));
+  await expect(saveCameraReference(BASE, fetcher, "arm", "front", png)).rejects.toThrow("Save reference failed: 400");
+});