fix(cameras): recover stalled browser streams

frontend/src/components/control/BrowserCameraPreview.test.tsx

@@ -19,30 +19,23 @@ test("the Rotation control exposes four clockwise values and emits a number", ()
   act(() => { select.value = "90"; select.dispatchEvent(new Event("change", { bubbles: true })); });
   expect(change).toHaveBeenCalledWith(90);
 });
-test("a quarter turn fits the whole non-square frame and uses actual stream dimensions", () => {
+test("a quarter turn fills the rotated recorded frame", () => {
   act(() => root.render(<BrowserCameraPreview deviceId="wrist" rotation={90} width={640} height={480} />));
   const video = container.querySelector("video")!, wrapper = video.parentElement!;
   expect(wrapper.style.aspectRatio).toBe("480 / 640");
-  expect(video.style.objectFit).toBe("contain");
+  expect(video.style.objectFit).toBe("cover");
   expect(video.style.maxWidth).toBe("none");
   expect(video.style.transform).toBe("translate(-50%, -50%) rotate(90deg)");
   expect(parseFloat(video.style.width)).toBeCloseTo(640 / 480 * 100);
   expect(parseFloat(video.style.height)).toBeCloseTo(480 / 640 * 100);
-  Object.defineProperty(video, "videoWidth", { value: 1280 });
-  Object.defineProperty(video, "videoHeight", { value: 720 });
-  act(() => video.dispatchEvent(new Event("loadedmetadata")));
-  expect(wrapper.style.aspectRatio).toBe("720 / 1280");
-  expect(parseFloat(video.style.width)).toBeCloseTo(1280 / 720 * 100);
-  expect(parseFloat(video.style.height)).toBeCloseTo(720 / 1280 * 100);
 });
-test("device reassignment does not reuse the old device's frame geometry", () => {
-  act(() => root.render(<BrowserCameraPreview deviceId="A" rotation={270} />));
+test("a stream at another size keeps the recorded aspect ratio", () => {
+  act(() => root.render(<BrowserCameraPreview deviceId="A" rotation={0} width={640} height={480} />));
   const video = container.querySelector("video")!;
-  Object.defineProperty(video, "videoWidth", { value: 1920 });
-  Object.defineProperty(video, "videoHeight", { value: 1080 });
+  Object.defineProperty(video, "videoWidth", { value: 1280 });
+  Object.defineProperty(video, "videoHeight", { value: 720 });
   act(() => video.dispatchEvent(new Event("loadedmetadata")));
-  act(() => root.render(<BrowserCameraPreview deviceId="B" rotation={180} width={640} height={480} />));
   expect(video.parentElement!.style.aspectRatio).toBe("640 / 480");
   expect(video.style.width).toBe("100%");
-  expect(video.style.transform).toContain("rotate(180deg)");
+  expect(video.style.objectFit).toBe("cover");
 });

frontend/src/components/control/BrowserCameraPreview.tsx

@@ -1,4 +1,4 @@
-import React, { useState } from "react";
+import React from "react";
 import { VideoOff } from "lucide-react";
 import { useCameraStream } from "@/hooks/useCameraStream";
 import { CameraRotation, cameraPreviewGeometry, cameraRotationError } from "@/lib/cameraConfig";
@@ -13,7 +13,11 @@ interface Props {
   fallback?: string;
 }
 
-/** Browser streams contain raw frames, so only browser previews use CSS rotation. */
+/**
+ * Browser streams contain raw frames, so only browser previews use CSS rotation.
+ * The box keeps the recorded capture size's shape even when the browser stream
+ * has another size, so the preview frames the scene the way recording will.
+ */
 export default function BrowserCameraPreview({
   deviceId,
   paused = false,
@@ -24,12 +28,10 @@ export default function BrowserCameraPreview({
   fallback,
 }: Props) {
   const rotationError = cameraRotationError({ rotation });
-  const { videoRef, hasError } = useCameraStream(deviceId, paused || !!rotationError);
-  const [metadata, setMetadata] = useState<{ deviceId: string; width: number; height: number } | null>(null);
-  const dimensions = metadata?.deviceId === deviceId ? metadata : { width, height };
+  const { videoRef, hasError } = useCameraStream(deviceId, paused || !!rotationError, { width, height });
   const geometry = rotationError
     ? { aspectRatio: `${width} / ${height}`, videoStyle: {} }
-    : cameraPreviewGeometry(dimensions.width, dimensions.height, rotation);
+    : cameraPreviewGeometry(width, height, rotation);
   const showVideo = !rotationError && !paused && deviceId && !hasError;
   return (
     <div
@@ -43,12 +45,6 @@ export default function BrowserCameraPreview({
           muted
           playsInline
           style={geometry.videoStyle}
-          onLoadedMetadata={(event) => {
-            const video = event.currentTarget;
-            if (video.videoWidth > 0 && video.videoHeight > 0) {
-              setMetadata({ deviceId, width: video.videoWidth, height: video.videoHeight });
-            }
-          }}
         />
       ) : (
         <div className="absolute inset-0 flex flex-col items-center justify-center">

frontend/src/components/control/TeleopCameraPanel.tsx

@@ -4,6 +4,8 @@ import { Button } from "@/components/ui/button";
 import { Label } from "@/components/ui/label";
 import { Switch } from "@/components/ui/switch";
 import { useRobots } from "@/hooks/useRobots";
+import { useAvailableCameras } from "@/hooks/useAvailableCameras";
+import { previewDeviceId } from "@/lib/cameraConfig";
 import CameraFeed from "./CameraFeed";
 
 /**
@@ -23,16 +25,18 @@ const TeleopCameraPanel: React.FC = () => {
   // attempt) — useful if a camera was unplugged and reconnected.
   const [reloadKey, setReloadKey] = useState(0);
   const { selectedRecord, isLoading: robotsLoading } = useRobots();
+  const { cameras: availableCameras } = useAvailableCameras({ enabled });
 
-  // Feeds come solely from the robot's configured cameras; each carries a stored
-  // browser device_id we stream directly. A configured camera whose device is
-  // currently absent still shows (name + failed-preview placeholder), so the
-  // user can tell it's expected but not detected.
+  // Feeds come solely from the robot's configured cameras, streamed by their
+  // stored browser device_id or, in a browser that doesn't know that ID, by
+  // their cv2 index. A configured camera whose device is currently absent still
+  // shows (name + failed-preview placeholder), so the user can tell it's
+  // expected but not detected.
   const configured = selectedRecord?.cameras ?? [];
   const feeds = configured.map((c) => ({
     key: c.id,
     name: c.name,
-    deviceId: c.device_id,
+    deviceId: previewDeviceId(c, availableCameras),
     rotation: c.rotation,
     width: c.width,
     height: c.height,

frontend/src/components/recording/CameraConfiguration.tsx

@@ -20,7 +20,13 @@ import { useToast } from "@/hooks/use-toast";
 import { useAvailableCameras } from "@/hooks/useAvailableCameras";
 import BrowserCameraPreview from "@/components/control/BrowserCameraPreview";
 import CameraRotationSelect from "@/components/control/CameraRotationSelect";
-import { CameraConfig, cameraRotation, cameraRotationError, rotatedDimensions } from "@/lib/cameraConfig";
+import {
+  CameraConfig,
+  cameraRotation,
+  cameraRotationError,
+  previewDeviceId,
+  rotatedDimensions,
+} from "@/lib/cameraConfig";
 export type { CameraConfig } from "@/lib/cameraConfig";
 
 // Sentinels distinguish "leave unset" (auto-detect / platform default) from an
@@ -298,6 +304,7 @@ const CameraConfiguration: React.FC<CameraConfigurationProps> = ({
               <CameraPreview
                 key={camera.id}
                 camera={camera}
+                deviceId={previewDeviceId(camera, availableCameras)}
                 paused={streamsPaused}
                 readOnly={readOnly}
                 onRemove={() => removeCamera(camera.id)}
@@ -324,6 +331,7 @@ const CameraConfiguration: React.FC<CameraConfigurationProps> = ({
 
 interface CameraPreviewProps {
   camera: CameraConfig;
+  deviceId: string;
   paused: boolean;
   readOnly: boolean;
   onRemove: () => void;
@@ -332,6 +340,7 @@ interface CameraPreviewProps {
 
 const CameraPreview: React.FC<CameraPreviewProps> = ({
   camera,
+  deviceId,
   paused,
   readOnly,
   onRemove,
@@ -343,12 +352,12 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
   return (
     <div className="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
       <BrowserCameraPreview
-        deviceId={camera.device_id}
+        deviceId={deviceId}
         paused={paused}
         rotation={camera.rotation}
         width={camera.width}
         height={camera.height}
-        fallback={paused ? "Preview paused" : camera.device_id ? undefined : "No browser match"}
+        fallback={paused ? "Preview paused" : deviceId ? undefined : "No browser match"}
       />
 
       {/* Camera Info */}

frontend/src/hooks/useCameraBindings.ts

@@ -5,6 +5,7 @@ import { ImageFeatures } from "@/lib/checkpointsApi";
 import {
   CameraRequest,
   CameraRotation,
+  liveCameraFor,
   PhysicalCamera,
   physicalCameraKey,
   rotatedDimensions,
@@ -43,10 +44,7 @@ export function useCameraBindings(
           (c) => c.name.toLowerCase() === expectedName.toLowerCase(),
         );
         if (!robotCam) continue;
-        const live =
-          (robotCam.device_id &&
-            availableCameras.find((c) => c.deviceId === robotCam.device_id)) ||
-          availableCameras.find((c) => c.index === robotCam.camera_index);
+        const live = liveCameraFor(robotCam, availableCameras);
         if (live) {
           next[expectedName] = live.index;
           changed = true;

frontend/src/hooks/useCameraStream.test.tsx

@@ -0,0 +1,61 @@
+// @vitest-environment jsdom
+import React, { act } from "react";
+import { createRoot, Root } from "react-dom/client";
+import { afterEach, beforeEach, expect, test, vi } from "vitest";
+import { useCameraStream } from "./useCameraStream";
+
+let root: Root;
+let container: HTMLDivElement;
+const getUserMedia = vi.fn();
+const stop = vi.fn();
+
+const Preview = () => {
+  const { videoRef } = useCameraStream("cam", false, { width: 640, height: 480 });
+  return <video ref={videoRef} />;
+};
+
+const flush = () => act(async () => {});
+
+beforeEach(() => {
+  Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
+  vi.useFakeTimers();
+  getUserMedia.mockReset().mockResolvedValue({ getTracks: () => [{ stop }] });
+  stop.mockReset();
+  Object.defineProperty(navigator, "mediaDevices", {
+    configurable: true,
+    value: { getUserMedia, addEventListener: vi.fn(), removeEventListener: vi.fn() },
+  });
+  vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined);
+  container = document.createElement("div");
+  root = createRoot(container);
+});
+
+afterEach(() => {
+  act(() => root.unmount());
+  vi.useRealTimers();
+  vi.restoreAllMocks();
+});
+
+const requestedSize = (call: number) => {
+  const { width, height } = getUserMedia.mock.calls[call][0].video;
+  return [width.ideal, height.ideal];
+};
+
+test("opens at the recorded size and keeps a stream that delivers frames", async () => {
+  act(() => root.render(<Preview />));
+  await flush();
+  Object.defineProperty(container.querySelector("video")!, "readyState", { value: HTMLMediaElement.HAVE_CURRENT_DATA });
+  await act(async () => vi.advanceTimersByTime(5000));
+  expect(getUserMedia).toHaveBeenCalledTimes(1);
+  expect(requestedSize(0)).toEqual([640, 480]);
+  expect(stop).not.toHaveBeenCalled();
+});
+
+test("reopens at 1280x720 when the recorded size never delivers a frame", async () => {
+  act(() => root.render(<Preview />));
+  await flush();
+  await act(async () => vi.advanceTimersByTime(2500));
+  expect(stop).toHaveBeenCalledTimes(1);
+  expect(getUserMedia).toHaveBeenCalledTimes(2);
+  expect(requestedSize(1)).toEqual([1280, 720]);
+});

frontend/src/hooks/useCameraStream.ts

@@ -15,15 +15,35 @@ import { useEffect, useRef, useState } from "react";
  *     device-exposure change (timing isn't guaranteed) — so we re-attempt on
  *     that event instead. Without this the hook would otherwise stay stuck on
  *     the error state until a full page reload.
+ *
+ * The stream is first requested at `size`, the capture size recording uses
+ * (browser default when omitted). A stream can also open cleanly yet never
+ * deliver a frame: some UVC cameras (e.g. Innomaker U20CAM on macOS) accept
+ * 640x480 in Firefox but stay silent until something starts them at a native
+ * resolution. If no frame arrives within STALL_TIMEOUT_MS we reopen at the
+ * next size in FALLBACK_SIZES, so cameras that stream at `size` are untouched.
  */
 const MAX_RETRIES = 4;
 const BASE_DELAY_MS = 300;
 // Errors worth an immediate retry. Permission denial (NotAllowedError), missing
 // device (NotFoundError) and unsatisfiable constraints (OverconstrainedError)
 // won't fix themselves on a retimed retry — those recover via `devicechange`.
 const TRANSIENT_ERRORS = new Set(["NotReadableError", "AbortError"]);
+const STALL_TIMEOUT_MS = 2500;
+const FALLBACK_SIZES: CameraSize[] = [
+  { width: 1280, height: 720 },
+  { width: 1920, height: 1080 },
+];
+
+export interface CameraSize {
+  width: number;
+  height: number;
+}
+
+const sizeConstraints = (size: CameraSize | undefined): MediaTrackConstraints =>
+  size ? { width: { ideal: size.width }, height: { ideal: size.height } } : {};
 
-export function useCameraStream(deviceId: string, paused: boolean) {
+export function useCameraStream(deviceId: string, paused: boolean, size?: CameraSize) {
   const videoRef = useRef<HTMLVideoElement>(null);
   const [hasError, setHasError] = useState(false);
   // Bumping this forces the stream effect to re-run (a clean retry).
@@ -53,43 +73,61 @@ export function useCameraStream(deviceId: string, paused: boolean) {
     let cancelled = false;
     let stream: MediaStream | null = null;
     let retryTimer: ReturnType<typeof setTimeout> | null = null;
+    let stallTimer: ReturnType<typeof setTimeout> | null = null;
     setHasError(false);
+    const attempts = [size, ...FALLBACK_SIZES];
 
-    const start = async (attempt: number) => {
+    const stopStream = () => {
+      if (stream) stream.getTracks().forEach((t) => t.stop());
+      stream = null;
+    };
+
+    const start = async (attempt: number, sizeIndex: number) => {
       try {
         stream = await navigator.mediaDevices.getUserMedia({
-          video: { deviceId: { exact: deviceId } },
+          video: { deviceId: { exact: deviceId }, ...sizeConstraints(attempts[sizeIndex]) },
         });
         if (cancelled) {
-          stream.getTracks().forEach((t) => t.stop());
+          stopStream();
           return;
         }
-        if (videoRef.current) {
-          videoRef.current.srcObject = stream;
-          await videoRef.current.play().catch(() => {});
+        const video = videoRef.current;
+        if (!video) return;
+        video.srcObject = stream;
+        video.play().catch(() => {});
+        if (sizeIndex < attempts.length - 1) {
+          stallTimer = setTimeout(() => {
+            if (cancelled || video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) return;
+            stopStream();
+            start(0, sizeIndex + 1);
+          }, STALL_TIMEOUT_MS);
         }
       } catch (err) {
         if (cancelled) return;
         const name = err instanceof DOMException ? err.name : "";
         if (attempt < MAX_RETRIES && TRANSIENT_ERRORS.has(name)) {
           // Exponential backoff: 300ms, 600ms, 1200ms, ...
           retryTimer = setTimeout(
-            () => start(attempt + 1),
+            () => start(attempt + 1, sizeIndex),
             BASE_DELAY_MS * 2 ** attempt
           );
         } else {
           setHasError(true);
         }
       }
     };
-    start(0);
+    start(0, 0);
 
     return () => {
       cancelled = true;
       if (retryTimer) clearTimeout(retryTimer);
-      if (stream) stream.getTracks().forEach((t) => t.stop());
+      if (stallTimer) clearTimeout(stallTimer);
+      stopStream();
     };
-  }, [deviceId, paused, retryKey]);
+    // Depend on the size's numbers, not the object, so a caller passing a fresh
+    // literal each render doesn't restart the stream.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [deviceId, paused, retryKey, size?.width, size?.height]);
 
   return { videoRef, hasError };
 }

frontend/src/lib/cameraConfig.test.ts

@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import { CameraConfig, cameraRotation, normalizeCameraConfig, physicalCameraKey, rotatedDimensions, rotationForCamera, serializeCameras } from "./cameraConfig";
+import { CameraConfig, cameraRotation, normalizeCameraConfig, physicalCameraKey, previewDeviceId, rotatedDimensions, rotationForCamera, serializeCameras } from "./cameraConfig";
 const camera: CameraConfig = { id: "wrist", name: "wrist", device_id: "usb-wrist", camera_index: 1, type: "opencv", width: 640, height: 480, fps: 30 };
 
 test("old profiles default to zero without changing capture dimensions", () => {
@@ -27,3 +27,11 @@ test("rotation follows a physical camera across index drift and policy reassignm
   expect(rotationForCamera(top, saved, overrides)).toBe(90);
   expect(rotationForCamera({ deviceId: "unsaved", index: 5 }, saved, {})).toBe(0);
 });
+test("a saved camera previews by its device ID, else by cv2 index in another browser", () => {
+  const firefox = [{ index: 0, deviceId: "ff-other" }, { index: 1, deviceId: "usb-wrist" }];
+  const safari = [{ index: 0, deviceId: "sf-other" }, { index: 1, deviceId: "sf-wrist" }];
+  expect(previewDeviceId(camera, firefox)).toBe("usb-wrist");
+  expect(previewDeviceId(camera, safari)).toBe("sf-wrist");
+  expect(previewDeviceId(camera, [])).toBe("usb-wrist");
+  expect(previewDeviceId({ ...camera, camera_index: 5 }, safari)).toBe("");
+});