From 08bc2ba24c5fb75880574399160a20bef2d037c8 Mon Sep 17 00:00:00 2001
From: sysradium <sysradium@users.noreply.github.com>
Date: Sat, 24 Oct 2026 23:49:06 +0200
Subject: [PATCH] fix(cameras): recover stalled browser streams

Request the recorded capture size first, then retry at fallback resolutions when the stream delivers no frames. Keep previews framed to the recorded aspect ratio and crop mismatched streams.
---
 .../control/BrowserCameraPreview.test.tsx     | 21 +++----
 .../control/BrowserCameraPreview.tsx          | 20 +++---
 .../components/control/TeleopCameraPanel.tsx  | 14 +++--
 .../recording/CameraConfiguration.tsx         | 15 ++++-
 frontend/src/hooks/useCameraBindings.ts       |  6 +-
 frontend/src/hooks/useCameraStream.test.tsx   | 61 +++++++++++++++++++
 frontend/src/hooks/useCameraStream.ts         | 60 ++++++++++++++----
 frontend/src/lib/cameraConfig.test.ts         | 10 ++-
 frontend/src/lib/cameraConfig.ts              | 31 +++++++++-
 9 files changed, 187 insertions(+), 51 deletions(-)
 create mode 100644 frontend/src/hooks/useCameraStream.test.tsx

diff --git a/frontend/src/components/control/BrowserCameraPreview.test.tsx b/frontend/src/components/control/BrowserCameraPreview.test.tsx
index 6617378ae..818c762a7 100644
--- a/frontend/src/components/control/BrowserCameraPreview.test.tsx
+++ b/frontend/src/components/control/BrowserCameraPreview.test.tsx
@@ -19,30 +19,23 @@ test("the Rotation control exposes four clockwise values and emits a number", ()
   act(() => { select.value = "90"; select.dispatchEvent(new Event("change", { bubbles: true })); });
   expect(change).toHaveBeenCalledWith(90);
 });
-test("a quarter turn fits the whole non-square frame and uses actual stream dimensions", () => {
+test("a quarter turn fills the rotated recorded frame", () => {
   act(() => root.render(<BrowserCameraPreview deviceId="wrist" rotation={90} width={640} height={480} />));
   const video = container.querySelector("video")!, wrapper = video.parentElement!;
   expect(wrapper.style.aspectRatio).toBe("480 / 640");
-  expect(video.style.objectFit).toBe("contain");
+  expect(video.style.objectFit).toBe("cover");
   expect(video.style.maxWidth).toBe("none");
   expect(video.style.transform).toBe("translate(-50%, -50%) rotate(90deg)");
   expect(parseFloat(video.style.width)).toBeCloseTo(640 / 480 * 100);
   expect(parseFloat(video.style.height)).toBeCloseTo(480 / 640 * 100);
-  Object.defineProperty(video, "videoWidth", { value: 1280 });
-  Object.defineProperty(video, "videoHeight", { value: 720 });
-  act(() => video.dispatchEvent(new Event("loadedmetadata")));
-  expect(wrapper.style.aspectRatio).toBe("720 / 1280");
-  expect(parseFloat(video.style.width)).toBeCloseTo(1280 / 720 * 100);
-  expect(parseFloat(video.style.height)).toBeCloseTo(720 / 1280 * 100);
 });
-test("device reassignment does not reuse the old device's frame geometry", () => {
-  act(() => root.render(<BrowserCameraPreview deviceId="A" rotation={270} />));
+test("a stream at another size keeps the recorded aspect ratio", () => {
+  act(() => root.render(<BrowserCameraPreview deviceId="A" rotation={0} width={640} height={480} />));
   const video = container.querySelector("video")!;
-  Object.defineProperty(video, "videoWidth", { value: 1920 });
-  Object.defineProperty(video, "videoHeight", { value: 1080 });
+  Object.defineProperty(video, "videoWidth", { value: 1280 });
+  Object.defineProperty(video, "videoHeight", { value: 720 });
   act(() => video.dispatchEvent(new Event("loadedmetadata")));
-  act(() => root.render(<BrowserCameraPreview deviceId="B" rotation={180} width={640} height={480} />));
   expect(video.parentElement!.style.aspectRatio).toBe("640 / 480");
   expect(video.style.width).toBe("100%");
-  expect(video.style.transform).toContain("rotate(180deg)");
+  expect(video.style.objectFit).toBe("cover");
 });
diff --git a/frontend/src/components/control/BrowserCameraPreview.tsx b/frontend/src/components/control/BrowserCameraPreview.tsx
index 0ec1a8a28..de6107d99 100644
--- a/frontend/src/components/control/BrowserCameraPreview.tsx
+++ b/frontend/src/components/control/BrowserCameraPreview.tsx
@@ -1,4 +1,4 @@
-import React, { useState } from "react";
+import React from "react";
 import { VideoOff } from "lucide-react";
 import { useCameraStream } from "@/hooks/useCameraStream";
 import { CameraRotation, cameraPreviewGeometry, cameraRotationError } from "@/lib/cameraConfig";
@@ -13,7 +13,11 @@ interface Props {
   fallback?: string;
 }
 
-/** Browser streams contain raw frames, so only browser previews use CSS rotation. */
+/**
+ * Browser streams contain raw frames, so only browser previews use CSS rotation.
+ * The box keeps the recorded capture size's shape even when the browser stream
+ * has another size, so the preview frames the scene the way recording will.
+ */
 export default function BrowserCameraPreview({
   deviceId,
   paused = false,
@@ -24,12 +28,10 @@ export default function BrowserCameraPreview({
   fallback,
 }: Props) {
   const rotationError = cameraRotationError({ rotation });
-  const { videoRef, hasError } = useCameraStream(deviceId, paused || !!rotationError);
-  const [metadata, setMetadata] = useState<{ deviceId: string; width: number; height: number } | null>(null);
-  const dimensions = metadata?.deviceId === deviceId ? metadata : { width, height };
+  const { videoRef, hasError } = useCameraStream(deviceId, paused || !!rotationError, { width, height });
   const geometry = rotationError
     ? { aspectRatio: `${width} / ${height}`, videoStyle: {} }
-    : cameraPreviewGeometry(dimensions.width, dimensions.height, rotation);
+    : cameraPreviewGeometry(width, height, rotation);
   const showVideo = !rotationError && !paused && deviceId && !hasError;
   return (
     <div
@@ -43,12 +45,6 @@ export default function BrowserCameraPreview({
           muted
           playsInline
           style={geometry.videoStyle}
-          onLoadedMetadata={(event) => {
-            const video = event.currentTarget;
-            if (video.videoWidth > 0 && video.videoHeight > 0) {
-              setMetadata({ deviceId, width: video.videoWidth, height: video.videoHeight });
-            }
-          }}
         />
       ) : (
         <div className="absolute inset-0 flex flex-col items-center justify-center">
diff --git a/frontend/src/components/control/TeleopCameraPanel.tsx b/frontend/src/components/control/TeleopCameraPanel.tsx
index e386554e4..4e61377dc 100644
--- a/frontend/src/components/control/TeleopCameraPanel.tsx
+++ b/frontend/src/components/control/TeleopCameraPanel.tsx
@@ -4,6 +4,8 @@ import { Button } from "@/components/ui/button";
 import { Label } from "@/components/ui/label";
 import { Switch } from "@/components/ui/switch";
 import { useRobots } from "@/hooks/useRobots";
+import { useAvailableCameras } from "@/hooks/useAvailableCameras";
+import { previewDeviceId } from "@/lib/cameraConfig";
 import CameraFeed from "./CameraFeed";
 
 /**
@@ -23,16 +25,18 @@ const TeleopCameraPanel: React.FC = () => {
   // attempt) — useful if a camera was unplugged and reconnected.
   const [reloadKey, setReloadKey] = useState(0);
   const { selectedRecord, isLoading: robotsLoading } = useRobots();
+  const { cameras: availableCameras } = useAvailableCameras({ enabled });
 
-  // Feeds come solely from the robot's configured cameras; each carries a stored
-  // browser device_id we stream directly. A configured camera whose device is
-  // currently absent still shows (name + failed-preview placeholder), so the
-  // user can tell it's expected but not detected.
+  // Feeds come solely from the robot's configured cameras, streamed by their
+  // stored browser device_id or, in a browser that doesn't know that ID, by
+  // their cv2 index. A configured camera whose device is currently absent still
+  // shows (name + failed-preview placeholder), so the user can tell it's
+  // expected but not detected.
   const configured = selectedRecord?.cameras ?? [];
   const feeds = configured.map((c) => ({
     key: c.id,
     name: c.name,
-    deviceId: c.device_id,
+    deviceId: previewDeviceId(c, availableCameras),
     rotation: c.rotation,
     width: c.width,
     height: c.height,
diff --git a/frontend/src/components/recording/CameraConfiguration.tsx b/frontend/src/components/recording/CameraConfiguration.tsx
index 7337c09c2..aef2d80b9 100644
--- a/frontend/src/components/recording/CameraConfiguration.tsx
+++ b/frontend/src/components/recording/CameraConfiguration.tsx
@@ -20,7 +20,13 @@ import { useToast } from "@/hooks/use-toast";
 import { useAvailableCameras } from "@/hooks/useAvailableCameras";
 import BrowserCameraPreview from "@/components/control/BrowserCameraPreview";
 import CameraRotationSelect from "@/components/control/CameraRotationSelect";
-import { CameraConfig, cameraRotation, cameraRotationError, rotatedDimensions } from "@/lib/cameraConfig";
+import {
+  CameraConfig,
+  cameraRotation,
+  cameraRotationError,
+  previewDeviceId,
+  rotatedDimensions,
+} from "@/lib/cameraConfig";
 export type { CameraConfig } from "@/lib/cameraConfig";
 
 // Sentinels distinguish "leave unset" (auto-detect / platform default) from an
@@ -298,6 +304,7 @@ const CameraConfiguration: React.FC<CameraConfigurationProps> = ({
               <CameraPreview
                 key={camera.id}
                 camera={camera}
+                deviceId={previewDeviceId(camera, availableCameras)}
                 paused={streamsPaused}
                 readOnly={readOnly}
                 onRemove={() => removeCamera(camera.id)}
@@ -324,6 +331,7 @@ const CameraConfiguration: React.FC<CameraConfigurationProps> = ({
 
 interface CameraPreviewProps {
   camera: CameraConfig;
+  deviceId: string;
   paused: boolean;
   readOnly: boolean;
   onRemove: () => void;
@@ -332,6 +340,7 @@ interface CameraPreviewProps {
 
 const CameraPreview: React.FC<CameraPreviewProps> = ({
   camera,
+  deviceId,
   paused,
   readOnly,
   onRemove,
@@ -343,12 +352,12 @@ const CameraPreview: React.FC<CameraPreviewProps> = ({
   return (
     <div className="bg-gray-900 rounded-lg border border-gray-700 overflow-hidden">
       <BrowserCameraPreview
-        deviceId={camera.device_id}
+        deviceId={deviceId}
         paused={paused}
         rotation={camera.rotation}
         width={camera.width}
         height={camera.height}
-        fallback={paused ? "Preview paused" : camera.device_id ? undefined : "No browser match"}
+        fallback={paused ? "Preview paused" : deviceId ? undefined : "No browser match"}
       />
 
       {/* Camera Info */}
diff --git a/frontend/src/hooks/useCameraBindings.ts b/frontend/src/hooks/useCameraBindings.ts
index f8081d332..74b3e329a 100644
--- a/frontend/src/hooks/useCameraBindings.ts
+++ b/frontend/src/hooks/useCameraBindings.ts
@@ -5,6 +5,7 @@ import { ImageFeatures } from "@/lib/checkpointsApi";
 import {
   CameraRequest,
   CameraRotation,
+  liveCameraFor,
   PhysicalCamera,
   physicalCameraKey,
   rotatedDimensions,
@@ -43,10 +44,7 @@ export function useCameraBindings(
           (c) => c.name.toLowerCase() === expectedName.toLowerCase(),
         );
         if (!robotCam) continue;
-        const live =
-          (robotCam.device_id &&
-            availableCameras.find((c) => c.deviceId === robotCam.device_id)) ||
-          availableCameras.find((c) => c.index === robotCam.camera_index);
+        const live = liveCameraFor(robotCam, availableCameras);
         if (live) {
           next[expectedName] = live.index;
           changed = true;
diff --git a/frontend/src/hooks/useCameraStream.test.tsx b/frontend/src/hooks/useCameraStream.test.tsx
new file mode 100644
index 000000000..eb6c1f328
--- /dev/null
+++ b/frontend/src/hooks/useCameraStream.test.tsx
@@ -0,0 +1,61 @@
+// @vitest-environment jsdom
+import React, { act } from "react";
+import { createRoot, Root } from "react-dom/client";
+import { afterEach, beforeEach, expect, test, vi } from "vitest";
+import { useCameraStream } from "./useCameraStream";
+
+let root: Root;
+let container: HTMLDivElement;
+const getUserMedia = vi.fn();
+const stop = vi.fn();
+
+const Preview = () => {
+  const { videoRef } = useCameraStream("cam", false, { width: 640, height: 480 });
+  return <video ref={videoRef} />;
+};
+
+const flush = () => act(async () => {});
+
+beforeEach(() => {
+  Object.assign(globalThis, { IS_REACT_ACT_ENVIRONMENT: true });
+  vi.useFakeTimers();
+  getUserMedia.mockReset().mockResolvedValue({ getTracks: () => [{ stop }] });
+  stop.mockReset();
+  Object.defineProperty(navigator, "mediaDevices", {
+    configurable: true,
+    value: { getUserMedia, addEventListener: vi.fn(), removeEventListener: vi.fn() },
+  });
+  vi.spyOn(HTMLMediaElement.prototype, "play").mockResolvedValue(undefined);
+  container = document.createElement("div");
+  root = createRoot(container);
+});
+
+afterEach(() => {
+  act(() => root.unmount());
+  vi.useRealTimers();
+  vi.restoreAllMocks();
+});
+
+const requestedSize = (call: number) => {
+  const { width, height } = getUserMedia.mock.calls[call][0].video;
+  return [width.ideal, height.ideal];
+};
+
+test("opens at the recorded size and keeps a stream that delivers frames", async () => {
+  act(() => root.render(<Preview />));
+  await flush();
+  Object.defineProperty(container.querySelector("video")!, "readyState", { value: HTMLMediaElement.HAVE_CURRENT_DATA });
+  await act(async () => vi.advanceTimersByTime(5000));
+  expect(getUserMedia).toHaveBeenCalledTimes(1);
+  expect(requestedSize(0)).toEqual([640, 480]);
+  expect(stop).not.toHaveBeenCalled();
+});
+
+test("reopens at 1280x720 when the recorded size never delivers a frame", async () => {
+  act(() => root.render(<Preview />));
+  await flush();
+  await act(async () => vi.advanceTimersByTime(2500));
+  expect(stop).toHaveBeenCalledTimes(1);
+  expect(getUserMedia).toHaveBeenCalledTimes(2);
+  expect(requestedSize(1)).toEqual([1280, 720]);
+});
diff --git a/frontend/src/hooks/useCameraStream.ts b/frontend/src/hooks/useCameraStream.ts
index 1f44cf18b..bbf1a6f0f 100644
--- a/frontend/src/hooks/useCameraStream.ts
+++ b/frontend/src/hooks/useCameraStream.ts
@@ -15,6 +15,13 @@ import { useEffect, useRef, useState } from "react";
  *     device-exposure change (timing isn't guaranteed) — so we re-attempt on
  *     that event instead. Without this the hook would otherwise stay stuck on
  *     the error state until a full page reload.
+ *
+ * The stream is first requested at `size`, the capture size recording uses
+ * (browser default when omitted). A stream can also open cleanly yet never
+ * deliver a frame: some UVC cameras (e.g. Innomaker U20CAM on macOS) accept
+ * 640x480 in Firefox but stay silent until something starts them at a native
+ * resolution. If no frame arrives within STALL_TIMEOUT_MS we reopen at the
+ * next size in FALLBACK_SIZES, so cameras that stream at `size` are untouched.
  */
 const MAX_RETRIES = 4;
 const BASE_DELAY_MS = 300;
@@ -22,8 +29,21 @@ const BASE_DELAY_MS = 300;
 // device (NotFoundError) and unsatisfiable constraints (OverconstrainedError)
 // won't fix themselves on a retimed retry — those recover via `devicechange`.
 const TRANSIENT_ERRORS = new Set(["NotReadableError", "AbortError"]);
+const STALL_TIMEOUT_MS = 2500;
+const FALLBACK_SIZES: CameraSize[] = [
+  { width: 1280, height: 720 },
+  { width: 1920, height: 1080 },
+];
+
+export interface CameraSize {
+  width: number;
+  height: number;
+}
+
+const sizeConstraints = (size: CameraSize | undefined): MediaTrackConstraints =>
+  size ? { width: { ideal: size.width }, height: { ideal: size.height } } : {};
 
-export function useCameraStream(deviceId: string, paused: boolean) {
+export function useCameraStream(deviceId: string, paused: boolean, size?: CameraSize) {
   const videoRef = useRef<HTMLVideoElement>(null);
   const [hasError, setHasError] = useState(false);
   // Bumping this forces the stream effect to re-run (a clean retry).
@@ -53,20 +73,34 @@ export function useCameraStream(deviceId: string, paused: boolean) {
     let cancelled = false;
     let stream: MediaStream | null = null;
     let retryTimer: ReturnType<typeof setTimeout> | null = null;
+    let stallTimer: ReturnType<typeof setTimeout> | null = null;
     setHasError(false);
+    const attempts = [size, ...FALLBACK_SIZES];
 
-    const start = async (attempt: number) => {
+    const stopStream = () => {
+      if (stream) stream.getTracks().forEach((t) => t.stop());
+      stream = null;
+    };
+
+    const start = async (attempt: number, sizeIndex: number) => {
       try {
         stream = await navigator.mediaDevices.getUserMedia({
-          video: { deviceId: { exact: deviceId } },
+          video: { deviceId: { exact: deviceId }, ...sizeConstraints(attempts[sizeIndex]) },
         });
         if (cancelled) {
-          stream.getTracks().forEach((t) => t.stop());
+          stopStream();
           return;
         }
-        if (videoRef.current) {
-          videoRef.current.srcObject = stream;
-          await videoRef.current.play().catch(() => {});
+        const video = videoRef.current;
+        if (!video) return;
+        video.srcObject = stream;
+        video.play().catch(() => {});
+        if (sizeIndex < attempts.length - 1) {
+          stallTimer = setTimeout(() => {
+            if (cancelled || video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) return;
+            stopStream();
+            start(0, sizeIndex + 1);
+          }, STALL_TIMEOUT_MS);
         }
       } catch (err) {
         if (cancelled) return;
@@ -74,7 +108,7 @@ export function useCameraStream(deviceId: string, paused: boolean) {
         if (attempt < MAX_RETRIES && TRANSIENT_ERRORS.has(name)) {
           // Exponential backoff: 300ms, 600ms, 1200ms, ...
           retryTimer = setTimeout(
-            () => start(attempt + 1),
+            () => start(attempt + 1, sizeIndex),
             BASE_DELAY_MS * 2 ** attempt
           );
         } else {
@@ -82,14 +116,18 @@ export function useCameraStream(deviceId: string, paused: boolean) {
         }
       }
     };
-    start(0);
+    start(0, 0);
 
     return () => {
       cancelled = true;
       if (retryTimer) clearTimeout(retryTimer);
-      if (stream) stream.getTracks().forEach((t) => t.stop());
+      if (stallTimer) clearTimeout(stallTimer);
+      stopStream();
     };
-  }, [deviceId, paused, retryKey]);
+    // Depend on the size's numbers, not the object, so a caller passing a fresh
+    // literal each render doesn't restart the stream.
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [deviceId, paused, retryKey, size?.width, size?.height]);
 
   return { videoRef, hasError };
 }
diff --git a/frontend/src/lib/cameraConfig.test.ts b/frontend/src/lib/cameraConfig.test.ts
index 8da735c36..ff3dce4a6 100644
--- a/frontend/src/lib/cameraConfig.test.ts
+++ b/frontend/src/lib/cameraConfig.test.ts
@@ -1,5 +1,5 @@
 import { expect, test } from "vitest";
-import { CameraConfig, cameraRotation, normalizeCameraConfig, physicalCameraKey, rotatedDimensions, rotationForCamera, serializeCameras } from "./cameraConfig";
+import { CameraConfig, cameraRotation, normalizeCameraConfig, physicalCameraKey, previewDeviceId, rotatedDimensions, rotationForCamera, serializeCameras } from "./cameraConfig";
 const camera: CameraConfig = { id: "wrist", name: "wrist", device_id: "usb-wrist", camera_index: 1, type: "opencv", width: 640, height: 480, fps: 30 };
 
 test("old profiles default to zero without changing capture dimensions", () => {
@@ -27,3 +27,11 @@ test("rotation follows a physical camera across index drift and policy reassignm
   expect(rotationForCamera(top, saved, overrides)).toBe(90);
   expect(rotationForCamera({ deviceId: "unsaved", index: 5 }, saved, {})).toBe(0);
 });
+test("a saved camera previews by its device ID, else by cv2 index in another browser", () => {
+  const firefox = [{ index: 0, deviceId: "ff-other" }, { index: 1, deviceId: "usb-wrist" }];
+  const safari = [{ index: 0, deviceId: "sf-other" }, { index: 1, deviceId: "sf-wrist" }];
+  expect(previewDeviceId(camera, firefox)).toBe("usb-wrist");
+  expect(previewDeviceId(camera, safari)).toBe("sf-wrist");
+  expect(previewDeviceId(camera, [])).toBe("usb-wrist");
+  expect(previewDeviceId({ ...camera, camera_index: 5 }, safari)).toBe("");
+});
diff --git a/frontend/src/lib/cameraConfig.ts b/frontend/src/lib/cameraConfig.ts
index 16b21a0f4..befcde3f0 100644
--- a/frontend/src/lib/cameraConfig.ts
+++ b/frontend/src/lib/cameraConfig.ts
@@ -89,7 +89,9 @@ export function cameraPreviewGeometry(width: number, height: number, rotation: C
     top: "50%",
     width: quarterTurn ? `${(width / height) * 100}%` : "100%",
     height: quarterTurn ? `${(height / width) * 100}%` : "100%",
-    objectFit: "contain",
+    // Crop, don't letterbox: a stream that fell back to another aspect ratio
+    // (see useCameraStream) should still show the recorded framing.
+    objectFit: "cover",
     maxWidth: "none",
     transform: `translate(-50%, -50%) rotate(${rotation}deg)`,
   };
@@ -101,6 +103,33 @@ export interface PhysicalCamera {
   deviceId: string;
 }
 
+/**
+ * The live camera for a saved one. Browser device IDs differ per browser and
+ * origin, so an ID saved in Firefox is unknown to Safari (or to the Vite dev
+ * origin); fall back to the cv2 index, which every browser shares.
+ */
+export function liveCameraFor<T extends PhysicalCamera>(
+  saved: Pick<CameraConfig, "device_id" | "camera_index">,
+  available: T[],
+): T | undefined {
+  return (
+    (saved.device_id ? available.find((c) => c.deviceId === saved.device_id) : undefined) ??
+    available.find((c) => c.index === saved.camera_index)
+  );
+}
+
+/**
+ * Browser device ID to preview a saved camera with. Until the live cameras
+ * have been listed, try the saved ID; afterwards "" means no camera matched.
+ */
+export function previewDeviceId(
+  saved: Pick<CameraConfig, "device_id" | "camera_index">,
+  available: PhysicalCamera[],
+): string {
+  if (available.length === 0) return saved.device_id;
+  return liveCameraFor(saved, available)?.deviceId ?? "";
+}
+
 export function physicalCameraKey(camera: PhysicalCamera): string {
   return camera.deviceId ? `device:${camera.deviceId}` : `index:${camera.index}`;
 }