From 49e4fa0e21b79983a2fb4f1f422875766c55435a Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Thu, 22 Oct 2026 14:53:23 +0800 Subject: [PATCH 1/9] feat(oob): forward the replay-related client params to the headset URL The WebXR client already reads immersiveMode, referenceSpace, replayPacing and showRecordingControls from its query string, but --setup-oob had no way to set them: the operator had to open the bookmark and edit the URL by hand, which is exactly what an unattended run cannot do. Follow the existing TELEOP_CLIENT_* convention (codec, panelHiddenAtStart) rather than adding a new mechanism. All four default to unset, so a URL built without them is byte-identical to today's. immersiveMode matters beyond replay: a PICO 4 Ultra rejects immersive-ar with NotSupportedError, so a headset that needs vr currently cannot be driven by --setup-oob at all. Signed-off-by: Emily Ma --- .../isaaccapture/cloudxr/oob_teleop_env.py | 29 ++++++- .../core/cloudxr/test_oob_teleop_env.py | 87 +++++++++++++++++++ 2 files changed, 114 insertions(+), 2 deletions(-) diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_env.py b/src/python/isaaccapture/cloudxr/oob_teleop_env.py index d4d56a34cb..c103b77967 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_env.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_env.py @@ -439,7 +439,8 @@ def client_ui_fields_from_env() -> dict: Keys match query params the WebXR client reads on page load (``serverIP``, ``port``, ``codec``, ``panelHiddenAtStart``, - ``reconnectMaxAttempts``, ``reconnectDelayMs``, + ``immersiveMode``, ``referenceSpace``, ``replayPacing``, + ``showRecordingControls``, ``reconnectMaxAttempts``, ``reconnectDelayMs``, ``streamAttachTimeoutMs``, ``warmupBeginTimeoutMs``, ``warmupEndTimeoutMs``). Deliberately excludes ``reconnectEnabled`` - OOB never sets it; the client's own reconnect loop is opt-in via a URL param the user supplies directly. @@ -454,6 +455,22 @@ def client_ui_fields_from_env() -> dict: out["panelHiddenAtStart"] = True elif ph in ("0", "false", "no", "off"): out["panelHiddenAtStart"] = False + # Free-form like codec: the client validates the value against its own + # allowed set, so a typo surfaces there rather than being silently dropped + # by a second allowlist here. + for env_name, key in ( + ("TELEOP_CLIENT_IMMERSIVE_MODE", "immersiveMode"), + ("TELEOP_CLIENT_REFERENCE_SPACE", "referenceSpace"), + ("TELEOP_CLIENT_REPLAY_PACING", "replayPacing"), + ): + val = os.environ.get(env_name, "").strip() + if val: + out[key] = val + rc = os.environ.get("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", "").strip().lower() + if rc in ("1", "true", "yes", "on"): + out["showRecordingControls"] = True + elif rc in ("0", "false", "no", "off"): + out["showRecordingControls"] = False # reconnectMaxAttempts/reconnectDelayMs are already set above by # client_reconnect_config_from_env(), which validates them and raises on bad input - do not # re-parse them here with this loop's silent-skip-on-ValueError semantics. @@ -505,7 +522,8 @@ def build_headset_bookmark_url( Set *oob_enable* to ``False`` to omit ``oobEnable`` (and the control token, which only authenticates hub operations). The remaining params — - ``serverIP``, ``port``, ``codec``, ``panelHiddenAtStart``, ``reconnectEnabled``, + ``serverIP``, ``port``, ``codec``, ``panelHiddenAtStart``, ``immersiveMode``, + ``referenceSpace``, ``replayPacing``, ``showRecordingControls``, ``reconnectEnabled``, ``reconnectMaxAttempts``, ``reconnectDelayMs``, ``streamAttachTimeoutMs``, ``warmupBeginTimeoutMs``, ``warmupEndTimeoutMs`` — are plain form overrides the client honours either way, so the headset still lands with the right @@ -535,6 +553,13 @@ def build_headset_bookmark_url( v = cfg.get("panelHiddenAtStart") if isinstance(v, bool): params["panelHiddenAtStart"] = "true" if v else "false" + for key in ("immersiveMode", "referenceSpace", "replayPacing"): + v = cfg.get(key) + if v is not None and str(v).strip() != "": + params[key] = str(v).strip() + v = cfg.get("showRecordingControls") + if isinstance(v, bool): + params["showRecordingControls"] = "true" if v else "false" v = cfg.get("reconnectEnabled") if isinstance(v, bool): params["reconnectEnabled"] = "true" if v else "false" diff --git a/tests/python/core/cloudxr/test_oob_teleop_env.py b/tests/python/core/cloudxr/test_oob_teleop_env.py index c9138c214a..9c23d1f12d 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_env.py +++ b/tests/python/core/cloudxr/test_oob_teleop_env.py @@ -49,6 +49,10 @@ def clear_teleop_env(monkeypatch: pytest.MonkeyPatch) -> None: "TELEOP_STREAM_PORT", "TELEOP_CLIENT_CODEC", "TELEOP_CLIENT_PANEL_HIDDEN_AT_START", + "TELEOP_CLIENT_IMMERSIVE_MODE", + "TELEOP_CLIENT_REFERENCE_SPACE", + "TELEOP_CLIENT_REPLAY_PACING", + "TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", "TELEOP_CLIENT_RECONNECT_MAX_ATTEMPTS", "TELEOP_CLIENT_RECONNECT_DELAY_MS", "TELEOP_CLIENT_STREAM_ATTACH_TIMEOUT_MS", @@ -210,6 +214,45 @@ def test_build_headset_bookmark_url_with_token() -> None: assert q["controlToken"] == ["secret123"] +def test_build_headset_bookmark_url_with_replay_overrides() -> None: + """Replay-related fields reach the headset URL so a run is reproducible without hand-editing it.""" + u = build_headset_bookmark_url( + web_client_base="https://h.test/", + stream_config={ + "serverIP": "10.0.0.1", + "port": 48322, + "immersiveMode": "vr", + "referenceSpace": "local", + "replayPacing": "frame", + "showRecordingControls": True, + }, + ) + q = parse_qs(urlparse(u).query) + assert q["immersiveMode"] == ["vr"] + assert q["referenceSpace"] == ["local"] + assert q["replayPacing"] == ["frame"] + assert q["showRecordingControls"] == ["true"] + + +def test_build_headset_bookmark_url_omits_unset_replay_overrides() -> None: + """Without the overrides the query is unchanged - existing callers keep today's URL.""" + q = parse_qs( + urlparse( + build_headset_bookmark_url( + web_client_base="https://h.test/", + stream_config={"serverIP": "10.0.0.1", "port": 48322}, + ) + ).query + ) + for key in ( + "immersiveMode", + "referenceSpace", + "replayPacing", + "showRecordingControls", + ): + assert key not in q + + def test_build_headset_bookmark_url_with_codec() -> None: """codec from stream_config is forwarded as a query param in the bookmark URL.""" u = build_headset_bookmark_url( @@ -372,6 +415,50 @@ def test_client_ui_fields_from_env_panel_hidden( assert fields["panelHiddenAtStart"] is True +@pytest.mark.parametrize( + ("env_name", "key", "value"), + [ + ("TELEOP_CLIENT_IMMERSIVE_MODE", "immersiveMode", "vr"), + ("TELEOP_CLIENT_REFERENCE_SPACE", "referenceSpace", "local"), + ("TELEOP_CLIENT_REPLAY_PACING", "replayPacing", "frame"), + ], +) +def test_client_ui_fields_from_env_passthrough_strings( + clear_teleop_env: None, + monkeypatch: pytest.MonkeyPatch, + env_name: str, + key: str, + value: str, +) -> None: + """Each string override reaches the matching client field verbatim.""" + monkeypatch.setenv(env_name, value) + assert client_ui_fields_from_env()[key] == value + + +def test_client_ui_fields_from_env_show_recording_controls( + clear_teleop_env: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """TELEOP_CLIENT_SHOW_RECORDING_CONTROLS parses both truthy and falsey spellings.""" + monkeypatch.setenv("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", "on") + assert client_ui_fields_from_env()["showRecordingControls"] is True + monkeypatch.setenv("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", "off") + assert client_ui_fields_from_env()["showRecordingControls"] is False + + +def test_client_ui_fields_from_env_unset_stays_absent( + clear_teleop_env: None, +) -> None: + """Unset overrides add no keys: an operator who sets none sees the URL they see today.""" + fields = client_ui_fields_from_env() + for key in ( + "immersiveMode", + "referenceSpace", + "replayPacing", + "showRecordingControls", + ): + assert key not in fields + + def test_client_ui_fields_from_env_never_sets_reconnect_enabled( clear_teleop_env: None, ) -> None: From 91c237e29edd603c5803c973a312580933377541 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Thu, 22 Oct 2026 15:00:13 +0800 Subject: [PATCH 2/9] fix(oob): pick the DevTools socket that actually holds the teleop tab A PICO 4 Ultra exposes one weblayer_devtools_remote_ socket per browser process. Prefix priority cannot separate them, so discovery returned whichever came first in /proc/net/unix - observed driving a new-tab page while the teleop tab sat on the second socket. Nothing reports it: CDP connects, the tab list is non-empty, and CONNECT is simply never clicked. Probe only the sockets that tie at the top of the priority order, and keep the one with an oobEnable= tab. Prefix order still decides on its own whenever it can, so the single-socket and mixed-prefix paths are unchanged and cost no extra adb calls. Finding none is the normal pre-am-start state and still returns the best-ranked socket. Signed-off-by: Emily Ma --- .../isaaccapture/cloudxr/oob_teleop_adb.py | 44 ++++++++++--- .../core/cloudxr/test_oob_teleop_adb.py | 62 +++++++++++++++++++ 2 files changed, 99 insertions(+), 7 deletions(-) diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py index 5bf23f2d00..e5ff46d0c7 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py @@ -1225,6 +1225,9 @@ def stop_coturn(proc: subprocess.Popen | None) -> None: def _discover_devtools_socket() -> str | None: """Return the bare name of the browser's DevTools abstract socket, or None. + When the device exposes more than one, the socket holding a teleop tab + wins; see the selection comment below. + Chromium-based Android browsers expose a Unix abstract socket matching ``@_devtools_remote[_]`` in ``/proc/net/unix``. Known prefixes in the wild: @@ -1255,14 +1258,41 @@ def _discover_devtools_socket() -> str | None: candidates.append(m.group(1)) if not candidates: return None - # Prefer teleop-relevant prefixes in this order, otherwise fall back to - # the first discovered socket. + # Prefer teleop-relevant prefixes in this order, otherwise keep discovery + # order. priority = ("weblayer", "com.oculus.browser", "chrome", "webview") - for prefix in priority: - for cand in candidates: - if cand.startswith(prefix): - return cand - return candidates[0] + + def rank(name: str) -> int: + return next( + (i for i, pre in enumerate(priority) if name.startswith(pre)), + len(priority), + ) + + ranked = sorted(candidates, key=rank) + # Prefix order still decides whenever it can. Only sockets that tie at the + # top are ambiguous, and only those are worth a round of adb probing. + tied = [c for c in ranked if rank(c) == rank(ranked[0])] + if len(tied) == 1: + return tied[0] + # A PICO 4 Ultra exposes one weblayer_devtools_remote_ per browser + # process, so the tie is a real one: observed with the teleop tab on the + # second socket while the first held only a new-tab page. Ask each in turn + # and keep the one that actually has a teleop tab. + for cand in tied: + try: + _adb_forward_cdp(cand, _CDP_LOCAL_PORT) + except (OobAdbError, subprocess.CalledProcessError, OSError) as exc: + log.debug("CDP: cannot forward to @%s: %s", cand, exc) + continue + if any( + "oobEnable=" in (t.get("url") or "") + for t in _cdp_list_tabs(_CDP_LOCAL_PORT) + ): + log.info("CDP: teleop tab found on @%s", cand) + return cand + # None has one yet. Normal before the page is opened, so hand back the + # best-ranked socket and let the caller go on to open it there. + return tied[0] def _adb_forward_cdp(socket_name: str, local_port: int) -> None: diff --git a/tests/python/core/cloudxr/test_oob_teleop_adb.py b/tests/python/core/cloudxr/test_oob_teleop_adb.py index efb131b5af..9c7eaef7b3 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_adb.py +++ b/tests/python/core/cloudxr/test_oob_teleop_adb.py @@ -1025,6 +1025,68 @@ def test_discover_devtools_socket_prefers_weblayer_over_generic( assert _discover_devtools_socket() == "weblayer_devtools_remote_777" +_TWO_WEBLAYER_SOCKETS = ( + "0: 00000003 00000000 00010000 0001 01 12345 @weblayer_devtools_remote_111\n" + "1: 00000003 00000000 00010000 0001 01 12346 @weblayer_devtools_remote_222\n" +) + + +@patch("cloudxr_py_test_ns.oob_teleop_adb._cdp_list_tabs") +@patch("cloudxr_py_test_ns.oob_teleop_adb._adb_forward_cdp") +@patch("cloudxr_py_test_ns.oob_teleop_adb._run_adb") +def test_discover_devtools_socket_picks_the_one_holding_the_teleop_tab( + mock_run_adb: MagicMock, + mock_forward: MagicMock, + mock_list_tabs: MagicMock, +) -> None: + """With several WebLayer sockets, the one with a teleop tab wins over socket order. + + A PICO 4 Ultra runs more than one WebLayer process; taking the first + silently drives the wrong browser. + """ + mock_run_adb.return_value = _TWO_WEBLAYER_SOCKETS + mock_list_tabs.side_effect = [ + [{"url": "about:blank"}], + [{"url": "https://h.test/?oobEnable=1&serverIP=10.0.0.1"}], + ] + assert _discover_devtools_socket() == "weblayer_devtools_remote_222" + assert [c.args[0] for c in mock_forward.call_args_list] == [ + "weblayer_devtools_remote_111", + "weblayer_devtools_remote_222", + ] + + +@patch("cloudxr_py_test_ns.oob_teleop_adb._cdp_list_tabs") +@patch("cloudxr_py_test_ns.oob_teleop_adb._adb_forward_cdp") +@patch("cloudxr_py_test_ns.oob_teleop_adb._run_adb") +def test_discover_devtools_socket_tie_without_teleop_tab_keeps_first( + mock_run_adb: MagicMock, + mock_forward: MagicMock, + mock_list_tabs: MagicMock, +) -> None: + """No teleop tab anywhere is the normal pre-``am start`` state, not a failure.""" + mock_run_adb.return_value = _TWO_WEBLAYER_SOCKETS + mock_list_tabs.return_value = [{"url": "about:blank"}] + assert _discover_devtools_socket() == "weblayer_devtools_remote_111" + + +@patch("cloudxr_py_test_ns.oob_teleop_adb._cdp_list_tabs") +@patch("cloudxr_py_test_ns.oob_teleop_adb._adb_forward_cdp") +@patch("cloudxr_py_test_ns.oob_teleop_adb._run_adb") +def test_discover_devtools_socket_tie_skips_a_socket_that_will_not_forward( + mock_run_adb: MagicMock, + mock_forward: MagicMock, + mock_list_tabs: MagicMock, +) -> None: + """A dead socket is skipped rather than aborting the search.""" + mock_run_adb.return_value = _TWO_WEBLAYER_SOCKETS + mock_forward.side_effect = [OobAdbError("gone"), None] + mock_list_tabs.return_value = [ + {"url": "https://h.test/?oobEnable=1&serverIP=10.0.0.1"} + ] + assert _discover_devtools_socket() == "weblayer_devtools_remote_222" + + @patch("cloudxr_py_test_ns.oob_teleop_adb._run_adb") def test_discover_devtools_socket_falls_back_to_first_candidate( mock_run_adb: MagicMock, From 1c3d5ef4baed950795acde95af750d11cb3a133c Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Thu, 22 Oct 2026 16:02:49 +0800 Subject: [PATCH 3/9] feat(oob): optionally preload a recording before CONNECT --setup-oob could open the page and click CONNECT, but the client's recording file input can only be filled through the browser's own filesystem, so a run still began with someone picking a file in the headset. TELEOP_OOB_RECORDING pushes the file over adb and selects it via DOM.setFileInputFiles, between page-ready and the CONNECT click. Before rather than after: the input is rendered from first paint by RecorderProvider, so waiting buys nothing and means reaching into a page the operator is already immersed in. Unset - the default - leaves the flow untouched. Starting the replay itself stays with the operator. Signed-off-by: Emily Ma --- .../isaaccapture/cloudxr/oob_teleop_adb.py | 72 ++++++++++++++++++- .../isaaccapture/cloudxr/oob_teleop_env.py | 11 +++ .../core/cloudxr/test_oob_teleop_env.py | 15 ++++ 3 files changed, 97 insertions(+), 1 deletion(-) diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py index e5ff46d0c7..15fc38661a 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py @@ -38,12 +38,14 @@ from inspect import isawaitable from dataclasses import dataclass from enum import Enum +from pathlib import Path from .oob_teleop_env import ( default_web_client_origin, parse_env_port, build_headset_bookmark_url, client_ui_fields_from_env, oob_progress, + oob_replay_recording_from_env, redact_control_token, resolve_control_token, resolve_lan_host_for_oob, @@ -1552,12 +1554,43 @@ async def send(ws, method: str, params: dict | None = None) -> None: log.info("CDP: sent reset-panel key (%s)", _RESET_PANEL_KEY) +_HEADSET_RECORDING_DIR = "/sdcard/Download" + + +def _push_recording_to_headset(host_path: str) -> str: + """Copy a recording onto the headset and return its on-device path. + + DOM.setFileInputFiles names a path in the *browser's* filesystem, so the + file has to exist on the headset first. + """ + src = Path(host_path) + if not src.is_file(): + raise OobAdbError(f"recording not found: {src}") + device_path = f"{_HEADSET_RECORDING_DIR}/{src.name}" + try: + _adb_run( + ["adb", "push", str(src), device_path], + capture_output=True, + text=True, + timeout=120, + check=True, + ) + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or exc.stdout or "").strip() or "(no adb output)" + raise OobAdbError( + f"adb push {src} -> {device_path} failed (rc={exc.returncode}): {detail}" + ) from exc + log.info("pushed recording to headset: %s", device_path) + return device_path + + async def _cdp_session_click_connect( ws_url: str, *, refresh_static_assets: bool = False, allow_navigation: bool = True, clear_stale_error: bool = False, + recording_device_path: str | None = None, on_dispatched: Callable[[], Awaitable[None] | None] | None = None, on_client_loaded: Callable[[], Awaitable[None] | None] | None = None, ) -> None: @@ -1775,6 +1808,34 @@ async def send(ws, method, params=None): if isawaitable(result): await result + # Load the recording before CONNECT, not after: the client's file input + # is in the page from first render, and selecting a file once the + # session is immersive means reaching into a page the operator is + # already inside. + if recording_device_path: + await send(ws, "DOM.enable") + doc = await send(ws, "DOM.getDocument", {"depth": -1}) + node = await send( + ws, + "DOM.querySelector", + { + "nodeId": doc["root"]["nodeId"], + "selector": "input[type=file]", + }, + ) + node_id = node.get("nodeId") + if not node_id: + raise OobAdbError( + "CDP: the teleop page has no recording file input; " + "the web client is older than the recorder UI" + ) + await send( + ws, + "DOM.setFileInputFiles", + {"nodeId": node_id, "files": [recording_device_path]}, + ) + log.info("CDP: loaded recording %s", recording_device_path) + # Extra grace period: the CONNECT button can become enabled before the # React store is fully mounted, which causes "XR is not available" # errors if clicked immediately. @@ -1964,6 +2025,7 @@ async def _find_and_click_teleop_tab( usb_local: bool = False, host_client: bool = False, refresh_static_assets: bool = False, + recording_device_path: str | None = None, on_dispatched: Callable[[], Awaitable[None] | None] | None = None, on_client_loaded: Callable[[], Awaitable[None] | None] | None = None, ) -> str: @@ -2110,6 +2172,7 @@ def _is_candidate_tab(tab: dict) -> bool: await _cdp_session_click_connect( ws_url, refresh_static_assets=refresh_static_assets, + recording_device_path=recording_device_path, on_dispatched=on_dispatched, on_client_loaded=on_client_loaded, ) @@ -2248,7 +2311,13 @@ async def run_oob_connect( ) from exc try: - # --- Steps 3+4: find the teleop tab and click CONNECT ------------------ + # --- Steps 3+4: find the teleop tab, load any recording, click CONNECT - + recording = oob_replay_recording_from_env() + recording_device_path = ( + await asyncio.to_thread(_push_recording_to_headset, recording) + if recording + else None + ) ws_url = await _find_and_click_teleop_tab( resolved_port=resolved_port, deadline=deadline, @@ -2256,6 +2325,7 @@ async def run_oob_connect( usb_local=usb_local, host_client=host_client, refresh_static_assets=usb_local or host_client, + recording_device_path=recording_device_path, on_dispatched=on_dispatched, on_client_loaded=on_client_loaded, ) diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_env.py b/src/python/isaaccapture/cloudxr/oob_teleop_env.py index c103b77967..631ad02545 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_env.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_env.py @@ -489,6 +489,17 @@ def client_ui_fields_from_env() -> dict: return out +def oob_replay_recording_from_env() -> str: + """Host path of a recording to load into the client, or ``""``. + + Set ``TELEOP_OOB_RECORDING`` to start a run with the recording already + selected: the file is pushed to the headset and chosen in the client's + recording file input, before CONNECT. Unset (the default) leaves the client + with no recording loaded, exactly as before. + """ + return os.environ.get("TELEOP_OOB_RECORDING", "").strip() + + def teleop_client_route_from_env() -> str: """Return the HashRouter fragment to append to the headset bookmark URL. diff --git a/tests/python/core/cloudxr/test_oob_teleop_env.py b/tests/python/core/cloudxr/test_oob_teleop_env.py index 9c23d1f12d..016c963c93 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_env.py +++ b/tests/python/core/cloudxr/test_oob_teleop_env.py @@ -25,6 +25,7 @@ default_initial_stream_config, default_web_client_origin, guess_lan_ipv4, + oob_replay_recording_from_env, versioned_web_client_url, print_hosted_client_line, print_oob_hub_startup_banner, @@ -53,6 +54,7 @@ def clear_teleop_env(monkeypatch: pytest.MonkeyPatch) -> None: "TELEOP_CLIENT_REFERENCE_SPACE", "TELEOP_CLIENT_REPLAY_PACING", "TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", + "TELEOP_OOB_RECORDING", "TELEOP_CLIENT_RECONNECT_MAX_ATTEMPTS", "TELEOP_CLIENT_RECONNECT_DELAY_MS", "TELEOP_CLIENT_STREAM_ATTACH_TIMEOUT_MS", @@ -956,3 +958,16 @@ def test_print_hosted_client_line_hides_credentials() -> None: assert "controlToken=" in out assert "turnCredential=" in out assert "port=48322" in out + + +def test_oob_replay_recording_unset_is_empty(clear_teleop_env: None) -> None: + """No recording configured is the default: nothing is pushed or selected.""" + assert oob_replay_recording_from_env() == "" + + +def test_oob_replay_recording_from_env( + clear_teleop_env: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """The configured host path is returned stripped.""" + monkeypatch.setenv("TELEOP_OOB_RECORDING", " /tmp/take-7.json ") + assert oob_replay_recording_from_env() == "/tmp/take-7.json" From d5f58fde538fe6abbd19d89b377c59f7f178b9a5 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Thu, 22 Oct 2026 16:02:50 +0800 Subject: [PATCH 4/9] docs(oob): document the new client and recording env vars Five additions to the environment table in the OOB reference, placed next to the TELEOP_CLIENT_* and TELEOP_OOB_* entries they extend. Signed-off-by: Emily Ma --- docs/source/references/oob_teleop_control.rst | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/source/references/oob_teleop_control.rst b/docs/source/references/oob_teleop_control.rst index 25d69ee59f..b5225dc181 100644 --- a/docs/source/references/oob_teleop_control.rst +++ b/docs/source/references/oob_teleop_control.rst @@ -574,6 +574,21 @@ Environment variables - Default video codec for headset bookmarks * - ``TELEOP_CLIENT_PANEL_HIDDEN_AT_START`` - Hide control panel on load (``true`` / ``false``) + * - ``TELEOP_CLIENT_IMMERSIVE_MODE`` + - WebXR session mode for headset bookmarks: ``vr`` or ``ar``. Needed on + headsets that reject ``immersive-ar``, such as the PICO 4 Ultra. + * - ``TELEOP_CLIENT_REFERENCE_SPACE`` + - XR reference space: ``auto``, ``local-floor``, ``local``, ``viewer`` or + ``unbounded``. Recording and replay are only comparable across runs when + this is pinned. + * - ``TELEOP_CLIENT_REPLAY_PACING`` + - Input replay pacing: ``frame`` or ``time``. ``frame`` advances one + recorded frame per XR frame callback, so a replay is reproducible + regardless of how fast the server renders. + * - ``TELEOP_CLIENT_SHOW_RECORDING_CONTROLS`` + - Show the in-XR recording controls (``true`` / ``false``), which carry the + replay buttons an operator uses after ``TELEOP_OOB_RECORDING`` has loaded + a file. * - ``TELEOP_CLIENT_ROUTE`` - HashRouter fragment appended to the bookmark URL. Default empty (no fragment — the web client picks its own landing route). Set @@ -608,6 +623,11 @@ Environment variables Before each new browser launch, ADB must confirm ``mWakefulness=Awake``; wear or unlock the headset (or cover its proximity sensor) when the service reports that it is asleep. + * - ``TELEOP_OOB_RECORDING`` + - Host path of a recorded session to start the run with. The file is + pushed to the headset and selected in the client's recording file input + before CONNECT, so the operator only has to start the replay. Unset by + default, so nothing is loaded. * - ``TELEOP_OOB_RECOVERY_TIMEOUT_SEC`` - Positive finite recovery-episode duration in seconds (default ``60``). Expiry changes to observation mode; it never stops the host. If this is From 6c342519f6fcf01a08a8628a49acdd8ffbb9bca7 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Thu, 22 Oct 2026 16:51:31 +0800 Subject: [PATCH 5/9] feat(oob): add --oob-* flags for the teleop page a run starts from A benchmark replays the same recording the same way on every run, which until now meant editing the headset URL by hand - something an automated run cannot do, and the one step that kept --setup-oob from being usable unattended. --oob-reference-space pin the XR reference space --oob-replay-pacing frame-paced replay, independent of render speed --oob-immersive-mode vr, for headsets that reject immersive-ar --oob-recording PATH to push, or 'latest' from the headset --oob-recording-controls show the in-XR replay buttons Each is the command-line spelling of a TELEOP_* variable the OOB code already reads, so the flag is the only new surface; omit them all and the URL is the one built today. They require --setup-oob, which is what opens the page they describe, and 'service start' passes them through to the detached 'run'. Selecting the recording is as far as this goes: the operator still starts the replay. Signed-off-by: Emily Ma --- docs/source/references/oob_teleop_control.rst | 20 ++++ .../isaaccapture/cloudxr/oob_teleop_adb.py | 36 ++++++- .../isaaccapture/cloudxr/service/__main__.py | 100 +++++++++++++++++- tests/python/core/cloudxr/test_service_cli.py | 61 +++++++++++ 4 files changed, 214 insertions(+), 3 deletions(-) diff --git a/docs/source/references/oob_teleop_control.rst b/docs/source/references/oob_teleop_control.rst index b5225dc181..8d30707e7b 100644 --- a/docs/source/references/oob_teleop_control.rst +++ b/docs/source/references/oob_teleop_control.rst @@ -38,6 +38,24 @@ This will: 4. Accept the self-signed certificate and click CONNECT automatically via Chrome DevTools Protocol (CDP) +Optionally, shape the page the run starts from. Each flag below is the +command-line spelling of the matching ``TELEOP_*`` variable in +:ref:`the environment table `; omit them all and the headset +URL is exactly the one built today: + +.. code-block:: bash + + python -m isaaccapture.cloudxr.service run --accept-eula --setup-oob \ + --oob-reference-space local \ + --oob-replay-pacing frame \ + --oob-recording latest \ + --oob-recording-controls + +``--oob-recording`` takes a host path, which is pushed to the headset, or +``latest``, which picks the newest recording already in its Download folder. +Either way the file is only *selected* in the client; starting the replay +remains an operator action. + Ordinary headset absence, an ``offline`` or ``unauthorized`` transport, and browser recovery failures leave the host running with a degraded OOB status. The lifecycle retries for 60 seconds at a 5-second cadence, then keeps @@ -546,6 +564,8 @@ exception: a saved device profile other than ``Custom`` is re-applied from the c profile table at startup, so profile default updates (frame rate, bitrate, codec) reach returning clients. Only ``Custom`` keeps manually edited values. +.. _oob-environment: + Environment variables --------------------- diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py index 15fc38661a..55f434a505 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py @@ -1555,6 +1555,40 @@ async def send(ws, method: str, params: dict | None = None) -> None: _HEADSET_RECORDING_DIR = "/sdcard/Download" +# The spelling that means "whatever the operator recorded last" instead of a path. +LATEST_RECORDING = "latest" + + +def _newest_recording_on_headset() -> str: + """Path of the most recent ``*.json`` in the headset's Download folder. + + Lets a run follow straight on from a recording session without the operator + having to look up the generated filename. + """ + out = _run_adb( + "newest recording", + [ + "adb", + "shell", + f"ls -t {_HEADSET_RECORDING_DIR}/*.json 2>/dev/null | head -n 1", + ], + timeout=10.0, + ) + path = (out or "").strip() + if not path or not path.startswith(_HEADSET_RECORDING_DIR): + raise OobAdbError( + f"no recording found in {_HEADSET_RECORDING_DIR} on the headset; " + "record one first, or pass a host path instead of 'latest'" + ) + log.info("newest recording on headset: %s", path) + return path + + +def _resolve_recording(spec: str) -> str: + """Turn the configured recording into a path on the headset.""" + if spec == LATEST_RECORDING: + return _newest_recording_on_headset() + return _push_recording_to_headset(spec) def _push_recording_to_headset(host_path: str) -> str: @@ -2314,7 +2348,7 @@ async def run_oob_connect( # --- Steps 3+4: find the teleop tab, load any recording, click CONNECT - recording = oob_replay_recording_from_env() recording_device_path = ( - await asyncio.to_thread(_push_recording_to_headset, recording) + await asyncio.to_thread(_resolve_recording, recording) if recording else None ) diff --git a/src/python/isaaccapture/cloudxr/service/__main__.py b/src/python/isaaccapture/cloudxr/service/__main__.py index 96fae05df2..b845560250 100644 --- a/src/python/isaaccapture/cloudxr/service/__main__.py +++ b/src/python/isaaccapture/cloudxr/service/__main__.py @@ -80,6 +80,76 @@ def _add_run_arguments(parser: argparse.ArgumentParser) -> None: "TELEOP_WEB_CLIENT_STATIC_DIR or ~/.cloudxr/static-client." ), ) + _add_oob_client_arguments(parser) + + +# Each flag below is the command-line spelling of a TELEOP_* variable the OOB +# code already reads. Omitting one leaves the variable untouched, so a command +# without them builds exactly the headset URL it builds today. +_OOB_CLIENT_FLAG_ENV = { + "oob_reference_space": "TELEOP_CLIENT_REFERENCE_SPACE", + "oob_replay_pacing": "TELEOP_CLIENT_REPLAY_PACING", + "oob_immersive_mode": "TELEOP_CLIENT_IMMERSIVE_MODE", + "oob_recording": "TELEOP_OOB_RECORDING", +} + + +def _add_oob_client_arguments(parser: argparse.ArgumentParser) -> None: + """Register the optional ``--setup-oob`` client overrides. + + These shape the teleop page a run starts from. They exist because a + benchmark has to replay the same input the same way every time, which + otherwise means editing the headset's URL by hand. + """ + parser.add_argument( + "--oob-reference-space", + choices=("auto", "local-floor", "local", "viewer", "unbounded"), + default=None, + help=( + "XR reference space for the teleop page. Recording and replay are " + "only comparable across runs when this is pinned. Requires --setup-oob." + ), + ) + parser.add_argument( + "--oob-replay-pacing", + choices=("frame", "time"), + default=None, + help=( + "Input replay pacing. 'frame' advances one recorded frame per XR " + "frame callback, so a replay is reproducible regardless of how fast " + "the server renders. Requires --setup-oob." + ), + ) + parser.add_argument( + "--oob-immersive-mode", + choices=("ar", "vr"), + default=None, + help=( + "WebXR session mode. Needed on headsets that reject immersive-ar, " + "such as the PICO 4 Ultra. Requires --setup-oob." + ), + ) + parser.add_argument( + "--oob-recording", + type=str, + default=None, + metavar="PATH|latest", + help=( + "Start the run with a recording already selected in the client. A " + "path is pushed to the headset; 'latest' picks the newest recording " + "already in its Download folder. The operator still starts the " + "replay. Requires --setup-oob." + ), + ) + parser.add_argument( + "--oob-recording-controls", + action="store_true", + default=False, + help=( + "Show the in-XR recording controls, which carry the replay buttons. " + "Requires --setup-oob." + ), + ) def _run_flags(args: argparse.Namespace) -> list[str]: @@ -94,9 +164,13 @@ def _run_flags(args: argparse.Namespace) -> list[str]: flags += ["--cloudxr-install-dir", args.cloudxr_install_dir] if args.cloudxr_env_config: flags += ["--cloudxr-env-config", args.cloudxr_env_config] - for name in ("setup_oob", "usb_local", "host_client"): - if getattr(args, name): + for name in ("setup_oob", "usb_local", "host_client", "oob_recording_controls"): + if getattr(args, name, False): flags.append("--" + name.replace("_", "-")) + for name in _OOB_CLIENT_FLAG_ENV: + value = getattr(args, name, None) + if value: + flags += ["--" + name.replace("_", "-"), value] return flags @@ -268,10 +342,32 @@ def _print_service_summary( ) +def _apply_oob_client_flags(args: argparse.Namespace) -> None: + """Turn the ``--oob-*`` client flags into the variables the OOB code reads. + + A flag wins over an inherited variable of the same name: it is the more + specific statement of intent for this one run. + """ + given = [ + name + for name in (*_OOB_CLIENT_FLAG_ENV, "oob_recording_controls") + if getattr(args, name, None) + ] + if given and not args.setup_oob: + _fail("--" + given[0].replace("_", "-") + " requires --setup-oob.") + for name, env_name in _OOB_CLIENT_FLAG_ENV.items(): + value = getattr(args, name, None) + if value: + os.environ[env_name] = value + if args.oob_recording_controls: + os.environ["TELEOP_CLIENT_SHOW_RECORDING_CONTROLS"] = "true" + + def _cmd_run(args: argparse.Namespace) -> int: """Run the service in the foreground until interrupted.""" if args.usb_local and not args.setup_oob: _fail("--usb-local requires --setup-oob.") + _apply_oob_client_flags(args) if args.usb_local and os.getenv("TELEOP_OOB_HUB_ONLY"): _fail( "TELEOP_OOB_HUB_ONLY is not compatible with --usb-local " diff --git a/tests/python/core/cloudxr/test_service_cli.py b/tests/python/core/cloudxr/test_service_cli.py index 26734b6ae8..6ecd8081d2 100644 --- a/tests/python/core/cloudxr/test_service_cli.py +++ b/tests/python/core/cloudxr/test_service_cli.py @@ -51,6 +51,25 @@ def test_install_dir_emitted_only_when_changed(self): default = os.path.expanduser("~/.cloudxr") assert cli._run_flags(_run_args(cloudxr_install_dir=default)) == [] + def test_oob_client_flags_round_trip(self): + """`start` must hand the detached `run` the same client setup it was given.""" + args = _run_args( + setup_oob=True, + oob_reference_space="local", + oob_replay_pacing="frame", + oob_recording="latest", + oob_recording_controls=True, + ) + flags = cli._run_flags(args) + assert flags[:2] == ["--setup-oob", "--oob-recording-controls"] + for pair in ( + ["--oob-reference-space", "local"], + ["--oob-replay-pacing", "frame"], + ["--oob-recording", "latest"], + ): + i = flags.index(pair[0]) + assert flags[i : i + 2] == pair + def test_accept_eula_is_never_baked_into_the_unit(self): """Acceptance is a marker file written at install time, not a unit flag.""" assert "--accept-eula" not in cli._run_flags(_run_args(accept_eula=True)) @@ -287,3 +306,45 @@ def test_colour_is_dropped_when_output_is_redirected(self, capsys): with patch("sys.stdout.isatty", return_value=False): cli._out("\033[33mplain\033[0m") assert capsys.readouterr().out == "plain\n" + + +class TestOobClientFlags: + """The --oob-* overrides, and the promise that omitting them changes nothing.""" + + def test_flags_set_the_variables_the_oob_code_reads(self, monkeypatch): + for env_name in cli._OOB_CLIENT_FLAG_ENV.values(): + monkeypatch.delenv(env_name, raising=False) + monkeypatch.delenv("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", raising=False) + cli._apply_oob_client_flags( + _run_args( + setup_oob=True, + oob_reference_space="local", + oob_replay_pacing="frame", + oob_immersive_mode="vr", + oob_recording="/tmp/take-7.json", + oob_recording_controls=True, + ) + ) + assert os.environ["TELEOP_CLIENT_REFERENCE_SPACE"] == "local" + assert os.environ["TELEOP_CLIENT_REPLAY_PACING"] == "frame" + assert os.environ["TELEOP_CLIENT_IMMERSIVE_MODE"] == "vr" + assert os.environ["TELEOP_OOB_RECORDING"] == "/tmp/take-7.json" + assert os.environ["TELEOP_CLIENT_SHOW_RECORDING_CONTROLS"] == "true" + + def test_omitting_them_touches_nothing(self, monkeypatch): + """The whole point: an existing command keeps building the URL it builds today.""" + for env_name in cli._OOB_CLIENT_FLAG_ENV.values(): + monkeypatch.delenv(env_name, raising=False) + monkeypatch.delenv("TELEOP_CLIENT_SHOW_RECORDING_CONTROLS", raising=False) + cli._apply_oob_client_flags(_run_args(setup_oob=True)) + for env_name in cli._OOB_CLIENT_FLAG_ENV.values(): + assert env_name not in os.environ + assert "TELEOP_CLIENT_SHOW_RECORDING_CONTROLS" not in os.environ + + def test_they_require_setup_oob(self, monkeypatch): + """Without --setup-oob nothing opens the page, so the override has no target.""" + monkeypatch.delenv("TELEOP_CLIENT_REFERENCE_SPACE", raising=False) + with pytest.raises(SystemExit): + cli._apply_oob_client_flags( + _run_args(setup_oob=False, oob_reference_space="local") + ) From 8e8fa02787bd6e4f601279ccf1e3f3418aff00f6 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Thu, 22 Oct 2026 17:31:08 +0800 Subject: [PATCH 6/9] perf(oob): push a recording once per run, not once per connect attempt Connecting retries until the application has an XR session, and each attempt re-opens the page and re-selects the file. A sim that spends ten minutes loading its scene therefore pushed the same recording dozens of times. Key the cache on the resolved path, size and mtime so a re-recorded file of the same name still lands. 'latest' is left uncached: it is one adb shell ls, and re-reading it means a recording made while waiting is picked up. Signed-off-by: Emily Ma --- .../isaaccapture/cloudxr/oob_teleop_adb.py | 17 ++++++++- .../core/cloudxr/test_oob_teleop_adb.py | 36 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py index 55f434a505..547d1f36df 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py @@ -1591,15 +1591,29 @@ def _resolve_recording(spec: str) -> str: return _push_recording_to_headset(spec) +# Recordings already pushed this run, keyed by what identifies the bytes. +# Connecting retries until the application has an XR session, and each attempt +# re-opens the page and re-selects the file; without this, a sim that takes ten +# minutes to load the scene pushes the same recording dozens of times. +_pushed_recordings: dict[tuple[str, int, int], str] = {} + + def _push_recording_to_headset(host_path: str) -> str: """Copy a recording onto the headset and return its on-device path. DOM.setFileInputFiles names a path in the *browser's* filesystem, so the - file has to exist on the headset first. + file has to exist on the headset first. An edited file is pushed again: + the key covers size and mtime, not just the name. """ src = Path(host_path) if not src.is_file(): raise OobAdbError(f"recording not found: {src}") + stat_result = src.stat() + key = (str(src.resolve()), stat_result.st_size, stat_result.st_mtime_ns) + cached = _pushed_recordings.get(key) + if cached: + log.debug("recording already on headset this run: %s", cached) + return cached device_path = f"{_HEADSET_RECORDING_DIR}/{src.name}" try: _adb_run( @@ -1614,6 +1628,7 @@ def _push_recording_to_headset(host_path: str) -> str: raise OobAdbError( f"adb push {src} -> {device_path} failed (rc={exc.returncode}): {detail}" ) from exc + _pushed_recordings[key] = device_path log.info("pushed recording to headset: %s", device_path) return device_path diff --git a/tests/python/core/cloudxr/test_oob_teleop_adb.py b/tests/python/core/cloudxr/test_oob_teleop_adb.py index 9c7eaef7b3..fcb9e780bd 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_adb.py +++ b/tests/python/core/cloudxr/test_oob_teleop_adb.py @@ -8,6 +8,7 @@ import asyncio import http.server import json +import os import subprocess import threading from concurrent.futures import ThreadPoolExecutor @@ -1773,3 +1774,38 @@ async def test_monitor_without_callback_still_logs_and_keeps_watching() -> None: task.cancel() with pytest.raises(asyncio.CancelledError): await task + + +class TestRecordingPush: + """Pushing a recording, and not pushing it again on every connect retry.""" + + def _reset(self): + adb_module._pushed_recordings.clear() + + @patch("cloudxr_py_test_ns.oob_teleop_adb._adb_run") + def test_pushes_once_and_reuses_the_device_path(self, mock_adb, tmp_path): + """Connecting retries until the sim has an XR session; one push covers them all.""" + self._reset() + rec = tmp_path / "take-7.json" + rec.write_text("{}", encoding="utf-8") + first = adb_module._push_recording_to_headset(str(rec)) + second = adb_module._push_recording_to_headset(str(rec)) + assert first == second == "/sdcard/Download/take-7.json" + assert mock_adb.call_count == 1 + + @patch("cloudxr_py_test_ns.oob_teleop_adb._adb_run") + def test_an_edited_recording_is_pushed_again(self, mock_adb, tmp_path): + """The key covers size and mtime, so re-recording to the same name still lands.""" + self._reset() + rec = tmp_path / "take-7.json" + rec.write_text("{}", encoding="utf-8") + adb_module._push_recording_to_headset(str(rec)) + rec.write_text('{"frames": []}', encoding="utf-8") + os.utime(rec, (0, 0)) + adb_module._push_recording_to_headset(str(rec)) + assert mock_adb.call_count == 2 + + def test_a_missing_recording_is_named_in_the_error(self, tmp_path): + self._reset() + with pytest.raises(OobAdbError, match="recording not found"): + adb_module._push_recording_to_headset(str(tmp_path / "nope.json")) From 5aa8e5fc3e9cb4cda01ba37f982e9ae6b49cd3fd Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Mon, 26 Oct 2026 00:33:16 -0700 Subject: [PATCH 7/9] fix(oob): re-scan DevTools sockets while waiting for the teleop tab run_oob_connect forwards the CDP port before the page exists, and _find_and_click_teleop_tab then polls only that socket. On a PICO the browser runs several WebLayer processes, and _discover_devtools_socket tells them apart by asking which one holds an oobEnable= tab - but _close_stale_teleop_tabs has just closed exactly those, so on a cold start every candidate answers empty and the tie falls back to rank order. When that guess is the new-tab process, the loop watches a socket the page never appears on and only gives up at the deadline. Re-run discovery every third pass of the wait loop and move the forward when it names a different socket. Once am start has painted the page the oobEnable= probe matches for real, so the loop corrects itself within a few seconds. On the headset this took cold starts that timed out after 60s down to about 3s, across browser restarts that changed the socket each time. Signed-off-by: Emily Ma --- .../isaaccapture/cloudxr/oob_teleop_adb.py | 30 ++++++++ .../core/cloudxr/test_oob_teleop_adb.py | 75 +++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py index 547d1f36df..bf8f822a00 100644 --- a/src/python/isaaccapture/cloudxr/oob_teleop_adb.py +++ b/src/python/isaaccapture/cloudxr/oob_teleop_adb.py @@ -2121,13 +2121,43 @@ def _is_candidate_tab(tab: dict) -> bool: ws_url: str | None = None am_start_retried = False + # Which socket _CDP_LOCAL_PORT currently points at, as far as the re-scan + # below is concerned. None until the first re-scan, so the first one always + # forwards even if it picks the socket we are already on - one extra adb + # call, and it keeps this independent of how the caller set the forward up. + forwarded_socket: str | None = None # Re-fire am start once if we don't see the tab in the first half # of the deadline. Cold-launch can swallow the first VIEW intent # (browser was being woken up), and a second intent reliably # navigates an already-warm tab. retry_at = time.monotonic() + (timeout / 2) + rescan_tick = 0 while ws_url is None and time.monotonic() < deadline: await asyncio.sleep(1.0) + # The forward was pinned to a socket chosen before the page existed, + # and on a PICO that choice is a guess. _discover_devtools_socket tells + # the browser's WebLayer processes apart by asking which one holds a + # tab with ``oobEnable=`` - but _close_stale_teleop_tabs has just closed + # exactly those tabs, so on a cold start every candidate answers empty + # and the tie falls back to rank order. That picks the wrong process + # roughly as often as the right one, and the loop below then watches a + # socket the page will never appear on until the deadline runs out. + # Re-running the probe now that ``am start`` has had a second to paint + # is self-correcting: once the page is up, ``oobEnable=`` matches for + # real. Every third pass, so this costs a couple of adb calls, not one + # per second. + rescan_tick += 1 + if rescan_tick % 3 == 0: + rescanned = await asyncio.to_thread(_discover_devtools_socket) + if rescanned and rescanned != forwarded_socket: + try: + await asyncio.to_thread( + _adb_forward_cdp, rescanned, _CDP_LOCAL_PORT + ) + except (OobAdbError, subprocess.CalledProcessError, OSError) as exc: + log.debug("CDP: re-forward to @%s failed: %s", rescanned, exc) + else: + forwarded_socket = rescanned for tab in await asyncio.to_thread(_cdp_list_tabs, _CDP_LOCAL_PORT): if "id" not in tab or not tab.get("webSocketDebuggerUrl"): continue diff --git a/tests/python/core/cloudxr/test_oob_teleop_adb.py b/tests/python/core/cloudxr/test_oob_teleop_adb.py index fcb9e780bd..e2487cce4a 100644 --- a/tests/python/core/cloudxr/test_oob_teleop_adb.py +++ b/tests/python/core/cloudxr/test_oob_teleop_adb.py @@ -1809,3 +1809,78 @@ def test_a_missing_recording_is_named_in_the_error(self, tmp_path): self._reset() with pytest.raises(OobAdbError, match="recording not found"): adb_module._push_recording_to_headset(str(tmp_path / "nope.json")) + + +# --------------------------------------------------------------------------- +# CDP: _find_and_click_teleop_tab re-scans the DevTools sockets +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_find_teleop_tab_follows_the_page_to_another_socket() -> None: + """Cold start on a PICO: the forward was set up before the page existed. + + _close_stale_teleop_tabs has just closed every ``oobEnable=`` tab, so + _discover_devtools_socket could only guess between the browser's WebLayer + processes, and here it guessed the one that only holds the new-tab page. + The wait loop must re-run discovery and move the forward to where the + teleop page actually opened instead of polling the wrong socket until + the deadline. + """ + tabs_by_socket = { + "weblayer_devtools_remote_111": [ + { + "id": "ntp", + "url": "https://browser.example/new", + "webSocketDebuggerUrl": "ws://ntp", + } + ], + "weblayer_devtools_remote_222": [ + { + "id": "teleop", + "url": "https://10.0.0.2:48322/client/?oobEnable=1", + "webSocketDebuggerUrl": "ws://teleop", + } + ], + } + forwarded = {"socket": "weblayer_devtools_remote_111"} + + def forward(socket_name: str, _port: int) -> None: + forwarded["socket"] = socket_name + + def list_tabs(_port: int) -> list[dict]: + return tabs_by_socket[forwarded["socket"]] + + real_sleep = asyncio.sleep + + async def no_wait(_seconds: float) -> None: + await real_sleep(0) + + async def immediate(fn, *args, **kwargs): + return fn(*args, **kwargs) + + with ( + patch.object(adb_module.asyncio, "to_thread", side_effect=immediate), + patch.object(adb_module.asyncio, "sleep", side_effect=no_wait), + patch.object( + adb_module, + "_discover_devtools_socket", + return_value="weblayer_devtools_remote_222", + ), + patch.object(adb_module, "_adb_forward_cdp", side_effect=forward), + patch.object(adb_module, "_cdp_list_tabs", side_effect=list_tabs), + patch.object(adb_module, "_cdp_session_click_connect") as click, + patch.object(adb_module, "run_adb_headset_bookmark") as relaunch, + ): + ws_url = await adb_module._find_and_click_teleop_tab( + resolved_port=48322, + deadline=time.monotonic() + 5.0, + timeout=60.0, + ) + + assert ws_url == "ws://teleop" + assert forwarded["socket"] == "weblayer_devtools_remote_222" + click.assert_awaited_once() + assert click.await_args.args == ("ws://teleop",) + # Found by moving the forward, not by the mid-timeout am start re-fire. + relaunch.assert_not_called() From ec9f9399d8cc73548582411539caa829c6d0f74e Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Mon, 26 Oct 2026 00:35:26 -0700 Subject: [PATCH 8/9] feat(oob): defer headset setup from the CLI until the app is ready The service already knows how to hold headset setup back: defer_oob_connect gates both the OOB lifecycle and the relay of headset signalling until connect_headset() is called. Only code that embeds the service can reach it, though. `service run --setup-oob` always starts the lifecycle as soon as the proxy listens, so when the OpenXR application starts after the service - the usual order, since the app needs the runtime first - the headset enters XR and connects to a runtime with nothing to stream, then cycles through the client's attach and warm-up timeouts and the lifecycle's retries until the app arrives. Expose the gate as --defer-oob-connect, and open it with GET /api/oob/v1/connect. The endpoint authenticates like the other hub endpoints, keeps their no-CORS response, is idempotent, and answers 409 on a service that was not started deferred rather than pretending to have done something. GET because the proxy's HTTP layer only parses GET; /api/oob/v1/config already writes through it the same way. Without the flag nothing changes: no gate is created and setup starts at once, as before. `service start` passes the flag through to the detached run. Signed-off-by: Emily Ma --- docs/source/references/oob_teleop_control.rst | 39 +++++++ .../isaaccapture/cloudxr/service/__main__.py | 30 ++++- src/python/isaaccapture/cloudxr/wss.py | 41 ++++++- tests/python/core/cloudxr/test_service_cli.py | 29 +++++ .../core/cloudxr/test_wss_oob_connect.py | 105 ++++++++++++++++++ 5 files changed, 240 insertions(+), 4 deletions(-) create mode 100644 tests/python/core/cloudxr/test_wss_oob_connect.py diff --git a/docs/source/references/oob_teleop_control.rst b/docs/source/references/oob_teleop_control.rst index 8d30707e7b..f6e17d38ab 100644 --- a/docs/source/references/oob_teleop_control.rst +++ b/docs/source/references/oob_teleop_control.rst @@ -56,6 +56,24 @@ URL is exactly the one built today: Either way the file is only *selected* in the client; starting the replay remains an operator action. +When the OpenXR application starts after the service, add +``--defer-oob-connect``. The runtime and the WSS listener start as usual, but +the headset is left alone - no page is opened, no CONNECT is clicked and no +headset signalling is relayed to the runtime - until +``GET /api/oob/v1/connect``. Request it once the application has created its +OpenXR session, and the headset connects to a stream that is already there +instead of retrying against one that is not: + +.. code-block:: bash + + CONTROL_TOKEN= python -m isaaccapture.cloudxr.service run --accept-eula --setup-oob --defer-oob-connect + # start the application; once it has its OpenXR session: + curl -k -H "X-Control-Token: " https://localhost:48322/api/oob/v1/connect + +Applications that embed the service get the same gate from +``CloudXRLauncher(run_embedded=True, setup_oob=True, defer_oob_connect=True)`` +and ``connect_headset()``. + Ordinary headset absence, an ``offline`` or ``unauthorized`` transport, and browser recovery failures leave the host running with a degraded OOB status. The lifecycle retries for 60 seconds at a 5-second cadence, then keeps @@ -516,6 +534,27 @@ Supported query keys: ``serverIP``, ``port``, ``panelHiddenAtStart``, ``codec``. Optional ``targetClientId`` restricts the push to a single headset (returns 404 if not connected). +``GET /api/oob/v1/connect`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Releases headset setup on a service started with ``--defer-oob-connect`` +(``defer_oob_connect=True``): the lifecycle opens the teleop page and clicks +CONNECT, and the proxy starts relaying headset signalling to the runtime. + +.. code-block:: bash + + curl -k -H "X-Control-Token: " https://localhost:48322/api/oob/v1/connect + +Example response: + +.. code-block:: json + + {"connectRequested": true, "alreadyRequested": false} + +Repeating the request is harmless and reports ``"alreadyRequested": true``. A +service that was not started deferred answers 409: its headset setup began at +startup. + Authentication -------------- diff --git a/src/python/isaaccapture/cloudxr/service/__main__.py b/src/python/isaaccapture/cloudxr/service/__main__.py index b845560250..c09542498a 100644 --- a/src/python/isaaccapture/cloudxr/service/__main__.py +++ b/src/python/isaaccapture/cloudxr/service/__main__.py @@ -80,6 +80,20 @@ def _add_run_arguments(parser: argparse.ArgumentParser) -> None: "TELEOP_WEB_CLIENT_STATIC_DIR or ~/.cloudxr/static-client." ), ) + parser.add_argument( + "--defer-oob-connect", + action="store_true", + default=False, + help=( + "Start the runtime and the WSS listener now, but hold headset setup " + "(opening the teleop page, clicking CONNECT, relaying the headset's " + "signalling) until GET /api/oob/v1/connect, authenticated like the " + "other /api/oob/v1 endpoints. For an OpenXR application that starts " + "after the service: the headset then connects to a stream that is " + "already there instead of waiting on one that is not. Requires " + "--setup-oob." + ), + ) _add_oob_client_arguments(parser) @@ -164,7 +178,13 @@ def _run_flags(args: argparse.Namespace) -> list[str]: flags += ["--cloudxr-install-dir", args.cloudxr_install_dir] if args.cloudxr_env_config: flags += ["--cloudxr-env-config", args.cloudxr_env_config] - for name in ("setup_oob", "usb_local", "host_client", "oob_recording_controls"): + for name in ( + "setup_oob", + "usb_local", + "host_client", + "defer_oob_connect", + "oob_recording_controls", + ): if getattr(args, name, False): flags.append("--" + name.replace("_", "-")) for name in _OOB_CLIENT_FLAG_ENV: @@ -329,6 +349,11 @@ def _print_service_summary( print_oob_hub_startup_banner( lan_host=oob_lan_host, web_client_base=hosted_client_url ) + if getattr(args, "defer_oob_connect", False): + _out( + "\033[33mHeadset setup is deferred: request /api/oob/v1/connect " + "(with the control token) once the application is running.\033[0m" + ) elif hosted_client_url is not None: print_hosted_client_line(hosted_client_url) else: @@ -367,6 +392,8 @@ def _cmd_run(args: argparse.Namespace) -> int: """Run the service in the foreground until interrupted.""" if args.usb_local and not args.setup_oob: _fail("--usb-local requires --setup-oob.") + if args.defer_oob_connect and not args.setup_oob: + _fail("--defer-oob-connect requires --setup-oob.") _apply_oob_client_flags(args) if args.usb_local and os.getenv("TELEOP_OOB_HUB_ONLY"): _fail( @@ -385,6 +412,7 @@ def _cmd_run(args: argparse.Namespace) -> int: usb_local=args.usb_local, host_client=args.host_client, recovery_config=getattr(args, "_oob_recovery_config", None), + defer_oob_connect=args.defer_oob_connect, ) except RuntimeError as exc: _fail(str(exc)) diff --git a/src/python/isaaccapture/cloudxr/wss.py b/src/python/isaaccapture/cloudxr/wss.py index a0a7d84d70..c50b69f245 100755 --- a/src/python/isaaccapture/cloudxr/wss.py +++ b/src/python/isaaccapture/cloudxr/wss.py @@ -239,6 +239,7 @@ def _is_oob_hub_http_path(path: str) -> bool: return path in ( "/api/oob/v1/state", "/api/oob/v1/config", + "/api/oob/v1/connect", ) @@ -305,8 +306,14 @@ def _json_response(status: int, phrase: str, body: dict) -> Response: ) -def _make_http_handler(backend_host, backend_port, hub=None, static_dir=None): - """Return the WSS HTTP request handler (OOB hub API, static ``/client/``, cert page).""" +def _make_http_handler( + backend_host, backend_port, hub=None, static_dir=None, connect_requested=None +): + """Return the WSS HTTP request handler (OOB hub API, static ``/client/``, cert page). + + *connect_requested* is the gate a service started with ``defer_oob_connect`` + holds headset setup behind; ``/api/oob/v1/connect`` opens it. + """ async def handle_http_request(connection, request): """Dispatch non-WebSocket HTTP: CORS preflight, OOB APIs, static client, or cert HTML.""" @@ -362,6 +369,30 @@ async def handle_http_request(connection, request): }.get(status, "Error") return _json_response(status, phrase, body) + if path == "/api/oob/v1/connect": + if not hub.check_token(token): + return _json_response( + 401, "Unauthorized", {"error": "Unauthorized"} + ) + # Without defer_oob_connect there is no gate: headset setup + # began with the service, and a request here cannot mean + # anything a caller should be allowed to assume worked. + if connect_requested is None: + return _json_response( + 409, + "Conflict", + {"error": "headset setup is not deferred on this service"}, + ) + already = connect_requested.is_set() + if not already: + log.info("OOB headset setup released by /api/oob/v1/connect") + connect_requested.set() + return _json_response( + 200, + "OK", + {"connectRequested": True, "alreadyRequested": already}, + ) + if hub is None and _is_oob_hub_http_path(path): return Response( 404, @@ -719,7 +750,11 @@ async def handler(ws) -> None: ) http_handler = _make_http_handler( - backend_host, backend_port, hub=hub, static_dir=_host_client_static_dir + backend_host, + backend_port, + hub=hub, + static_dir=_host_client_static_dir, + connect_requested=connect_requested, ) async with ws_serve( diff --git a/tests/python/core/cloudxr/test_service_cli.py b/tests/python/core/cloudxr/test_service_cli.py index 6ecd8081d2..7853be46dc 100644 --- a/tests/python/core/cloudxr/test_service_cli.py +++ b/tests/python/core/cloudxr/test_service_cli.py @@ -70,6 +70,11 @@ def test_oob_client_flags_round_trip(self): i = flags.index(pair[0]) assert flags[i : i + 2] == pair + def test_defer_oob_connect_round_trips(self): + """A detached `run` must hold headset setup back just like a foreground one.""" + args = _run_args(setup_oob=True, defer_oob_connect=True) + assert cli._run_flags(args) == ["--setup-oob", "--defer-oob-connect"] + def test_accept_eula_is_never_baked_into_the_unit(self): """Acceptance is a marker file written at install time, not a unit flag.""" assert "--accept-eula" not in cli._run_flags(_run_args(accept_eula=True)) @@ -133,6 +138,30 @@ def test_foreground_fatal_with_stalled_writer_returns_error(tmp_path, capsys): service.stop.assert_called_once() +def test_defer_oob_connect_requires_setup_oob(capsys): + with pytest.raises(SystemExit): + cli._cmd_run(_run_args(defer_oob_connect=True)) + assert "--defer-oob-connect requires --setup-oob" in capsys.readouterr().err + + +@pytest.mark.parametrize("defer", [False, True]) +def test_defer_oob_connect_reaches_the_service(tmp_path, defer): + """The flag maps onto the service's existing defer_oob_connect gate, default off.""" + argv = ["run", "--setup-oob", "--cloudxr-install-dir", str(tmp_path)] + if defer: + argv.append("--defer-oob-connect") + args = cli._build_parser().parse_args(argv) + with ( + patch.object(cli, "_oob_preflight", return_value=None), + patch.object( + cli, "CloudXRService", side_effect=RuntimeError("stop here") + ) as service, + ): + with pytest.raises(SystemExit): + cli._cmd_run(args) + assert service.call_args.kwargs["defer_oob_connect"] is defer + + class TestStartEula: """The EULA gate on start.""" diff --git a/tests/python/core/cloudxr/test_wss_oob_connect.py b/tests/python/core/cloudxr/test_wss_oob_connect.py new file mode 100644 index 0000000000..f36dacb491 --- /dev/null +++ b/tests/python/core/cloudxr/test_wss_oob_connect.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""``/api/oob/v1/connect``: releasing a deferred headset setup over HTTP.""" + +from __future__ import annotations + +import asyncio +import json + +import pytest +from cloudxr_py_test_ns.oob_teleop_hub import OOBControlHub +from cloudxr_py_test_ns.wss import CORS_HEADERS, _make_http_handler, add_cors_headers + +TOKEN = "test-control-token" + + +class FakeRequest: + def __init__(self, path: str, headers: dict[str, str] | None = None) -> None: + self.path = path + self.headers = headers or {} + + +def _handler(connect_requested: asyncio.Event | None): + hub = OOBControlHub(control_token=TOKEN) + return _make_http_handler( + "localhost", 49100, hub=hub, connect_requested=connect_requested + ) + + +@pytest.mark.asyncio +async def test_connect_opens_the_deferred_gate() -> None: + gate = asyncio.Event() + handler = _handler(gate) + + response = await handler( + None, FakeRequest("/api/oob/v1/connect", {"X-Control-Token": TOKEN}) + ) + + assert response.status_code == 200 + assert json.loads(response.body) == { + "connectRequested": True, + "alreadyRequested": False, + } + assert gate.is_set() + + +@pytest.mark.asyncio +async def test_connect_is_idempotent() -> None: + """An orchestrator that retries must not be told it failed.""" + gate = asyncio.Event() + handler = _handler(gate) + request = FakeRequest(f"/api/oob/v1/connect?token={TOKEN}") + + await handler(None, request) + response = await handler(None, request) + + assert response.status_code == 200 + assert json.loads(response.body)["alreadyRequested"] is True + assert gate.is_set() + + +@pytest.mark.asyncio +async def test_connect_requires_the_control_token() -> None: + gate = asyncio.Event() + handler = _handler(gate) + + for headers in ({}, {"X-Control-Token": "wrong"}): + response = await handler(None, FakeRequest("/api/oob/v1/connect", headers)) + assert response.status_code == 401 + + assert not gate.is_set() + + +@pytest.mark.asyncio +async def test_connect_without_deferral_is_a_conflict() -> None: + """Without defer_oob_connect setup already ran; say so rather than pretend.""" + handler = _handler(None) + + response = await handler( + None, FakeRequest("/api/oob/v1/connect", {"X-Control-Token": TOKEN}) + ) + + assert response.status_code == 409 + + +@pytest.mark.asyncio +async def test_connect_response_has_no_cors_headers() -> None: + """Like the other hub endpoints, it must not be usable cross-origin.""" + handler = _handler(asyncio.Event()) + request = FakeRequest("/api/oob/v1/connect", {"X-Control-Token": TOKEN}) + + response = await handler(None, request) + add_cors_headers(None, request, response) + + assert not any(k in response.headers for k in CORS_HEADERS) + + +@pytest.mark.asyncio +async def test_connect_is_not_served_without_a_hub() -> None: + handler = _make_http_handler("localhost", 49100, hub=None) + + response = await handler(None, FakeRequest("/api/oob/v1/connect")) + + assert response.status_code == 404 From 4364ba7ad83f0f535f734964f153d48d35ab5604 Mon Sep 17 00:00:00 2001 From: Emily Ma Date: Mon, 26 Oct 2026 02:42:11 -0700 Subject: [PATCH 9/9] feat(oob): add --oob-browser-package to choose the headset browser On PICO, headset_browser_package() prefers com.pico.browser.overseas when it is installed. That package is a regular app, and PICO OS intercepts a VR launch from a regular app that does not declare hand tracking: while the headset is in hand-tracking mode it shows "This app requires the use of controllers for interaction" and only enters VR once someone answers. Nobody is there to answer on an unattended run, so CONNECT is dispatched and the XR session never starts. com.pico.browser ships as a system app, which the same check skips. Both are shells over the system WebLayer, so the page behaves the same; on a PICO 4 Ultra in hand-tracking mode the system browser entered VR 1s after CONNECT with no dialog, where the overseas one waited on it. TELEOP_HEADSET_BROWSER_PACKAGE already overrides the choice. Give it the same command-line spelling as the other --oob-* client flags, and document the variable, which the environment table did not list. The per-vendor default is unchanged. Signed-off-by: Emily Ma --- docs/source/references/oob_teleop_control.rst | 10 +++++++++ .../isaaccapture/cloudxr/service/__main__.py | 13 ++++++++++++ tests/python/core/cloudxr/test_service_cli.py | 21 +++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/docs/source/references/oob_teleop_control.rst b/docs/source/references/oob_teleop_control.rst index f6e17d38ab..ecb039d80a 100644 --- a/docs/source/references/oob_teleop_control.rst +++ b/docs/source/references/oob_teleop_control.rst @@ -687,6 +687,16 @@ Environment variables pushed to the headset and selected in the client's recording file input before CONNECT, so the operator only has to start the replay. Unset by default, so nothing is loaded. + * - ``TELEOP_HEADSET_BROWSER_PACKAGE`` + - Android package of the headset browser that opens the teleop page + (``--oob-browser-package``). Unset, the vendor's browser is picked: + ``com.oculus.browser`` on Quest, ``com.pico.browser.overseas`` if + installed and otherwise ``com.pico.browser`` on PICO. On PICO OS, a VR + launch from an app that is neither a system app nor declares hand + tracking is held behind a "This app requires the use of controllers" + dialog whenever the headset is in hand-tracking mode, which an + unattended run cannot answer. ``com.pico.browser`` ships as a system + app and is not held. * - ``TELEOP_OOB_RECOVERY_TIMEOUT_SEC`` - Positive finite recovery-episode duration in seconds (default ``60``). Expiry changes to observation mode; it never stops the host. If this is diff --git a/src/python/isaaccapture/cloudxr/service/__main__.py b/src/python/isaaccapture/cloudxr/service/__main__.py index c09542498a..fbe0877f23 100644 --- a/src/python/isaaccapture/cloudxr/service/__main__.py +++ b/src/python/isaaccapture/cloudxr/service/__main__.py @@ -105,6 +105,7 @@ def _add_run_arguments(parser: argparse.ArgumentParser) -> None: "oob_replay_pacing": "TELEOP_CLIENT_REPLAY_PACING", "oob_immersive_mode": "TELEOP_CLIENT_IMMERSIVE_MODE", "oob_recording": "TELEOP_OOB_RECORDING", + "oob_browser_package": "TELEOP_HEADSET_BROWSER_PACKAGE", } @@ -155,6 +156,18 @@ def _add_oob_client_arguments(parser: argparse.ArgumentParser) -> None: "replay. Requires --setup-oob." ), ) + parser.add_argument( + "--oob-browser-package", + type=str, + default=None, + metavar="PACKAGE", + help=( + "Android package of the headset browser that opens the teleop page, " + "instead of the one picked per vendor. On PICO, com.pico.browser is " + "the system browser, which the OS lets enter VR without asking for " + "controllers. Requires --setup-oob." + ), + ) parser.add_argument( "--oob-recording-controls", action="store_true", diff --git a/tests/python/core/cloudxr/test_service_cli.py b/tests/python/core/cloudxr/test_service_cli.py index 7853be46dc..3d73e54dd4 100644 --- a/tests/python/core/cloudxr/test_service_cli.py +++ b/tests/python/core/cloudxr/test_service_cli.py @@ -70,6 +70,14 @@ def test_oob_client_flags_round_trip(self): i = flags.index(pair[0]) assert flags[i : i + 2] == pair + def test_oob_browser_package_round_trips(self): + args = _run_args(setup_oob=True, oob_browser_package="com.pico.browser") + assert cli._run_flags(args) == [ + "--setup-oob", + "--oob-browser-package", + "com.pico.browser", + ] + def test_defer_oob_connect_round_trips(self): """A detached `run` must hold headset setup back just like a foreground one.""" args = _run_args(setup_oob=True, defer_oob_connect=True) @@ -138,6 +146,19 @@ def test_foreground_fatal_with_stalled_writer_returns_error(tmp_path, capsys): service.stop.assert_called_once() +def test_oob_browser_package_sets_the_override_the_adb_code_reads(monkeypatch): + monkeypatch.delenv("TELEOP_HEADSET_BROWSER_PACKAGE", raising=False) + args = _run_args(setup_oob=True, oob_browser_package="com.pico.browser") + cli._apply_oob_client_flags(args) + assert os.environ["TELEOP_HEADSET_BROWSER_PACKAGE"] == "com.pico.browser" + + +def test_oob_browser_package_requires_setup_oob(capsys): + with pytest.raises(SystemExit): + cli._apply_oob_client_flags(_run_args(oob_browser_package="com.pico.browser")) + assert "--oob-browser-package requires --setup-oob" in capsys.readouterr().err + + def test_defer_oob_connect_requires_setup_oob(capsys): with pytest.raises(SystemExit): cli._cmd_run(_run_args(defer_oob_connect=True))