sensing: stream sensors to LiveKit from the hardware encoder

src/plugins/sensing/CMakeLists.txt

@@ -188,12 +188,64 @@ message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}")
 # the tegra lib dir ships no libnvjpeg to link against.
 endif()
 
+# ==============================================================================
+# LiveKit video output (optional)
+# ==============================================================================
+# Publishes each selected sensor as a LiveKit camera track. Off by default so a build that does not
+# stream needs nothing new. The SDK is a prebuilt package (headers and shared libraries), pinned to a
+# release and checked against its hash; point LIVEKIT_SDK_DIR at an unpacked copy to use another.
+option(SENSING_LIVEKIT "Add LiveKit video streaming to the SENSING plugin (downloads the LiveKit C++ SDK)" OFF)
+if(SENSING_LIVEKIT)
+    if(NOT LIVEKIT_SDK_DIR)
+        if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+            set(_livekit_arch "arm64")
+            set(_livekit_hash "e3a97f71943c0bf9fa4ebd70927ca2906b31dcd6c14b42312b1225b4b868e4ad")
+        elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|AMD64")
+            set(_livekit_arch "x64")
+            set(_livekit_hash "5e0feb2e519c6ff131dd2d1c45149115057ae4e21a324fdb249e1789d5614e7b")
+        else()
+            message(FATAL_ERROR "No prebuilt LiveKit SDK for ${CMAKE_SYSTEM_PROCESSOR}; set LIVEKIT_SDK_DIR.")
+        endif()
+        include(FetchContent)
+        FetchContent_Declare(
+            livekit_sdk
+            URL https://github.com/livekit/client-sdk-cpp/releases/download/v1.12.4/livekit-sdk-ubuntu-24.04-${_livekit_arch}-1.12.4.tar.gz
+            URL_HASH SHA256=${_livekit_hash}
+        )
+        FetchContent_MakeAvailable(livekit_sdk)
+        set(LIVEKIT_SDK_DIR "${livekit_sdk_SOURCE_DIR}")
+    endif()
+    # The package ships no licence files: install the SDK's own, pinned to the same release.
+    foreach(_notice LICENSE NOTICE)
+        if(_notice STREQUAL "LICENSE")
+            set(_notice_hash "09e8a9bcec8067104652c168685ab0931e7868f9c8284b66f5ae6edae5f1130b")
+        else()
+            set(_notice_hash "6d0f7c5e0deb284362296c5da110dd353831da2e67ca77cb5abc051b951b0fac")
+        endif()
+        FetchContent_Declare(
+            livekit_sdk_${_notice}
+            URL https://raw.githubusercontent.com/livekit/client-sdk-cpp/v1.12.4/${_notice}
+            URL_HASH SHA256=${_notice_hash}
+            DOWNLOAD_NO_EXTRACT TRUE
+        )
+        FetchContent_MakeAvailable(livekit_sdk_${_notice})
+        set(SENSING_LIVEKIT_${_notice}_FILE "${livekit_sdk_${_notice}_SOURCE_DIR}/${_notice}" CACHE INTERNAL "")
+    endforeach()
+    find_package(LiveKit CONFIG REQUIRED PATHS "${LIVEKIT_SDK_DIR}" NO_DEFAULT_PATH)
+    message(STATUS "SENSING plugin: LiveKit SDK=${LIVEKIT_SDK_DIR}")
+    # Cache, so tests/ (a sibling directory) can find the same SDK.
+    set(SENSING_LIVEKIT_SDK_DIR "${LIVEKIT_SDK_DIR}" CACHE INTERNAL "")
+endif()
+
 # ==============================================================================
 # Targets
 # ==============================================================================
 message(STATUS "Building SENSING camera plugin")
 
 add_subdirectory(cuda_ipc)
+if(SENSING_LIVEKIT)
+    add_subdirectory(livekit)
+endif()
 add_subdirectory(core)
 add_subdirectory(app)
 add_subdirectory(tools)

src/plugins/sensing/README.md

@@ -25,6 +25,7 @@ That guide owns the operating instructions; this README maps the source tree.
   host-side container setup.
 - [`configs/`](configs/): the platform configuration and its provenance.
 - [`core/mock_*`](core/): the mock backend (see below).
+- [`livekit/`](livekit/): the LiveKit SDK behind a small interface, for streaming (see below).
 
 ## Platform configuration provenance
 
@@ -101,3 +102,101 @@ Environment, read once at construction:
 
 A pass on the mock says nothing about SIPL, the ISP output format, NvSci buffer lifetime, the
 V4L2 encoder or real frame rates. Those are only checked on an Orin with SENSING cameras.
+
+## LiveKit streaming
+
+`livekit=<track>` on an `--add-stream` publishes that sensor as a LiveKit camera track, from the real
+cameras or the mock alike: the code is the same on both backends. It is off unless built with
+`-DSENSING_LIVEKIT=ON`, which downloads the pinned LiveKit C++ SDK package (Ubuntu 24.04, x64 or
+arm64, checked against its hash) at configure time; set `LIVEKIT_SDK_DIR` to use an unpacked copy
+instead.
+
+```bash
+camera_plugin_sensing --livekit-url=wss://livekit.example.org --livekit-token-file=join.token \
+    --add-stream=sensor=0,livekit=sensing-left --add-stream=sensor=1,livekit=sensing-right
+```
+
+- **Hardware-encoded passthrough.** Each track carries H.264 that this machine's hardware encoder
+  produced, published through the SDK's pre-encoded source (`EncodedVideoSource`, encoder backend
+  `PreEncoded`): the SDK forwards each access unit and does not encode. The encoder is the plugin's
+  `JetsonEncoder` class, so it is the V4L2 engine on a Jetson and NVENC through libavcodec on the
+  mock. There is no software encoder and no fallback: if the hardware encoder cannot start,
+  construction fails. The status line names the encoder and the passthrough mode.
+- **Token.** The operator supplies it in a file, never on the command line. It names the room and
+  the participant and must allow publishing; viewers use a separate subscribe-only token. It is not
+  printed.
+- **Tracks.** One H.264 track per sensor, no simulcast. The default size is the sensor's configured
+  size. `--livekit-scale-div=N` reduces every track by an integer divisor that must divide the
+  sensor exactly and leave even sizes (2 gives 1280x992 from 2560x1984); it is refused otherwise,
+  never rounded or stretched, and the encoder runs at the reduced size. `--livekit-fps` caps the
+  publication rate on the capture clock (default 30). `--livekit-bitrate` sets the encoder's target
+  (default 8 Mbit/s; the VBR ceiling is half as much again), unlike the recording's `--bitrate`.
+- **Frames.** `update()` takes a GPU snapshot of the camera frame and finishes that copy before
+  returning, because the camera may rewrite its slot at the second following poll. A worker per
+  sensor then reduces it if asked, hands it to the hardware encoder and publishes each encoded unit
+  as one access unit, with its own stamp. A slow network drops frames before encoding and cannot
+  hold up capture or the other sensor: one snapshot is being encoded, one waits, and a newer capture
+  replaces the waiting one.
+- **Feedback.** A viewer joining, or packet loss, makes the SDK ask for a keyframe: the next frame
+  is an IDR with SPS/PPS. A frame the SDK refuses breaks the reference chain, so the next frame is
+  an IDR and nothing is published in between. Bitrate targets from congestion control are clamped
+  to the configured range and applied to the encoder: the Jetson engine changes rate while running;
+  the mock reopens its session at the new rate, which starts with an IDR.
+- **Metadata.** Each frame carries the camera's sequence as its frame ID, the capture time on the host
+  clock (CLOCK_MONOTONIC microseconds) as its user timestamp, and the sensor's capture TSC as eight
+  little-endian bytes of user data. Two separately published tracks are not exposure-synchronised;
+  the shared TSC is what pairs them.
+- **Recording.** A stream with `output=` and `livekit=` runs two encoders, one for the file at the
+  recording's settings and one for the track at the live settings. `--mcap-filename` metadata is
+  separate and unchanged.
+- **Failure.** Losing the room, or a failed publish, ends capture with an error; there is no
+  fallback to files or another transport. The SDK's own reconnection runs first; on loopback, losing
+  the server was reported after 15 to 30 s.
+
+**Verified.** On a desktop GPU against a local `livekit-server`, with a second participant decoding
+what arrives: tracks, sizes and picture content, frame IDs and capture TSCs, a viewer joining late
+and rejoining, keyframe requests, recovery after a refused frame, the end-of-stream drain, lower
+bitrate targets, and the transport's stats naming the passthrough source. The arm64 SDK package has
+the same pre-encoded API headers as the x64 one.
+
+**Not verified.** Nothing here ran on an Orin: not real cameras, not the V4L2 encoder feeding
+LiveKit, not the arm64 package, and not the `JETSON` build with LiveKit. The Jetson encoder's
+keyframe, rate and drain code is written but was not compiled off the rig. That the H.264 the
+Jetson engine produces (High profile, Level 5.2) is accepted by a WebRTC receiver is likewise
+unproven; the mock's NVENC output, with the same settings, decoded in the tests.
+
+**Receiver page.** [`livekit/receiver/index.html`](livekit/receiver/index.html) is a static page that
+uses the LiveKit browser SDK (from a CDN, integrity-checked): enter the server URL and a subscriber
+token, Connect, and each subscribed video track appears under its name with its size, in its own
+aspect ratio. There is no backend or token generator; it is installed beside the plugin under
+`receiver/` and can be opened from disk or any static server.
+
+**Capacity on a Jetson (not measured).** The numbers below are calculated encoder workloads from the
+module datasheets, not achieved frame rates; nothing has run on an Orin. They assume head images at
+2560x1984 and wrist images at 1920x1536.
+
+| Images | Rate | Encoded pixels/s | Assessment |
+| --- | --- | --- | --- |
+| Head stereo, native | 30 fps | 305 million | plausible first target |
+| Head stereo and both wrists, native | 30 fps | 482 million | depends on the preset; little headroom |
+| All four at half width and height | 30 fps | 120 million | more headroom |
+| All four, native | 60 fps | 963 million | not a supported target |
+
+The datasheets list H.264 ceilings of 930/530/280 MPixel/s (UHP/HP/HQ) for the AGX Orin 64GB module,
+730/420/220 for the 32GB module and 680/470/220 for Orin NX; four native images at 30 fps already
+exceed the NX and 32GB HP ceilings. Orin Nano has no NVENC, so it cannot meet the hardware-encoder
+requirement. The encoder selects the V4L2 `FAST` hardware preset explicitly (a control separate from
+max-performance clocks); four native images may need `ULTRAFAST`, to be measured. A stream with both
+`output=` and `livekit=` runs two encoder instances on the one hardware engine, so budget the sum of
+their pixel rates. Sustained rates also depend on power mode and cooling. A frame the encoder has no
+input buffer for is dropped before any conversion work. Moving frames without the intermediate
+copies (a direct NV12 or DMA-BUF path) is a separate change to make only if measurements call for it.
+
+`cmake --install` copies `liblivekit.so` and `liblivekit_ffi.so` beside the plugin, which finds them
+through `$ORIGIN`, and the SDK's own `LICENSE` and `NOTICE` (fetched at configure time, hash
+checked) under `licenses/livekit-sdk/`. `liblivekit_ffi.so` bundles third-party components, WebRTC
+among them, whose notices the SDK package does not ship: they need a compliance review before the
+binaries are distributed.
+
+Tests that need a LiveKit service start a local `livekit-server` when `LIVEKIT_SERVER` names its
+binary, and are skipped without it; the rest need only a GPU.

src/plugins/sensing/app/CMakeLists.txt

@@ -23,3 +23,19 @@ install(TARGETS camera_plugin_sensing
 install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/../configs/"
     DESTINATION plugins/sensing_camera/configs
 )
+
+# The plugin links the LiveKit runtime, whose RUNPATH points into the build tree; ship it beside the
+# binary, where INSTALL_RPATH=$ORIGIN (and liblivekit.so's own $ORIGIN) finds it.
+if(SENSING_LIVEKIT)
+    install(FILES
+        "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit.so"
+        "${SENSING_LIVEKIT_SDK_DIR}/lib/liblivekit_ffi.so"
+        DESTINATION plugins/sensing_camera
+    )
+    install(FILES "${SENSING_LIVEKIT_LICENSE_FILE}" "${SENSING_LIVEKIT_NOTICE_FILE}"
+        DESTINATION plugins/sensing_camera/licenses/livekit-sdk
+    )
+    install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../livekit/receiver/index.html"
+        DESTINATION plugins/sensing_camera/receiver
+    )
+endif()