sensing: add a mock backend so the plugin runs without a Jetson

README.md

@@ -80,4 +80,5 @@ To get started, please refer to Isaac Lab's [Installation](https://isaac-sim.git
 
 The default license is [Apache-2.0](LICENSE.md).
 EgoPost is licensed under [OpenMDW-1.1](src/python/isaaccapture_egopost/LICENSE.md).
-Third-party materials may carry their own licenses.
+Third-party materials may carry their own licenses; for example, the sample GIFs under
+[`src/plugins/sensing/mock_assets/`](src/plugins/sensing/mock_assets/) are Animated Noto Emoji under CC BY 4.0.

src/plugins/sensing/CMakeLists.txt

@@ -4,22 +4,29 @@
 # ==============================================================================
 # SENSING Camera Plugin (Jetson SIPL capture + V4L2 H.264 encode)
 # ==============================================================================
-# Jetson-only: SIPL and the V4L2 M2M encoder exist only on L4T.
+# Two backends sit behind SiplCamera and JetsonEncoder:
+#   JETSON  SIPL capture and the V4L2 M2M encoder, which exist only on L4T.
+#   MOCK    synthetic frames and an NVENC encoder (libavcodec), for any machine with an
+#           NVIDIA GPU and the CUDA toolkit. Everything above those two classes
+#           (SensingCamera, sinks, CUDA IPC, the app) is the same code.
+# SENSING_BACKEND=AUTO picks JETSON on aarch64 and MOCK elsewhere.
 #
 # Four leaf targets, one per directory (cmake/cmake-structure.md rule 1):
 #   cuda_ipc/  sensing_cuda_ipc  — CUDA-only IPC publisher + wire format
 #   core/      sensing_core      — SIPL capture, encoder, sinks
 #   app/       camera_plugin_sensing
 #   tools/     sensing_ipc_testsrc — links cuda_ipc only, no SIPL, no encoder
 #
-# The Jetson SDK checks below gate the whole plugin, tools/ included.
+# The Jetson SDK checks below gate the whole plugin, tools/ included, when the
+# backend is JETSON.
 #
 # Prerequisites:
 #   src/plugins/sensing/setup_container.sh   # fetches the Multimedia API,
 #                                            # locates jetson_sipl_api
 #
 # Configure with:
-#   cmake -B build -DBUILD_PLUGIN_SENSING=ON
+#   cmake -B build -DBUILD_PLUGIN_SENSING=ON                          # Jetson, or mock elsewhere
+#   cmake -B build -DBUILD_PLUGIN_SENSING=ON -DSENSING_BACKEND=MOCK   # force the mock
 # ==============================================================================
 
 enable_language(CUDA)
@@ -31,12 +38,32 @@ endif()
 
 find_package(CUDAToolkit REQUIRED)
 
+set(SENSING_BACKEND "AUTO" CACHE STRING "SENSING backend: AUTO, JETSON (SIPL + V4L2 NVENC) or MOCK (synthetic)")
+set_property(CACHE SENSING_BACKEND PROPERTY STRINGS AUTO JETSON MOCK)
+if(SENSING_BACKEND STREQUAL "AUTO")
+    # A Jetson that lacks the SDK should still get the setup instructions below,
+    # not a silent mock, so only non-aarch64 hosts fall back.
+    if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
+        set(SENSING_BACKEND_RESOLVED "JETSON" CACHE INTERNAL "")
+    else()
+        set(SENSING_BACKEND_RESOLVED "MOCK" CACHE INTERNAL "")
+    endif()
+elseif(SENSING_BACKEND STREQUAL "JETSON" OR SENSING_BACKEND STREQUAL "MOCK")
+    set(SENSING_BACKEND_RESOLVED "${SENSING_BACKEND}" CACHE INTERNAL "")
+else()
+    message(FATAL_ERROR "SENSING_BACKEND must be AUTO, JETSON or MOCK, got '${SENSING_BACKEND}'")
+endif()
+# Cache, not a plain variable, so tests/ (a sibling directory) can gate on it too.
+message(STATUS "SENSING plugin backend: ${SENSING_BACKEND_RESOLVED}")
+
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|arm64")
     set(_tegra_lib_dirs "/usr/lib/aarch64-linux-gnu/nvidia" "/usr/lib/aarch64-linux-gnu/tegra")
 else()
     set(_tegra_lib_dirs "/usr/lib/x86_64-linux-gnu/nvidia")
 endif()
 
+if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON")
+# Jetson-only SDK checks.
 # ==============================================================================
 # SIPL
 # ==============================================================================
@@ -159,6 +186,7 @@ message(STATUS "SENSING plugin: V4L2=${V4L2_LIBRARY}")
 # No libnvjpeg here on purpose: on R39 libnvbufsurface has no libjpeg
 # dependency at all (readelf -d lists only EGL, cuda, dl, pthread, rt, c), and
 # the tegra lib dir ships no libnvjpeg to link against.
+endif()
 
 # ==============================================================================
 # Targets

src/plugins/sensing/README.md

@@ -24,6 +24,7 @@ That guide owns the operating instructions; this README maps the source tree.
 - [`docker/`](docker/): example image, image-side group configuration, and
   host-side container setup.
 - [`configs/`](configs/): the platform configuration and its provenance.
+- [`core/mock_*`](core/): the mock backend (see below).
 
 ## Platform configuration provenance
 
@@ -44,3 +45,59 @@ Keep the file byte-identical to the vendor copy. Compare new vendor drops
 before replacing it; do not hand-edit the board wiring. Its CSI ports, I2C
 buses, deserializer addresses, GPIOs, and link modes must match the installed
 SIPL drivers.
+
+## Mock backend: run the plugin without a Jetson
+
+`SENSING_BACKEND` selects what sits behind `SiplCamera` and `JetsonEncoder`:
+
+| Value | Capture | Encoder | Needs |
+| --- | --- | --- | --- |
+| `JETSON` | SIPL | V4L2 NVENC | L4T SDKs |
+| `MOCK` | animated GIF frames | NVENC through libavcodec | NVIDIA GPU, CUDA toolkit, EGL headers (`libegl-dev`), `libavcodec-dev` |
+| `AUTO` (default) | `JETSON` on aarch64, `MOCK` elsewhere | | |
+
+```bash
+cmake -B build -DBUILD_PLUGIN_SENSING=ON -DSENSING_BACKEND=MOCK
+cmake --build build --target camera_plugin_sensing sensing_mock_tests
+./build/src/plugins/sensing/app/camera_plugin_sensing --list-sensors
+```
+
+Everything above those two classes is the code that runs on the rig: `SensingCamera`, the
+sinks, the YUV-to-RGBA and RGBA-to-NV12 kernels, and the CUDA IPC publisher. The mock
+resolves `--list-sensors` and the link masks from the vendor JSON, builds a synthetic NV12
+surface from a GIF frame, and runs the real conversion on it. Both sensors share a capture
+TSC, as the rig's fsync does.
+
+The mock encoder produces real, decodable H.264 on the desktop GPU's NVENC (libavcodec's `h264_nvenc`),
+configured as the Jetson encoder is: High profile, Level 5.2, VBR with a peak, an IDR every GOP with
+SPS/PPS in front of it, no B-frames. It reproduces what callers depend on: a fixed pool of input
+buffers that makes `submit()` refuse frames when the encoder falls behind, capture stamps that come
+back at microsecond resolution, and the tail returned as one unit at end of stream. Bitstream bytes,
+unit sizes and latency are NVENC's, not the Jetson engine's.
+
+Every sensor shows an animated GIF chosen by its **module name** (never its pipeline index, which the
+link mask renumbers), from the bundled Animated Noto Emoji in [`mock_assets/`](mock_assets/) (CC BY 4.0;
+see its README): `S56C_HEAD` the robot, `SHF3L_LEFT` the waving hand, `SHF3L_RIGHT` the hand flipped left
+to right. The two sensors of `S56C_HEAD` are its left and right eyes; their pictures are offset against
+each other by a fixed 2% of the width (illustrative, not calibrated geometry). A module not in that
+table is an error. The GIF plays on the capture clock, looping, scaled
+to cover the sensor's configured size and centre-cropped so it is never stretched (bilinear when
+enlarging, area-averaged when reducing). Transparent pixels show a plain gray. The GIF's own size and
+aspect do not matter; the frame size always comes from the platform config. A GIF that cannot be loaded,
+the bundled one included, is an error: there is no fallback.
+
+[`mock_assets/mock_rig.json`](mock_assets/mock_rig.json) describes a three-camera rig for the mock only
+(not a SIPL config): one S56C stereo head camera with two 2560x1984 @ 30 eyes, and two SHF3L wrist
+cameras at 1920x1536 @ 30. Use `--platform-config=.../mock_rig.json --config-name=MOCK_S56C_2_SHF3L
+--link-masks='0x0000 0x0111'`.
+
+Environment, read once at construction:
+
+| Variable | Effect |
+| --- | --- |
+| `SENSING_MOCK_FPS` | override the platform config's frame rate |
+| `SENSING_MOCK_SCALE_DIV` | divide the reported width and height (kept even); useful for tests |
+| `SENSING_MOCK_FAIL_AFTER` | every sensor fails after N frames, as a link drop would |
+
+A pass on the mock says nothing about SIPL, the ISP output format, NvSci buffer lifetime, the
+V4L2 encoder or real frame rates. Those are only checked on an Orin with SENSING cameras.

src/plugins/sensing/core/CMakeLists.txt

@@ -1,17 +1,70 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# SIPL capture, the V4L2 M2M encoder wrapper, and the output sinks. Everything
-# that needs the Jetson SDKs lives here, so tools/ stays buildable without them.
-add_library(sensing_core STATIC
+# Capture, encoder and output sinks. The JETSON backend needs the Jetson SDKs,
+# so everything that does lives in this directory and tools/ stays buildable
+# without them. The MOCK backend replaces SiplCamera and JetsonEncoder with
+# synthetic ones that need only CUDA.
+set(SENSING_COMMON_SOURCES
     sensing_camera.cpp
     frame_sink.cpp
     rawdata_writer.cpp
-    sipl_camera.cpp
     yuv_to_rgba.cu
-    jetson_encoder.cpp
     rgba_to_nv12.cu
-    ${JETSON_MMAPI_SOURCES}
+)
+
+if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON")
+    set(SENSING_BACKEND_SOURCES
+        sipl_camera.cpp
+        jetson_encoder.cpp
+        ${JETSON_MMAPI_SOURCES}
+    )
+else()
+    # yuv_to_rgba.cu includes cudaEGL.h, which needs the EGL headers even though
+    # nothing here opens an EGL display.
+    find_path(SENSING_EGL_INCLUDE_DIR EGL/egl.h)
+    if(NOT SENSING_EGL_INCLUDE_DIR)
+        message(FATAL_ERROR "The SENSING mock backend needs the EGL headers (cudaEGL.h includes them). "
+                            "Install libegl-dev.")
+    endif()
+    # Parses the vendor JSON itself, so a MOCK build has no SIPL query library.
+    find_package(nlohmann_json 3 QUIET)
+    if(NOT nlohmann_json_FOUND)
+        include(FetchContent)
+        FetchContent_Declare(
+            nlohmann_json
+            GIT_REPOSITORY https://github.com/nlohmann/json.git
+            GIT_TAG v3.11.3
+            GIT_SHALLOW TRUE
+        )
+        set(JSON_BuildTests OFF CACHE INTERNAL "")
+        isaac_teleop_third_party_scope_begin()
+        FetchContent_MakeAvailable(nlohmann_json)
+        isaac_teleop_third_party_scope_end()
+    endif()
+    # Decodes the GIFs the mock camera can show. A single public-domain header, pinned to a commit and
+    # checked against its hash, so nothing else of the project is cloned.
+    include(FetchContent)
+    FetchContent_Declare(
+        stb_image
+        URL https://raw.githubusercontent.com/nothings/stb/003501f6d7a8668ce8b6d875e34dd97ad0d5d5b5/stb_image.h
+        URL_HASH SHA256=594c2fe35d49488b4382dbfaec8f98366defca819d916ac95becf3e75f4200b3
+        DOWNLOAD_NO_EXTRACT TRUE
+    )
+    FetchContent_MakeAvailable(stb_image)
+
+    set(SENSING_BACKEND_SOURCES
+        mock_sipl_camera.cpp
+        mock_gif_source.cpp
+        mock_platform_config.cpp
+        mock_nv12_source.cu
+        mock_jetson_encoder.cpp
+    )
+endif()
+
+add_library(sensing_core STATIC
+    ${SENSING_COMMON_SOURCES}
+    ${SENSING_BACKEND_SOURCES}
 )
 add_library(sensing::core ALIAS sensing_core)
 
@@ -27,19 +80,6 @@ set_target_properties(sensing_core PROPERTIES
 target_include_directories(sensing_core
     PUBLIC
         ${CMAKE_CURRENT_SOURCE_DIR}/..
-    PRIVATE
-        # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay
-        # private: they drag in NvMedia and X11-style macros that collide with
-        # mcap's StatusCode, which is what sensing_types.hpp exists to avoid.
-        ${SIPL_INCLUDE_DIRS}
-        ${EGL_INCLUDE_DIRS}
-        ${JETSON_MMAPI_DIR}/include
-)
-
-# The Multimedia API reference sources carry legacy warnings.
-target_compile_options(sensing_core PRIVATE
-    $<$<COMPILE_LANGUAGE:CXX>:-Wno-reorder -Wno-uninitialized -Wno-unused-variable
-                              -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare>
 )
 
 target_link_libraries(sensing_core
@@ -48,10 +88,40 @@ target_link_libraries(sensing_core
         sensing::cuda_ipc
         isaacteleop_schema
     PRIVATE
-        ${SIPL_LIBRARIES}
-        ${EGL_LIBRARIES}
-        ${NVBUFSURFACE_LIBRARY}
-        ${V4L2_LIBRARY}
         mcap::mcap
         oxr::oxr_utils
 )
+
+if(SENSING_BACKEND_RESOLVED STREQUAL "JETSON")
+    target_include_directories(sensing_core
+        PRIVATE
+            # sipl_camera.hpp pimpls the SIPL and NvSci types away, so these stay
+            # private: they drag in NvMedia and X11-style macros that collide with
+            # mcap's StatusCode, which is what sensing_types.hpp exists to avoid.
+            ${SIPL_INCLUDE_DIRS}
+            ${EGL_INCLUDE_DIRS}
+            ${JETSON_MMAPI_DIR}/include
+    )
+
+    # The Multimedia API reference sources carry legacy warnings.
+    target_compile_options(sensing_core PRIVATE
+        $<$<COMPILE_LANGUAGE:CXX>:-Wno-reorder -Wno-uninitialized -Wno-unused-variable
+                                  -Wno-unused-parameter -Wno-missing-field-initializers -Wno-sign-compare>
+    )
+
+    target_link_libraries(sensing_core
+        PRIVATE
+            ${SIPL_LIBRARIES}
+            ${EGL_LIBRARIES}
+            ${NVBUFSURFACE_LIBRARY}
+            ${V4L2_LIBRARY}
+    )
+else()
+    # The mock encoder produces real H.264 with libavcodec's h264_nvenc (development package:
+    # libavcodec-dev; the desktop driver supplies NVENC).
+    find_package(PkgConfig REQUIRED)
+    pkg_check_modules(AVCODEC REQUIRED IMPORTED_TARGET libavcodec libavutil)
+    target_link_libraries(sensing_core PRIVATE nlohmann_json::nlohmann_json PkgConfig::AVCODEC)
+    target_include_directories(sensing_core SYSTEM PRIVATE ${stb_image_SOURCE_DIR})
+    target_compile_definitions(sensing_core PRIVATE SENSING_MOCK_ASSET_DIR="${CMAKE_CURRENT_SOURCE_DIR}/../mock_assets")
+endif()