feat(webcam): add AI background cutout, blur, and custom wallpapers (#291)

.github/workflows/build.yml

@@ -353,6 +353,16 @@ jobs:
       - name: Build Metal compositor addon
         run: npm run build:native:compositor:mac
 
+      # Third step this job has to spell out, same reason as the two above: it is in
+      # `npm run build:mac`, which this job does not run — it needs `--dir` plus a
+      # hand-rolled DMG. Windows and Linux get it free from `build:win` / `build:linux`.
+      # Unlike the compositor addon, a missing ONNX Runtime does not fail the pack: the
+      # camera-background control just does nothing, on every shipped Mac, with nothing
+      # in CI raising a word. No-ops on x64 — upstream publishes no osx-x64 asset, so the
+      # script says so and exits 0 rather than failing that build.
+      - name: Stage ONNX Runtime
+        run: npm run fetch:onnxruntime
+
       - name: Package .app bundle
         run: npx electron-builder --mac --${{ matrix.arch }} --dir --publish never
         env:

.github/workflows/ci.yml

@@ -159,9 +159,19 @@ jobs:
       # [env] block (cargo has no [target.<cfg>.env] — the macOS section in that file
       # is inert and cargo warns "unused key"), so before that change this job pointed
       # bindgen at the win64 tree and could never have gone green.
+      #
+      # Without a library on ORT_DYLIB_PATH, `runtime_available()` is false and every
+      # segmentation test returns early — the suite goes green having exercised no
+      # inference at all, which is exactly how the `ort`-panics-when-absent bug got
+      # in. Staging it here is what makes `the_whole_loop_produces_a_mask_from_
+      # compose_frame_alone` a real test on this runner instead of a skipped one.
+      # ~30 MB, next to nothing beside `brew install ffmpeg` above.
+      - name: Stage ONNX Runtime
+        run: node scripts/fetch-onnxruntime.mjs
       - name: cargo test (compositor, aarch64-apple-darwin)
         env:
           MAC_FFMPEG_DIR: /opt/homebrew/opt/ffmpeg
+          ORT_DYLIB_PATH: ${{ github.workspace }}/electron/native/bin/darwin-arm64/libonnxruntime.dylib
         run: |
           cd crates
           cargo test -p openscreen-compositor --lib --tests
@@ -266,11 +276,21 @@ jobs:
           # et rendre une erreur qui ne designe pas la cause non plus.
           test -n "$libclang" || { echo "libclang introuvable apres l'installation"; exit 1; }
           echo "LIBCLANG_PATH=$(dirname "$libclang")" >> "$GITHUB_ENV"
+      # Meme raison que sur le job macOS : sans bibliotheque sur ORT_DYLIB_PATH,
+      # `runtime_available()` est faux et chaque test de segmentation rend la main
+      # tout de suite — la suite passe au vert sans avoir exerce la moindre
+      # inference, ce qui est exactement par ou le bug « ort panique quand elle
+      # manque » est entre. C'est ce qui fait de
+      # `the_whole_loop_produces_a_mask_from_compose_frame_alone` un vrai test ici
+      # plutot qu'un test saute. Builtins node uniquement, comme fetch:ffmpeg:sdk.
+      - name: Stage ONNX Runtime
+        run: node scripts/fetch-onnxruntime.mjs
       - name: cargo test (compositor)
         env:
           # Les .so ffmpeg vendorises ne sont dans aucun chemin systeme : sans ca
           # le binaire de test se lance puis meurt sur `libavformat.so.62`.
           LD_LIBRARY_PATH: ${{ github.workspace }}/crates/thirdparty/ffmpeg-linux64-lgpl-shared/lib
+          ORT_DYLIB_PATH: ${{ github.workspace }}/electron/native/bin/linux-x64/libonnxruntime.so
           # Fait ECHOUER `cpu_backend_linux.rs` s'il n'obtient pas le backend CPU,
           # au lieu de le sauter en silence comme sur un poste sans lavapipe.
           OPENSCREEN_REQUIRE_CPU_BACKEND: "1"

THIRD-PARTY-NOTICES.md

@@ -49,6 +49,42 @@ distributed by their own registries, not redistributed inside our binaries.
 - The speech model (`ggml-*.bin`) is **not** bundled — it is downloaded into the
   user's data directory on first use by `electron/stt/modelManager.ts`.
 
+## ONNX Runtime (Windows and Apple Silicon macOS)
+
+- **Component**: `onnxruntime.dll` / `libonnxruntime.dylib`, under
+  `resources/electron/native/bin/<platform>-<arch>/`.
+- **License**: MIT — <https://github.com/microsoft/onnxruntime>.
+- Not built here: the pinned upstream release archive is downloaded, SHA-256
+  verified and unpacked by `scripts/fetch-onnxruntime.mjs`, which also checks the
+  archive's own LICENSE really is MIT before vendoring anything.
+- **Why it ships**: the native compositor segments the webcam subject with it, on
+  the CPU execution provider, to drive the camera background cutout/blur/custom
+  modes. The `gpu_cuda*` builds are deliberately not used — they are an order of
+  magnitude larger and carry NVIDIA redistribution terms.
+- **Not on Intel macOS**: upstream publishes no `osx-x86_64` asset from 1.27 on,
+  so the x64 DMG ships without it and the camera background effects are simply
+  absent there. Not shipped on Linux either, where the compositor has no capture
+  path for the mask yet.
+- The segmentation model it runs is a separate component, immediately below.
+
+## MediaPipe Selfie Segmentation — model weights
+
+- **Components**: `selfie_segmentation.tflite`,
+  `selfie_segmentation_landscape.tflite` and the `selfie_segmentation_landscape.onnx`
+  derived from them, shipped inside `app.asar` under `dist/mediapipe/`.
+- **License**: Apache-2.0 — <https://google.github.io/mediapipe/solutions/selfie_segmentation>.
+  Copyright The MediaPipe Authors.
+- The `.onnx` is a **derived work**, generated from the vendored `.tflite` by
+  `scripts/convert-selfie-segmentation-to-onnx.py`. No third-party weights are
+  downloaded at build time.
+- **Why it is listed here**: these weights are redistributed inside the installer,
+  and Apache-2.0 §4 asks that the attribution travel with them. The provenance note
+  in `public/mediapipe/selfie_segmentation/README.md` does not — electron-builder's
+  `"!*.md"` filter strips it from the package — so this file is the only copy a user
+  ever receives.
+- The MediaPipe **JavaScript** solution and its two ~5.6 MB WASM builds are no longer
+  bundled: inference moved into the native compositor, and nothing loaded them.
+
 ## Microsoft OpenMP runtime — `vcomp140.dll` (Windows only)
 
 - **Component**: `resources/electron/native/bin/win32-x64/vcomp140.dll`.

biome.json

@@ -3,7 +3,7 @@
 	"vcs": { "enabled": true, "clientKind": "git", "useIgnoreFile": true },
 	"files": {
 		"ignoreUnknown": false,
-		"includes": ["**", "!**/*.css", "!**/design/**", "!**/.worktrees/**"]
+		"includes": ["**", "!**/*.css", "!**/design/**", "!**/.worktrees/**", "!**/public/mediapipe/**"]
 	},
 	"formatter": {
 		"enabled": true,

crates/Cargo.toml

@@ -34,6 +34,26 @@ image = { version = "0.25", default-features = false, features = ["jpeg", "png"]
 wgpu = { version = "24", features = ["wgsl"] }
 pollster = "0.4"
 cosmic-text = "0.19"
+# Inference for the webcam segmentation mask. CPU execution provider only: measured on the
+# target integrated GPU it costs +0.47 ms/frame against DirectML's +1.03, its cost does not
+# scale with input resolution, and choosing it deletes the whole D3D11<->D3D12 interop
+# (technical-documentation/engineering/webcam-segmentation.md).
+#
+# Behind the `segmentation` feature and OFF by default: `download-binaries` fetches the
+# ONNX Runtime libs at build time, which is a packaging decision (nix, AUR, MS Store, CI)
+# that has not been taken yet. The default build is unchanged.
+# `load-dynamic` et NON `download-binaries` : ce dernier tire une build STATIQUE d'ONNX
+# Runtime avec DirectML dedans (DirectML.lib, DXCORE.lib et les DmlOperator* apparaissent
+# dans la ligne de lien) — exactement la dépendance que le choix de l'EP CPU sert à
+# supprimer. En chargement dynamique, la lib est résolue à l'exécution, ce qui laisse le
+# packaging la stager par plateforme comme il le fait déjà pour whisper-stt.
+ort = { version = "2.0.0-rc.13", default-features = false, features = [
+    "std",
+    "ndarray",
+    "load-dynamic",
+    "api-27",
+] }
+ndarray = "0.16"
 
 [workspace.dependencies.windows]
 version = "0.58"

crates/compositor/Cargo.toml

@@ -12,8 +12,20 @@ path = "src/lib.rs"
 bindgen = "0.70"
 cc = "1"
 
+[features]
+default = ["segmentation"]
+# Segmentation IA de la webcam via ONNX Runtime (EP CPU).
+#
+# Activée par défaut, ce qui ne coûte rien au build : `ort` est lié en `load-dynamic`, donc
+# aucune bibliothèque n'est nécessaire pour COMPILER. Elle l'est pour tourner — absente,
+# `Segmenter::load` échoue, le compositeur écrit une ligne et dessine la webcam telle quelle.
+# La désactiver reste possible pour une build qui ne veut pas du tout du code d'inférence.
+segmentation = ["dep:ort", "dep:ndarray"]
+
 [dependencies]
 anyhow.workspace = true
+ort = { workspace = true, optional = true }
+ndarray = { workspace = true, optional = true }
 serde.workspace = true
 serde_json.workspace = true
 image.workspace = true