From 3d7b9c0c0bd49ee6e057e6c4a6609ed0f2d1dfc1 Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Mon, 24 Aug 2026 21:12:49 +0200 Subject: [PATCH 1/7] feat(linux): capture mouse clicks on Wayland via evdev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wayland exposes no portal for mouse buttons, so cursor telemetry on Linux was always "move" and the cursor click-bounce never fired. Read left-button presses from evdev (/dev/input/event*) instead — the coinciding cursor sample is tagged "click", matching what the macOS and Windows helpers already do. Needs the user in the `input` group (the nodes are root:input); degrades silently to all-"move" otherwise, with a one-line warning. Scoped to BTN_LEFT only, never keystrokes; OPENSCREEN_DISABLE_CLICK_CAPTURE=1 disables it. Co-Authored-By: Claude Opus 4.8 --- .../pipeWireCursorAccumulator.test.ts | 14 ++- .../recording/pipeWireCursorAccumulator.ts | 11 +- .../pipeWireCursorRecordingSession.ts | 7 +- electron/native/README.md | 15 ++- electron/native/pipewire-capture/Cargo.lock | 52 ++++++++ electron/native/pipewire-capture/Cargo.toml | 1 + .../native/pipewire-capture/src/events.rs | 28 +++++ electron/native/pipewire-capture/src/input.rs | 119 ++++++++++++++++++ electron/native/pipewire-capture/src/main.rs | 50 +++++++- 9 files changed, 282 insertions(+), 15 deletions(-) create mode 100644 electron/native/pipewire-capture/src/input.rs diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts index 69ae80e3a..d0b93adff 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts @@ -56,13 +56,25 @@ describe("PipeWireCursorAccumulator", () => { expect(point.timeMs).toBe(500); }); - it("reports every sample as a move, because Wayland exposes no buttons", () => { + it("defaults a sample with no interaction to a move", () => { + // The helper omits interactionType on the common case, so the accumulator + // owns the "move" fallback. This is what the user not being in the `input` + // group looks like: every sample arrives bare. const accumulator = new PipeWireCursorAccumulator(100); accumulator.reset(0); accumulator.addSample(sample(10, 5, 5)); expect(accumulator.toRecordingData().samples[0].interactionType).toBe("move"); }); + it("preserves a click the helper read from evdev", () => { + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(0); + accumulator.addSample(sample(10, 5, 5, { interactionType: "click" })); + accumulator.addSample(sample(20, 6, 6)); + const { samples } = accumulator.toRecordingData(); + expect(samples.map((s) => s.interactionType)).toEqual(["click", "move"]); + }); + it("re-bases onto the video's start and drops what came before it", () => { // This is the single-session case. Cursor samples start flowing as soon // as the helper does, but the video's frame 0 is only stamped once the diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts index 84a13130e..4fa434093 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts @@ -57,6 +57,11 @@ export type PipeWireHelperEvent = visible: boolean; assetId?: string; asset?: PipeWireCursorAssetPayload; + /** `"click"` on the sample coinciding with a left-button press the + * helper read from evdev; absent on a plain move (see the helper's + * input.rs). The helper never emits the `"move"` default — that word + * is filled in below so it lives in exactly one place. */ + interactionType?: "move" | "click"; } | { event: "audio-source"; @@ -166,8 +171,10 @@ export class PipeWireCursorAccumulator { cx: clamp(payload.x / width, 0, 1), cy: clamp(payload.y / height, 0, 1), visible: payload.visible, - // Wayland exposes no click events to an unprivileged process. - interactionType: "move", + // The portal never reports a button; the helper tags a sample "click" + // only when it read a left-button press from evdev (needs the user in + // the `input` group). Everything else — the common case — is a move. + interactionType: payload.interactionType ?? "move", ...(payload.assetId ? { assetId: payload.assetId } : {}), }); diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts index 16815071c..7dd222ced 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts @@ -21,8 +21,11 @@ import type { CursorRecordingSession } from "./session"; * * Two consequences the caller should know about: * - * * `interactionType` is always "move". Wayland exposes no portal for mouse - * buttons and /dev/input/event* is root:input, so clicks are unobtainable. + * * `interactionType` is "move" unless the helper could read left-button + * presses from evdev — which needs the user in the `input` group, because + * Wayland exposes no portal for mouse buttons and /dev/input/event* is + * root:input. When it can, the coinciding sample is tagged "click"; when it + * cannot, every sample is a move, as before. See the helper's input.rs. * * The helper raises its own portal picker. On Wayland, Electron's * `desktopCapturer` already raised one, so the user currently picks a source * twice. Merging the two is the job of the capture stage that will reuse this diff --git a/electron/native/README.md b/electron/native/README.md index 8ff2e3cdd..4b12c7061 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -89,8 +89,8 @@ Encoder selection: by default the helper keeps the existing sink-writer path fir Frame input path: the helper feeds the encoder from the GPU when it can. On that path it copies the WGC frame across a keyed-mutex bridge to a second D3D11 device, converts BGRA to NV12 with the D3D11 video processor, and submits an allocator-owned DXGI sample to the hardware H.264 encoder, so no frame ever passes through system memory. The alternative is the original path: a staging texture, `Map(D3D11_MAP_READ)`, and a row-by-row copy into an `IMFMediaBuffer` — which is where a driver stall costs a recording (issue #252). The GPU path is a preference, never a requirement: it is skipped outright for `preferSoftwareEncoder` and for inline webcam PiP (both need the frame in system memory), and it degrades to the CPU path on its own if the encoding device, the NV12 video processor, the shared bridge texture, the DXGI sample allocator, or the hardware sink writer is unavailable. The GPU path is OFF by default: it fixed #252 on the machine that reproduces it and broke recording outright in #336, and its fallbacks only cover failures during `initialize()`, not one that appears once frames are flowing. Set `OPENSCREEN_WGC_ENABLE_DXGI_INPUT=1` to turn it on. Because the two paths land on different encoders and hardware MFTs default to constant bitrate, the GPU path asks for VBR through `ICodecAPI`; without it a static screen spends the full configured budget (measured 16.9 Mbps against 1.95 for the same desktop). -The helper reports the outcome through the `encoder-selection` stdout event (`video` is `default`, `software-preferred`, or `software-fallback`; `videoInput` is `dxgi-nv12` or `cpu-rgb32`; `container` is `fragmented-mp4` or `mp4`; all three report what the encoder settled on rather than what was asked for). On the GPU path the helper also prints one `[frame-drops] gpu_bridge_contended=` line to stderr at stop: a frame the bridge was too busy to take is skipped rather than failing the recording, and a large count there is the first thing to look at in a report about missing frames. When the app sees `software-fallback` — the default encoder failed and the helper switched on its own — it shows a small dismissible notice in the recording HUD with a "Don't show again" option, because software encoding can raise CPU usage. An explicit `software-preferred` selection shows no notice, and the event stays available for diagnostics either way. - +The helper reports the outcome through the `encoder-selection` stdout event (`video` is `default`, `software-preferred`, or `software-fallback`; `videoInput` is `dxgi-nv12` or `cpu-rgb32`; `container` is `fragmented-mp4` or `mp4`; all three report what the encoder settled on rather than what was asked for). On the GPU path the helper also prints one `[frame-drops] gpu_bridge_contended=` line to stderr at stop: a frame the bridge was too busy to take is skipped rather than failing the recording, and a large count there is the first thing to look at in a report about missing frames. When the app sees `software-fallback` — the default encoder failed and the helper switched on its own — it shows a small dismissible notice in the recording HUD with a "Don't show again" option, because software encoding can raise CPU usage. An explicit `software-preferred` selection shows no notice, and the event stays available for diagnostics either way. + At startup the helper also emits `capture-adapter`, naming the GPU its D3D device landed on and the one actually driving the captured display, each with its LUID, plus one `[adapters]` line per enumerated adapter on stderr. `createD3DDevice` asks for the *default* adapter and nothing checks that it is the one driving the display; when they differ every frame crosses an adapter boundary before the caller touches it. The LUIDs are there because the descriptions are not enough to tell: an IddCx virtual display driver renders through the physical GPU and inherits its description string while being a separate DXGI adapter, so the configuration this diagnostic exists to catch is precisely the one where both names are identical and only the LUIDs differ (measured: `NVIDIA Quadro RTX 4000` at LUID `0:24084` driving the display, the same string at `0:12889146` for the virtual adapter). `monitorLookup` says which of three things happened: `ok`, `no-output-claims-it` (the enumeration finished and nothing owns the captured monitor, which is what an active virtual display looks like), or `unavailable` (`EnumOutputs` refused, as it does in session 0 — the outputs were never inspected, so the absence means nothing about the hardware). Encoder diagnostic on final sink-writer failure: when the final sink-writer attempt fails (`MFCreateSinkWriterFromMediaSink` on the fragmented container, `MFCreateSinkWriterFromURL` on the plain one; the message names which), the helper logs the registered H.264 video encoder MFT count (via `MFTEnumEx`), the registered AAC encoder count when audio was requested, and the hex HRESULT. If no H.264 encoder is registered, it additionally emits the four-bullet actionable error (missing Media Feature Pack / GPU driver registration / empty `HKLM:\SOFTWARE\Microsoft\Windows Media Foundation\Transforms` / reboot). If an H.264 encoder IS registered but the sink writer still failed, it logs a hint pointing at invalid output path, missing MP4 mux, or GPU driver incompatibility. There is still no fail-fast pre-flight gate because `MFTEnumEx` and the sink writer can disagree about which H.264 encoders are available in non-interactive / Session 0 contexts. @@ -209,9 +209,14 @@ electron/native/bin/linux-x64/openscreen-pipewire-helper '{"probeOnly":true}' ### Known gaps -- **Mouse clicks are unobtainable.** Wayland exposes no portal for input events - and `/dev/input/event*` is `root:input`, so every sample's `interactionType` is - `"move"`. +- **Mouse clicks need the `input` group.** Wayland exposes no portal for input + events, so the helper reads left-button presses straight from evdev + (`/dev/input/event*`). Those nodes are `root:input`, so a user outside the + `input` group gets no readable device and every sample's `interactionType` + stays `"move"` — the same as before. When a device is readable, the coinciding + sample is tagged `"click"`. Scope is deliberately narrow: `BTN_LEFT` only, + never keystrokes (see `pipewire-capture/src/input.rs`), and + `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` turns it off entirely. - **The user picks a source twice.** Electron's `desktopCapturer` raises its own portal dialog for the video, and this helper raises a second one for the cursor. Collapsing them requires one portal session serving both, which is why the diff --git a/electron/native/pipewire-capture/Cargo.lock b/electron/native/pipewire-capture/Cargo.lock index f8fa27a58..17c2abd0a 100644 --- a/electron/native/pipewire-capture/Cargo.lock +++ b/electron/native/pipewire-capture/Cargo.lock @@ -238,6 +238,18 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -415,6 +427,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "evdev" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b686663ba7f08d92880ff6ba22170f1df4e83629341cba34cf82cd65ebea99" +dependencies = [ + "bitvec", + "cfg-if", + "libc", + "nix", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -475,6 +499,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.33" @@ -834,6 +864,7 @@ dependencies = [ "base64", "bindgen", "cc", + "evdev", "png", "pollster", "serde", @@ -974,6 +1005,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.7" @@ -1213,6 +1250,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.27.0" @@ -1463,6 +1506,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xdg-home" version = "1.3.0" diff --git a/electron/native/pipewire-capture/Cargo.toml b/electron/native/pipewire-capture/Cargo.toml index 604da16af..862b848bd 100644 --- a/electron/native/pipewire-capture/Cargo.toml +++ b/electron/native/pipewire-capture/Cargo.toml @@ -50,6 +50,7 @@ serde_json = "1" png = "0.17" base64 = "0.22" sha2 = "0.10" +evdev = "0.13" [build-dependencies] cc = "1" diff --git a/electron/native/pipewire-capture/src/events.rs b/electron/native/pipewire-capture/src/events.rs index 085ea56aa..ce98ac87e 100644 --- a/electron/native/pipewire-capture/src/events.rs +++ b/electron/native/pipewire-capture/src/events.rs @@ -97,6 +97,12 @@ pub enum Event { asset_id: Option, #[serde(skip_serializing_if = "Option::is_none")] asset: Option, + /// `"click"` on the sample that coincides with a left-button press read + /// from evdev (see `input.rs`), absent otherwise. Omitted rather than + /// defaulted to `"move"` so the accumulator keeps that fallback in one + /// place and the wire stays quiet on the common case. + #[serde(skip_serializing_if = "Option::is_none")] + interaction_type: Option, }, /// Which capture node each audio source was linked to. /// @@ -303,12 +309,32 @@ mod tests { visible: true, asset_id: None, asset: None, + interaction_type: None, }); assert_eq!(value["event"], "cursor-sample"); assert_eq!(value["x"], 100); assert_eq!(value["visible"], true); assert!(value.get("assetId").is_none()); assert!(value.get("asset").is_none()); + // A plain move stays silent about its interaction so the accumulator's + // "move" default is the single source of that word. + assert!(value.get("interactionType").is_none()); + } + + #[test] + fn cursor_samples_report_a_click_when_tagged() { + let value = parse_one(&Event::CursorSample { + timestamp_ms: 12, + x: 100, + y: 200, + width: 1920, + height: 1080, + visible: true, + asset_id: None, + asset: None, + interaction_type: Some("click".to_owned()), + }); + assert_eq!(value["interactionType"], "click"); } #[test] @@ -329,6 +355,7 @@ mod tests { hotspot_x: 4, hotspot_y: 3, }), + interaction_type: None, }); assert_eq!(value["assetId"], "abc"); assert_eq!(value["asset"]["imageDataUrl"], "data:image/png;base64,AA=="); @@ -359,6 +386,7 @@ mod tests { visible: true, asset_id: None, asset: None, + interaction_type: None, }); assert_eq!(value["timestampMs"], 1234, "a sample's capture time must not be overwritten"); } diff --git a/electron/native/pipewire-capture/src/input.rs b/electron/native/pipewire-capture/src/input.rs new file mode 100644 index 000000000..a94f2188c --- /dev/null +++ b/electron/native/pipewire-capture/src/input.rs @@ -0,0 +1,119 @@ +//! Left mouse-button telemetry on Wayland, read from evdev. +//! +//! WHY THIS EXISTS. Wayland deliberately denies an unprivileged process any view +//! of global input: the ScreenCast portal reports cursor POSITION as frame +//! metadata but never button state, and the only portal that streams input +//! (`InputCapture`) *grabs* it, redirecting clicks away from the app being +//! recorded — useless while the user is demoing. The one remaining source is the +//! kernel's evdev interface (`/dev/input/event*`). Reading it needs membership in +//! the `input` group (the nodes are `root:input`), which is the user's own, +//! out-of-band act of consent — the Wayland equivalent of the button state the +//! macOS and Windows helpers already read from their native APIs. +//! +//! SCOPE AND PRIVACY. A pointer node can also deliver keystrokes on a combined +//! keyboard+mouse device. This reader inspects ONLY `EV_KEY` events whose code is +//! `BTN_LEFT`, and only their press edge; it never reads, stores, or forwards any +//! other key code, and it only ever opens devices that advertise `BTN_LEFT` in +//! the first place. Set `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` to turn it off +//! entirely even where the permission exists. + +use std::sync::mpsc::Sender; +use std::thread; + +use evdev::{Device, EventType, KeyCode}; + +use crate::Message; + +const DISABLE_ENV: &str = "OPENSCREEN_DISABLE_CLICK_CAPTURE"; + +/// True when this evdev event is the press edge of the left mouse button. +/// +/// Extracted as a pure function so the decision is unit-testable without a real +/// device: a release (`value == 0`), an autorepeat (`value == 2`), and every +/// non-`BTN_LEFT` code — including every keyboard key — must NOT count. +pub fn is_left_button_press(event_type: EventType, code: u16, value: i32) -> bool { + event_type == EventType::KEY && code == KeyCode::BTN_LEFT.0 && value == 1 +} + +/// Opens every readable pointer device that reports `BTN_LEFT` and spawns a +/// reader thread per device. Returns whether at least one was opened — the caller +/// uses that to tell the log why Linux clicks are or are not being captured. +/// +/// Never fails: an unreadable node (the common case, when the user is not in the +/// `input` group) is skipped by `evdev::enumerate`, and no readable node at all +/// simply means every sample stays `"move"`, exactly as before this existed. +pub fn spawn_readers(sender: &Sender) -> bool { + if std::env::var_os(DISABLE_ENV).is_some() { + return false; + } + let mut opened = 0usize; + for (_path, device) in evdev::enumerate() { + if !device_reports_left_button(&device) { + continue; + } + opened += 1; + let forward = sender.clone(); + thread::spawn(move || read_device(device, forward)); + } + opened > 0 +} + +fn device_reports_left_button(device: &Device) -> bool { + device + .supported_keys() + .is_some_and(|keys| keys.contains(KeyCode::BTN_LEFT)) +} + +/// Blocks reading `device`, forwarding one `PointerButton` message per left-button +/// press. Returns when the device errors (e.g. unplugged) or the loop's channel +/// has closed, so the thread cannot outlive the recording it serves. +fn read_device(mut device: Device, sender: Sender) { + loop { + let events = match device.fetch_events() { + Ok(events) => events, + Err(_) => return, + }; + for event in events { + if is_left_button_press(event.event_type(), event.code(), event.value()) + && sender.send(Message::PointerButton).is_err() + { + return; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const BTN_LEFT: u16 = KeyCode::BTN_LEFT.0; + const BTN_RIGHT: u16 = KeyCode::BTN_RIGHT.0; + + #[test] + fn a_left_button_press_is_a_click() { + assert!(is_left_button_press(EventType::KEY, BTN_LEFT, 1)); + } + + #[test] + fn a_left_button_release_is_not() { + assert!(!is_left_button_press(EventType::KEY, BTN_LEFT, 0)); + } + + #[test] + fn a_left_button_autorepeat_is_not() { + // A held button emits value 2; only the 0->1 edge is a click. + assert!(!is_left_button_press(EventType::KEY, BTN_LEFT, 2)); + } + + #[test] + fn a_right_button_press_is_not_a_left_click() { + assert!(!is_left_button_press(EventType::KEY, BTN_RIGHT, 1)); + } + + #[test] + fn a_key_matching_btn_lefts_code_on_another_axis_is_not_a_click() { + // Same numeric code but a relative-motion event, not a key — must miss. + assert!(!is_left_button_press(EventType::RELATIVE, BTN_LEFT, 1)); + } +} diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 8ec4078c2..e8f8e4298 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -20,15 +20,19 @@ //! the cursor-only session Stage 1 shipped, which is what //! `PipeWireCursorRecordingSession` still uses. //! -//! WHAT IT CANNOT DO. Mouse buttons. Wayland exposes no portal for input -//! events, and /dev/input/event* is root:input. Every sample is therefore a -//! "move"; there is no click detection to be had here at any effort level. +//! MOUSE BUTTONS. Not from the portal — Wayland exposes no portal for input +//! events. The one source left is evdev (/dev/input/event*), which is root:input +//! and so needs the user in the `input` group. When that permission exists the +//! helper reads left-button presses and tags the coinciding sample "click"; when +//! it does not, every sample is a "move", as before. See `input.rs` for the +//! reader and its deliberately narrow scope (BTN_LEFT only, never keystrokes). mod bitmap; mod capture; mod encoder; mod events; mod ffmpeg; +mod input; mod portal; mod shim; @@ -208,6 +212,10 @@ struct AudioSourceConfig { enum Message { Portal(Box>), Stream(StreamEvent), + /// A left mouse-button press observed on evdev — the next cursor sample is + /// tagged `"click"`. See [`input`] for why this is the only way to see a + /// button on Wayland, and for its permission and privacy model. + PointerButton, /// Arm a deferred session: connect to PipeWire and start encoding. Record, Pause, @@ -283,6 +291,18 @@ fn main() { let (sender, receiver) = mpsc::channel::(); spawn_stdin_reader(sender.clone()); spawn_portal(sender.clone(), cursor_mode); + // Left-button telemetry from evdev. Absent when the user is not in the + // `input` group; a session that paints no cursor has no use for it either. + // Warn in that case so a "clicks do nothing on Linux" report is answerable + // from the log alone rather than looking like a capture bug. + if !input::spawn_readers(&sender) && cursor_mode.reports_cursor() { + let _ = emitter.emit(&Event::Warning { + code: "click-capture-unavailable".to_owned(), + message: "no readable /dev/input pointer device — add this user to the 'input' \ + group to record click telemetry; cursor samples will otherwise all be moves" + .to_owned(), + }); + } let session = RunConfig { tick, @@ -568,6 +588,8 @@ fn run( let mut cursor: Option = None; let mut known_assets: HashSet = HashSet::new(); let mut pending_asset: Option = None; + // Set by a `PointerButton` message, consumed by the next emitted sample. + let mut pending_click = false; let mut reported_cursor_meta = false; // Allocated up front so the PipeWire callback has somewhere to put frames // from the very first buffer; `None` in cursor-only mode, which is also what @@ -597,6 +619,12 @@ fn run( match receiver.recv_timeout(config.tick) { Ok(Message::Stop) => break, + // Latched, not emitted here: a bare press carries no position, so it + // waits for the next sample (which does) to become a `"click"`. + Ok(Message::PointerButton) => { + pending_click = true; + } + Ok(Message::Pause) => { paused = true; if let Some(capture) = capture.as_mut() { @@ -1052,14 +1080,14 @@ fn run( // A new sprite ships immediately; positions respect the sample // interval so a 120fps compositor cannot flood stdout. if asset_is_new || last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset); + emit_sample(emitter, &cursor, size, &mut pending_asset, &mut pending_click); last_emit = Instant::now(); } } Err(RecvTimeoutError::Timeout) => { if cursor.is_some() && last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset); + emit_sample(emitter, &cursor, size, &mut pending_asset, &mut pending_click); last_emit = Instant::now(); } // The heartbeat that keeps the output at a constant frame rate @@ -1153,11 +1181,22 @@ fn emit_sample( cursor: &Option, size: Option<(i32, i32)>, pending_asset: &mut Option, + pending_click: &mut bool, ) { let (Some(state), Some((width, height))) = (cursor, size) else { return; }; let visible = state.x >= 0 && state.y >= 0 && state.x < width && state.y < height; + // A click observed since the last sample rides out on this one. Cleared only + // when a sample is actually emitted, so a press that lands before the stream + // is live tags the first real sample rather than being dropped; at the sample + // cadence the cursor has not moved enough for the position to be wrong. + let interaction_type = if *pending_click { + *pending_click = false; + Some("click".to_owned()) + } else { + None + }; let _ = emitter.emit(&Event::CursorSample { timestamp_ms: timestamp_ms(), x: state.x, @@ -1167,6 +1206,7 @@ fn emit_sample( visible, asset_id: state.asset_id.clone(), asset: pending_asset.take(), + interaction_type, }); } From eae6762643174f3448aafaceed3e85529e14e9bb Mon Sep 17 00:00:00 2001 From: Etienne Lescot Date: Thu, 27 Aug 2026 19:20:41 +0200 Subject: [PATCH 2/7] fix(linux): gate evdev readers on cursor mode, and document the input group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The readers were spawned before the cursor mode was consulted (`!spawn_readers(&sender) && cursor_mode.reports_cursor()`), so every /dev/input/event* node was opened and one blocking thread per pointer device ran even in `embedded` and `hidden` — modes where `emit_sample` returns before it ever reads `pending_click`. Swapped the operands so the mode gates the spawn, which is what the comment above it already described. `spawn_readers` also returned false both for "no readable device" and for "disabled by OPENSCREEN_DISABLE_CLICK_CAPTURE", so opting out emitted the `click-capture-unavailable` warning recommending the `input` group the operator had just declined. It now returns a three-state `ClickCapture`, and the call site warns for `NoDevice` only; the warning text is unchanged. The `input` group requirement and the env-var opt-out were written down only in the contributor-facing electron/native/README.md. Added both to website/docs/installation.md (the Linux requirements row plus a "Mouse clicks on Wayland" subsection). The capability table is deliberately untouched: click effects are still gated off for Linux in supportsCursorClickEffects(), so this is a recording-side requirement only. Also restored electron/native/README.md lines 92-93 to LF. An unrelated paragraph on encoder selection had been rewritten as CRLF, which put a phantom hunk in the diff. Co-Authored-By: Claude Opus 5 --- electron/native/README.md | 4 +-- electron/native/pipewire-capture/src/input.rs | 31 ++++++++++++++++--- electron/native/pipewire-capture/src/main.rs | 15 ++++++--- website/docs/installation.md | 14 ++++++++- 4 files changed, 51 insertions(+), 13 deletions(-) diff --git a/electron/native/README.md b/electron/native/README.md index 4b12c7061..9bd69ae81 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -89,8 +89,8 @@ Encoder selection: by default the helper keeps the existing sink-writer path fir Frame input path: the helper feeds the encoder from the GPU when it can. On that path it copies the WGC frame across a keyed-mutex bridge to a second D3D11 device, converts BGRA to NV12 with the D3D11 video processor, and submits an allocator-owned DXGI sample to the hardware H.264 encoder, so no frame ever passes through system memory. The alternative is the original path: a staging texture, `Map(D3D11_MAP_READ)`, and a row-by-row copy into an `IMFMediaBuffer` — which is where a driver stall costs a recording (issue #252). The GPU path is a preference, never a requirement: it is skipped outright for `preferSoftwareEncoder` and for inline webcam PiP (both need the frame in system memory), and it degrades to the CPU path on its own if the encoding device, the NV12 video processor, the shared bridge texture, the DXGI sample allocator, or the hardware sink writer is unavailable. The GPU path is OFF by default: it fixed #252 on the machine that reproduces it and broke recording outright in #336, and its fallbacks only cover failures during `initialize()`, not one that appears once frames are flowing. Set `OPENSCREEN_WGC_ENABLE_DXGI_INPUT=1` to turn it on. Because the two paths land on different encoders and hardware MFTs default to constant bitrate, the GPU path asks for VBR through `ICodecAPI`; without it a static screen spends the full configured budget (measured 16.9 Mbps against 1.95 for the same desktop). -The helper reports the outcome through the `encoder-selection` stdout event (`video` is `default`, `software-preferred`, or `software-fallback`; `videoInput` is `dxgi-nv12` or `cpu-rgb32`; `container` is `fragmented-mp4` or `mp4`; all three report what the encoder settled on rather than what was asked for). On the GPU path the helper also prints one `[frame-drops] gpu_bridge_contended=` line to stderr at stop: a frame the bridge was too busy to take is skipped rather than failing the recording, and a large count there is the first thing to look at in a report about missing frames. When the app sees `software-fallback` — the default encoder failed and the helper switched on its own — it shows a small dismissible notice in the recording HUD with a "Don't show again" option, because software encoding can raise CPU usage. An explicit `software-preferred` selection shows no notice, and the event stays available for diagnostics either way. - +The helper reports the outcome through the `encoder-selection` stdout event (`video` is `default`, `software-preferred`, or `software-fallback`; `videoInput` is `dxgi-nv12` or `cpu-rgb32`; `container` is `fragmented-mp4` or `mp4`; all three report what the encoder settled on rather than what was asked for). On the GPU path the helper also prints one `[frame-drops] gpu_bridge_contended=` line to stderr at stop: a frame the bridge was too busy to take is skipped rather than failing the recording, and a large count there is the first thing to look at in a report about missing frames. When the app sees `software-fallback` — the default encoder failed and the helper switched on its own — it shows a small dismissible notice in the recording HUD with a "Don't show again" option, because software encoding can raise CPU usage. An explicit `software-preferred` selection shows no notice, and the event stays available for diagnostics either way. + At startup the helper also emits `capture-adapter`, naming the GPU its D3D device landed on and the one actually driving the captured display, each with its LUID, plus one `[adapters]` line per enumerated adapter on stderr. `createD3DDevice` asks for the *default* adapter and nothing checks that it is the one driving the display; when they differ every frame crosses an adapter boundary before the caller touches it. The LUIDs are there because the descriptions are not enough to tell: an IddCx virtual display driver renders through the physical GPU and inherits its description string while being a separate DXGI adapter, so the configuration this diagnostic exists to catch is precisely the one where both names are identical and only the LUIDs differ (measured: `NVIDIA Quadro RTX 4000` at LUID `0:24084` driving the display, the same string at `0:12889146` for the virtual adapter). `monitorLookup` says which of three things happened: `ok`, `no-output-claims-it` (the enumeration finished and nothing owns the captured monitor, which is what an active virtual display looks like), or `unavailable` (`EnumOutputs` refused, as it does in session 0 — the outputs were never inspected, so the absence means nothing about the hardware). Encoder diagnostic on final sink-writer failure: when the final sink-writer attempt fails (`MFCreateSinkWriterFromMediaSink` on the fragmented container, `MFCreateSinkWriterFromURL` on the plain one; the message names which), the helper logs the registered H.264 video encoder MFT count (via `MFTEnumEx`), the registered AAC encoder count when audio was requested, and the hex HRESULT. If no H.264 encoder is registered, it additionally emits the four-bullet actionable error (missing Media Feature Pack / GPU driver registration / empty `HKLM:\SOFTWARE\Microsoft\Windows Media Foundation\Transforms` / reboot). If an H.264 encoder IS registered but the sink writer still failed, it logs a hint pointing at invalid output path, missing MP4 mux, or GPU driver incompatibility. There is still no fail-fast pre-flight gate because `MFTEnumEx` and the sink writer can disagree about which H.264 encoders are available in non-interactive / Session 0 contexts. diff --git a/electron/native/pipewire-capture/src/input.rs b/electron/native/pipewire-capture/src/input.rs index a94f2188c..96ce9dc9c 100644 --- a/electron/native/pipewire-capture/src/input.rs +++ b/electron/native/pipewire-capture/src/input.rs @@ -35,16 +35,33 @@ pub fn is_left_button_press(event_type: EventType, code: u16, value: i32) -> boo event_type == EventType::KEY && code == KeyCode::BTN_LEFT.0 && value == 1 } +/// What [`spawn_readers`] settled on — the caller uses it to tell the log why +/// Linux clicks are or are not being captured. +/// +/// The two ways of ending up without clicks are NOT the same event: no readable +/// node is a permission the operator may still want to grant, while `DISABLE_ENV` +/// is one they explicitly declined — recommending the `input` group there answers +/// a question nobody asked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClickCapture { + /// Turned off by `OPENSCREEN_DISABLE_CLICK_CAPTURE`; nothing was opened. + Disabled, + /// No readable `/dev/input` node reports `BTN_LEFT` — the common case, when + /// the user is not in the `input` group. + NoDevice, + /// At least one device is open, with a reader thread running on it. + Active, +} + /// Opens every readable pointer device that reports `BTN_LEFT` and spawns a -/// reader thread per device. Returns whether at least one was opened — the caller -/// uses that to tell the log why Linux clicks are or are not being captured. +/// reader thread per device. /// /// Never fails: an unreadable node (the common case, when the user is not in the /// `input` group) is skipped by `evdev::enumerate`, and no readable node at all /// simply means every sample stays `"move"`, exactly as before this existed. -pub fn spawn_readers(sender: &Sender) -> bool { +pub fn spawn_readers(sender: &Sender) -> ClickCapture { if std::env::var_os(DISABLE_ENV).is_some() { - return false; + return ClickCapture::Disabled; } let mut opened = 0usize; for (_path, device) in evdev::enumerate() { @@ -55,7 +72,11 @@ pub fn spawn_readers(sender: &Sender) -> bool { let forward = sender.clone(); thread::spawn(move || read_device(device, forward)); } - opened > 0 + if opened > 0 { + ClickCapture::Active + } else { + ClickCapture::NoDevice + } } fn device_reports_left_button(device: &Device) -> bool { diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index e8f8e4298..e48c38dce 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -291,11 +291,16 @@ fn main() { let (sender, receiver) = mpsc::channel::(); spawn_stdin_reader(sender.clone()); spawn_portal(sender.clone(), cursor_mode); - // Left-button telemetry from evdev. Absent when the user is not in the - // `input` group; a session that paints no cursor has no use for it either. - // Warn in that case so a "clicks do nothing on Linux" report is answerable - // from the log alone rather than looking like a capture bug. - if !input::spawn_readers(&sender) && cursor_mode.reports_cursor() { + // Left-button telemetry from evdev. Gated on the cursor mode FIRST: a session + // that paints no cursor emits no samples to tag, so opening every pointer node + // and blocking a thread on each would buy nothing. Warn only when the devices + // are unreadable — never when the operator opted out through the env var, + // which would recommend the permission they just declined — so a "clicks do + // nothing on Linux" report is answerable from the log alone rather than + // looking like a capture bug. + if cursor_mode.reports_cursor() + && input::spawn_readers(&sender) == input::ClickCapture::NoDevice + { let _ = emitter.emit(&Event::Warning { code: "click-capture-unavailable".to_owned(), message: "no readable /dev/input pointer device — add this user to the 'input' \ diff --git a/website/docs/installation.md b/website/docs/installation.md index d71f0197c..1a4272625 100644 --- a/website/docs/installation.md +++ b/website/docs/installation.md @@ -24,7 +24,7 @@ Download the latest installer for your platform from the [download page](/downlo |---|---|---| | **Windows** | Windows 10 version 1903 (build 18362) or later, Intel 8th Gen / AMD Ryzen 2000 series or newer | Windows 11, Intel 12th Gen / AMD Ryzen 4000 series or newer | | **macOS** | macOS 12.3 (Monterey) — required by ScreenCaptureKit for native capture | macOS 14 or later | -| **Linux** | `xdg-desktop-portal` and PipeWire for native capture and system audio (default on Ubuntu 22.04+, Fedora 34+) — recording still works without them through the [browser-capture fallback](#platform-differences), with fewer capabilities | Same, kept up to date | +| **Linux** | `xdg-desktop-portal` and PipeWire for native capture and system audio (default on Ubuntu 22.04+, Fedora 34+) — recording still works without them through the [browser-capture fallback](#platform-differences), with fewer capabilities. Recording mouse clicks on Wayland additionally needs your user in the `input` group — see [Mouse clicks on Wayland](#mouse-clicks-on-wayland) | Same, kept up to date | | **RAM** | 8 GB | 16 GB | :::note Older integrated graphics on Windows @@ -111,6 +111,18 @@ Home Manager users can use `openscreen.homeManagerModules.default` with the same You may need to grant screen-recording permission depending on your desktop environment. +### Mouse clicks on Wayland + +Wayland exposes no portal for input events, so OpenScreen reads left-button presses straight from the kernel's evdev interface (`/dev/input/event*`) instead. Those device nodes are owned by `root:input`, so a recording only distinguishes a click from ordinary cursor movement when your user is in the `input` group: + +```bash +sudo usermod -aG input $USER +``` + +Log out and back in for the new group to take effect. Nothing breaks without it — recording works exactly as it did before, and every cursor sample is simply recorded as a move. + +The scope is deliberately narrow: only the left mouse button (`BTN_LEFT`) is ever read, never keystrokes. To turn the reader off entirely even where the permission exists, set `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` in the environment OpenScreen is launched from. + ## Platform differences The editing tools are the same everywhere — zooms, backgrounds, crop/trim/speed, annotations, transcription, captions, and projects. Every export format works on every platform; what differs is **capture**, and how fast MP4 encodes on Linux: From b64414e180c801277acaa386758bd32ec52c490c Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Fri, 28 Aug 2026 10:13:12 +0200 Subject: [PATCH 3/7] fix(cursor): enable click-bounce effect on Linux now that evdev captures clicks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `supportsCursorClickEffects()` gated Linux off on the premise that an unprivileged Wayland process can never observe mouse buttons — true before the evdev capture path landed. Since that path (the pipewire helper's input.rs) tags the coinciding sample `interactionType: "click"`, the accumulator preserves it, the `.cursor.json` sidecar carries it, and the compositor's shared `frame_geometry.rs` scales the cursor through `CursorTrack::bounce()` with no Linux-specific branch — so the effect already fires on Linux at the default strength. The gate only hid the tuning slider, leaving the effect on with no way to see, tune, or disable it, and the doc comment factually wrong. Flip the gate to true so the Click Bounce control renders on Linux, and rewrite the doc comment to describe the real path and the `input`-group requirement. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/cursor/cursorCapabilities.ts | 36 ++++++++++++++-------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/lib/cursor/cursorCapabilities.ts b/src/lib/cursor/cursorCapabilities.ts index 271372b04..5b6440789 100644 --- a/src/lib/cursor/cursorCapabilities.ts +++ b/src/lib/cursor/cursorCapabilities.ts @@ -1,26 +1,26 @@ -import { getPlatform } from "@/utils/platformUtils"; - /** * Can this platform tell us when a mouse button was pressed? * - * Not a preference and not a feature flag — a hard limit of the display server. - * On Wayland an unprivileged process cannot observe mouse buttons at all: the - * ScreenCast portal reports pointer POSITION as frame metadata and nothing else, - * there is no portal for input events, and `/dev/input/event*` is `root:input`. - * So the Linux capture helper stamps every sample `interactionType: "move"` - * (see `pipeWireCursorAccumulator.ts`), the compositor's `CursorTrack.clicks` - * vector is always empty, and `CursorTrack::bounce()` returns exactly 1.0 — - * which `frame_geometry.rs` multiplies the cursor size by, leaving it unchanged - * for every possible slider value. + * True on every platform we support. macOS and Windows read real button state in + * their native cursor helpers (`macNativeCursorRecordingSession.ts`, + * `windowsNativeRecordingSession.ts`). * - * macOS and Windows both read real button state in their native cursor helpers - * (`macNativeCursorRecordingSession.ts`, `windowsNativeRecordingSession.ts`), so - * the effect works there. + * On Linux/Wayland the ScreenCast portal still reports pointer POSITION as frame + * metadata and nothing else — there is no portal for button events. But the + * capture helper now reads left-button presses directly from `/dev/input/event*` + * via evdev and tags the coinciding sample `interactionType: "click"` (the + * pipewire helper's `input.rs`; the accumulator preserves the tag in + * `pipeWireCursorAccumulator.ts`). Those clicks travel in the `.cursor.json` + * sidecar, populate the compositor's `CursorTrack.clicks` vector (`cursor.rs`), + * and drive `CursorTrack::bounce()` through the shared geometry in + * `frame_geometry.rs` — the exact same path macOS and Windows take, with no + * Linux-specific branch. * - * Same shape and same intent as `supportsWebcamReactiveZoom` in - * `compositeLayout.ts`: a control that provably cannot change a pixel is - * dropped rather than shown doing nothing. + * Reading `/dev/input` requires the recording user to be in the `input` group. + * When they are not, no click is captured and every sample stays a plain move, + * so the effect has nothing to act on — a recording-side permission matter, not + * a reason to hide the control (see `website/docs/installation.md`). */ export function supportsCursorClickEffects(): boolean { - return getPlatform() !== "linux"; + return true; } From 6beb5c11f49ba3454c960d0a71b5b36fc5f663b8 Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Fri, 28 Aug 2026 10:43:51 +0200 Subject: [PATCH 4/7] fix(linux): ignore clicks until the stream is live, dropping the portal "Share" press MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The evdev readers arm milliseconds after `spawn_portal` puts up the ScreenCast picker, while it is still on screen. The physical left-click that dismisses it — the click on the picker's own "Share" button — was captured as a `BTN_LEFT` press and latched into `pending_click`, which `emit_sample` clears only on the next emitted sample. Since samples do not flow until after the portal is answered, that press rode out on the recording's first sample as a phantom `"click"` at t≈0, bouncing the cursor at the very start of every Wayland recording. Gate the latch on a new `streaming` flag, set when the pw_stream reaches `streaming` — the moment mutter actually starts handing us frames, and the exact edge past which a press lands on recorded content rather than the picker. This also survives the Record-before-portal-answered ordering, where gating on `armed` alone would still admit the Share click. Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/native/pipewire-capture/src/main.rs | 29 ++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index e48c38dce..7424cb0d5 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -595,6 +595,14 @@ fn run( let mut pending_asset: Option = None; // Set by a `PointerButton` message, consumed by the next emitted sample. let mut pending_click = false; + // The pw_stream has reached `streaming`, i.e. mutter has actually started + // handing us frames. Presses are ignored until then: before it, the only + // thing on screen is the portal's own picker, and the click that dismisses + // it (its "Share" button) would otherwise latch and ride out on the + // recording's first sample as a phantom click at t≈0. mutter enables its + // capture source on STREAMING, so this is the exact edge at which a press + // starts landing on content the recording contains. + let mut streaming = false; let mut reported_cursor_meta = false; // Allocated up front so the PipeWire callback has somewhere to put frames // from the very first buffer; `None` in cursor-only mode, which is also what @@ -626,8 +634,13 @@ fn run( // Latched, not emitted here: a bare press carries no position, so it // waits for the next sample (which does) to become a `"click"`. + // Dropped before the stream is live (see `streaming`): a press that + // lands while the picker is still up is the click on its "Share" + // button, not content, and must not tag the first real sample. Ok(Message::PointerButton) => { - pending_click = true; + if streaming { + pending_click = true; + } } Ok(Message::Pause) => { @@ -998,6 +1011,11 @@ fn run( ("error", error.clone().into()), ]), }); + // Once frames are flowing, presses land on recorded content; the + // picker (and the "Share" click that dismissed it) is behind us. + if state == "streaming" { + streaming = true; + } if let Some(error) = error { let _ = emitter.emit(&Event::Warning { code: "stream-error".to_owned(), @@ -1192,10 +1210,11 @@ fn emit_sample( return; }; let visible = state.x >= 0 && state.y >= 0 && state.x < width && state.y < height; - // A click observed since the last sample rides out on this one. Cleared only - // when a sample is actually emitted, so a press that lands before the stream - // is live tags the first real sample rather than being dropped; at the sample - // cadence the cursor has not moved enough for the position to be wrong. + // A click observed since the last sample rides out on this one, cleared only + // when a sample is actually emitted — at the sample cadence the cursor has not + // moved enough for the position to be wrong. Presses before the stream is live + // are never latched (see the `PointerButton` arm), so this cannot carry the + // portal picker's own "Share" click into the recording. let interaction_type = if *pending_click { *pending_click = false; Some("click".to_owned()) From 8a5831b5caf608a64e041bf161865e2c3ea035ed Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Fri, 28 Aug 2026 10:43:51 +0200 Subject: [PATCH 5/7] docs(linux): note that touchpad tap-to-click is not captured evdev sees a physical clickpad press (a real BTN_LEFT) but not a tap: libinput synthesises tap-to-click above the kernel device and never writes it back, so there is nothing at the evdev layer the helper reads. Document the limitation for users under "Mouse clicks on Wayland", and at the device_reports_left_button site where a touchpad passes the BTN_LEFT check yet silently drops taps. Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/native/pipewire-capture/src/input.rs | 7 +++++++ website/docs/installation.md | 2 ++ 2 files changed, 9 insertions(+) diff --git a/electron/native/pipewire-capture/src/input.rs b/electron/native/pipewire-capture/src/input.rs index 96ce9dc9c..7de31e390 100644 --- a/electron/native/pipewire-capture/src/input.rs +++ b/electron/native/pipewire-capture/src/input.rs @@ -79,6 +79,13 @@ pub fn spawn_readers(sender: &Sender) -> ClickCapture { } } +/// A touchpad advertises `BTN_LEFT` for a physical clickpad press, so it passes +/// this check and its node is opened — but tap-to-click never arrives here. +/// libinput consumes the raw `BTN_TOUCH`/`ABS_MT_*` stream and synthesises the +/// button for its own clients without writing `BTN_LEFT` back to the kernel +/// device, so a tap is invisible at the evdev layer we read. The consequence is +/// documented for users under "Mouse clicks on Wayland" in the installation docs: +/// on a touchpad only hard presses are captured, taps are not. fn device_reports_left_button(device: &Device) -> bool { device .supported_keys() diff --git a/website/docs/installation.md b/website/docs/installation.md index 1a4272625..f5deef83d 100644 --- a/website/docs/installation.md +++ b/website/docs/installation.md @@ -123,6 +123,8 @@ Log out and back in for the new group to take effect. Nothing breaks without it The scope is deliberately narrow: only the left mouse button (`BTN_LEFT`) is ever read, never keystrokes. To turn the reader off entirely even where the permission exists, set `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` in the environment OpenScreen is launched from. +**Touchpads:** only a physical click — pressing the pad down until it depresses — is recorded. **Tap-to-click is not**, because your compositor's input stack (libinput) synthesises those taps for its own use and never writes them back to the kernel device that OpenScreen reads, so there is nothing at the evdev layer to see. A mouse, or a touchpad with tap-to-click turned off, records every click. + ## Platform differences The editing tools are the same everywhere — zooms, backgrounds, crop/trim/speed, annotations, transcription, captions, and projects. Every export format works on every platform; what differs is **capture**, and how fast MP4 encodes on Linux: From ee8b09de32491691d923d0c92b18997cf40718d2 Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Fri, 28 Aug 2026 10:57:13 +0200 Subject: [PATCH 6/7] fix(linux): harden the evdev click reader (EINTR, hotplug, device loss, double-click) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the three remaining review comments on the click-capture path: 1. Transient errors and hotplug. `read_device` treated every `fetch_events` error as a terminal unplug and ended the thread silently, so a stray `EINTR` killed click capture with nothing in the log, and a mouse plugged in after startup was never picked up. `EINTR` is now retried; a genuine device error sends `PointerDeviceLost`, which the loop surfaces as a warning; and `spawn_readers` leaves a daemon thread re-scanning `/dev/input` on an interval so a device attached mid-recording is adopted (deduped by node path). 2. Overstated scope claim. The module header said it "only ever opens devices that advertise BTN_LEFT" — but `evdev::enumerate` opens every readable node to inspect it. Reworded to say what is true: non-BTN_LEFT nodes are dropped immediately without a single event read from them. 3. Double-click collapse and late timestamps. `pending_click` was a bool consumed by the next throttled sample, so two presses inside one sample window became one bounce, and every click was stamped up to a sample interval late. Presses now carry their evdev-read time in `PointerButton(u64)` and emit a dedicated click sample immediately — one per press, stamped when the button went down. Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/native/pipewire-capture/src/input.rs | 85 ++++++++++++++----- electron/native/pipewire-capture/src/main.rs | 79 +++++++++++------ 2 files changed, 117 insertions(+), 47 deletions(-) diff --git a/electron/native/pipewire-capture/src/input.rs b/electron/native/pipewire-capture/src/input.rs index 7de31e390..dd47ef73e 100644 --- a/electron/native/pipewire-capture/src/input.rs +++ b/electron/native/pipewire-capture/src/input.rs @@ -13,19 +13,32 @@ //! SCOPE AND PRIVACY. A pointer node can also deliver keystrokes on a combined //! keyboard+mouse device. This reader inspects ONLY `EV_KEY` events whose code is //! `BTN_LEFT`, and only their press edge; it never reads, stores, or forwards any -//! other key code, and it only ever opens devices that advertise `BTN_LEFT` in -//! the first place. Set `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` to turn it off -//! entirely even where the permission exists. - +//! other key code. Enumerating the devices does briefly open each readable +//! `/dev/input/event*` node to inspect its capability bits, but any node that +//! does not advertise `BTN_LEFT` is dropped immediately, without a single event +//! ever being read from it — only `BTN_LEFT` devices get a reader. Set +//! `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` to turn it off entirely even where the +//! permission exists. + +use std::collections::HashSet; +use std::io::ErrorKind; +use std::path::PathBuf; use std::sync::mpsc::Sender; use std::thread; +use std::time::Duration; use evdev::{Device, EventType, KeyCode}; +use crate::events::timestamp_ms; use crate::Message; const DISABLE_ENV: &str = "OPENSCREEN_DISABLE_CLICK_CAPTURE"; +/// How often the hotplug watcher re-scans `/dev/input` for pointer devices that +/// appeared after startup. A few seconds is imperceptible for a device the user +/// just plugged in and costs a cheap directory walk. +const RESCAN_INTERVAL: Duration = Duration::from_secs(3); + /// True when this evdev event is the press edge of the left mouse button. /// /// Extracted as a pure function so the decision is unit-testable without a real @@ -53,30 +66,52 @@ pub enum ClickCapture { Active, } -/// Opens every readable pointer device that reports `BTN_LEFT` and spawns a -/// reader thread per device. +/// Opens every readable pointer device that reports `BTN_LEFT`, spawns a reader +/// thread per device, and leaves a daemon thread re-scanning for devices plugged +/// in later. /// /// Never fails: an unreadable node (the common case, when the user is not in the /// `input` group) is skipped by `evdev::enumerate`, and no readable node at all -/// simply means every sample stays `"move"`, exactly as before this existed. +/// simply means every sample stays `"move"`, exactly as before this existed. The +/// returned value reflects the INITIAL scan only — a device attached afterwards +/// is adopted by the watcher without changing it. pub fn spawn_readers(sender: &Sender) -> ClickCapture { if std::env::var_os(DISABLE_ENV).is_some() { return ClickCapture::Disabled; } - let mut opened = 0usize; - for (_path, device) in evdev::enumerate() { - if !device_reports_left_button(&device) { + // Paths already given a reader. Only ever grows, so a device is never + // double-read; the one case it misses is a device that unplugs and returns + // on the SAME node path — a replug usually lands on a fresh `eventNN`, which + // is not in the set and so is picked up. + let mut opened: HashSet = HashSet::new(); + scan_once(sender, &mut opened); + let result = if opened.is_empty() { + ClickCapture::NoDevice + } else { + ClickCapture::Active + }; + // Hotplug: the one-shot scan above cannot see a mouse attached mid-recording, + // so a daemon thread re-scans and starts readers for nodes it has not seen. + // Detached, like the reader threads — it ends when the process does. + let watch_sender = sender.clone(); + thread::spawn(move || loop { + thread::sleep(RESCAN_INTERVAL); + scan_once(&watch_sender, &mut opened); + }); + result +} + +/// Spawns a reader for every `BTN_LEFT` device not already in `opened`, recording +/// each newly opened node's path. Shared by the initial scan and the watcher. +fn scan_once(sender: &Sender, opened: &mut HashSet) { + for (path, device) in evdev::enumerate() { + if opened.contains(&path) || !device_reports_left_button(&device) { continue; } - opened += 1; + opened.insert(path); let forward = sender.clone(); thread::spawn(move || read_device(device, forward)); } - if opened > 0 { - ClickCapture::Active - } else { - ClickCapture::NoDevice - } } /// A touchpad advertises `BTN_LEFT` for a physical clickpad press, so it passes @@ -93,17 +128,27 @@ fn device_reports_left_button(device: &Device) -> bool { } /// Blocks reading `device`, forwarding one `PointerButton` message per left-button -/// press. Returns when the device errors (e.g. unplugged) or the loop's channel -/// has closed, so the thread cannot outlive the recording it serves. +/// press, each stamped with the press time so a click is not backdated to the +/// next cursor sample. Returns when the device fails terminally (e.g. unplugged) +/// or the loop's channel has closed, so the thread cannot outlive the recording +/// it serves. A transient `EINTR` is retried, not mistaken for an unplug. fn read_device(mut device: Device, sender: Sender) { loop { let events = match device.fetch_events() { Ok(events) => events, - Err(_) => return, + // A signal interrupted the blocking read — not a device failure. + Err(err) if err.kind() == ErrorKind::Interrupted => continue, + // A real error, typically the device unplugging. Report it rather + // than ending silently, so click capture going quiet mid-recording + // is answerable from the log; the watcher re-adopts it on a replug. + Err(err) => { + let _ = sender.send(Message::PointerDeviceLost(err.to_string())); + return; + } }; for event in events { if is_left_button_press(event.event_type(), event.code(), event.value()) - && sender.send(Message::PointerButton).is_err() + && sender.send(Message::PointerButton(timestamp_ms())).is_err() { return; } diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 7424cb0d5..6db7db1ab 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -212,10 +212,16 @@ struct AudioSourceConfig { enum Message { Portal(Box>), Stream(StreamEvent), - /// A left mouse-button press observed on evdev — the next cursor sample is - /// tagged `"click"`. See [`input`] for why this is the only way to see a - /// button on Wayland, and for its permission and privacy model. - PointerButton, + /// A left mouse-button press observed on evdev, carrying the press time in + /// milliseconds so the emitted click sample is stamped when it happened + /// rather than at the next throttled sample. See [`input`] for why this is + /// the only way to see a button on Wayland, and for its permission and + /// privacy model. + PointerButton(u64), + /// A pointer reader thread stopped on a device error (typically an unplug); + /// the string is the OS error. Surfaced as a warning so click capture going + /// quiet mid-recording is not silent. + PointerDeviceLost(String), /// Arm a deferred session: connect to PipeWire and start encoding. Record, Pause, @@ -593,8 +599,6 @@ fn run( let mut cursor: Option = None; let mut known_assets: HashSet = HashSet::new(); let mut pending_asset: Option = None; - // Set by a `PointerButton` message, consumed by the next emitted sample. - let mut pending_click = false; // The pw_stream has reached `streaming`, i.e. mutter has actually started // handing us frames. Presses are ignored until then: before it, the only // thing on screen is the portal's own picker, and the click that dismisses @@ -632,17 +636,38 @@ fn run( match receiver.recv_timeout(config.tick) { Ok(Message::Stop) => break, - // Latched, not emitted here: a bare press carries no position, so it - // waits for the next sample (which does) to become a `"click"`. - // Dropped before the stream is live (see `streaming`): a press that - // lands while the picker is still up is the click on its "Share" - // button, not content, and must not tag the first real sample. - Ok(Message::PointerButton) => { + // Emit a click sample straight away at the current cursor position, + // stamped with the press time the reader captured: immediate so it is + // not backdated to the next throttled sample, and one sample per press + // so a rapid double-click reads as two clicks rather than collapsing + // into one. Dropped before the stream is live (see `streaming`): a + // press while the picker is still up is the click on its "Share" + // button, not content. + Ok(Message::PointerButton(press_ms)) => { if streaming { - pending_click = true; + emit_sample( + emitter, + &cursor, + size, + &mut pending_asset, + Some(press_ms), + ); } } + // A reader lost its device (typically an unplug). Report it — if it + // was the only pointer, clicks stop until one is (re)connected, which + // the input watcher will pick up. + Ok(Message::PointerDeviceLost(reason)) => { + let _ = emitter.emit(&Event::Warning { + code: "click-capture-device-lost".to_owned(), + message: format!( + "a pointer device stopped delivering clicks ({reason}); if it was the \ + only one, clicks are not captured until a device is reconnected" + ), + }); + } + Ok(Message::Pause) => { paused = true; if let Some(capture) = capture.as_mut() { @@ -1103,14 +1128,14 @@ fn run( // A new sprite ships immediately; positions respect the sample // interval so a 120fps compositor cannot flood stdout. if asset_is_new || last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset, &mut pending_click); + emit_sample(emitter, &cursor, size, &mut pending_asset, None); last_emit = Instant::now(); } } Err(RecvTimeoutError::Timeout) => { if cursor.is_some() && last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset, &mut pending_click); + emit_sample(emitter, &cursor, size, &mut pending_asset, None); last_emit = Instant::now(); } // The heartbeat that keeps the output at a constant frame rate @@ -1199,30 +1224,30 @@ fn finish_capture( } } +/// Emits one cursor sample at the current position. `click` is `Some(press_ms)` +/// for a left-button press — the sample is then tagged `"click"` and stamped with +/// that press time — or `None` for an ordinary throttled position sample stamped +/// now. Either way a pending new sprite rides out on it. fn emit_sample( emitter: &mut Emitter, cursor: &Option, size: Option<(i32, i32)>, pending_asset: &mut Option, - pending_click: &mut bool, + click: Option, ) { let (Some(state), Some((width, height))) = (cursor, size) else { return; }; let visible = state.x >= 0 && state.y >= 0 && state.x < width && state.y < height; - // A click observed since the last sample rides out on this one, cleared only - // when a sample is actually emitted — at the sample cadence the cursor has not - // moved enough for the position to be wrong. Presses before the stream is live - // are never latched (see the `PointerButton` arm), so this cannot carry the - // portal picker's own "Share" click into the recording. - let interaction_type = if *pending_click { - *pending_click = false; - Some("click".to_owned()) - } else { - None + // A click carries its own press time so the bounce lands when the button went + // down, not up to a sample interval later; the position is the latest known, + // which at the sample cadence has not moved enough to be wrong. + let (timestamp, interaction_type) = match click { + Some(press_ms) => (press_ms, Some("click".to_owned())), + None => (timestamp_ms(), None), }; let _ = emitter.emit(&Event::CursorSample { - timestamp_ms: timestamp_ms(), + timestamp_ms: timestamp, x: state.x, y: state.y, width, From f7d5d4b4bc8c08fa55555a4fd1f34d74456b1b19 Mon Sep 17 00:00:00 2001 From: Benjamin Freeman Date: Fri, 28 Aug 2026 23:57:27 +0200 Subject: [PATCH 7/7] fix(linux): address CodeRabbit re-review of the click-capture path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Reconnect on the same node path. `read_device` now removes its path from the shared `opened` set on every exit, so a device that unplugs and returns on the same `/dev/input/eventN` is re-adopted by the next scan instead of skipped for the rest of the recording. The set is an Arc>; check-and-insert is one locked step so two scans can't both adopt a path. - Drop pre-stream presses by time, not just a flag. `PointerButton` and `StreamEvent::State` arrive on different channels, so a "Share"-button press read before streaming could be dequeued after a bare `streaming` flag flipped true. Replace the flag with `streaming_since: Option` and emit a press only when its read time is >= that instant — the older press is dropped regardless of delivery order. - Stop click capture on disconnect. `streaming_since` is cleared on the `unconnected` transition, so presses after the stream drops no longer emit click samples against a stale cursor position. - Update the installation capability table: Linux click effects now work on Wayland with the `input` group, matching the "Mouse clicks on Wayland" section. Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/native/pipewire-capture/src/input.rs | 49 +++++++++++++------ electron/native/pipewire-capture/src/main.rs | 46 +++++++++-------- website/docs/installation.md | 2 +- 3 files changed, 61 insertions(+), 36 deletions(-) diff --git a/electron/native/pipewire-capture/src/input.rs b/electron/native/pipewire-capture/src/input.rs index dd47ef73e..a1d41c90d 100644 --- a/electron/native/pipewire-capture/src/input.rs +++ b/electron/native/pipewire-capture/src/input.rs @@ -24,6 +24,7 @@ use std::collections::HashSet; use std::io::ErrorKind; use std::path::PathBuf; use std::sync::mpsc::Sender; +use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; @@ -79,13 +80,13 @@ pub fn spawn_readers(sender: &Sender) -> ClickCapture { if std::env::var_os(DISABLE_ENV).is_some() { return ClickCapture::Disabled; } - // Paths already given a reader. Only ever grows, so a device is never - // double-read; the one case it misses is a device that unplugs and returns - // on the SAME node path — a replug usually lands on a fresh `eventNN`, which - // is not in the set and so is picked up. - let mut opened: HashSet = HashSet::new(); - scan_once(sender, &mut opened); - let result = if opened.is_empty() { + // Paths with a LIVE reader. Shared with the reader threads: each removes its + // own path when it exits, so a device that unplugs and reconnects on the SAME + // node path is adopted again by the next scan. A set that only grew skipped + // such a replug for the rest of the recording. + let opened: Arc>> = Arc::new(Mutex::new(HashSet::new())); + scan_once(sender, &opened); + let result = if opened.lock().unwrap().is_empty() { ClickCapture::NoDevice } else { ClickCapture::Active @@ -94,23 +95,28 @@ pub fn spawn_readers(sender: &Sender) -> ClickCapture { // so a daemon thread re-scans and starts readers for nodes it has not seen. // Detached, like the reader threads — it ends when the process does. let watch_sender = sender.clone(); + let watch_opened = Arc::clone(&opened); thread::spawn(move || loop { thread::sleep(RESCAN_INTERVAL); - scan_once(&watch_sender, &mut opened); + scan_once(&watch_sender, &watch_opened); }); result } -/// Spawns a reader for every `BTN_LEFT` device not already in `opened`, recording -/// each newly opened node's path. Shared by the initial scan and the watcher. -fn scan_once(sender: &Sender, opened: &mut HashSet) { +/// Spawns a reader for every `BTN_LEFT` device not already being read, recording +/// each newly opened node's path. Shared by the initial scan and the watcher; the +/// check-and-insert is one locked step so two scans cannot both adopt one path. +fn scan_once(sender: &Sender, opened: &Arc>>) { for (path, device) in evdev::enumerate() { - if opened.contains(&path) || !device_reports_left_button(&device) { + if !device_reports_left_button(&device) { continue; } - opened.insert(path); + if !opened.lock().unwrap().insert(path.clone()) { + continue; // already has a live reader + } let forward = sender.clone(); - thread::spawn(move || read_device(device, forward)); + let owned = Arc::clone(opened); + thread::spawn(move || read_device(device, path, forward, owned)); } } @@ -132,7 +138,18 @@ fn device_reports_left_button(device: &Device) -> bool { /// next cursor sample. Returns when the device fails terminally (e.g. unplugged) /// or the loop's channel has closed, so the thread cannot outlive the recording /// it serves. A transient `EINTR` is retried, not mistaken for an unplug. -fn read_device(mut device: Device, sender: Sender) { +/// +/// On EVERY exit it drops `path` from `opened`, so a device reconnecting on the +/// same node path is re-adopted by the next scan. +fn read_device( + mut device: Device, + path: PathBuf, + sender: Sender, + opened: Arc>>, +) { + let release = || { + opened.lock().unwrap().remove(&path); + }; loop { let events = match device.fetch_events() { Ok(events) => events, @@ -142,6 +159,7 @@ fn read_device(mut device: Device, sender: Sender) { // than ending silently, so click capture going quiet mid-recording // is answerable from the log; the watcher re-adopts it on a replug. Err(err) => { + release(); let _ = sender.send(Message::PointerDeviceLost(err.to_string())); return; } @@ -150,6 +168,7 @@ fn read_device(mut device: Device, sender: Sender) { if is_left_button_press(event.event_type(), event.code(), event.value()) && sender.send(Message::PointerButton(timestamp_ms())).is_err() { + release(); return; } } diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 6db7db1ab..8afe2af88 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -599,14 +599,17 @@ fn run( let mut cursor: Option = None; let mut known_assets: HashSet = HashSet::new(); let mut pending_asset: Option = None; - // The pw_stream has reached `streaming`, i.e. mutter has actually started - // handing us frames. Presses are ignored until then: before it, the only - // thing on screen is the portal's own picker, and the click that dismisses - // it (its "Share" button) would otherwise latch and ride out on the - // recording's first sample as a phantom click at t≈0. mutter enables its - // capture source on STREAMING, so this is the exact edge at which a press - // starts landing on content the recording contains. - let mut streaming = false; + // Wall-clock ms at which the pw_stream last reached `streaming` (mutter began + // handing us frames), or `None` when it is not streaming. A press counts only + // if its own read time is at or after this: before streaming the only thing on + // screen is the portal picker, and the click that dismisses it (its "Share" + // button) would otherwise ride out on the first sample as a phantom click at + // t≈0. Comparing timestamps rather than a bare flag closes two gaps: the press + // and the stream-state arrive on DIFFERENT channels, so a pre-stream press can + // be dequeued after the flag flips (it is still dropped, its time is older); + // and clearing it on disconnect stops clicks emitting against a stale cursor + // after capture has stopped. + let mut streaming_since: Option = None; let mut reported_cursor_meta = false; // Allocated up front so the PipeWire callback has somewhere to put frames // from the very first buffer; `None` in cursor-only mode, which is also what @@ -640,18 +643,13 @@ fn run( // stamped with the press time the reader captured: immediate so it is // not backdated to the next throttled sample, and one sample per press // so a rapid double-click reads as two clicks rather than collapsing - // into one. Dropped before the stream is live (see `streaming`): a - // press while the picker is still up is the click on its "Share" - // button, not content. + // into one. Counted only if the press happened at or after streaming + // began (see `streaming_since`): a press while the picker is still up — + // the click on its "Share" button — is older, so it is dropped even if + // its message is delivered after the stream-state one. Ok(Message::PointerButton(press_ms)) => { - if streaming { - emit_sample( - emitter, - &cursor, - size, - &mut pending_asset, - Some(press_ms), - ); + if streaming_since.is_some_and(|since| press_ms >= since) { + emit_sample(emitter, &cursor, size, &mut pending_asset, Some(press_ms)); } } @@ -1038,8 +1036,16 @@ fn run( }); // Once frames are flowing, presses land on recorded content; the // picker (and the "Share" click that dismissed it) is behind us. + // Stamped so a press read before this instant is dropped by time, + // and cleared on disconnect so clicks stop emitting against a stale + // cursor after capture ends. Only set on the FIRST streaming edge so + // a transient renegotiation `paused`→`streaming` does not re-arm it. if state == "streaming" { - streaming = true; + if streaming_since.is_none() { + streaming_since = Some(timestamp_ms()); + } + } else if state == "unconnected" { + streaming_since = None; } if let Some(error) = error { let _ = emitter.emit(&Event::Warning { diff --git a/website/docs/installation.md b/website/docs/installation.md index f5deef83d..5d2c7545f 100644 --- a/website/docs/installation.md +++ b/website/docs/installation.md @@ -132,7 +132,7 @@ The editing tools are the same everywhere — zooms, backgrounds, crop/trim/spee | | macOS | Windows | Linux | |---|---|---|---| | Capture pipeline | Native (ScreenCaptureKit) | Native (Windows Graphics Capture) | Browser pipeline | -| Custom cursor themes / click effects | ✅ | ✅ | ❌ (position-only, used for auto-zoom) | +| Custom cursor themes / click effects | ✅ | ✅ | ✅ on Wayland — click capture needs the `input` group ([details](#mouse-clicks-on-wayland)) | | Webcam | Native capture | Native capture | Browser capture (still works as PiP) | | System audio | macOS 13+; permission prompt on 14.2+; not available on macOS 12 and below | Works out of the box | Needs PipeWire (default on Ubuntu 22.04+, Fedora 34+) | | MP4 export | ✅ | ✅ | ✅ (software encode) |