diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts index 69ae80e3a..d0b93adff 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.test.ts @@ -56,13 +56,25 @@ describe("PipeWireCursorAccumulator", () => { expect(point.timeMs).toBe(500); }); - it("reports every sample as a move, because Wayland exposes no buttons", () => { + it("defaults a sample with no interaction to a move", () => { + // The helper omits interactionType on the common case, so the accumulator + // owns the "move" fallback. This is what the user not being in the `input` + // group looks like: every sample arrives bare. const accumulator = new PipeWireCursorAccumulator(100); accumulator.reset(0); accumulator.addSample(sample(10, 5, 5)); expect(accumulator.toRecordingData().samples[0].interactionType).toBe("move"); }); + it("preserves a click the helper read from evdev", () => { + const accumulator = new PipeWireCursorAccumulator(100); + accumulator.reset(0); + accumulator.addSample(sample(10, 5, 5, { interactionType: "click" })); + accumulator.addSample(sample(20, 6, 6)); + const { samples } = accumulator.toRecordingData(); + expect(samples.map((s) => s.interactionType)).toEqual(["click", "move"]); + }); + it("re-bases onto the video's start and drops what came before it", () => { // This is the single-session case. Cursor samples start flowing as soon // as the helper does, but the video's frame 0 is only stamped once the diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts index 84a13130e..4fa434093 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorAccumulator.ts @@ -57,6 +57,11 @@ export type PipeWireHelperEvent = visible: boolean; assetId?: string; asset?: PipeWireCursorAssetPayload; + /** `"click"` on the sample coinciding with a left-button press the + * helper read from evdev; absent on a plain move (see the helper's + * input.rs). The helper never emits the `"move"` default — that word + * is filled in below so it lives in exactly one place. */ + interactionType?: "move" | "click"; } | { event: "audio-source"; @@ -166,8 +171,10 @@ export class PipeWireCursorAccumulator { cx: clamp(payload.x / width, 0, 1), cy: clamp(payload.y / height, 0, 1), visible: payload.visible, - // Wayland exposes no click events to an unprivileged process. - interactionType: "move", + // The portal never reports a button; the helper tags a sample "click" + // only when it read a left-button press from evdev (needs the user in + // the `input` group). Everything else — the common case — is a move. + interactionType: payload.interactionType ?? "move", ...(payload.assetId ? { assetId: payload.assetId } : {}), }); diff --git a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts index 16815071c..7dd222ced 100644 --- a/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/pipeWireCursorRecordingSession.ts @@ -21,8 +21,11 @@ import type { CursorRecordingSession } from "./session"; * * Two consequences the caller should know about: * - * * `interactionType` is always "move". Wayland exposes no portal for mouse - * buttons and /dev/input/event* is root:input, so clicks are unobtainable. + * * `interactionType` is "move" unless the helper could read left-button + * presses from evdev — which needs the user in the `input` group, because + * Wayland exposes no portal for mouse buttons and /dev/input/event* is + * root:input. When it can, the coinciding sample is tagged "click"; when it + * cannot, every sample is a move, as before. See the helper's input.rs. * * The helper raises its own portal picker. On Wayland, Electron's * `desktopCapturer` already raised one, so the user currently picks a source * twice. Merging the two is the job of the capture stage that will reuse this diff --git a/electron/native/README.md b/electron/native/README.md index 8ff2e3cdd..9bd69ae81 100644 --- a/electron/native/README.md +++ b/electron/native/README.md @@ -209,9 +209,14 @@ electron/native/bin/linux-x64/openscreen-pipewire-helper '{"probeOnly":true}' ### Known gaps -- **Mouse clicks are unobtainable.** Wayland exposes no portal for input events - and `/dev/input/event*` is `root:input`, so every sample's `interactionType` is - `"move"`. +- **Mouse clicks need the `input` group.** Wayland exposes no portal for input + events, so the helper reads left-button presses straight from evdev + (`/dev/input/event*`). Those nodes are `root:input`, so a user outside the + `input` group gets no readable device and every sample's `interactionType` + stays `"move"` — the same as before. When a device is readable, the coinciding + sample is tagged `"click"`. Scope is deliberately narrow: `BTN_LEFT` only, + never keystrokes (see `pipewire-capture/src/input.rs`), and + `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` turns it off entirely. - **The user picks a source twice.** Electron's `desktopCapturer` raises its own portal dialog for the video, and this helper raises a second one for the cursor. Collapsing them requires one portal session serving both, which is why the diff --git a/electron/native/pipewire-capture/Cargo.lock b/electron/native/pipewire-capture/Cargo.lock index f8fa27a58..17c2abd0a 100644 --- a/electron/native/pipewire-capture/Cargo.lock +++ b/electron/native/pipewire-capture/Cargo.lock @@ -238,6 +238,18 @@ version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "bitvec" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -415,6 +427,18 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "evdev" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25b686663ba7f08d92880ff6ba22170f1df4e83629341cba34cf82cd65ebea99" +dependencies = [ + "bitvec", + "cfg-if", + "libc", + "nix", +] + [[package]] name = "event-listener" version = "5.4.2" @@ -475,6 +499,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures-channel" version = "0.3.33" @@ -834,6 +864,7 @@ dependencies = [ "base64", "bindgen", "cc", + "evdev", "png", "pollster", "serde", @@ -974,6 +1005,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.8.7" @@ -1213,6 +1250,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.27.0" @@ -1463,6 +1506,15 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "xdg-home" version = "1.3.0" diff --git a/electron/native/pipewire-capture/Cargo.toml b/electron/native/pipewire-capture/Cargo.toml index 604da16af..862b848bd 100644 --- a/electron/native/pipewire-capture/Cargo.toml +++ b/electron/native/pipewire-capture/Cargo.toml @@ -50,6 +50,7 @@ serde_json = "1" png = "0.17" base64 = "0.22" sha2 = "0.10" +evdev = "0.13" [build-dependencies] cc = "1" diff --git a/electron/native/pipewire-capture/src/events.rs b/electron/native/pipewire-capture/src/events.rs index 085ea56aa..ce98ac87e 100644 --- a/electron/native/pipewire-capture/src/events.rs +++ b/electron/native/pipewire-capture/src/events.rs @@ -97,6 +97,12 @@ pub enum Event { asset_id: Option, #[serde(skip_serializing_if = "Option::is_none")] asset: Option, + /// `"click"` on the sample that coincides with a left-button press read + /// from evdev (see `input.rs`), absent otherwise. Omitted rather than + /// defaulted to `"move"` so the accumulator keeps that fallback in one + /// place and the wire stays quiet on the common case. + #[serde(skip_serializing_if = "Option::is_none")] + interaction_type: Option, }, /// Which capture node each audio source was linked to. /// @@ -303,12 +309,32 @@ mod tests { visible: true, asset_id: None, asset: None, + interaction_type: None, }); assert_eq!(value["event"], "cursor-sample"); assert_eq!(value["x"], 100); assert_eq!(value["visible"], true); assert!(value.get("assetId").is_none()); assert!(value.get("asset").is_none()); + // A plain move stays silent about its interaction so the accumulator's + // "move" default is the single source of that word. + assert!(value.get("interactionType").is_none()); + } + + #[test] + fn cursor_samples_report_a_click_when_tagged() { + let value = parse_one(&Event::CursorSample { + timestamp_ms: 12, + x: 100, + y: 200, + width: 1920, + height: 1080, + visible: true, + asset_id: None, + asset: None, + interaction_type: Some("click".to_owned()), + }); + assert_eq!(value["interactionType"], "click"); } #[test] @@ -329,6 +355,7 @@ mod tests { hotspot_x: 4, hotspot_y: 3, }), + interaction_type: None, }); assert_eq!(value["assetId"], "abc"); assert_eq!(value["asset"]["imageDataUrl"], "data:image/png;base64,AA=="); @@ -359,6 +386,7 @@ mod tests { visible: true, asset_id: None, asset: None, + interaction_type: None, }); assert_eq!(value["timestampMs"], 1234, "a sample's capture time must not be overwritten"); } diff --git a/electron/native/pipewire-capture/src/input.rs b/electron/native/pipewire-capture/src/input.rs new file mode 100644 index 000000000..a1d41c90d --- /dev/null +++ b/electron/native/pipewire-capture/src/input.rs @@ -0,0 +1,211 @@ +//! Left mouse-button telemetry on Wayland, read from evdev. +//! +//! WHY THIS EXISTS. Wayland deliberately denies an unprivileged process any view +//! of global input: the ScreenCast portal reports cursor POSITION as frame +//! metadata but never button state, and the only portal that streams input +//! (`InputCapture`) *grabs* it, redirecting clicks away from the app being +//! recorded — useless while the user is demoing. The one remaining source is the +//! kernel's evdev interface (`/dev/input/event*`). Reading it needs membership in +//! the `input` group (the nodes are `root:input`), which is the user's own, +//! out-of-band act of consent — the Wayland equivalent of the button state the +//! macOS and Windows helpers already read from their native APIs. +//! +//! SCOPE AND PRIVACY. A pointer node can also deliver keystrokes on a combined +//! keyboard+mouse device. This reader inspects ONLY `EV_KEY` events whose code is +//! `BTN_LEFT`, and only their press edge; it never reads, stores, or forwards any +//! other key code. Enumerating the devices does briefly open each readable +//! `/dev/input/event*` node to inspect its capability bits, but any node that +//! does not advertise `BTN_LEFT` is dropped immediately, without a single event +//! ever being read from it — only `BTN_LEFT` devices get a reader. Set +//! `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` to turn it off entirely even where the +//! permission exists. + +use std::collections::HashSet; +use std::io::ErrorKind; +use std::path::PathBuf; +use std::sync::mpsc::Sender; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +use evdev::{Device, EventType, KeyCode}; + +use crate::events::timestamp_ms; +use crate::Message; + +const DISABLE_ENV: &str = "OPENSCREEN_DISABLE_CLICK_CAPTURE"; + +/// How often the hotplug watcher re-scans `/dev/input` for pointer devices that +/// appeared after startup. A few seconds is imperceptible for a device the user +/// just plugged in and costs a cheap directory walk. +const RESCAN_INTERVAL: Duration = Duration::from_secs(3); + +/// True when this evdev event is the press edge of the left mouse button. +/// +/// Extracted as a pure function so the decision is unit-testable without a real +/// device: a release (`value == 0`), an autorepeat (`value == 2`), and every +/// non-`BTN_LEFT` code — including every keyboard key — must NOT count. +pub fn is_left_button_press(event_type: EventType, code: u16, value: i32) -> bool { + event_type == EventType::KEY && code == KeyCode::BTN_LEFT.0 && value == 1 +} + +/// What [`spawn_readers`] settled on — the caller uses it to tell the log why +/// Linux clicks are or are not being captured. +/// +/// The two ways of ending up without clicks are NOT the same event: no readable +/// node is a permission the operator may still want to grant, while `DISABLE_ENV` +/// is one they explicitly declined — recommending the `input` group there answers +/// a question nobody asked. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClickCapture { + /// Turned off by `OPENSCREEN_DISABLE_CLICK_CAPTURE`; nothing was opened. + Disabled, + /// No readable `/dev/input` node reports `BTN_LEFT` — the common case, when + /// the user is not in the `input` group. + NoDevice, + /// At least one device is open, with a reader thread running on it. + Active, +} + +/// Opens every readable pointer device that reports `BTN_LEFT`, spawns a reader +/// thread per device, and leaves a daemon thread re-scanning for devices plugged +/// in later. +/// +/// Never fails: an unreadable node (the common case, when the user is not in the +/// `input` group) is skipped by `evdev::enumerate`, and no readable node at all +/// simply means every sample stays `"move"`, exactly as before this existed. The +/// returned value reflects the INITIAL scan only — a device attached afterwards +/// is adopted by the watcher without changing it. +pub fn spawn_readers(sender: &Sender) -> ClickCapture { + if std::env::var_os(DISABLE_ENV).is_some() { + return ClickCapture::Disabled; + } + // Paths with a LIVE reader. Shared with the reader threads: each removes its + // own path when it exits, so a device that unplugs and reconnects on the SAME + // node path is adopted again by the next scan. A set that only grew skipped + // such a replug for the rest of the recording. + let opened: Arc>> = Arc::new(Mutex::new(HashSet::new())); + scan_once(sender, &opened); + let result = if opened.lock().unwrap().is_empty() { + ClickCapture::NoDevice + } else { + ClickCapture::Active + }; + // Hotplug: the one-shot scan above cannot see a mouse attached mid-recording, + // so a daemon thread re-scans and starts readers for nodes it has not seen. + // Detached, like the reader threads — it ends when the process does. + let watch_sender = sender.clone(); + let watch_opened = Arc::clone(&opened); + thread::spawn(move || loop { + thread::sleep(RESCAN_INTERVAL); + scan_once(&watch_sender, &watch_opened); + }); + result +} + +/// Spawns a reader for every `BTN_LEFT` device not already being read, recording +/// each newly opened node's path. Shared by the initial scan and the watcher; the +/// check-and-insert is one locked step so two scans cannot both adopt one path. +fn scan_once(sender: &Sender, opened: &Arc>>) { + for (path, device) in evdev::enumerate() { + if !device_reports_left_button(&device) { + continue; + } + if !opened.lock().unwrap().insert(path.clone()) { + continue; // already has a live reader + } + let forward = sender.clone(); + let owned = Arc::clone(opened); + thread::spawn(move || read_device(device, path, forward, owned)); + } +} + +/// A touchpad advertises `BTN_LEFT` for a physical clickpad press, so it passes +/// this check and its node is opened — but tap-to-click never arrives here. +/// libinput consumes the raw `BTN_TOUCH`/`ABS_MT_*` stream and synthesises the +/// button for its own clients without writing `BTN_LEFT` back to the kernel +/// device, so a tap is invisible at the evdev layer we read. The consequence is +/// documented for users under "Mouse clicks on Wayland" in the installation docs: +/// on a touchpad only hard presses are captured, taps are not. +fn device_reports_left_button(device: &Device) -> bool { + device + .supported_keys() + .is_some_and(|keys| keys.contains(KeyCode::BTN_LEFT)) +} + +/// Blocks reading `device`, forwarding one `PointerButton` message per left-button +/// press, each stamped with the press time so a click is not backdated to the +/// next cursor sample. Returns when the device fails terminally (e.g. unplugged) +/// or the loop's channel has closed, so the thread cannot outlive the recording +/// it serves. A transient `EINTR` is retried, not mistaken for an unplug. +/// +/// On EVERY exit it drops `path` from `opened`, so a device reconnecting on the +/// same node path is re-adopted by the next scan. +fn read_device( + mut device: Device, + path: PathBuf, + sender: Sender, + opened: Arc>>, +) { + let release = || { + opened.lock().unwrap().remove(&path); + }; + loop { + let events = match device.fetch_events() { + Ok(events) => events, + // A signal interrupted the blocking read — not a device failure. + Err(err) if err.kind() == ErrorKind::Interrupted => continue, + // A real error, typically the device unplugging. Report it rather + // than ending silently, so click capture going quiet mid-recording + // is answerable from the log; the watcher re-adopts it on a replug. + Err(err) => { + release(); + let _ = sender.send(Message::PointerDeviceLost(err.to_string())); + return; + } + }; + for event in events { + if is_left_button_press(event.event_type(), event.code(), event.value()) + && sender.send(Message::PointerButton(timestamp_ms())).is_err() + { + release(); + return; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const BTN_LEFT: u16 = KeyCode::BTN_LEFT.0; + const BTN_RIGHT: u16 = KeyCode::BTN_RIGHT.0; + + #[test] + fn a_left_button_press_is_a_click() { + assert!(is_left_button_press(EventType::KEY, BTN_LEFT, 1)); + } + + #[test] + fn a_left_button_release_is_not() { + assert!(!is_left_button_press(EventType::KEY, BTN_LEFT, 0)); + } + + #[test] + fn a_left_button_autorepeat_is_not() { + // A held button emits value 2; only the 0->1 edge is a click. + assert!(!is_left_button_press(EventType::KEY, BTN_LEFT, 2)); + } + + #[test] + fn a_right_button_press_is_not_a_left_click() { + assert!(!is_left_button_press(EventType::KEY, BTN_RIGHT, 1)); + } + + #[test] + fn a_key_matching_btn_lefts_code_on_another_axis_is_not_a_click() { + // Same numeric code but a relative-motion event, not a key — must miss. + assert!(!is_left_button_press(EventType::RELATIVE, BTN_LEFT, 1)); + } +} diff --git a/electron/native/pipewire-capture/src/main.rs b/electron/native/pipewire-capture/src/main.rs index 8ec4078c2..8afe2af88 100644 --- a/electron/native/pipewire-capture/src/main.rs +++ b/electron/native/pipewire-capture/src/main.rs @@ -20,15 +20,19 @@ //! the cursor-only session Stage 1 shipped, which is what //! `PipeWireCursorRecordingSession` still uses. //! -//! WHAT IT CANNOT DO. Mouse buttons. Wayland exposes no portal for input -//! events, and /dev/input/event* is root:input. Every sample is therefore a -//! "move"; there is no click detection to be had here at any effort level. +//! MOUSE BUTTONS. Not from the portal — Wayland exposes no portal for input +//! events. The one source left is evdev (/dev/input/event*), which is root:input +//! and so needs the user in the `input` group. When that permission exists the +//! helper reads left-button presses and tags the coinciding sample "click"; when +//! it does not, every sample is a "move", as before. See `input.rs` for the +//! reader and its deliberately narrow scope (BTN_LEFT only, never keystrokes). mod bitmap; mod capture; mod encoder; mod events; mod ffmpeg; +mod input; mod portal; mod shim; @@ -208,6 +212,16 @@ struct AudioSourceConfig { enum Message { Portal(Box>), Stream(StreamEvent), + /// A left mouse-button press observed on evdev, carrying the press time in + /// milliseconds so the emitted click sample is stamped when it happened + /// rather than at the next throttled sample. See [`input`] for why this is + /// the only way to see a button on Wayland, and for its permission and + /// privacy model. + PointerButton(u64), + /// A pointer reader thread stopped on a device error (typically an unplug); + /// the string is the OS error. Surfaced as a warning so click capture going + /// quiet mid-recording is not silent. + PointerDeviceLost(String), /// Arm a deferred session: connect to PipeWire and start encoding. Record, Pause, @@ -283,6 +297,23 @@ fn main() { let (sender, receiver) = mpsc::channel::(); spawn_stdin_reader(sender.clone()); spawn_portal(sender.clone(), cursor_mode); + // Left-button telemetry from evdev. Gated on the cursor mode FIRST: a session + // that paints no cursor emits no samples to tag, so opening every pointer node + // and blocking a thread on each would buy nothing. Warn only when the devices + // are unreadable — never when the operator opted out through the env var, + // which would recommend the permission they just declined — so a "clicks do + // nothing on Linux" report is answerable from the log alone rather than + // looking like a capture bug. + if cursor_mode.reports_cursor() + && input::spawn_readers(&sender) == input::ClickCapture::NoDevice + { + let _ = emitter.emit(&Event::Warning { + code: "click-capture-unavailable".to_owned(), + message: "no readable /dev/input pointer device — add this user to the 'input' \ + group to record click telemetry; cursor samples will otherwise all be moves" + .to_owned(), + }); + } let session = RunConfig { tick, @@ -568,6 +599,17 @@ fn run( let mut cursor: Option = None; let mut known_assets: HashSet = HashSet::new(); let mut pending_asset: Option = None; + // Wall-clock ms at which the pw_stream last reached `streaming` (mutter began + // handing us frames), or `None` when it is not streaming. A press counts only + // if its own read time is at or after this: before streaming the only thing on + // screen is the portal picker, and the click that dismisses it (its "Share" + // button) would otherwise ride out on the first sample as a phantom click at + // t≈0. Comparing timestamps rather than a bare flag closes two gaps: the press + // and the stream-state arrive on DIFFERENT channels, so a pre-stream press can + // be dequeued after the flag flips (it is still dropped, its time is older); + // and clearing it on disconnect stops clicks emitting against a stale cursor + // after capture has stopped. + let mut streaming_since: Option = None; let mut reported_cursor_meta = false; // Allocated up front so the PipeWire callback has somewhere to put frames // from the very first buffer; `None` in cursor-only mode, which is also what @@ -597,6 +639,33 @@ fn run( match receiver.recv_timeout(config.tick) { Ok(Message::Stop) => break, + // Emit a click sample straight away at the current cursor position, + // stamped with the press time the reader captured: immediate so it is + // not backdated to the next throttled sample, and one sample per press + // so a rapid double-click reads as two clicks rather than collapsing + // into one. Counted only if the press happened at or after streaming + // began (see `streaming_since`): a press while the picker is still up — + // the click on its "Share" button — is older, so it is dropped even if + // its message is delivered after the stream-state one. + Ok(Message::PointerButton(press_ms)) => { + if streaming_since.is_some_and(|since| press_ms >= since) { + emit_sample(emitter, &cursor, size, &mut pending_asset, Some(press_ms)); + } + } + + // A reader lost its device (typically an unplug). Report it — if it + // was the only pointer, clicks stop until one is (re)connected, which + // the input watcher will pick up. + Ok(Message::PointerDeviceLost(reason)) => { + let _ = emitter.emit(&Event::Warning { + code: "click-capture-device-lost".to_owned(), + message: format!( + "a pointer device stopped delivering clicks ({reason}); if it was the \ + only one, clicks are not captured until a device is reconnected" + ), + }); + } + Ok(Message::Pause) => { paused = true; if let Some(capture) = capture.as_mut() { @@ -965,6 +1034,19 @@ fn run( ("error", error.clone().into()), ]), }); + // Once frames are flowing, presses land on recorded content; the + // picker (and the "Share" click that dismissed it) is behind us. + // Stamped so a press read before this instant is dropped by time, + // and cleared on disconnect so clicks stop emitting against a stale + // cursor after capture ends. Only set on the FIRST streaming edge so + // a transient renegotiation `paused`→`streaming` does not re-arm it. + if state == "streaming" { + if streaming_since.is_none() { + streaming_since = Some(timestamp_ms()); + } + } else if state == "unconnected" { + streaming_since = None; + } if let Some(error) = error { let _ = emitter.emit(&Event::Warning { code: "stream-error".to_owned(), @@ -1052,14 +1134,14 @@ fn run( // A new sprite ships immediately; positions respect the sample // interval so a 120fps compositor cannot flood stdout. if asset_is_new || last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset); + emit_sample(emitter, &cursor, size, &mut pending_asset, None); last_emit = Instant::now(); } } Err(RecvTimeoutError::Timeout) => { if cursor.is_some() && last_emit.elapsed() >= config.sample_interval { - emit_sample(emitter, &cursor, size, &mut pending_asset); + emit_sample(emitter, &cursor, size, &mut pending_asset, None); last_emit = Instant::now(); } // The heartbeat that keeps the output at a constant frame rate @@ -1148,18 +1230,30 @@ fn finish_capture( } } +/// Emits one cursor sample at the current position. `click` is `Some(press_ms)` +/// for a left-button press — the sample is then tagged `"click"` and stamped with +/// that press time — or `None` for an ordinary throttled position sample stamped +/// now. Either way a pending new sprite rides out on it. fn emit_sample( emitter: &mut Emitter, cursor: &Option, size: Option<(i32, i32)>, pending_asset: &mut Option, + click: Option, ) { let (Some(state), Some((width, height))) = (cursor, size) else { return; }; let visible = state.x >= 0 && state.y >= 0 && state.x < width && state.y < height; + // A click carries its own press time so the bounce lands when the button went + // down, not up to a sample interval later; the position is the latest known, + // which at the sample cadence has not moved enough to be wrong. + let (timestamp, interaction_type) = match click { + Some(press_ms) => (press_ms, Some("click".to_owned())), + None => (timestamp_ms(), None), + }; let _ = emitter.emit(&Event::CursorSample { - timestamp_ms: timestamp_ms(), + timestamp_ms: timestamp, x: state.x, y: state.y, width, @@ -1167,6 +1261,7 @@ fn emit_sample( visible, asset_id: state.asset_id.clone(), asset: pending_asset.take(), + interaction_type, }); } diff --git a/src/lib/cursor/cursorCapabilities.ts b/src/lib/cursor/cursorCapabilities.ts index 271372b04..5b6440789 100644 --- a/src/lib/cursor/cursorCapabilities.ts +++ b/src/lib/cursor/cursorCapabilities.ts @@ -1,26 +1,26 @@ -import { getPlatform } from "@/utils/platformUtils"; - /** * Can this platform tell us when a mouse button was pressed? * - * Not a preference and not a feature flag — a hard limit of the display server. - * On Wayland an unprivileged process cannot observe mouse buttons at all: the - * ScreenCast portal reports pointer POSITION as frame metadata and nothing else, - * there is no portal for input events, and `/dev/input/event*` is `root:input`. - * So the Linux capture helper stamps every sample `interactionType: "move"` - * (see `pipeWireCursorAccumulator.ts`), the compositor's `CursorTrack.clicks` - * vector is always empty, and `CursorTrack::bounce()` returns exactly 1.0 — - * which `frame_geometry.rs` multiplies the cursor size by, leaving it unchanged - * for every possible slider value. + * True on every platform we support. macOS and Windows read real button state in + * their native cursor helpers (`macNativeCursorRecordingSession.ts`, + * `windowsNativeRecordingSession.ts`). * - * macOS and Windows both read real button state in their native cursor helpers - * (`macNativeCursorRecordingSession.ts`, `windowsNativeRecordingSession.ts`), so - * the effect works there. + * On Linux/Wayland the ScreenCast portal still reports pointer POSITION as frame + * metadata and nothing else — there is no portal for button events. But the + * capture helper now reads left-button presses directly from `/dev/input/event*` + * via evdev and tags the coinciding sample `interactionType: "click"` (the + * pipewire helper's `input.rs`; the accumulator preserves the tag in + * `pipeWireCursorAccumulator.ts`). Those clicks travel in the `.cursor.json` + * sidecar, populate the compositor's `CursorTrack.clicks` vector (`cursor.rs`), + * and drive `CursorTrack::bounce()` through the shared geometry in + * `frame_geometry.rs` — the exact same path macOS and Windows take, with no + * Linux-specific branch. * - * Same shape and same intent as `supportsWebcamReactiveZoom` in - * `compositeLayout.ts`: a control that provably cannot change a pixel is - * dropped rather than shown doing nothing. + * Reading `/dev/input` requires the recording user to be in the `input` group. + * When they are not, no click is captured and every sample stays a plain move, + * so the effect has nothing to act on — a recording-side permission matter, not + * a reason to hide the control (see `website/docs/installation.md`). */ export function supportsCursorClickEffects(): boolean { - return getPlatform() !== "linux"; + return true; } diff --git a/website/docs/installation.md b/website/docs/installation.md index d71f0197c..5d2c7545f 100644 --- a/website/docs/installation.md +++ b/website/docs/installation.md @@ -24,7 +24,7 @@ Download the latest installer for your platform from the [download page](/downlo |---|---|---| | **Windows** | Windows 10 version 1903 (build 18362) or later, Intel 8th Gen / AMD Ryzen 2000 series or newer | Windows 11, Intel 12th Gen / AMD Ryzen 4000 series or newer | | **macOS** | macOS 12.3 (Monterey) — required by ScreenCaptureKit for native capture | macOS 14 or later | -| **Linux** | `xdg-desktop-portal` and PipeWire for native capture and system audio (default on Ubuntu 22.04+, Fedora 34+) — recording still works without them through the [browser-capture fallback](#platform-differences), with fewer capabilities | Same, kept up to date | +| **Linux** | `xdg-desktop-portal` and PipeWire for native capture and system audio (default on Ubuntu 22.04+, Fedora 34+) — recording still works without them through the [browser-capture fallback](#platform-differences), with fewer capabilities. Recording mouse clicks on Wayland additionally needs your user in the `input` group — see [Mouse clicks on Wayland](#mouse-clicks-on-wayland) | Same, kept up to date | | **RAM** | 8 GB | 16 GB | :::note Older integrated graphics on Windows @@ -111,6 +111,20 @@ Home Manager users can use `openscreen.homeManagerModules.default` with the same You may need to grant screen-recording permission depending on your desktop environment. +### Mouse clicks on Wayland + +Wayland exposes no portal for input events, so OpenScreen reads left-button presses straight from the kernel's evdev interface (`/dev/input/event*`) instead. Those device nodes are owned by `root:input`, so a recording only distinguishes a click from ordinary cursor movement when your user is in the `input` group: + +```bash +sudo usermod -aG input $USER +``` + +Log out and back in for the new group to take effect. Nothing breaks without it — recording works exactly as it did before, and every cursor sample is simply recorded as a move. + +The scope is deliberately narrow: only the left mouse button (`BTN_LEFT`) is ever read, never keystrokes. To turn the reader off entirely even where the permission exists, set `OPENSCREEN_DISABLE_CLICK_CAPTURE=1` in the environment OpenScreen is launched from. + +**Touchpads:** only a physical click — pressing the pad down until it depresses — is recorded. **Tap-to-click is not**, because your compositor's input stack (libinput) synthesises those taps for its own use and never writes them back to the kernel device that OpenScreen reads, so there is nothing at the evdev layer to see. A mouse, or a touchpad with tap-to-click turned off, records every click. + ## Platform differences The editing tools are the same everywhere — zooms, backgrounds, crop/trim/speed, annotations, transcription, captions, and projects. Every export format works on every platform; what differs is **capture**, and how fast MP4 encodes on Linux: @@ -118,7 +132,7 @@ The editing tools are the same everywhere — zooms, backgrounds, crop/trim/spee | | macOS | Windows | Linux | |---|---|---|---| | Capture pipeline | Native (ScreenCaptureKit) | Native (Windows Graphics Capture) | Browser pipeline | -| Custom cursor themes / click effects | ✅ | ✅ | ❌ (position-only, used for auto-zoom) | +| Custom cursor themes / click effects | ✅ | ✅ | ✅ on Wayland — click capture needs the `input` group ([details](#mouse-clicks-on-wayland)) | | Webcam | Native capture | Native capture | Browser capture (still works as PiP) | | System audio | macOS 13+; permission prompt on 14.2+; not available on macOS 12 and below | Works out of the box | Needs PipeWire (default on Ubuntu 22.04+, Fedora 34+) | | MP4 export | ✅ | ✅ | ✅ (software encode) |