From 3db80129f0743c1e9e51e959af5f9083f65e8077 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Thu, 22 Oct 2026 19:15:21 +0200 Subject: [PATCH 01/16] perf(recording): cut Windows readback copies and encode from textures Read Windows capture back through a staging ring with DO_NOT_WAIT instead of a blocking Map plus a second full-frame copy, and let the fragmented/instant M4S muxer encode from the captured D3D11 texture through Media Foundation with the existing FFmpeg encoder as fallback. Also removes the unused win_segmented muxers and the never-invoked DiskSpaceCallback plumbing. --- crates/enc-ffmpeg/src/mux/segmented_stream.rs | 446 ++++++--- crates/enc-ffmpeg/src/video/h264.rs | 7 + crates/enc-mediafoundation/src/video/h264.rs | 132 ++- crates/mediafoundation-ffmpeg/src/h264.rs | 126 ++- crates/mediafoundation-ffmpeg/src/lib.rs | 4 +- crates/recording/src/capture_pipeline.rs | 12 +- .../output_pipeline/macos_fragmented_m4s.rs | 16 +- crates/recording/src/output_pipeline/mod.rs | 10 - .../src/output_pipeline/oop_fragmented_m4s.rs | 8 +- .../output_pipeline/oop_fragmented_m4s_win.rs | 28 +- crates/recording/src/output_pipeline/win.rs | 22 +- .../src/output_pipeline/win_fragmented_m4s.rs | 362 +++++++- .../src/output_pipeline/win_segmented.rs | 822 ----------------- .../output_pipeline/win_segmented_camera.rs | 849 ------------------ .../src/sources/screen_capture/windows.rs | 2 +- crates/scap-direct3d/src/lib.rs | 519 +++++++++-- crates/scap-ffmpeg/examples/cli.rs | 225 ++++- crates/scap-ffmpeg/src/direct3d.rs | 217 +---- 18 files changed, 1613 insertions(+), 2194 deletions(-) delete mode 100644 crates/recording/src/output_pipeline/win_segmented.rs delete mode 100644 crates/recording/src/output_pipeline/win_segmented_camera.rs diff --git a/crates/enc-ffmpeg/src/mux/segmented_stream.rs b/crates/enc-ffmpeg/src/mux/segmented_stream.rs index 0ea15ad1566..aa7eb4d669a 100644 --- a/crates/enc-ffmpeg/src/mux/segmented_stream.rs +++ b/crates/enc-ffmpeg/src/mux/segmented_stream.rs @@ -1,6 +1,7 @@ use super::fragment_metadata::read_fragment_metadata; use super::write_behind::{MuxSink, MuxTarget}; -use cap_media_info::VideoInfo; +use crate::base::PacketSink; +use cap_media_info::{VideoInfo, ensure_even}; use ffmpeg::{format, frame}; use serde::Serialize; use std::{ @@ -16,16 +17,7 @@ use crate::video::h264::{ }; const INIT_SEGMENT_NAME: &str = "init.mp4"; - -#[derive(Debug, Clone)] -pub struct DiskSpaceWarning { - pub available_mb: u64, - pub threshold_mb: u64, - pub path: String, - pub is_critical: bool, -} - -pub type DiskSpaceCallback = Arc; +const STREAM_TIME_BASE: ffmpeg::Rational = ffmpeg::Rational(1, 10_000_000); /// Shared so a write-behind muxer can install the callback without waiting /// for the writer thread, which may be blocked on storage. @@ -95,10 +87,6 @@ pub struct SegmentedVideoEncoder { encoder: H264Encoder, mux: MuxSink, segment_tx: SegmentSender, - - segment_duration: Duration, - - disk_space_callback: Option, } struct SegmentMux { @@ -199,6 +187,10 @@ pub enum InitError { Encoder(#[from] H264EncoderError), #[error("IO: {0}")] Io(#[from] std::io::Error), + #[error("The encoder did not expose an H.264 sequence header for init.mp4")] + MissingExtradata, + #[error("Failed to allocate the H.264 extradata buffer")] + Extradata, } #[derive(thiserror::Error, Debug)] @@ -245,6 +237,92 @@ impl Default for SegmentedVideoEncoderConfig { } } +fn init_segment_path(base_path: &Path) -> PathBuf { + base_path.join(INIT_SEGMENT_NAME) +} + +fn validate_init_segment(init_path: &Path) -> Result<(), String> { + if !init_path.exists() { + return Err(format!( + "init.mp4 is missing at {}. M4S segments will be unplayable without it.", + init_path.display() + )); + } + + match std::fs::metadata(init_path) { + Ok(metadata) => { + let size = metadata.len(); + if size < 100 { + return Err(format!( + "init.mp4 at {} is too small ({} bytes). It may be corrupted.", + init_path.display(), + size + )); + } + Ok(()) + } + Err(e) => Err(format!( + "Cannot read init.mp4 metadata at {}: {e}", + init_path.display() + )), + } +} + +fn open_dash_output( + base_path: &Path, + segment_duration: Duration, + timed_fragments: bool, +) -> Result { + let manifest_path = base_path.join("dash_manifest.mpd"); + + #[cfg(target_os = "windows")] + let manifest_path_str = manifest_path.to_string_lossy().replace('\\', "/"); + #[cfg(not(target_os = "windows"))] + let manifest_path_str = manifest_path.to_string_lossy().to_string(); + + let mut output = super::dash_output::create(&manifest_path_str)?; + + let init_seg_str = INIT_SEGMENT_NAME; + let media_seg_str = "segment_$Number%03d$.m4s"; + + unsafe { + let opts = output.as_mut_ptr(); + + let set_opt = |key: &str, value: &str| { + let k = CString::new(key).unwrap(); + let v = CString::new(value).unwrap(); + ffmpeg::ffi::av_opt_set((*opts).priv_data, k.as_ptr(), v.as_ptr(), 0); + }; + + set_opt("init_seg_name", init_seg_str); + set_opt("media_seg_name", media_seg_str); + set_opt("seg_duration", &segment_duration.as_secs_f64().to_string()); + set_opt("use_timeline", "1"); + set_opt("use_template", "1"); + set_opt("single_file", "0"); + set_opt("hls_playlist", "1"); + if timed_fragments { + set_opt("frag_type", "duration"); + set_opt("frag_duration", "2"); + set_opt("streaming", "1"); + } + } + + Ok(output) +} + +fn dash_codec_info(video_config: VideoInfo, stream_size: (u32, u32)) -> CodecInfo { + CodecInfo { + width: stream_size.0, + height: stream_size.1, + frame_rate_num: video_config.frame_rate.0, + frame_rate_den: video_config.frame_rate.1, + time_base_num: video_config.time_base.0, + time_base_den: video_config.time_base.1, + pixel_format: format!("{:?}", video_config.pixel_format), + } +} + impl SegmentedVideoEncoder { pub fn init( base_path: PathBuf, @@ -287,43 +365,16 @@ impl SegmentedVideoEncoder { ) -> Result { std::fs::create_dir_all(&base_path)?; - let manifest_path = base_path.join("dash_manifest.mpd"); - - #[cfg(target_os = "windows")] - let manifest_path_str = manifest_path.to_string_lossy().replace('\\', "/"); - #[cfg(not(target_os = "windows"))] - let manifest_path_str = manifest_path.to_string_lossy().to_string(); - - let mut output = super::dash_output::create(&manifest_path_str)?; - - let init_seg_str = INIT_SEGMENT_NAME; - let media_seg_str = "segment_$Number%03d$.m4s"; - - unsafe { - let opts = output.as_mut_ptr(); - - let set_opt = |key: &str, value: &str| { - let k = CString::new(key).unwrap(); - let v = CString::new(value).unwrap(); - ffmpeg::ffi::av_opt_set((*opts).priv_data, k.as_ptr(), v.as_ptr(), 0); - }; + let stream_size = config + .output_size + .map(|(width, height)| (ensure_even(width), ensure_even(height))) + .unwrap_or(( + ensure_even(video_config.width), + ensure_even(video_config.height), + )); - set_opt("init_seg_name", init_seg_str); - set_opt("media_seg_name", media_seg_str); - set_opt( - "seg_duration", - &config.segment_duration.as_secs_f64().to_string(), - ); - set_opt("use_timeline", "1"); - set_opt("use_template", "1"); - set_opt("single_file", "0"); - set_opt("hls_playlist", "1"); - if timed_fragments { - set_opt("frag_type", "duration"); - set_opt("frag_duration", "2"); - set_opt("streaming", "1"); - } - } + let manifest_path = base_path.join("dash_manifest.mpd"); + let mut output = open_dash_output(&base_path, config.segment_duration, timed_fragments)?; let mut builder = H264EncoderBuilder::new(video_config) .with_preset(config.preset) @@ -369,15 +420,7 @@ impl SegmentedVideoEncoder { "FFmpeg DASH muxer state after write_header()" ); - let codec_info = CodecInfo { - width: video_config.width, - height: video_config.height, - frame_rate_num: video_config.frame_rate.0, - frame_rate_den: video_config.frame_rate.1, - time_base_num: video_config.time_base.0, - time_base_den: video_config.time_base.1, - pixel_format: format!("{:?}", video_config.pixel_format), - }; + let codec_info = dash_codec_info(video_config, stream_size); tracing::info!( path = %base_path.display(), @@ -405,8 +448,6 @@ impl SegmentedVideoEncoder { encoder, mux: MuxSink::Direct(mux), segment_tx, - segment_duration: config.segment_duration, - disk_space_callback: None, }; if write_behind { @@ -425,10 +466,6 @@ impl SegmentedVideoEncoder { Ok(instance) } - pub fn set_disk_space_callback(&mut self, callback: DiskSpaceCallback) { - self.disk_space_callback = Some(callback); - } - pub fn set_segment_callback(&mut self, tx: std::sync::mpsc::Sender) { *self .segment_tx @@ -512,50 +549,16 @@ impl SegmentedVideoEncoder { Some(&mut self.encoder) } - pub fn base_path(&self) -> &Path { - &self.base_path - } - - pub fn segment_duration(&self) -> Duration { - self.segment_duration - } - pub fn current_index(&self) -> u32 { self.mux.with_target_ref(|mux| mux.current_index) } pub fn init_segment_path(&self) -> PathBuf { - self.base_path.join(INIT_SEGMENT_NAME) + init_segment_path(&self.base_path) } pub fn validate_init_segment(&self) -> Result<(), String> { - let init_path = self.init_segment_path(); - - if !init_path.exists() { - return Err(format!( - "init.mp4 is missing at {}. M4S segments will be unplayable without it.", - init_path.display() - )); - } - - match std::fs::metadata(&init_path) { - Ok(metadata) => { - let size = metadata.len(); - if size < 100 { - return Err(format!( - "init.mp4 at {} is too small ({} bytes). It may be corrupted.", - init_path.display(), - size - )); - } - Ok(()) - } - Err(e) => Err(format!( - "Cannot read init.mp4 metadata at {}: {}", - init_path.display(), - e - )), - } + validate_init_segment(&self.init_segment_path()) } } @@ -872,6 +875,156 @@ impl SegmentMux { } } +/// Muxes externally encoded H.264 access units into the same DASH segments as +/// [`SegmentedVideoEncoder`], for encoders that cannot hand over raw frames (the Windows Media +/// Foundation encoder takes D3D11 textures and emits compressed samples). +pub struct SegmentedPacketMuxer { + mux: MuxSink, + stream_index: usize, + segment_tx: SegmentSender, + last_written_pts: Option, + consecutive_pts_bumps: u64, +} + +impl SegmentedPacketMuxer { + /// `extradata` must be the H.264 sequence header (SPS/PPS) in Annex B; `init.mp4` is written + /// during construction and cannot carry a codec configuration discovered later. + pub fn new( + base_path: PathBuf, + video_config: VideoInfo, + stream_size: (u32, u32), + segment_duration: Duration, + timed_fragments: bool, + bitrate: u32, + extradata: &[u8], + ) -> Result { + if extradata.is_empty() { + return Err(InitError::MissingExtradata); + } + + let stream_size = (ensure_even(stream_size.0), ensure_even(stream_size.1)); + + std::fs::create_dir_all(&base_path)?; + + let mut output = open_dash_output(&base_path, segment_duration, timed_fragments)?; + + let h264 = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::H264) + .ok_or(ffmpeg::Error::DecoderNotFound)?; + let mut stream = output.add_stream(h264)?; + let stream_index = stream.index(); + stream.set_time_base(STREAM_TIME_BASE); + + unsafe { + let codecpar = (*stream.as_mut_ptr()).codecpar; + (*codecpar).codec_type = ffmpeg::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO; + (*codecpar).codec_id = ffmpeg::ffi::AVCodecID::AV_CODEC_ID_H264; + (*codecpar).width = stream_size.0 as i32; + (*codecpar).height = stream_size.1 as i32; + (*codecpar).bit_rate = i64::from(bitrate); + (*codecpar).format = ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NV12 as i32; + + let stream_ptr = stream.as_mut_ptr(); + (*stream_ptr).avg_frame_rate = ffmpeg::ffi::AVRational { + num: video_config.frame_rate.0, + den: video_config.frame_rate.1, + }; + (*stream_ptr).r_frame_rate = ffmpeg::ffi::AVRational { + num: video_config.frame_rate.0, + den: video_config.frame_rate.1, + }; + + let size = extradata.len() + ffmpeg::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize; + let buffer = ffmpeg::ffi::av_mallocz(size).cast::(); + if buffer.is_null() { + return Err(InitError::Extradata); + } + std::ptr::copy_nonoverlapping(extradata.as_ptr(), buffer, extradata.len()); + (*codecpar).extradata = buffer; + (*codecpar).extradata_size = extradata.len() as i32; + } + + output.write_header()?; + + let mut mux = SegmentMux { + base_path: base_path.clone(), + output, + current_index: 1, + completed_segments: Vec::new(), + frames_since_segment_scan: 0, + codec_info: dash_codec_info(video_config, stream_size), + segment_tx: SegmentSender::default(), + init_notified: false, + initial_manifest_pending: true, + }; + mux.write_initial_manifest(); + + Ok(Self { + segment_tx: mux.segment_tx.clone(), + mux: MuxSink::Direct(mux), + stream_index, + last_written_pts: None, + consecutive_pts_bumps: 0, + }) + } + + pub fn set_segment_callback(&mut self, tx: std::sync::mpsc::Sender) { + *self + .segment_tx + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(tx); + self.mux.with_target(SegmentMux::try_notify_init_segment); + } + + pub fn write_packet( + &mut self, + mut packet: ffmpeg::Packet, + source_time_base: ffmpeg::Rational, + ) -> Result<(), ffmpeg::Error> { + packet.set_stream(self.stream_index); + + let destination_time_base = self.mux.stream_time_base(self.stream_index); + packet.rescale_ts(source_time_base, destination_time_base); + + // The mov/DASH muxer rejects duplicate pts/dts. MF timestamps normally advance in step with + // the stream time base, but a re-quantized or stuck source clock can repeat one; advance the + // tie by a tick so muxing never fails on it. + if let Some(pts) = packet.pts() { + let pts = match self.last_written_pts { + Some(last) if pts <= last => { + self.consecutive_pts_bumps += 1; + if self.consecutive_pts_bumps == 30 + || self.consecutive_pts_bumps.is_multiple_of(300) + { + tracing::warn!( + consecutive_bumps = self.consecutive_pts_bumps, + "Media Foundation sample times are not advancing; the muxer is tie-bumping every packet (stuck source clock?)" + ); + } + last + 1 + } + _ => { + self.consecutive_pts_bumps = 0; + pts + } + }; + self.last_written_pts = Some(pts); + packet.set_pts(Some(pts)); + packet.set_dts(Some(pts)); + } + + self.mux.write_packet(packet)?; + self.mux.frame_queued(); + + Ok(()) + } + + pub fn finish(&mut self) -> Result<(), FinishError> { + self.mux + .finish(|_| Ok(()), |mux, flush_result| mux.finish(flush_result)) + .map_err(std::io::Error::from)? + } +} + #[cfg(test)] mod tests { use super::*; @@ -899,6 +1052,95 @@ mod tests { frame } + #[test] + fn packet_muxer_writes_the_same_init_and_manifest_contract() { + ffmpeg::init().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let base_path = temp.path().to_path_buf(); + let video_info = test_video_info(); + let capture_config = VideoInfo { + width: 640, + height: 480, + ..video_info + }; + + let mut encoder = H264EncoderBuilder::new(video_info) + .with_preset(H264Preset::Ultrafast) + .build_standalone() + .unwrap(); + let extradata = encoder.extradata(); + assert!(!extradata.is_empty(), "encoder must expose SPS/PPS"); + + let mut muxer = SegmentedPacketMuxer::new( + base_path.clone(), + capture_config, + (video_info.width, video_info.height), + Duration::from_secs(2), + false, + 300_000, + &extradata, + ) + .unwrap(); + + let mut packets = Vec::new(); + for index in 0..90u64 { + encoder + .encode_frame( + create_test_frame(320, 240), + Duration::from_micros(index * 33_333), + |encoded| { + packets.push(encoded); + Ok(()) + }, + ) + .unwrap(); + } + + let source_time_base = encoder.time_base(); + for encoded in packets { + let mut packet = ffmpeg::Packet::copy(&encoded.data); + packet.set_pts(Some(encoded.pts)); + packet.set_dts(Some(encoded.dts)); + packet.set_duration(encoded.duration); + if encoded.is_keyframe { + packet.set_flags(ffmpeg::packet::Flags::KEY); + } + muxer + .write_packet(packet, source_time_base) + .expect("write packet"); + } + muxer.finish().unwrap(); + + let init_path = base_path.join(INIT_SEGMENT_NAME); + validate_init_segment(&init_path).unwrap(); + let init = std::fs::read(&init_path).unwrap(); + assert!( + init.windows(4).any(|window| window == b"avcC"), + "init.mp4 must carry the H.264 decoder configuration" + ); + + let input = ffmpeg::format::input(&init_path).expect("open init.mp4"); + let stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .expect("video stream"); + let parameters = stream.parameters(); + let (declared_width, declared_height) = + unsafe { ((*parameters.as_ptr()).width, (*parameters.as_ptr()).height) }; + assert_eq!((declared_width, declared_height), (320, 240)); + + let manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(base_path.join("manifest.json")).unwrap()) + .unwrap(); + assert_eq!(manifest["is_complete"].as_bool(), Some(true)); + assert!( + !manifest["segments"].as_array().unwrap().is_empty(), + "the DASH muxer must publish at least one media segment" + ); + assert_eq!(manifest["codec_info"]["width"].as_u64(), Some(320)); + assert_eq!(manifest["codec_info"]["height"].as_u64(), Some(240)); + } + #[test] fn video_manifest_and_events_match_muxer_playlist() { ffmpeg::init().unwrap(); diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index 0042b4610af..46299035428 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -1663,6 +1663,13 @@ fn get_bitrate(width: u32, height: u32, frame_rate: f32, bpp: f32) -> usize { ((pixels_per_second * bpp as f64) as usize).min(MAX_BITRATE) } +/// Bitrate this module's encoders target for `bpp`, so a hardware encoder fed straight from the +/// capture texture keeps the recording quality of the software path. +pub fn bitrate_for_bpp(width: u32, height: u32, frame_rate: ffmpeg::Rational, bpp: f32) -> u32 { + let fps = frame_rate.0 as f32 / frame_rate.1.max(1) as f32; + get_bitrate(width, height, fps, bpp) as u32 +} + const MAX_BITRATE: usize = 1_000_000_000; #[cfg(test)] diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 2e496e47f64..de011af6d66 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -10,6 +10,7 @@ use std::{ }, time::{Duration, Instant}, }; +use tracing::debug; use windows::{ Foundation::TimeSpan, Graphics::SizeInt32, @@ -22,13 +23,14 @@ use windows::{ Media::MediaFoundation::{ self, IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, IMFSample, IMFTransform, MF_E_INVALIDMEDIATYPE, MF_E_NO_MORE_TYPES, - MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_FLAG_NONE, MF_EVENT_TYPE, - MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, - MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, - MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, + MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_FLAG_NO_WAIT, + MF_EVENT_FLAG_NONE, MF_EVENT_TYPE, MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, + MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, + MF_MT_MAX_KEYFRAME_SPACING, MF_MT_MPEG_SEQUENCE_HEADER, MF_MT_PIXEL_ASPECT_RATIO, + MF_MT_SUBTYPE, MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType, MFCreateSample, MFMediaType_Video, MFT_ENUM_FLAG, MFT_ENUM_FLAG_HARDWARE, - MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_FLUSH, + MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_DRAIN, MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_OF_STREAM, MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY, @@ -38,6 +40,8 @@ use windows::{ core::{Error, Interface}, }; +/// DASH segments are cut on keyframes; 2 s matches the software encoder's GOP. +const MUXED_KEYFRAME_INTERVAL_SECS: u32 = 2; const MAX_CONSECUTIVE_EMPTY_SAMPLES: u8 = 20; const MAX_INPUT_WITHOUT_OUTPUT: u32 = 30; const MAX_PROCESS_INPUT_FAILURES: u32 = 5; @@ -241,17 +245,10 @@ impl H264Encoder { input_resolution: SizeInt32, output_resolution: SizeInt32, frame_rate: u32, - bitrate_multipler: f32, + bitrate: u32, flags: MFT_ENUM_FLAG, enable_hardware_transforms: bool, ) -> Result { - let bitrate = calculate_bitrate( - output_resolution.Width as u32, - output_resolution.Height as u32, - frame_rate, - bitrate_multipler, - ); - let transform = EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags) .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)? @@ -357,6 +354,10 @@ impl H264Encoder { output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + output_type.SetUINT32( + &MF_MT_MAX_KEYFRAME_SPACING, + frame_rate.saturating_mul(MUXED_KEYFRAME_INTERVAL_SECS), + )?; MFSetAttributeSize( &attributes, &MF_MT_FRAME_SIZE, @@ -443,6 +444,29 @@ impl H264Encoder { output_resolution: SizeInt32, frame_rate: u32, bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_and_bitrate( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + calculate_bitrate( + output_resolution.Width as u32, + output_resolution.Height as u32, + frame_rate, + bitrate_multipler, + ), + ) + } + + pub fn new_with_scaled_output_and_bitrate( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate: u32, ) -> Result { Self::new_with_scaled_output_with_flags( d3d_device, @@ -450,7 +474,7 @@ impl H264Encoder { input_resolution, output_resolution, frame_rate, - bitrate_multipler, + bitrate, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY, true, ) @@ -470,7 +494,12 @@ impl H264Encoder { input_resolution, output_resolution, frame_rate, - bitrate_multipler, + calculate_bitrate( + output_resolution.Width as u32, + output_resolution.Height as u32, + frame_rate, + bitrate_multipler, + ), MFT_ENUM_FLAG_TRANSCODE_ONLY, false, ) @@ -518,6 +547,28 @@ impl H264Encoder { &self.output_type } + /// SPS/PPS in Annex B, as the encoder exposes them on its output type. The DASH `init.mp4` is + /// written before the first sample, so a caller muxing compressed samples must have this up + /// front. + pub fn sequence_header(&self) -> Option> { + let attributes: IMFAttributes = self.output_type.cast().ok()?; + let size = unsafe { attributes.GetBlobSize(&MF_MT_MPEG_SEQUENCE_HEADER) }.ok()?; + if size == 0 { + return None; + } + + let mut header = vec![0u8; size as usize]; + let mut written = 0u32; + unsafe { + attributes + .GetBlob(&MF_MT_MPEG_SEQUENCE_HEADER, &mut header, Some(&mut written)) + .ok()?; + } + header.truncate(written as usize); + + Some(header) + } + pub fn validate(&self) -> Result<(), NewVideoEncoderError> { unsafe { self.transform @@ -667,6 +718,57 @@ impl H264Encoder { } } + let _ = self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0); + + let drain_deadline = Instant::now() + Duration::from_millis(250); + let mut drained = 0u64; + while Instant::now() < drain_deadline { + let event = match self.event_generator.GetEvent(MF_EVENT_FLAG_NO_WAIT) { + Ok(event) => event, + Err(_) => { + std::thread::sleep(Duration::from_millis(1)); + continue; + } + }; + let event_type = match event.GetType() { + Ok(event_type) => event_type, + Err(_) => break, + }; + + match MF_EVENT_TYPE(event_type as i32) { + MediaFoundation::METransformHaveOutput => { + let mut status = 0; + let output_buffer = MFT_OUTPUT_DATA_BUFFER { + dwStreamID: self.output_stream_id, + ..Default::default() + }; + let mut output_buffers = [output_buffer]; + + match self + .transform + .ProcessOutput(0, &mut output_buffers, &mut status) + { + Ok(()) => { + if let Some(sample) = output_buffers[0].pSample.take() { + on_sample(sample)?; + drained += 1; + } + } + Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => break, + Err(error) => { + debug!(%error, "Media Foundation drain output stopped early"); + break; + } + } + } + MediaFoundation::METransformDrainComplete => break, + _ => {} + } + } + if drained > 0 { + debug!(drained, "Drained Media Foundation samples before shutdown"); + } + self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; self.transform diff --git a/crates/mediafoundation-ffmpeg/src/h264.rs b/crates/mediafoundation-ffmpeg/src/h264.rs index f1077860e9d..199dccbe15b 100644 --- a/crates/mediafoundation-ffmpeg/src/h264.rs +++ b/crates/mediafoundation-ffmpeg/src/h264.rs @@ -27,6 +27,8 @@ impl Default for MuxerConfig { } } +/// Writes Media Foundation H.264 samples into an mp4/mov output. Production callers are the +/// Windows MP4 hardware paths in `recording::output_pipeline::win` (studio and camera). pub struct H264StreamMuxer { stream_index: usize, time_base: ffmpeg::Rational, @@ -99,7 +101,7 @@ impl H264StreamMuxer { return Err("Muxer is already finished".into()); } - let mut packet = self.mf_sample_to_avpacket(sample)?; + let mut packet = sample_to_packet(sample, self.time_base, self.stream_index)?; packet.rescale_ts( self.time_base, @@ -145,44 +147,9 @@ impl H264StreamMuxer { packet.write_interleaved(output)?; - Ok(()) - } - - fn mf_sample_to_avpacket(&self, sample: &IMFSample) -> windows::core::Result { - let len = unsafe { sample.GetTotalLength()? }; - let mut packet = ffmpeg::Packet::new(len as usize); - - { - let buffer = unsafe { sample.ConvertToContiguousBuffer()? }; - let data = buffer.lock()?; - - packet - .data_mut() - .unwrap() - .copy_from_slice(&data[0..len as usize]); - } - - let pts = unsafe { sample.GetSampleTime() } - .ok() - .map(|v| mf_from_mf_time(self.time_base, v)); - packet.set_pts(pts); - packet.set_dts(pts); + self.frame_count += 1; - let duration = unsafe { sample.GetSampleDuration() } - .ok() - .map(|v| mf_from_mf_time(self.time_base, v)) - .unwrap_or_default(); - packet.set_duration(duration); - - if let Ok(t) = unsafe { sample.GetUINT32(&MFSampleExtension_CleanPoint) } - && t != 0 - { - packet.set_flags(packet::Flags::KEY); - } - - packet.set_stream(self.stream_index); - - Ok(packet) + Ok(()) } pub fn finish(&mut self) -> Result<(), ffmpeg::Error> { @@ -206,7 +173,74 @@ impl H264StreamMuxer { } } -const MF_TIMEBASE: ffmpeg::Rational = ffmpeg::Rational(1, 10_000_000); +/// Media Foundation sample times are in 100 ns ticks. +pub const MF_TIMEBASE: ffmpeg::Rational = ffmpeg::Rational(1, 10_000_000); + +pub fn sample_to_packet( + sample: &IMFSample, + time_base: Rational, + stream_index: usize, +) -> windows::core::Result { + let len = unsafe { sample.GetTotalLength()? }; + let mut packet = ffmpeg::Packet::new(len as usize); + + { + let buffer = unsafe { sample.ConvertToContiguousBuffer()? }; + let data = buffer.lock()?; + + packet + .data_mut() + .unwrap() + .copy_from_slice(&data[0..len as usize]); + } + + let pts = unsafe { sample.GetSampleTime() } + .ok() + .map(|v| mf_from_mf_time(time_base, v)); + packet.set_pts(pts); + packet.set_dts(pts); + + let duration = unsafe { sample.GetSampleDuration() } + .ok() + .map(|v| mf_from_mf_time(time_base, v)) + .unwrap_or_default(); + packet.set_duration(duration); + + let clean_point = + unsafe { sample.GetUINT32(&MFSampleExtension_CleanPoint) }.is_ok_and(|value| value != 0); + if clean_point || contains_idr_nal(packet.data().unwrap_or_default()) { + packet.set_flags(packet::Flags::KEY); + } + + packet.set_stream(stream_index); + + Ok(packet) +} + +/// Some Media Foundation encoders leave `MFSampleExtension_CleanPoint` unset. The DASH muxer +/// splits segments on keyframes, so fall back to the bitstream: an IDR NAL (type 5) after an +/// Annex B start code. +fn contains_idr_nal(data: &[u8]) -> bool { + let mut index = 0; + while index + 3 < data.len() { + if data[index] == 0 + && data[index + 1] == 0 + && (data[index + 2] == 1 + || (data[index + 2] == 0 && index + 4 < data.len() && data[index + 3] == 1)) + { + let nal = if data[index + 2] == 1 { + index + 3 + } else { + index + 4 + }; + if data.get(nal).is_some_and(|byte| byte & 0x1F == 5) { + return true; + } + } + index += 1; + } + false +} fn mf_from_mf_time(tb: Rational, stime: i64) -> i64 { unsafe { av_rescale_q(stime, MF_TIMEBASE.into(), tb.into()) } @@ -253,3 +287,17 @@ pub fn set_fragmented_mp4_options( Ok(()) } + +#[cfg(test)] +mod tests { + use super::contains_idr_nal; + + #[test] + fn detects_idr_after_three_and_four_byte_start_codes() { + assert!(contains_idr_nal(&[0, 0, 1, 0x65])); + assert!(contains_idr_nal(&[0, 0, 0, 1, 0x65])); + assert!(!contains_idr_nal(&[0, 0, 1, 0x41])); + assert!(!contains_idr_nal(&[0, 0, 1])); + assert!(!contains_idr_nal(&[])); + } +} diff --git a/crates/mediafoundation-ffmpeg/src/lib.rs b/crates/mediafoundation-ffmpeg/src/lib.rs index 1080a2c25c0..ac7ac10b601 100644 --- a/crates/mediafoundation-ffmpeg/src/lib.rs +++ b/crates/mediafoundation-ffmpeg/src/lib.rs @@ -4,4 +4,6 @@ mod audio; mod h264; pub use audio::AudioExt; -pub use h264::{H264StreamMuxer, MuxerConfig, set_fragmented_mp4_options}; +pub use h264::{ + H264StreamMuxer, MF_TIMEBASE, MuxerConfig, sample_to_packet, set_fragmented_mp4_options, +}; diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index d935e0e30f2..7c5da374acd 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -251,6 +251,8 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { .parent() .map(|p| p.join("display")) .unwrap_or_else(|| output_path.with_file_name("display")); + let d3d_device = screen_capture.d3d_device.clone(); + let pixel_format = screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(); let bpp = if ultra { H264EncoderBuilder::ULTRA_BPP @@ -301,7 +303,6 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { bpp, output_size, shared_pause_state, - disk_space_callback: None, segment_tx: None, ..Default::default() }) @@ -318,7 +319,9 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { bpp, output_size, shared_pause_state, - disk_space_callback: None, + d3d_device: Some(d3d_device), + pixel_format: Some(pixel_format), + encoder_preferences: encoder_preferences.clone(), segment_tx: None, }) .await @@ -356,6 +359,7 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { start_gate: Option, segment_tx: Option>, ) -> anyhow::Result { + let d3d_device = screen_capture.d3d_device.clone(); OutputPipeline::builder(segments_dir) .with_video::(screen_capture) .with_timestamps(start_time) @@ -367,7 +371,9 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { bpp: H264EncoderBuilder::INSTANT_MODE_BPP, output_size: Some(output_size), shared_pause_state: None, - disk_space_callback: None, + d3d_device: Some(d3d_device), + pixel_format: Some(screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi()), + encoder_preferences: EncoderPreferences::new(), segment_tx, }) .await diff --git a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs index cd36e3bff3b..a4cb8cf9f23 100644 --- a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs @@ -10,7 +10,7 @@ use crate::{ use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::segmented_stream::{ - DiskSpaceCallback, SegmentCompletedEvent, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, + SegmentCompletedEvent, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, }; use cap_media_info::{AudioInfo, VideoInfo}; use std::{ @@ -165,7 +165,6 @@ pub struct MacOSFragmentedM4SMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, } @@ -177,7 +176,6 @@ pub struct MacOSFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, } @@ -190,7 +188,6 @@ impl Default for MacOSFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, } } @@ -232,7 +229,6 @@ impl Muxer for MacOSFragmentedM4SMuxer { pause, frame_drops: FrameDropTracker::new(SharedHealthSender::new(), "muxer:macos-fragmented"), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), }) @@ -305,9 +301,6 @@ impl MacOSFragmentedM4SMuxer { encoder_config, self.timed_fragments, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } if let Some(tx) = &self.segment_tx { encoder.set_segment_callback(tx.clone()); } @@ -553,7 +546,6 @@ pub struct MacOSFragmentedM4SCameraMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, } @@ -564,7 +556,6 @@ pub struct MacOSFragmentedM4SCameraMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, } @@ -576,7 +567,6 @@ impl Default for MacOSFragmentedM4SCameraMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, } } @@ -621,7 +611,6 @@ impl Muxer for MacOSFragmentedM4SCameraMuxer { "muxer:macos-fragmented-camera", ), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), }) @@ -696,9 +685,6 @@ impl MacOSFragmentedM4SCameraMuxer { encoder_config, true, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } if let Some(tx) = &self.segment_tx { encoder.set_segment_callback(tx.clone()); } diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rs index 25ecb5c52aa..c93c84217e4 100644 --- a/crates/recording/src/output_pipeline/mod.rs +++ b/crates/recording/src/output_pipeline/mod.rs @@ -47,16 +47,6 @@ mod win; #[cfg(windows)] pub use win::*; -#[cfg(windows)] -mod win_segmented; -#[cfg(windows)] -pub use win_segmented::*; - -#[cfg(windows)] -mod win_segmented_camera; -#[cfg(windows)] -pub use win_segmented_camera::*; - #[cfg(windows)] mod win_fragmented_m4s; #[cfg(windows)] diff --git a/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs b/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs index 9afbb643412..0ee7c7991db 100644 --- a/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs @@ -16,7 +16,7 @@ use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::fragment_manifest::FragmentManifestTracker; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::h264_packet::EncodePacketError; -use cap_enc_ffmpeg::segmented_stream::{DiskSpaceCallback, SegmentCompletedEvent}; +use cap_enc_ffmpeg::segmented_stream::SegmentCompletedEvent; use cap_media_info::{AudioInfo, VideoInfo}; use std::{ path::PathBuf, @@ -117,7 +117,6 @@ pub struct OutOfProcessFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, pub max_respawns: u32, } @@ -130,7 +129,6 @@ impl Default for OutOfProcessFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, max_respawns: DEFAULT_MAX_RESPAWNS, } @@ -148,7 +146,6 @@ pub struct OutOfProcessFragmentedM4SMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, max_respawns: u32, @@ -189,7 +186,6 @@ impl Muxer for OutOfProcessFragmentedM4SMuxer { pause, frame_drops: FrameDropTracker::new(SharedHealthSender::new(), "muxer:macos-oop"), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), max_respawns: config.max_respawns, @@ -270,7 +266,6 @@ impl OutOfProcessFragmentedM4SMuxer { let preset = self.preset; let bpp = self.bpp; let output_size = self.output_size; - let disk_space_callback = self.disk_space_callback.clone(); let segment_tx = self.segment_tx.clone(); let health_tx = self.health_tx.clone(); let max_respawns = self.max_respawns; @@ -363,7 +358,6 @@ impl OutOfProcessFragmentedM4SMuxer { }; let mut disk_monitor = DiskSpaceMonitor::new(); - let _ = disk_space_callback; disk_monitor.poll(&base_path, &health_tx); if ready_tx.send(Ok(())).is_err() { diff --git a/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs b/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs index d90b19cf7c9..1867f248607 100644 --- a/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs +++ b/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs @@ -15,7 +15,7 @@ use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::fragment_manifest::FragmentManifestTracker; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::h264_packet::EncodePacketError; -use cap_enc_ffmpeg::segmented_stream::{DiskSpaceCallback, SegmentCompletedEvent}; +use cap_enc_ffmpeg::segmented_stream::SegmentCompletedEvent; use cap_media_info::{AudioInfo, VideoInfo}; use std::{ path::PathBuf, @@ -122,7 +122,6 @@ pub struct WindowsOOPFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, pub max_respawns: u32, } @@ -135,7 +134,6 @@ impl Default for WindowsOOPFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, max_respawns: DEFAULT_MAX_RESPAWNS, } @@ -152,7 +150,6 @@ pub struct WindowsOOPFragmentedM4SMuxer { state: Option, pause: SharedPauseState, frame_drops: FrameDropTracker, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, max_respawns: u32, @@ -193,7 +190,6 @@ impl Muxer for WindowsOOPFragmentedM4SMuxer { state: None, pause, frame_drops: FrameDropTracker::new(SharedHealthSender::new(), "muxer:windows-oop"), - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), max_respawns: config.max_respawns, @@ -264,7 +260,6 @@ impl WindowsOOPFragmentedM4SMuxer { let preset = self.preset; let bpp = self.bpp; let output_size = self.output_size; - let disk_space_callback = self.disk_space_callback.clone(); let segment_tx = self.segment_tx.clone(); let health_tx = self.health_tx.clone(); let max_respawns = self.max_respawns; @@ -357,7 +352,6 @@ impl WindowsOOPFragmentedM4SMuxer { }; let mut disk_monitor = DiskSpaceMonitor::new(); - let _ = disk_space_callback; disk_monitor.poll(&base_path, &health_tx); if ready_tx.send(Ok(())).is_err() { @@ -370,6 +364,7 @@ impl WindowsOOPFragmentedM4SMuxer { let mut last_timestamp: Option = None; let mut first_timestamp: Option = None; let mut total_frames = 0u64; + let mut readback_misses = 0u64; let mut duplicated_frames = 0u64; let mut slow_convert_count = 0u32; let mut slow_encode_count = 0u32; @@ -471,7 +466,24 @@ impl WindowsOOPFragmentedM4SMuxer { (Some(f), ts) } Err(e) => { - warn!("Windows OOP failed to convert D3D11 frame: {e:?}"); + let readback_miss = matches!( + e, + scap_ffmpeg::AsFFmpegError::NotReady + | scap_ffmpeg::AsFFmpegError::StagingBusy + ); + if !readback_miss { + warn!("Windows OOP failed to convert D3D11 frame: {e:?}"); + } else { + readback_misses += 1; + if readback_misses == 1 + || readback_misses.is_multiple_of(90) + { + warn!( + readback_misses, + "Screen readback is missing deadlines; the encoder is duplicating the last frame" + ); + } + } duplicated_frames += 1; (last_ffmpeg_frame.as_ref().map(reference_video_frame), ts) } diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index faab33d6e76..8d2fc226df9 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -418,6 +418,7 @@ impl Muxer for WindowsMuxer { let mut pacer = FramePacer::new(config.frame_rate); let mut last_ffmpeg_frame: Option = None; let mut first_timestamp: Option = None; + let mut readback_misses = 0u64; loop { let (ffmpeg_frame, ts) = match pacer.next(&video_rx, &pause_flag) { @@ -427,7 +428,24 @@ impl Muxer for WindowsMuxer { (Some(f), timestamp) } Err(e) => { - warn!("Failed to convert frame: {e:?}"); + let readback_miss = matches!( + e, + scap_ffmpeg::AsFFmpegError::NotReady + | scap_ffmpeg::AsFFmpegError::StagingBusy + ); + if !readback_miss { + warn!("Failed to convert frame: {e:?}"); + } else { + readback_misses += 1; + if readback_misses == 1 + || readback_misses.is_multiple_of(90) + { + warn!( + readback_misses, + "Screen readback is missing deadlines; the encoder is duplicating the last frame" + ); + } + } ( last_ffmpeg_frame.as_ref().map(reference_video_frame), timestamp, @@ -587,7 +605,7 @@ impl AudioMuxer for WindowsMuxer { } } -fn duration_to_timespan(duration: Duration) -> TimeSpan { +pub(crate) fn duration_to_timespan(duration: Duration) -> TimeSpan { const TICKS_PER_SEC: u64 = 10_000_000; const NANOS_PER_TICK: u32 = 100; diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index b517b575903..4e61d2300ed 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -4,17 +4,19 @@ use super::core::{ }; use super::frame_pacing::{FramePacer, Paced}; use super::mjpeg::MjpegDecoder; -use super::win::reference_video_frame; +use super::win::{duration_to_timespan, reference_video_frame}; use crate::{ AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer, + capture_pipeline::EncoderPreferences, output_pipeline::{NativeCameraFrame, camera_frame_to_ffmpeg}, screen_capture, }; use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::segmented_stream::{ - DiskSpaceCallback, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, + SegmentCompletedEvent, SegmentedPacketMuxer, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, }; +use cap_enc_mediafoundation::H264Encoder; use cap_media_info::{AudioInfo, Pixel, VideoInfo}; use std::{ path::PathBuf, @@ -27,6 +29,10 @@ use std::{ time::Duration, }; use tracing::*; +use windows::{ + Graphics::SizeInt32, + Win32::Graphics::{Direct3D11::ID3D11Device, Dxgi::Common::DXGI_FORMAT}, +}; const DEFAULT_MUXER_BUFFER_SIZE: usize = 240; @@ -132,14 +138,9 @@ impl FinishableEncoderState for EncoderState { } fn finish_encoder(&self, timestamp: Duration, finish_label: &str) -> anyhow::Result<()> { - match self.encoder.lock() { - Ok(mut encoder) => encoder - .finish_with_timestamp(timestamp) - .map_err(|error| anyhow!("{finish_label}: {error:#}")), - Err(_) => Err(anyhow!( - "{finish_label}: encoder mutex poisoned - recording may be corrupt or incomplete" - )), - } + self.backend + .finish_with_timestamp(timestamp) + .map_err(|error| anyhow!("{finish_label}: {error:#}")) } } @@ -166,6 +167,16 @@ impl FinishableEncoderState for CameraEncoderState { } } +fn normalize_timestamp(ts: Duration, first: &mut Option) -> Duration { + match *first { + Some(first_ts) => ts.checked_sub(first_ts).unwrap_or(Duration::ZERO), + None => { + *first = Some(ts); + Duration::ZERO + } + } +} + fn finish_segmented_encoder_impl( state: &mut impl FinishableEncoderState, timestamp: Duration, @@ -196,10 +207,36 @@ fn finish_segmented_encoder_impl( struct EncoderState { video_tx: SyncSender>, - encoder: Arc>, + backend: EncoderBackend, encoder_handle: Option>>, } +enum EncoderBackend { + Ffmpeg(Arc>), + MediaFoundation(Arc>), +} + +impl EncoderBackend { + fn finish_with_timestamp(&self, timestamp: Duration) -> anyhow::Result<()> { + match self { + Self::Ffmpeg(encoder) => match encoder.lock() { + Ok(mut encoder) => encoder + .finish_with_timestamp(timestamp) + .map_err(|error| anyhow!("{error:#}")), + Err(_) => Err(anyhow!( + "encoder mutex poisoned - recording may be corrupt or incomplete" + )), + }, + Self::MediaFoundation(muxer) => match muxer.lock() { + Ok(mut muxer) => muxer.finish().map_err(|error| anyhow!("{error:#}")), + Err(_) => Err(anyhow!( + "muxer mutex poisoned - recording may be corrupt or incomplete" + )), + }, + } + } +} + pub struct WindowsFragmentedM4SMuxer { timed_fragments: bool, base_path: PathBuf, @@ -208,13 +245,14 @@ pub struct WindowsFragmentedM4SMuxer { preset: H264Preset, bpp: f32, output_size: Option<(u32, u32)>, + d3d_device: Option, + pixel_format: Option, + encoder_preferences: EncoderPreferences, state: Option, pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, - segment_tx: - Option>, + segment_tx: Option>, health_tx: SharedHealthSender, } @@ -225,9 +263,10 @@ pub struct WindowsFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, - pub segment_tx: - Option>, + pub d3d_device: Option, + pub pixel_format: Option, + pub encoder_preferences: EncoderPreferences, + pub segment_tx: Option>, } impl Default for WindowsFragmentedM4SMuxerConfig { @@ -239,7 +278,9 @@ impl Default for WindowsFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, + d3d_device: None, + pixel_format: None, + encoder_preferences: EncoderPreferences::new(), segment_tx: None, } } @@ -277,6 +318,9 @@ impl Muxer for WindowsFragmentedM4SMuxer { preset: config.preset, bpp: config.bpp, output_size: config.output_size, + d3d_device: config.d3d_device, + pixel_format: config.pixel_format, + encoder_preferences: config.encoder_preferences, state: None, pause, frame_drops: FrameDropTracker::new( @@ -284,7 +328,6 @@ impl Muxer for WindowsFragmentedM4SMuxer { "muxer:windows-fragmented", ), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), }; @@ -325,6 +368,10 @@ impl Muxer for WindowsFragmentedM4SMuxer { impl WindowsFragmentedM4SMuxer { fn start_encoder(&mut self) -> anyhow::Result<()> { + if self.start_media_foundation_encoder()? { + return Ok(()); + } + let buffer_size = get_muxer_buffer_size(); debug!( buffer_size = buffer_size, @@ -349,9 +396,6 @@ impl WindowsFragmentedM4SMuxer { encoder_config, self.timed_fragments, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } if let Some(tx) = self.segment_tx.take() { encoder.set_segment_callback(tx); } @@ -379,20 +423,11 @@ impl WindowsFragmentedM4SMuxer { let mut slow_encode_count = 0u32; let mut total_frames = 0u64; let mut duplicated_frames = 0u64; + let mut readback_misses = 0u64; let mut disk_monitor = DiskSpaceMonitor::new(); let mut disk_exhausted = false; const SLOW_THRESHOLD_MS: u128 = 5; - let normalize_timestamp = - |ts: Duration, first: &mut Option| -> Duration { - if let Some(first_ts) = *first { - ts.checked_sub(first_ts).unwrap_or(Duration::ZERO) - } else { - *first = Some(ts); - Duration::ZERO - } - }; - let encode_frame_fn = |ffmpeg_frame: ffmpeg::frame::Video, timestamp: Duration, slow_encode_count: &mut u32, @@ -453,7 +488,22 @@ impl WindowsFragmentedM4SMuxer { (Some(f), ts) } Err(e) => { - warn!("Failed to convert D3D11 frame: {e:?}"); + let readback_miss = matches!( + e, + scap_ffmpeg::AsFFmpegError::NotReady + | scap_ffmpeg::AsFFmpegError::StagingBusy + ); + if !readback_miss { + warn!("Failed to convert D3D11 frame: {e:?}"); + } else { + readback_misses += 1; + if readback_misses == 1 || readback_misses.is_multiple_of(90) { + warn!( + readback_misses, + "Screen readback is missing deadlines; the encoder is duplicating the last frame" + ); + } + } duplicated_frames += 1; (last_ffmpeg_frame.as_ref().map(reference_video_frame), ts) } @@ -564,7 +614,7 @@ impl WindowsFragmentedM4SMuxer { self.state = Some(EncoderState { video_tx, - encoder, + backend: EncoderBackend::Ffmpeg(encoder), encoder_handle: Some(encoder_handle), }); @@ -577,6 +627,241 @@ impl WindowsFragmentedM4SMuxer { Ok(()) } + + fn start_media_foundation_encoder(&mut self) -> anyhow::Result { + let (Some(d3d_device), Some(pixel_format)) = (self.d3d_device.clone(), self.pixel_format) + else { + return Ok(false); + }; + + if self.encoder_preferences.should_force_software() + || matches!( + cap_frame_converter::detect_primary_gpu().map(|gpu| gpu.vendor), + Some(cap_frame_converter::GpuVendor::Amd) + ) + { + return Ok(false); + } + + let (video_tx, video_rx) = sync_channel::>( + get_muxer_buffer_size(), + ); + let (ready_tx, ready_rx) = + sync_channel::>, String>>(1); + + let video_config = self.video_config; + let output_size = self.output_size; + let bpp = self.bpp; + let segment_duration = self.segment_duration; + let timed_fragments = self.timed_fragments; + let base_path = self.base_path.clone(); + let pause_flag = self.pause.flag(); + let segment_tx = self.segment_tx.clone(); + let encoder_preferences = self.encoder_preferences.clone(); + let health_tx = self.health_tx.clone(); + + let encoder_handle = std::thread::Builder::new() + .name("win-m4s-mf-encoder".to_string()) + .spawn(move || { + cap_mediafoundation_utils::thread_init(); + + let disk_base_path = base_path.clone(); + let setup = + (|| -> anyhow::Result<(H264Encoder, Arc>)> { + let input_size = SizeInt32 { + Width: video_config.width as i32, + Height: video_config.height as i32, + }; + let (bitrate_width, bitrate_height) = + output_size.unwrap_or((video_config.width, video_config.height)); + let output_size = output_size + .map(|(width, height)| SizeInt32 { + Width: width as i32, + Height: height as i32, + }) + .unwrap_or(input_size); + let frame_rate = video_config.fps(); + let bitrate = cap_enc_ffmpeg::h264::bitrate_for_bpp( + bitrate_width, + bitrate_height, + video_config.frame_rate, + bpp, + ); + + let encoder = H264Encoder::new_with_scaled_output_and_bitrate( + &d3d_device, + pixel_format, + input_size, + output_size, + frame_rate, + bitrate, + )?; + encoder.validate().map_err(|error| anyhow!("{error}"))?; + let extradata = encoder.sequence_header().ok_or_else(|| { + anyhow!("no H.264 sequence header before the first frame") + })?; + let mut muxer = SegmentedPacketMuxer::new( + base_path, + video_config, + (output_size.Width as u32, output_size.Height as u32), + segment_duration, + timed_fragments, + encoder.bitrate(), + &extradata, + )?; + if let Some(tx) = segment_tx { + muxer.set_segment_callback(tx); + } + + Ok((encoder, Arc::new(Mutex::new(muxer)))) + })(); + + let (mut encoder, muxer) = match setup { + Ok(setup) => setup, + Err(error) => { + let _ = ready_tx.send(Err(format!("{error:#}"))); + return Ok(()); + } + }; + + if ready_tx.send(Ok(muxer.clone())).is_err() { + return Ok(()); + } + + let mut pacer = FramePacer::new(video_config.fps()); + let mut last_frame: Option = None; + let mut first_timestamp: Option = None; + let mut failed_writes = 0u64; + let mut disk_monitor = DiskSpaceMonitor::new(); + + let result = encoder.run( + Arc::new(AtomicBool::default()), + || loop { + if matches!( + disk_monitor.poll(&disk_base_path, &health_tx), + super::core::DiskSpacePollResult::Exhausted { .. } + | super::core::DiskSpacePollResult::Stopped + ) { + return Ok(None); + } + + let (texture, timestamp) = match pacer.next(&video_rx, &pause_flag) { + Paced::Frame(frame, timestamp) => { + let texture = frame.texture().clone(); + // Keep the frame itself, not just its texture: the capture frame + // owns the surface, and dropping it lets WGC recycle the buffer + // while a repeat is still fed to the encoder. + last_frame = Some(frame); + (texture, timestamp) + } + Paced::Repeat(timestamp) => { + let Some(frame) = &last_frame else { + continue; + }; + (frame.texture().clone(), timestamp) + } + Paced::End => return Ok(None), + }; + let normalized = normalize_timestamp(timestamp, &mut first_timestamp); + + return Ok(Some((texture, duration_to_timespan(normalized)))); + }, + |sample| { + let packet = match cap_mediafoundation_ffmpeg::sample_to_packet( + &sample, + cap_mediafoundation_ffmpeg::MF_TIMEBASE, + 0, + ) { + Ok(packet) => packet, + Err(error) => { + warn!("Failed to convert Media Foundation sample: {error}"); + return Ok(()); + } + }; + + let Ok(mut muxer) = muxer.lock() else { + return Ok(()); + }; + if let Err(error) = + muxer.write_packet(packet, cap_mediafoundation_ffmpeg::MF_TIMEBASE) + { + failed_writes += 1; + if failed_writes == 1 || failed_writes.is_multiple_of(300) { + warn!( + failed_writes, + "Failed to write Media Foundation sample: {error}" + ); + } + } + Ok(()) + }, + ); + + match result { + Ok(status) => { + debug!( + frames = status.total_frames_encoded, + "Windows M4S Media Foundation encoder finished" + ); + Ok(()) + } + Err(error) => { + if error.should_fallback() { + encoder_preferences.force_software_only(); + } + error!( + "Media Foundation encoder stopped; the rest of the video stream is dropped: {error}" + ); + health_tx.emit(super::core::PipelineHealthEvent::MuxerCrashed { + reason: format!("Media Foundation encoder stopped: {error}"), + }); + + // Keep the channel connected while the recording runs: a closed channel + // makes send_video_frame fail and aborts the whole mux-video task, losing + // audio and the segments already written. Draining lets the pipeline stop + // normally and finish() finalize what exists. + while let Ok(message) = video_rx.recv() { + if message.is_none() { + break; + } + } + + Ok(()) + } + } + })?; + + match ready_rx.recv() { + Ok(Ok(muxer)) => { + self.state = Some(EncoderState { + video_tx, + backend: EncoderBackend::MediaFoundation(muxer), + encoder_handle: Some(encoder_handle), + }); + self.started = true; + info!( + path = %self.base_path.display(), + "Started Windows M4S fragmented video encoder (Media Foundation, texture input)" + ); + Ok(true) + } + Ok(Err(reason)) => { + warn!( + %reason, + "Media Foundation texture encoding unavailable; falling back to FFmpeg" + ); + let _ = encoder_handle.join(); + Ok(false) + } + Err(_) => { + warn!( + "Media Foundation encoder thread ended before signalling readiness; falling back to FFmpeg" + ); + let _ = encoder_handle.join(); + Ok(false) + } + } + } } impl VideoMuxer for WindowsFragmentedM4SMuxer { @@ -629,7 +914,6 @@ pub struct WindowsFragmentedM4SCameraMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, health_tx: SharedHealthSender, } @@ -639,7 +923,6 @@ pub struct WindowsFragmentedM4SCameraMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, } impl Default for WindowsFragmentedM4SCameraMuxerConfig { @@ -650,7 +933,6 @@ impl Default for WindowsFragmentedM4SCameraMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, } } } @@ -694,7 +976,6 @@ impl Muxer for WindowsFragmentedM4SCameraMuxer { "muxer:windows-fragmented-camera", ), started: false, - disk_space_callback: config.disk_space_callback, health_tx: SharedHealthSender::new(), }; @@ -794,15 +1075,12 @@ impl WindowsFragmentedM4SCameraMuxer { prefer_videotoolbox_hw_input: false, }; - let mut encoder = SegmentedVideoEncoder::init_with_timed_fragments( + let encoder = SegmentedVideoEncoder::init_with_timed_fragments( self.base_path.clone(), self.video_config, encoder_config, true, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } let encoder = Arc::new(Mutex::new(encoder)); let encoder_clone = encoder.clone(); diff --git a/crates/recording/src/output_pipeline/win_segmented.rs b/crates/recording/src/output_pipeline/win_segmented.rs deleted file mode 100644 index 1edaf97daf1..00000000000 --- a/crates/recording/src/output_pipeline/win_segmented.rs +++ /dev/null @@ -1,822 +0,0 @@ -use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; -use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation, screen_capture}; -use anyhow::{Context, anyhow}; -use cap_media_info::{AudioInfo, VideoInfo}; -use serde::Serialize; -use std::{ - path::PathBuf, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc::{SyncSender, sync_channel}, - }, - thread::JoinHandle, - time::Duration, -}; -use tracing::*; -use windows::{ - Foundation::TimeSpan, - Graphics::SizeInt32, - Win32::Graphics::{Direct3D11::ID3D11Device, Dxgi::Common::DXGI_FORMAT}, -}; - -#[derive(Debug, Clone)] -pub struct SegmentInfo { - pub path: PathBuf, - pub index: u32, - pub duration: Duration, - pub file_size: Option, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -struct SegmentState { - video_tx: SyncSender>, - output: Arc>, - encoder_handle: Option>>, -} - -struct PauseTracker { - flag: Arc, - paused_at: Option, - offset: Duration, -} - -struct FrameDropTracker { - count: u32, - last_warning: std::time::Instant, -} - -impl FrameDropTracker { - fn new() -> Self { - Self { - count: 0, - last_warning: std::time::Instant::now(), - } - } - - fn record_drop(&mut self) { - self.count += 1; - if self.count >= 30 && self.last_warning.elapsed() > Duration::from_secs(5) { - warn!( - "Dropped {} screen frames due to encoder backpressure", - self.count - ); - self.count = 0; - self.last_warning = std::time::Instant::now(); - } - } - - fn reset(&mut self) { - if self.count > 0 { - trace!("Frame drop count at segment boundary: {}", self.count); - } - self.count = 0; - } -} - -impl PauseTracker { - fn new(flag: Arc) -> Self { - Self { - flag, - paused_at: None, - offset: Duration::ZERO, - } - } - - fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { - if self.flag.load(Ordering::Acquire) { - if self.paused_at.is_none() { - self.paused_at = Some(timestamp); - } - return Ok(None); - } - - if let Some(start) = self.paused_at.take() { - let delta = match timestamp.checked_sub(start) { - Some(d) => d, - None => { - warn!( - resume_at = ?start, - current = ?timestamp, - "Timestamp anomaly: frame timestamp went backward during unpause (clock skew?), treating as zero delta" - ); - Duration::ZERO - } - }; - - self.offset = match self.offset.checked_add(delta) { - Some(o) => o, - None => { - warn!( - offset = ?self.offset, - delta = ?delta, - "Timestamp anomaly: pause offset overflow, clamping to MAX" - ); - Duration::MAX - } - }; - } - - let adjusted = match timestamp.checked_sub(self.offset) { - Some(t) => t, - None => { - warn!( - timestamp = ?timestamp, - offset = ?self.offset, - "Timestamp anomaly: adjusted timestamp underflow (clock skew?), using zero" - ); - Duration::ZERO - } - }; - - Ok(Some(adjusted)) - } -} - -pub struct WindowsSegmentedMuxer { - base_path: PathBuf, - segment_duration: Duration, - current_index: u32, - segment_start_time: Option, - completed_segments: Vec, - - current_state: Option, - - video_config: VideoInfo, - pixel_format: DXGI_FORMAT, - d3d_device: ID3D11Device, - frame_rate: u32, - bitrate_multiplier: f32, - output_size: Option, - encoder_preferences: crate::capture_pipeline::EncoderPreferences, - - pause: PauseTracker, - frame_drops: FrameDropTracker, -} - -pub struct WindowsSegmentedMuxerConfig { - pub pixel_format: DXGI_FORMAT, - pub d3d_device: ID3D11Device, - pub frame_rate: u32, - pub bitrate_multiplier: f32, - pub output_size: Option, - pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, - pub segment_duration: Duration, -} - -impl Muxer for WindowsSegmentedMuxer { - type Config = WindowsSegmentedMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - _audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result - where - Self: Sized, - { - let video_config = - video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; - - std::fs::create_dir_all(&output_path) - .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; - - Ok(Self { - base_path: output_path, - segment_duration: config.segment_duration, - current_index: 0, - segment_start_time: None, - completed_segments: Vec::new(), - current_state: None, - video_config, - pixel_format: config.pixel_format, - d3d_device: config.d3d_device, - frame_rate: config.frame_rate, - bitrate_multiplier: config.bitrate_multiplier, - output_size: config.output_size, - encoder_preferences: config.encoder_preferences, - pause: PauseTracker::new(pause_flag), - frame_drops: FrameDropTracker::new(), - }) - } - - fn stop(&mut self) { - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.send(None) - { - trace!("Screen encoder channel already closed during stop: {e}"); - } - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let mut finish_error = None; - let mut final_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Screen encoder thread", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - finish_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write screen trailer: {error:#}"); - finish_error = Some(match finish_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&segment_path); - - if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - final_segment = Some(SegmentInfo { - path: segment_path, - index: self.current_index, - duration: final_duration, - file_size, - }); - } - } - - if let Some(error) = finish_error { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - if let Some(segment) = final_segment { - self.completed_segments.push(segment); - } - - if let Err(error) = self.finalize_manifest() { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - Ok(Ok(())) - } -} - -impl WindowsSegmentedMuxer { - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn finalize_manifest(&self) -> anyhow::Result<()> { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - fragmentation::atomic_write_json(&manifest_path, &manifest).map_err(|error| { - anyhow!( - "Failed to write final manifest to {}: {error}", - manifest_path.display() - ) - }) - } - - fn create_segment(&mut self) -> anyhow::Result<()> { - let segment_path = self.current_segment_path(); - let input_size = SizeInt32 { - Width: self.video_config.width as i32, - Height: self.video_config.height as i32, - }; - let output_size = self.output_size.unwrap_or(input_size); - - let queue_depth = ((self.frame_rate as f32 / 30.0 * 5.0).ceil() as usize).clamp(3, 12); - let (video_tx, video_rx) = - sync_channel::>(queue_depth); - let (ready_tx, ready_rx) = sync_channel::>(1); - let output = ffmpeg::format::output(&segment_path)?; - let output = Arc::new(Mutex::new(output)); - - let d3d_device = self.d3d_device.clone(); - let pixel_format = self.pixel_format; - let frame_rate = self.frame_rate; - let bitrate_multiplier = self.bitrate_multiplier; - let video_config = self.video_config; - let encoder_preferences = self.encoder_preferences.clone(); - let output_clone = output.clone(); - - let encoder_handle = std::thread::Builder::new() - .name(format!("segment-encoder-{}", self.current_index)) - .spawn(move || { - cap_mediafoundation_utils::thread_init(); - - let encoder = (|| { - let fallback = |reason: Option| { - encoder_preferences.force_software_only(); - if let Some(reason) = reason.as_ref() { - error!("Falling back to software H264 encoder: {reason}"); - } else { - info!("Falling back to software H264 encoder"); - } - - let fallback_width = if output_size.Width > 0 { - output_size.Width as u32 - } else { - video_config.width - }; - let fallback_height = if output_size.Height > 0 { - output_size.Height as u32 - } else { - video_config.height - }; - - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return Err(anyhow!( - "ScreenSoftwareEncoder: failed to lock output mutex: {}", - poisoned - )); - } - }; - - cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) - .with_output_size(fallback_width, fallback_height) - .and_then(|builder| builder.build(&mut output_guard)) - .map(either::Right) - .map_err(|e| anyhow!("ScreenSoftwareEncoder/{e}")) - }; - - if encoder_preferences.should_force_software() { - return fallback(None); - } - - match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - pixel_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ) { - Ok(encoder) => { - if let Err(e) = encoder.validate() { - return fallback(Some(format!( - "Hardware encoder validation failed: {e}" - ))); - } - - let width = match u32::try_from(output_size.Width) { - Ok(width) if width > 0 => width, - _ => { - return fallback(Some(format!( - "Invalid output width: {}", - output_size.Width - ))); - } - }; - - let height = match u32::try_from(output_size.Height) { - Ok(height) if height > 0 => height, - _ => { - return fallback(Some(format!( - "Invalid output height: {}", - output_size.Height - ))); - } - }; - - let muxer = { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return fallback(Some(format!( - "Failed to lock output mutex: {poisoned}" - ))); - } - }; - - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width, - height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented: false, - frag_duration_us: 0, - }, - ) - }; - - match muxer { - Ok(muxer) => Ok(either::Left((encoder, muxer))), - Err(err) => fallback(Some(err.to_string())), - } - } - Err(err) => fallback(Some(err.to_string())), - } - })(); - - let encoder = match encoder { - Ok(encoder) => { - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } - encoder - } - Err(e) => { - error!("Encoder setup failed: {:#}", e); - let _ = ready_tx.send(Err(anyhow!("{e}"))); - return Err(anyhow!("{e}")); - } - }; - - match encoder { - either::Left((mut encoder, mut muxer)) => { - trace!("Running native encoder for segment"); - let mut first_timestamp: Option = None; - let result = encoder.run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more frames available for segment"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - let frame_time = duration_to_timespan(relative); - - Ok(Some((frame.texture().clone(), frame_time))) - }, - |output_sample| { - let mut output = match output_clone.lock() { - Ok(guard) => guard, - Err(e) => { - error!("Failed to lock output mutex: {e}"); - return Err(windows::core::Error::new( - windows::core::HRESULT(0x80004005u32 as i32), - format!("Mutex poisoned: {e}"), - )); - } - }; - - if let Err(e) = muxer.write_sample(&output_sample, &mut output) { - warn!("WriteSample failed: {e}"); - } - - Ok(()) - }, - ); - - match result { - Ok(health_status) => { - debug!( - "Hardware encoder completed for segment: {} frames encoded", - health_status.total_frames_encoded - ); - Ok(()) - } - Err(e) => { - if e.should_fallback() { - error!( - "Hardware encoder failed with recoverable error, marking for software fallback: {}", - e - ); - encoder_preferences.force_software_only(); - } - Err(anyhow!("Hardware encoder error: {}", e)) - } - } - } - either::Right(mut encoder) => { - while let Ok(Some((frame, time))) = video_rx.recv() { - let Ok(mut output) = output_clone.lock() else { - continue; - }; - - frame - .as_ffmpeg() - .context("frame as_ffmpeg") - .and_then(|frame| { - encoder - .queue_frame(frame, time, &mut output) - .context("queue_frame") - })?; - } - - Ok(()) - } - } - })?; - - ready_rx - .recv() - .map_err(|_| anyhow!("Encoder thread ended unexpectedly"))??; - - output.lock().unwrap().write_header()?; - - self.current_state = Some(SegmentState { - video_tx, - output, - encoder_handle: Some(encoder_handle), - }); - - Ok(()) - } - - fn rotate_segment(&mut self, timestamp: Duration) -> anyhow::Result<()> { - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let mut rotation_error = None; - let mut rotated_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during rotation: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Screen encoder thread during rotation", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - rotation_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Err(error); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write rotated screen trailer: {error:#}"); - rotation_error = Some(match rotation_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&completed_segment_path); - - let file_size = std::fs::metadata(&completed_segment_path) - .ok() - .map(|m| m.len()); - - rotated_segment = Some(SegmentInfo { - path: completed_segment_path, - index: self.current_index, - duration: segment_duration, - file_size, - }); - } - - if let Some(error) = rotation_error { - self.write_in_progress_manifest(); - return Err(error); - } - - if let Some(segment) = rotated_segment { - self.completed_segments.push(segment); - } - - self.write_manifest(); - self.frame_drops.reset(); - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - self.create_segment()?; - self.write_in_progress_manifest(); - - info!( - "Rotated to segment {} at {:?}", - self.current_index, timestamp - ); - - Ok(()) - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } -} - -impl VideoMuxer for WindowsSegmentedMuxer { - type VideoFrame = screen_capture::VideoFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { - return Ok(()); - }; - - if self.current_state.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - self.create_segment()?; - self.write_in_progress_manifest(); - } - - if self.segment_start_time.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - } - - let segment_elapsed = - adjusted_timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(adjusted_timestamp)?; - } - - if let Some(state) = &self.current_state - && let Err(e) = state - .video_tx - .try_send(Some((frame.frame, adjusted_timestamp))) - { - match e { - std::sync::mpsc::TrySendError::Full(_) => { - self.frame_drops.record_drop(); - } - std::sync::mpsc::TrySendError::Disconnected(_) => { - return Err(anyhow!("Screen encoder channel disconnected")); - } - } - } - - Ok(()) - } -} - -impl AudioMuxer for WindowsSegmentedMuxer { - fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { - Ok(()) - } -} - -fn duration_to_timespan(duration: Duration) -> TimeSpan { - const TICKS_PER_SEC: u64 = 10_000_000; - const NANOS_PER_TICK: u32 = 100; - - let secs_ticks = duration.as_secs().saturating_mul(TICKS_PER_SEC); - let nanos_ticks = (duration.subsec_nanos() / NANOS_PER_TICK) as u64; - let total_ticks = secs_ticks.saturating_add(nanos_ticks); - let clamped = total_ticks.min(i64::MAX as u64); - - TimeSpan { - Duration: clamped as i64, - } -} diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs deleted file mode 100644 index c377d33888a..00000000000 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ /dev/null @@ -1,849 +0,0 @@ -use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; -use crate::output_pipeline::win::{CameraBuffers, NativeCameraFrame, upload_mf_buffer_to_texture}; -use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation}; -use anyhow::{Context, anyhow}; -use cap_media_info::{AudioInfo, VideoInfo}; -use serde::Serialize; -use std::{ - path::PathBuf, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc::{SyncSender, sync_channel}, - }, - thread::JoinHandle, - time::Duration, -}; -use tracing::*; -use windows::{Foundation::TimeSpan, Graphics::SizeInt32}; - -#[derive(Debug, Clone)] -struct SegmentInfo { - path: PathBuf, - index: u32, - duration: Duration, - file_size: Option, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -struct SegmentState { - video_tx: SyncSender>, - output: Arc>, - encoder_handle: Option>>, -} - -struct PauseTracker { - flag: Arc, - paused_at: Option, - offset: Duration, -} - -struct FrameDropTracker { - count: u32, - last_warning: std::time::Instant, -} - -impl FrameDropTracker { - fn new() -> Self { - Self { - count: 0, - last_warning: std::time::Instant::now(), - } - } - - fn record_drop(&mut self) { - self.count += 1; - if self.count >= 30 && self.last_warning.elapsed() > Duration::from_secs(5) { - warn!( - "Dropped {} camera frames due to encoder backpressure", - self.count - ); - self.count = 0; - self.last_warning = std::time::Instant::now(); - } - } - - fn reset(&mut self) { - if self.count > 0 { - trace!( - "Camera frame drop count at segment boundary: {}", - self.count - ); - } - self.count = 0; - } -} - -impl PauseTracker { - fn new(flag: Arc) -> Self { - Self { - flag, - paused_at: None, - offset: Duration::ZERO, - } - } - - fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { - if self.flag.load(Ordering::Acquire) { - if self.paused_at.is_none() { - self.paused_at = Some(timestamp); - } - return Ok(None); - } - - if let Some(start) = self.paused_at.take() { - let delta = match timestamp.checked_sub(start) { - Some(d) => d, - None => { - warn!( - resume_at = ?start, - current = ?timestamp, - "Timestamp anomaly: frame timestamp went backward during unpause (clock skew?), treating as zero delta" - ); - Duration::ZERO - } - }; - - self.offset = match self.offset.checked_add(delta) { - Some(o) => o, - None => { - warn!( - offset = ?self.offset, - delta = ?delta, - "Timestamp anomaly: pause offset overflow, clamping to MAX" - ); - Duration::MAX - } - }; - } - - let adjusted = match timestamp.checked_sub(self.offset) { - Some(t) => t, - None => { - warn!( - timestamp = ?timestamp, - offset = ?self.offset, - "Timestamp anomaly: adjusted timestamp underflow (clock skew?), using zero" - ); - Duration::ZERO - } - }; - - Ok(Some(adjusted)) - } -} - -pub struct WindowsSegmentedCameraMuxer { - base_path: PathBuf, - segment_duration: Duration, - current_index: u32, - segment_start_time: Option, - completed_segments: Vec, - - current_state: Option, - - video_config: VideoInfo, - output_height: Option, - encoder_preferences: crate::capture_pipeline::EncoderPreferences, - - pause: PauseTracker, - frame_drops: FrameDropTracker, -} - -pub struct WindowsSegmentedCameraMuxerConfig { - pub output_height: Option, - pub segment_duration: Duration, - pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, -} - -impl Default for WindowsSegmentedCameraMuxerConfig { - fn default() -> Self { - Self { - output_height: None, - segment_duration: Duration::from_secs(3), - encoder_preferences: crate::capture_pipeline::EncoderPreferences::new(), - } - } -} - -impl Muxer for WindowsSegmentedCameraMuxer { - type Config = WindowsSegmentedCameraMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - _audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result - where - Self: Sized, - { - let video_config = - video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; - - std::fs::create_dir_all(&output_path) - .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; - - Ok(Self { - base_path: output_path, - segment_duration: config.segment_duration, - current_index: 0, - segment_start_time: None, - completed_segments: Vec::new(), - current_state: None, - video_config, - output_height: config.output_height, - encoder_preferences: config.encoder_preferences, - pause: PauseTracker::new(pause_flag), - frame_drops: FrameDropTracker::new(), - }) - } - - fn stop(&mut self) { - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.send(None) - { - trace!("Camera encoder channel already closed during stop: {e}"); - } - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let mut finish_error = None; - let mut final_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Camera encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Camera encoder thread", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - finish_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write camera trailer: {error:#}"); - finish_error = Some(match finish_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&segment_path); - - if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - final_segment = Some(SegmentInfo { - path: segment_path, - index: self.current_index, - duration: final_duration, - file_size, - }); - } - } - - if let Some(error) = finish_error { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - if let Some(segment) = final_segment { - self.completed_segments.push(segment); - } - - if let Err(error) = self.finalize_manifest() { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - Ok(Ok(())) - } -} - -impl WindowsSegmentedCameraMuxer { - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn finalize_manifest(&self) -> anyhow::Result<()> { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - fragmentation::atomic_write_json(&manifest_path, &manifest).map_err(|error| { - anyhow!( - "Failed to write final manifest to {}: {error}", - manifest_path.display() - ) - }) - } - - fn create_segment(&mut self, first_frame: &NativeCameraFrame) -> anyhow::Result<()> { - use crate::output_pipeline::{mjpeg::MjpegDecoder, win::camera_frame_to_ffmpeg}; - - let segment_path = self.current_segment_path(); - - let input_size = SizeInt32 { - Width: self.video_config.width as i32, - Height: self.video_config.height as i32, - }; - - let output_height = self.output_height.unwrap_or(self.video_config.height); - let output_width = (self.video_config.width * output_height) / self.video_config.height; - let output_width = output_width & !1; - let output_height = output_height & !1; - - let output_size = SizeInt32 { - Width: output_width as i32, - Height: output_height as i32, - }; - - let frame_rate = self.video_config.fps(); - let bitrate_multiplier = 0.2f32; - let input_format = first_frame.dxgi_format(); - let video_config = self.video_config; - let encoder_preferences = self.encoder_preferences.clone(); - - let (video_tx, video_rx) = sync_channel::>(30); - let (ready_tx, ready_rx) = sync_channel::>(1); - let output = ffmpeg::format::output(&segment_path)?; - let output = Arc::new(Mutex::new(output)); - let output_clone = output.clone(); - - let encoder_handle = std::thread::Builder::new() - .name(format!("camera-segment-encoder-{}", self.current_index)) - .spawn(move || { - cap_mediafoundation_utils::thread_init(); - - let d3d_device = match crate::capture_pipeline::create_d3d_device() { - Ok(device) => device, - Err(e) => { - let _ = ready_tx.send(Err(anyhow!("Failed to create D3D device: {e}"))); - return Err(anyhow!("Failed to create D3D device: {e}")); - } - }; - - let encoder = (|| { - let fallback = |reason: Option| { - encoder_preferences.force_software_only(); - if let Some(reason) = reason.as_ref() { - error!( - "Falling back to software H264 encoder for camera segment: {reason}" - ); - } else { - info!("Using software H264 encoder for camera segment"); - } - - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return Err(anyhow!( - "CameraSegmentSoftwareEncoder: failed to lock output mutex: {}", - poisoned - )); - } - }; - - cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) - .with_output_size(output_width, output_height) - .and_then(|builder| builder.build(&mut output_guard)) - .map(either::Right) - .map_err(|e| anyhow!("CameraSegmentSoftwareEncoder/{e}")) - }; - - if encoder_preferences.should_force_software() { - return fallback(None); - } - - match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - input_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ) { - Ok(encoder) => { - if let Err(e) = encoder.validate() { - return fallback(Some(format!( - "Camera hardware encoder validation failed: {e}" - ))); - } - - let muxer = { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return fallback(Some(format!( - "Failed to lock output mutex: {poisoned}" - ))); - } - }; - - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: output_width, - height: output_height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented: false, - frag_duration_us: 0, - }, - ) - }; - - match muxer { - Ok(muxer) => Ok(either::Left((encoder, muxer))), - Err(err) => fallback(Some(err.to_string())), - } - } - Err(err) => fallback(Some(err.to_string())), - } - })(); - - let encoder = match encoder { - Ok(encoder) => { - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } - encoder - } - Err(e) => { - error!("Camera segment encoder setup failed: {:#}", e); - let _ = ready_tx.send(Err(anyhow!("{e}"))); - return Err(anyhow!("{e}")); - } - }; - - match encoder { - either::Left((mut encoder, mut muxer)) => { - info!( - "Camera segment encoder started (hardware): {:?} {}x{} -> NV12 {}x{} @ {}fps", - input_format, - input_size.Width, - input_size.Height, - output_size.Width, - output_size.Height, - frame_rate - ); - - let mut first_timestamp: Option = None; - let mut camera_buffers = CameraBuffers::new(); - - let result = encoder.run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more camera frames available for segment"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - - let texture = upload_mf_buffer_to_texture(&d3d_device, &frame, &mut camera_buffers)?; - Ok(Some((texture, duration_to_timespan(relative)))) - }, - |output_sample| { - let mut output = output_clone.lock().map_err(|e| { - windows::core::Error::new( - windows::core::HRESULT(-1), - format!("Mutex poisoned: {e}"), - ) - })?; - muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| { - windows::core::Error::new( - windows::core::HRESULT(-1), - format!("WriteSample: {e}"), - ) - }) - }, - ); - - match result { - Ok(health_status) => { - info!( - "Camera segment encoder finished (hardware): {} frames encoded", - health_status.total_frames_encoded - ); - Ok(()) - } - Err(e) => { - if e.should_fallback() { - error!( - "Camera hardware encoder failed with recoverable error, marking for software fallback: {}", - e - ); - encoder_preferences.force_software_only(); - } - Err(anyhow!("Camera hardware encoder error: {}", e)) - } - } - } - either::Right(mut encoder) => { - info!( - "Camera segment encoder started (software): {}x{} -> {}x{} @ {}fps", - video_config.width, - video_config.height, - output_width, - output_height, - frame_rate - ); - - let mut first_timestamp: Option = None; - let mut mjpeg_decoder = MjpegDecoder::default(); - - while let Ok(Some((frame, timestamp))) = video_rx.recv() { - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - - let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame, &mut mjpeg_decoder) { - Ok(f) => f, - Err(e) => { - warn!("Failed to convert camera frame: {e}"); - continue; - } - }; - - let Ok(mut output_guard) = output_clone.lock() else { - continue; - }; - - if let Err(e) = - encoder.queue_frame(ffmpeg_frame, relative, &mut output_guard) - { - warn!("Failed to queue camera frame: {e}"); - } - } - - if let Ok(mut output_guard) = output_clone.lock() - && let Err(e) = encoder.flush(&mut output_guard) - { - warn!("Failed to flush software encoder: {e}"); - } - - Ok(()) - } - } - })?; - - ready_rx - .recv() - .map_err(|_| anyhow!("Camera encoder thread ended unexpectedly"))??; - - output.lock().unwrap().write_header()?; - - self.current_state = Some(SegmentState { - video_tx, - output, - encoder_handle: Some(encoder_handle), - }); - - Ok(()) - } - - fn rotate_segment( - &mut self, - timestamp: Duration, - next_frame: &NativeCameraFrame, - ) -> anyhow::Result<()> { - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let mut rotation_error = None; - let mut rotated_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Camera encoder channel already closed during rotation: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Camera encoder thread during rotation", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - rotation_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Err(error); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write rotated camera trailer: {error:#}"); - rotation_error = Some(match rotation_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&completed_segment_path); - - let file_size = std::fs::metadata(&completed_segment_path) - .ok() - .map(|m| m.len()); - - rotated_segment = Some(SegmentInfo { - path: completed_segment_path, - index: self.current_index, - duration: segment_duration, - file_size, - }); - } - - if let Some(error) = rotation_error { - self.write_in_progress_manifest(); - return Err(error); - } - - if let Some(segment) = rotated_segment { - self.completed_segments.push(segment); - } - - self.write_manifest(); - self.frame_drops.reset(); - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - self.create_segment(next_frame)?; - self.write_in_progress_manifest(); - - info!( - "Camera rotated to segment {} at {:?}", - self.current_index, timestamp - ); - - Ok(()) - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } -} - -impl VideoMuxer for WindowsSegmentedCameraMuxer { - type VideoFrame = NativeCameraFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { - return Ok(()); - }; - - if self.current_state.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - self.create_segment(&frame)?; - self.write_in_progress_manifest(); - } - - if self.segment_start_time.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - } - - let segment_elapsed = - adjusted_timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(adjusted_timestamp, &frame)?; - } - - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.try_send(Some((frame, adjusted_timestamp))) - { - match e { - std::sync::mpsc::TrySendError::Full(_) => { - self.frame_drops.record_drop(); - } - std::sync::mpsc::TrySendError::Disconnected(_) => { - return Err(anyhow!("Camera encoder channel disconnected")); - } - } - } - - Ok(()) - } -} - -impl AudioMuxer for WindowsSegmentedCameraMuxer { - fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { - Ok(()) - } -} - -fn duration_to_timespan(duration: Duration) -> TimeSpan { - const TICKS_PER_SEC: u64 = 10_000_000; - const NANOS_PER_TICK: u32 = 100; - - let secs_ticks = duration.as_secs().saturating_mul(TICKS_PER_SEC); - let nanos_ticks = (duration.subsec_nanos() / NANOS_PER_TICK) as u64; - let total_ticks = secs_ticks.saturating_add(nanos_ticks); - let clamped = total_ticks.min(i64::MAX as u64); - - TimeSpan { - Duration: clamped as i64, - } -} diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 36d5d448386..2148b35253e 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -130,7 +130,7 @@ impl ScreenFrame { } } - pub fn as_ffmpeg(&self) -> Result { + pub fn as_ffmpeg(&self) -> Result { match self { ScreenFrame::Captured(frame) => { use scap_ffmpeg::AsFFmpeg; diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index dbe42566731..c2753e339f9 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -7,7 +7,7 @@ pub use windows_version::WindowsVersion; use std::{ sync::{ Arc, Condvar, Mutex, - atomic::{AtomicBool, AtomicUsize, Ordering}, + atomic::{AtomicBool, AtomicU32, Ordering}, mpsc::RecvError, }, time::{Duration, Instant}, @@ -27,7 +27,8 @@ use windows::{ Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, - D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, D3D11_MAP_READ, + D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_FLAG, + D3D11_MAP_FLAG_DO_NOT_WAIT, D3D11_MAP_READ, D3D11_MAP_WRITE, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Multithread, ID3D11Texture2D, @@ -37,7 +38,7 @@ use windows::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC, }, - DXGI_ERROR_UNSUPPORTED, IDXGIDevice, + DXGI_ERROR_UNSUPPORTED, DXGI_ERROR_WAS_STILL_DRAWING, IDXGIDevice, }, }, System::WinRT::Direct3D11::{ @@ -72,6 +73,11 @@ impl PixelFormat { } const STAGING_POOL_SIZE: usize = 3; +const STAGING_POOL_LIMIT: usize = STAGING_POOL_SIZE + 1; +const STAGING_PADDING_ROWS: u32 = 1; +const READBACK_RETRY_INTERVAL: Duration = Duration::from_micros(250); +const READBACK_MISS_FALLBACK: u32 = 2; +const READBACK_BLOCKING_COOLDOWN_FRAMES: u32 = 150; struct CallbackActivity { active: Mutex, @@ -152,87 +158,287 @@ impl Drop for CallbackActivityGuard { } } +#[derive(Debug, thiserror::Error)] +pub enum ReadbackError { + #[error("D3D11: {0}")] + D3D11(#[from] windows::core::Error), + #[error("every staging texture is in flight")] + StagingBusy, + #[error("the GPU did not finish the readback before the deadline")] + NotReady, + #[error("failed to allocate the frame buffer reference")] + FrameBufferAllocation, +} + struct PooledStagingTexture { texture: ID3D11Texture2D, width: u32, height: u32, } +struct StagingPoolInner { + free: Vec, + live: usize, +} + pub struct StagingTexturePool { - textures: Mutex>, + inner: Mutex, d3d_device: ID3D11Device, + d3d_context: ID3D11DeviceContext, pixel_format: PixelFormat, - next_index: AtomicUsize, + readback_misses: AtomicU32, + blocking_frames: AtomicU32, } impl StagingTexturePool { - fn new(d3d_device: ID3D11Device, pixel_format: PixelFormat) -> Self { + fn new( + d3d_device: ID3D11Device, + d3d_context: ID3D11DeviceContext, + pixel_format: PixelFormat, + ) -> Self { Self { - textures: Mutex::new(Vec::with_capacity(STAGING_POOL_SIZE)), + inner: Mutex::new(StagingPoolInner { + free: Vec::with_capacity(STAGING_POOL_SIZE), + live: 0, + }), d3d_device, + d3d_context, pixel_format, - next_index: AtomicUsize::new(0), + readback_misses: AtomicU32::new(0), + blocking_frames: AtomicU32::new(0), } } - fn get_or_create_texture( - &self, - width: u32, - height: u32, - ) -> windows::core::Result { - let mut textures = self - .textures + fn take_blocking_readback(&self) -> bool { + self.blocking_frames + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |remaining| { + remaining.checked_sub(1) + }) + .is_ok() + } + + fn reset_readback_misses(&self) { + self.readback_misses.store(0, Ordering::Relaxed); + } + + fn note_readback_miss(&self) { + let misses = self.readback_misses.fetch_add(1, Ordering::Relaxed) + 1; + if misses == READBACK_MISS_FALLBACK { + self.readback_misses.store(0, Ordering::Relaxed); + self.blocking_frames + .store(READBACK_BLOCKING_COOLDOWN_FRAMES, Ordering::Relaxed); + tracing::warn!( + cooldown_frames = READBACK_BLOCKING_COOLDOWN_FRAMES, + "GPU readback missed its deadline twice in a row; blocking for a cooldown instead of duplicating frames" + ); + } + } + + fn acquire(self: &Arc, width: u32, height: u32) -> Result { + let mut inner = self + .inner .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let index = self.next_index.fetch_add(1, Ordering::Relaxed) % STAGING_POOL_SIZE; - - if let Some(pooled) = textures.get(index) - && pooled.width == width - && pooled.height == height + if let Some(index) = inner + .free + .iter() + .position(|pooled| pooled.width == width && pooled.height == height) { - return Ok(pooled.texture.clone()); - } - - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: width, - Height: height, - MipLevels: 1, - ArraySize: 1, - Format: self.pixel_format.as_dxgi(), - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_STAGING, - BindFlags: 0, - CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, - MiscFlags: 0, - }; + let pooled = inner.free.swap_remove(index); + return Ok(StagingLease { + pool: self.clone(), + texture: pooled.texture, + width, + height, + }); + } - let mut texture = None; - unsafe { - self.d3d_device - .CreateTexture2D(&texture_desc, None, Some(&mut texture))?; - }; + if inner.live >= STAGING_POOL_LIMIT { + return Err(ReadbackError::StagingBusy); + } - let texture = texture.ok_or_else(windows::core::Error::from_win32)?; + let texture = create_staging_texture( + &self.d3d_device, + &self.d3d_context, + self.pixel_format, + width, + height, + )?; + inner.live += 1; + + Ok(StagingLease { + pool: self.clone(), + texture, + width, + height, + }) + } + + fn recycle(&self, texture: &ID3D11Texture2D, width: u32, height: u32) { + let mut inner = self + .inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); - if index < textures.len() { - textures[index] = PooledStagingTexture { + if inner.free.len() < STAGING_POOL_SIZE { + inner.free.push(PooledStagingTexture { texture: texture.clone(), width, height, - }; - } else { - textures.push(PooledStagingTexture { + }); + } else if let Some(slot) = inner + .free + .iter_mut() + .find(|pooled| pooled.width != width || pooled.height != height) + { + // Capture size changed: keep the cache on the current geometry instead of allocating a + // fresh texture per frame while the stale sizes sit unused. + *slot = PooledStagingTexture { texture: texture.clone(), width, height, - }); + }; + // The replaced texture is dropped; the recycled one was an outstanding lease, so the + // pool now owns one fewer texture. + inner.live -= 1; + } else { + inner.live -= 1; + } + } +} + +struct StagingLease { + pool: Arc, + texture: ID3D11Texture2D, + width: u32, + height: u32, +} + +impl Drop for StagingLease { + fn drop(&mut self) { + self.pool.recycle(&self.texture, self.width, self.height); + } +} + +fn create_staging_texture( + device: &ID3D11Device, + context: &ID3D11DeviceContext, + pixel_format: PixelFormat, + width: u32, + height: u32, +) -> Result { + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height + STAGING_PADDING_ROWS, + MipLevels: 1, + ArraySize: 1, + Format: pixel_format.as_dxgi(), + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32 | D3D11_CPU_ACCESS_WRITE.0 as u32, + MiscFlags: 0, + }; + + let mut texture = None; + unsafe { + device.CreateTexture2D(&texture_desc, None, Some(&mut texture))?; + }; + let texture = texture.ok_or_else(windows::core::Error::from_win32)?; + zero_padding_row(context, &texture, height)?; + + Ok(texture) +} + +// ffmpeg may read up to AV_INPUT_BUFFER_PADDING_SIZE bytes past the last pixel and requires them +// to be zero. The staging texture carries one extra row for that, zeroed once because copies only +// ever touch the image rows. +fn zero_padding_row( + context: &ID3D11DeviceContext, + texture: &ID3D11Texture2D, + image_height: u32, +) -> Result<(), ReadbackError> { + let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { + context.Map(texture, 0, D3D11_MAP_WRITE, 0, Some(&mut mapped_resource))?; + + let padding = (mapped_resource.pData as *mut u8) + .add(mapped_resource.RowPitch as usize * image_height as usize); + std::ptr::write_bytes(padding, 0, mapped_resource.RowPitch as usize); + + context.Unmap(texture, 0); + }; + + Ok(()) +} + +pub struct MappedFrame { + lease: StagingLease, + d3d_context: ID3D11DeviceContext, + data: *mut u8, + row_pitch: u32, + width: u32, + height: u32, + pixel_format: PixelFormat, +} + +// SAFETY: the mapped staging memory belongs to the GPU resource, not to a thread; the only thread +// that touches the mapping owns this value, and the driver serializes the context (the capturer +// enables multithread protection). The AVBuffer built from it may be released on another thread, +// which only runs `Unmap` and returns the texture to the pool. +unsafe impl Send for MappedFrame {} + +impl std::fmt::Debug for MappedFrame { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MappedFrame") + .field("width", &self.width) + .field("height", &self.height) + .field("row_pitch", &self.row_pitch) + .finish() + } +} + +impl Drop for MappedFrame { + fn drop(&mut self) { + unsafe { + self.d3d_context.Unmap(&self.lease.texture, 0); } + } +} - Ok(texture) +impl MappedFrame { + pub fn width(&self) -> u32 { + self.width + } + + pub fn height(&self) -> u32 { + self.height + } + + pub fn row_pitch(&self) -> u32 { + self.row_pitch + } + + pub fn pixel_format(&self) -> PixelFormat { + self.pixel_format + } + + pub fn data_ptr(&self) -> *mut u8 { + self.data + } + + pub fn data_len(&self) -> usize { + self.row_pitch as usize * (self.height as usize + STAGING_PADDING_ROWS as usize) + } + + pub fn as_slice(&self) -> &[u8] { + unsafe { + std::slice::from_raw_parts(self.data, self.height as usize * self.row_pitch as usize) + } } } @@ -486,6 +692,7 @@ impl Capturer { let staging_pool = Arc::new(StagingTexturePool::new( d3d_device.clone(), + d3d_context.clone(), settings.pixel_format, )); @@ -768,82 +975,202 @@ impl Frame { &self.d3d_context } - pub fn as_buffer(&self) -> windows::core::Result> { - let staging_texture = self - .staging_pool - .get_or_create_texture(self.width, self.height)?; + /// Copies the frame into a staging texture and maps it, blocking until the GPU copy is done. + /// Use [`Self::map_readback`] on latency-sensitive paths. + pub fn as_buffer(&self) -> Result, ReadbackError> { + let lease = self.staging_pool.acquire(self.width, self.height)?; + self.copy_into_staging(&lease); + let mapped = self.map_staging(lease, None)?; - unsafe { - self.d3d_context - .CopyResource(&staging_texture, &self.texture); + Ok(FrameBuffer { + mapped, + frame: std::marker::PhantomData, + }) + } + + /// Copies the frame into a staging texture and maps it without waiting for the GPU copy + /// (`D3D11_MAP_FLAG_DO_NOT_WAIT`). Returns [`ReadbackError::NotReady`] when the copy has not + /// completed within `budget`, so the caller can reuse the previous frame instead of stalling + /// its thread behind the GPU. + pub fn map_readback(&self, budget: Duration) -> Result { + let lease = self.staging_pool.acquire(self.width, self.height)?; + self.copy_into_staging(&lease); + + // A GPU that cannot meet the deadline consistently would otherwise freeze the recording on + // repeated duplicates. After two consecutive misses, block for a cooldown so the encoder + // keeps receiving fresh frames at the slower readback rate, then probe the deadline again. + if self.staging_pool.take_blocking_readback() { + let mapped = self.map_staging(lease, None)?; + self.staging_pool.reset_readback_misses(); + return Ok(mapped); + } + + let deadline = Instant::now().checked_add(budget); + match self.map_staging(lease, deadline) { + Ok(mapped) => { + self.staging_pool.reset_readback_misses(); + Ok(mapped) + } + Err(ReadbackError::NotReady) => { + self.staging_pool.note_readback_miss(); + Err(ReadbackError::NotReady) + } + Err(error) => Err(error), + } + } + + fn copy_into_staging(&self, lease: &StagingLease) { + let source_box = D3D11_BOX { + left: 0, + top: 0, + front: 0, + right: self.width, + bottom: self.height, + back: 1, }; - let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); unsafe { - self.d3d_context.Map( - &staging_texture, + self.d3d_context.CopySubresourceRegion( + &lease.texture, 0, - D3D11_MAP_READ, 0, - Some(&mut mapped_resource), - )?; - }; + 0, + 0, + &self.texture, + 0, + Some(&source_box), + ); + self.d3d_context.Flush(); + } + } - let data = unsafe { - std::slice::from_raw_parts( - mapped_resource.pData.cast(), - (self.height * mapped_resource.RowPitch) as usize, - ) - }; + fn map_staging( + &self, + lease: StagingLease, + deadline: Option, + ) -> Result { + let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); - Ok(FrameBuffer { - data, + if let Some(deadline) = deadline { + loop { + match unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + D3D11_MAP_FLAG_DO_NOT_WAIT.0 as u32, + Some(&mut mapped_resource), + ) + } { + Ok(()) => break, + Err(error) if error.code() == DXGI_ERROR_WAS_STILL_DRAWING => { + let now = Instant::now(); + if now >= deadline { + return Err(ReadbackError::NotReady); + } + std::thread::sleep( + READBACK_RETRY_INTERVAL.min(deadline.saturating_duration_since(now)), + ); + } + Err(error) => return Err(error.into()), + } + } + } else { + unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + 0, + Some(&mut mapped_resource), + )?; + } + } + + Ok(MappedFrame { + lease, + d3d_context: self.d3d_context.clone(), + data: mapped_resource.pData as *mut u8, + row_pitch: mapped_resource.RowPitch, width: self.width, height: self.height, - stride: mapped_resource.RowPitch, pixel_format: self.pixel_format, - staging_texture, - d3d_context: self.d3d_context.clone(), }) } } pub struct FrameBuffer<'a> { - data: &'a [u8], - width: u32, - height: u32, - stride: u32, - pixel_format: PixelFormat, - staging_texture: ID3D11Texture2D, - d3d_context: ID3D11DeviceContext, -} - -impl Drop for FrameBuffer<'_> { - fn drop(&mut self) { - unsafe { - self.d3d_context.Unmap(&self.staging_texture, 0); - } - } + mapped: MappedFrame, + frame: std::marker::PhantomData<&'a Frame>, } impl FrameBuffer<'_> { pub fn width(&self) -> u32 { - self.width + self.mapped.width() } pub fn height(&self) -> u32 { - self.height + self.mapped.height() } pub fn stride(&self) -> u32 { - self.stride + self.mapped.row_pitch() } pub fn data(&self) -> &[u8] { - self.data + self.mapped.as_slice() } pub fn pixel_format(&self) -> PixelFormat { - self.pixel_format + self.mapped.pixel_format() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn warp_pool() -> Arc { + let mut device = None; + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_WARP, + HMODULE::default(), + D3D11_CREATE_DEVICE_FLAG::default(), + None, + D3D11_SDK_VERSION, + Some(&mut device), + None, + None, + ) + .unwrap(); + } + + let device = device.unwrap(); + let context = unsafe { device.GetImmediateContext().unwrap() }; + + Arc::new(StagingTexturePool::new( + device, + context, + PixelFormat::R8G8B8A8Unorm, + )) + } + + #[test] + fn pool_survives_capture_size_changes() { + let pool = warp_pool(); + let sizes = [(320, 240), (640, 480), (800, 600), (1024, 768), (1280, 720)]; + + for (index, (width, height)) in sizes.into_iter().enumerate() { + let lease = pool + .acquire(width, height) + .unwrap_or_else(|error| panic!("acquire size {index}: {error}")); + drop(lease); + + let inner = pool.inner.lock().unwrap(); + assert_eq!(inner.live, inner.free.len(), "live after size {index}"); + assert!(inner.live <= STAGING_POOL_LIMIT); + } } } diff --git a/crates/scap-ffmpeg/examples/cli.rs b/crates/scap-ffmpeg/examples/cli.rs index 12aa005c2df..c2e80034c38 100644 --- a/crates/scap-ffmpeg/examples/cli.rs +++ b/crates/scap-ffmpeg/examples/cli.rs @@ -1,32 +1,61 @@ +//! Windows/macOS capture demo. On Windows, `--bench [--seconds N]` A/Bs the old two-transfer +//! readback against the staging ring + `DO_NOT_WAIT` + zero-copy path on live frames. WGC only +//! produces frames while the captured content changes, so keep something moving during the run. + #[tokio::main] pub async fn main() { #[cfg(windows)] { use scap_direct3d::*; use scap_targets::Display; - use std::time::Duration; + use std::{ + sync::mpsc, + time::{Duration, Instant}, + }; + + let args: Vec = std::env::args().collect(); + let bench = args.iter().any(|arg| arg == "--bench"); + let seconds = args + .iter() + .position(|arg| arg == "--seconds") + .and_then(|index| args.get(index + 1)) + .and_then(|value| value.parse::().ok()) + .unwrap_or(15.0) + .clamp(1.0, 120.0); let display = Display::primary(); + let frame_sender = bench.then(|| { + let (tx, rx) = mpsc::sync_channel::(4); + (tx, rx) + }); + let frame_tx = frame_sender.as_ref().map(|(tx, _)| tx.clone()); let mut capturer = Capturer::new( display.raw_handle().try_as_capture_item().unwrap(), Settings { is_border_required: Some(true), is_cursor_capture_enabled: Some(true), pixel_format: PixelFormat::R8G8B8A8Unorm, + fps: Some(60), ..Default::default() }, - |frame| { - use scap_ffmpeg::AsFFmpeg; + move |frame| { + if let Some(tx) = &frame_tx { + let _ = tx.try_send(frame); + return Ok(()); + } - let ff_frame = frame.as_ffmpeg()?; + use scap_ffmpeg::AsFFmpeg; - println!( - "Frame: {}x{} format={:?}", - ff_frame.width(), - ff_frame.height(), - ff_frame.format() - ); + match frame.as_ffmpeg() { + Ok(ff_frame) => println!( + "Frame: {}x{} format={:?}", + ff_frame.width(), + ff_frame.height(), + ff_frame.format() + ), + Err(error) => eprintln!("Frame conversion failed: {error}"), + } Ok(()) }, @@ -37,11 +66,183 @@ pub async fn main() { capturer.start().unwrap(); - std::thread::sleep(Duration::from_secs(3)); + if let Some((_, frame_rx)) = frame_sender { + readback_benchmark(&display, frame_rx, seconds); + } else { + std::thread::sleep(Duration::from_secs(3)); + } capturer.stop().unwrap(); - std::thread::sleep(Duration::from_secs(3)); + std::thread::sleep(Duration::from_secs(1)); + + fn readback_benchmark(display: &Display, frame_rx: mpsc::Receiver, seconds: f64) { + use scap_ffmpeg::AsFFmpeg; + + let size = display.physical_size().unwrap(); + let height = size.height() as usize; + let row_length = size.width() as usize * 4; + let frame_bytes = (height * row_length) as u64; + + let mut old = Samples::default(); + let mut new = Samples::default(); + let mut old_frame = None; + let started = Instant::now(); + + while started.elapsed() < Duration::from_secs_f64(seconds) { + let Ok(frame) = frame_rx.recv_timeout(Duration::from_secs(1)) else { + break; + }; + + let mut run_old = || { + let started = Instant::now(); + let buffer = frame.as_buffer().ok()?; + let source_stride = buffer.stride() as usize; + let destination = old_frame.get_or_insert_with(|| { + ffmpeg::frame::Video::new( + ffmpeg::format::Pixel::RGBA, + size.width() as u32, + size.height() as u32, + ) + }); + let destination_stride = destination.stride(0); + let destination_data = destination.data_mut(0); + + for row in 0..height { + unsafe { + std::ptr::copy_nonoverlapping( + buffer.data().as_ptr().add(row * source_stride), + destination_data.as_mut_ptr().add(row * destination_stride), + row_length, + ); + } + } + + Some(started.elapsed()) + }; + + let run_new = || { + let started = Instant::now(); + match frame.as_ffmpeg() { + Ok(_) => Some((started.elapsed(), false)), + Err(scap_ffmpeg::AsFFmpegError::NotReady) => { + Some((started.elapsed(), true)) + } + Err(error) => { + eprintln!("as_ffmpeg failed: {error}"); + None + } + } + }; + + let old_first = old.count.is_multiple_of(2); + let (old_elapsed, new_elapsed) = if old_first { + (run_old(), run_new()) + } else { + let new_elapsed = run_new(); + (run_old(), new_elapsed) + }; + + if let Some(elapsed) = old_elapsed { + old.record(elapsed); + } else { + old.failures += 1; + } + + match new_elapsed { + Some((elapsed, dropped)) => { + if dropped { + new.dropped += 1; + } else { + new.record(elapsed); + } + } + None => new.failures += 1, + } + } + + println!( + "{}x{} RGBA ({:.1} MiB/frame), {seconds:.0}s", + size.width(), + size.height(), + frame_bytes as f64 / (1024.0 * 1024.0) + ); + println!("old (blocking two-transfer path): {old}"); + println!("new (ring + DO_NOT_WAIT + zero copy): {new}"); + println!( + "{{\"old\":{},\"new\":{},\"frame_bytes\":{frame_bytes}}}", + old.json(), + new.json() + ); + } + + #[derive(Default)] + struct Samples { + count: u64, + dropped: u64, + failures: u64, + elapsed: Vec, + } + + impl Samples { + fn record(&mut self, elapsed: Duration) { + self.count += 1; + self.elapsed.push(elapsed); + } + + fn mean_ms(&self) -> f64 { + if self.elapsed.is_empty() { + return 0.0; + } + self.elapsed + .iter() + .map(|elapsed| elapsed.as_secs_f64()) + .sum::() + * 1000.0 + / self.elapsed.len() as f64 + } + + fn percentile_ms(&self, percentile: f64) -> f64 { + if self.elapsed.is_empty() { + return 0.0; + } + let mut sorted = self.elapsed.clone(); + sorted.sort_unstable(); + let index = ((sorted.len() as f64 - 1.0) * percentile).round() as usize; + sorted[index].as_secs_f64() * 1000.0 + } + + fn json(&self) -> String { + format!( + "{{\"frames\":{},\"dropped\":{},\"failures\":{},\"mean_ms\":{:.3},\"p50_ms\":{:.3},\"p95_ms\":{:.3},\"p99_ms\":{:.3},\"max_ms\":{:.3}}}", + self.count, + self.dropped, + self.failures, + self.mean_ms(), + self.percentile_ms(0.50), + self.percentile_ms(0.95), + self.percentile_ms(0.99), + self.percentile_ms(1.0) + ) + } + } + + impl std::fmt::Display for Samples { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "frames={} dropped={} failures={} mean={:.2}ms p50={:.2}ms p95={:.2}ms p99={:.2}ms max={:.2}ms", + self.count, + self.dropped, + self.failures, + self.mean_ms(), + self.percentile_ms(0.50), + self.percentile_ms(0.95), + self.percentile_ms(0.99), + self.percentile_ms(1.0) + ) + } + } } #[cfg(target_os = "macos")] diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 762af9c58b9..5174ac03f44 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -1,158 +1,63 @@ use ffmpeg::format::Pixel; -use scap_direct3d::PixelFormat; +use scap_direct3d::{MappedFrame, PixelFormat, ReadbackError}; +use std::ffi::c_void; +use std::time::Duration; -pub type AsFFmpegError = windows::core::Error; +pub type AsFFmpegError = ReadbackError; -#[inline] -fn copy_frame_data( - src_bytes: &[u8], - src_stride: usize, - dest_bytes: &mut [u8], - dest_stride: usize, - row_length: usize, - height: usize, -) { - debug_assert!(height > 0, "height must be positive"); - debug_assert!( - src_bytes.len() - >= (height - 1) - .saturating_mul(src_stride) - .saturating_add(row_length), - "source buffer too small" - ); - debug_assert!( - dest_bytes.len() - >= (height - 1) - .saturating_mul(dest_stride) - .saturating_add(row_length), - "destination buffer too small" - ); +/// Half a frame interval at 60 fps. A 4K GPU-to-CPU copy completes well inside it on PCIe, so a +/// frame that misses the deadline means the GPU is busy; the caller reuses the previous frame +/// rather than stalling the encoder thread behind `Map`. +const READBACK_BUDGET: Duration = Duration::from_millis(8); - if src_stride == row_length && dest_stride == row_length { - let total_bytes = row_length.saturating_mul(height); - unsafe { - std::ptr::copy_nonoverlapping(src_bytes.as_ptr(), dest_bytes.as_mut_ptr(), total_bytes); - } - } else { - for i in 0..height { - unsafe { - std::ptr::copy_nonoverlapping( - src_bytes.as_ptr().add(i * src_stride), - dest_bytes.as_mut_ptr().add(i * dest_stride), - row_length, - ); - } - } - } -} - -struct FramePool { - key: (Pixel, u32, u32), - linesize: usize, - pool: *mut ffmpeg::ffi::AVBufferPool, -} - -impl Drop for FramePool { - fn drop(&mut self) { - // Frees the pool once every buffer still referenced by an in-flight frame is returned. - unsafe { ffmpeg::ffi::av_buffer_pool_uninit(&mut self.pool) }; +impl super::AsFFmpeg for scap_direct3d::Frame { + fn as_ffmpeg(&self) -> Result { + let mapped = self.map_readback(READBACK_BUDGET)?; + let format = mapped.pixel_format().as_ffmpeg(); + wrap_mapped_frame(mapped, format) } } -thread_local! { - static FRAME_POOL: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +unsafe extern "C" fn release_mapped_frame(opaque: *mut c_void, _data: *mut u8) { + drop(unsafe { Box::from_raw(opaque.cast::()) }); } -// A fresh multi-megabyte allocation per frame is served by fresh OS pages on Windows, so every -// capture paid page-fault and zeroing costs several times the copy itself (~2ms per 1080p frame). -fn pooled_frame(format: Pixel, width: u32, height: u32) -> ffmpeg::frame::Video { - debug_assert!(matches!(format, Pixel::RGBA | Pixel::BGRA)); - FRAME_POOL.with_borrow_mut(|slot| { - if slot - .as_ref() - .is_none_or(|pool| pool.key != (format, width, height)) - { - *slot = None; - let linesize = (width as usize).next_multiple_of(32) * 4; - let size = - linesize * height as usize + ffmpeg::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize; - let pool = unsafe { ffmpeg::ffi::av_buffer_pool_init(size, None) }; - if pool.is_null() { - return ffmpeg::frame::Video::new(format, width, height); - } - *slot = Some(FramePool { - key: (format, width, height), - linesize, - pool, - }); - } - let pool = slot.as_ref().expect("frame pool initialized above"); - unsafe { - let buffer = ffmpeg::ffi::av_buffer_pool_get(pool.pool); - if buffer.is_null() { - return ffmpeg::frame::Video::new(format, width, height); - } - let mut frame = ffmpeg::frame::Video::empty(); - frame.set_format(format); - frame.set_width(width); - frame.set_height(height); - let raw = frame.as_mut_ptr(); - (*raw).buf[0] = buffer; - (*raw).data[0] = (*buffer).data; - (*raw).linesize[0] = pool.linesize as i32; - frame +/// Wraps the mapped staging texture into an `AVFrame` whose backing `AVBufferRef` owns the mapping. +/// The pixels are never copied: the encoder reads the staging texture directly, and the last frame +/// reference unmaps it and returns it to the pool. +fn wrap_mapped_frame( + mapped: MappedFrame, + format: Pixel, +) -> Result { + let data = mapped.data_ptr(); + let size = mapped.data_len(); + let row_pitch = mapped.row_pitch() as i32; + let width = mapped.width(); + let height = mapped.height(); + + let mut frame = ffmpeg::frame::Video::empty(); + + unsafe { + let opaque = Box::into_raw(Box::new(mapped)).cast::(); + let buffer = + ffmpeg::ffi::av_buffer_create(data, size, Some(release_mapped_frame), opaque, 0); + + if buffer.is_null() { + drop(Box::from_raw(opaque.cast::())); + return Err(ReadbackError::FrameBufferAllocation); } - }) -} - -impl super::AsFFmpeg for scap_direct3d::Frame { - fn as_ffmpeg(&self) -> Result { - let buffer = self.as_buffer()?; - - let width = self.width() as usize; - let height = self.height() as usize; - let src_bytes = buffer.data(); - let src_stride = buffer.stride() as usize; - let row_length = width * 4; + frame.set_format(format); + frame.set_width(width); + frame.set_height(height); - match self.pixel_format() { - PixelFormat::R8G8B8A8Unorm => { - let mut ff_frame = pooled_frame(Pixel::RGBA, self.width(), self.height()); - - let dest_stride = ff_frame.stride(0); - let dest_bytes = ff_frame.data_mut(0); - - copy_frame_data( - src_bytes, - src_stride, - dest_bytes, - dest_stride, - row_length, - height, - ); - - Ok(ff_frame) - } - PixelFormat::B8G8R8A8Unorm => { - let mut ff_frame = pooled_frame(Pixel::BGRA, self.width(), self.height()); - - let dest_stride = ff_frame.stride(0); - let dest_bytes = ff_frame.data_mut(0); - - copy_frame_data( - src_bytes, - src_stride, - dest_bytes, - dest_stride, - row_length, - height, - ); - - Ok(ff_frame) - } - } + let raw = frame.as_mut_ptr(); + (*raw).buf[0] = buffer; + (*raw).data[0] = data; + (*raw).linesize[0] = row_pitch; } + + Ok(frame) } pub trait PixelFormatExt { @@ -167,31 +72,3 @@ impl PixelFormatExt for PixelFormat { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn pooled_frames_recycle_released_buffers() { - let first = pooled_frame(Pixel::BGRA, 1001, 3); - assert_eq!( - (first.format(), first.width(), first.height()), - (Pixel::BGRA, 1001, 3) - ); - assert!(first.stride(0) >= 1001 * 4); - assert!(first.data(0).len() >= first.stride(0) * 3); - let held = first.data(0).as_ptr(); - - let second = pooled_frame(Pixel::BGRA, 1001, 3); - assert_ne!(second.data(0).as_ptr(), held); - - drop(first); - let third = pooled_frame(Pixel::BGRA, 1001, 3); - assert_eq!(third.data(0).as_ptr(), held); - - let resized = pooled_frame(Pixel::BGRA, 640, 2); - assert_eq!((resized.width(), resized.height()), (640, 2)); - drop((second, third)); - } -} From 3fb93818fa1ea56112f05a8456a8707f1fbde4b8 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 13:28:20 +0200 Subject: [PATCH 02/16] fix(recording): bound Windows readback and harden MF setup - Keep the readback budget authoritative by default; CAP_READBACK_BLOCKING opts into an unbounded Map. - Negotiate MFT types inside the candidate loop and handle WARP without VIDEO_SUPPORT. - Validate harness --path values and count failures only for exercised paths. --- crates/enc-mediafoundation/src/d3d.rs | 13 +- crates/enc-mediafoundation/src/video/h264.rs | 345 +++++++++++-------- crates/scap-direct3d/src/lib.rs | 147 ++++---- crates/scap-ffmpeg/examples/cli.rs | 83 +++-- crates/scap-ffmpeg/src/direct3d.rs | 5 +- 5 files changed, 361 insertions(+), 232 deletions(-) diff --git a/crates/enc-mediafoundation/src/d3d.rs b/crates/enc-mediafoundation/src/d3d.rs index 77ae38543bd..7d9406c8dad 100644 --- a/crates/enc-mediafoundation/src/d3d.rs +++ b/crates/enc-mediafoundation/src/d3d.rs @@ -7,8 +7,8 @@ use windows::Win32::Graphics::{ D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_UNKNOWN, D3D_DRIVER_TYPE_WARP, }, Direct3D11::{ - D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG, D3D11_SDK_VERSION, - D3D11CreateDevice, ID3D11Device, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, D3D11CreateDevice, ID3D11Device, }, Dxgi::{DXGI_ERROR_UNSUPPORTED, IDXGIDevice}, }; @@ -70,7 +70,7 @@ fn create_d3d_device_on_pinned_adapter( pub fn create_d3d_device() -> Result { let mut device = None; let flags = { - let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; if cfg!(feature = "d3ddebug") { flags |= D3D11_CREATE_DEVICE_DEBUG; } @@ -97,7 +97,12 @@ pub fn create_d3d_device() -> Result { if let Err(error) = &result && error.code() == DXGI_ERROR_UNSUPPORTED { - result = create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device); + // WARP rejects VIDEO_SUPPORT; a WARP device cannot hardware-encode anyway. + result = create_d3d_device_with_type( + D3D_DRIVER_TYPE_WARP, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + &mut device, + ); } result?; device.ok_or_else(windows::core::Error::from_win32) diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index de011af6d66..9fb7f169ea2 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -10,7 +10,7 @@ use std::{ }, time::{Duration, Instant}, }; -use tracing::debug; +use tracing::{debug, warn}; use windows::{ Foundation::TimeSpan, Graphics::SizeInt32, @@ -249,14 +249,9 @@ impl H264Encoder { flags: MFT_ENUM_FLAG, enable_hardware_transforms: bool, ) -> Result { - let transform = + let devices = EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags) - .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)? - .first() - .cloned() - .ok_or(NewVideoEncoderError::NoVideoEncoderDevice)? - .create_transform() - .map_err(NewVideoEncoderError::EncoderTransform)?; + .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?; let video_processor = VideoProcessor::new( d3d_device.clone(), @@ -286,107 +281,57 @@ impl H264Encoder { .map_err(NewVideoEncoderError::DeviceManager)? }; - let event_generator: IMFMediaEventGenerator = transform - .cast() - .map_err(NewVideoEncoderError::EventGenerator)?; - let attributes = unsafe { - transform - .GetAttributes() - .map_err(NewVideoEncoderError::EventGenerator)? - }; - unsafe { - attributes - .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + let configure = |transform: &IMFTransform| -> Result< + (IMFMediaEventGenerator, u32, u32, IMFMediaType), + NewVideoEncoderError, + > { + let event_generator: IMFMediaEventGenerator = transform + .cast() .map_err(NewVideoEncoderError::EventGenerator)?; - attributes - .SetUINT32( - &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, - enable_hardware_transforms as u32, - ) - .map_err(NewVideoEncoderError::EventGenerator)?; - }; - let mut number_of_input_streams = 0; - let mut number_of_output_streams = 0; - unsafe { - transform - .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams) - .map_err(NewVideoEncoderError::EventGenerator)? - }; - let (input_stream_ids, output_stream_ids) = { - let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; - let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; - let result = - unsafe { transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) }; - match result { - Ok(_) => {} - Err(error) => { - if error.code() == E_NOTIMPL { - for i in 0..number_of_input_streams { - input_stream_ids[i as usize] = i; - } - for i in 0..number_of_output_streams { - output_stream_ids[i as usize] = i; + let mut number_of_input_streams = 0; + let mut number_of_output_streams = 0; + unsafe { + transform + .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams) + .map_err(NewVideoEncoderError::EventGenerator)? + }; + let (input_stream_ids, output_stream_ids) = { + let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; + let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; + let result = unsafe { + transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) + }; + match result { + Ok(_) => {} + Err(error) => { + if error.code() == E_NOTIMPL { + for i in 0..number_of_input_streams { + input_stream_ids[i as usize] = i; + } + for i in 0..number_of_output_streams { + output_stream_ids[i as usize] = i; + } + } else { + return Err(NewVideoEncoderError::ConfigureStreams(error)); } - } else { - return Err(NewVideoEncoderError::ConfigureStreams(error)); } } - } - (input_stream_ids, output_stream_ids) - }; - let input_stream_id = input_stream_ids[0]; - let output_stream_id = output_stream_ids[0]; - - unsafe { - let temp = media_device_manager.clone(); - transform - .ProcessMessage( - MFT_MESSAGE_SET_D3D_MANAGER, - std::mem::transmute::(temp), - ) - .map_err(NewVideoEncoderError::EncoderTransform)?; - }; - - let output_type = (|| unsafe { - let output_type = MFCreateMediaType()?; - let attributes: IMFAttributes = output_type.cast()?; - output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; - output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; - output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; - output_type.SetUINT32( - &MF_MT_MAX_KEYFRAME_SPACING, - frame_rate.saturating_mul(MUXED_KEYFRAME_INTERVAL_SECS), - )?; - MFSetAttributeSize( - &attributes, - &MF_MT_FRAME_SIZE, - output_resolution.Width as u32, - output_resolution.Height as u32, - )?; - MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; - MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; - output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; - output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; - transform.SetOutputType(output_stream_id, &output_type, 0)?; - Ok(output_type) - })() - .map_err(NewVideoEncoderError::OutputType)?; - - let input_type: Option = (|| unsafe { - let mut count = 0; - loop { - let result = transform.GetInputAvailableType(input_stream_id, count); - if let Err(error) = &result - && error.code() == MF_E_NO_MORE_TYPES - { - break Ok(None); - } - - let input_type = result?; - let attributes: IMFAttributes = input_type.cast()?; - input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; - input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; + (input_stream_ids, output_stream_ids) + }; + let input_stream_id = input_stream_ids[0]; + let output_stream_id = output_stream_ids[0]; + + let output_type = (|| unsafe { + let output_type = MFCreateMediaType()?; + let attributes: IMFAttributes = output_type.cast()?; + output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; + output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + output_type.SetUINT32( + &MF_MT_MAX_KEYFRAME_SPACING, + frame_rate.saturating_mul(MUXED_KEYFRAME_INTERVAL_SECS), + )?; MFSetAttributeSize( &attributes, &MF_MT_FRAME_SIZE, @@ -394,47 +339,167 @@ impl H264Encoder { output_resolution.Height as u32, )?; MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; - let result = transform.SetInputType( - input_stream_id, - &input_type, - MFT_SET_TYPE_TEST_ONLY.0 as u32, - ); - if let Err(error) = &result - && error.code() == MF_E_INVALIDMEDIATYPE - { - count += 1; - continue; + MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; + output_type + .SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; + transform.SetOutputType(output_stream_id, &output_type, 0)?; + transform.GetOutputCurrentType(output_stream_id) + })() + .map_err(NewVideoEncoderError::OutputType)?; + + let input_type: Option = (|| unsafe { + let mut count = 0; + loop { + let result = transform.GetInputAvailableType(input_stream_id, count); + if let Err(error) = &result + && error.code() == MF_E_NO_MORE_TYPES + { + break Ok(None); + } + + let input_type = result?; + let attributes: IMFAttributes = input_type.cast()?; + input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + let result = transform.SetInputType( + input_stream_id, + &input_type, + MFT_SET_TYPE_TEST_ONLY.0 as u32, + ); + if let Err(error) = &result + && error.code() == MF_E_INVALIDMEDIATYPE + { + count += 1; + continue; + } + result?; + break Ok(Some(input_type)); } - result?; - break Ok(Some(input_type)); + })() + .map_err(NewVideoEncoderError::InputType)?; + if let Some(input_type) = input_type { + unsafe { transform.SetInputType(input_stream_id, &input_type, 0) } + .map_err(NewVideoEncoderError::InputType)?; + } else { + return Err(NewVideoEncoderError::InputType(Error::new( + MF_E_TRANSFORM_TYPE_NOT_SET, + "No suitable input type found! Try a different set of encoding settings.", + ))); } - })() - .map_err(NewVideoEncoderError::InputType)?; - if let Some(input_type) = input_type { - unsafe { transform.SetInputType(input_stream_id, &input_type, 0) } - .map_err(NewVideoEncoderError::InputType)?; - } else { - return Err(NewVideoEncoderError::InputType(Error::new( - MF_E_TRANSFORM_TYPE_NOT_SET, - "No suitable input type found! Try a different set of encoding settings.", - ))); - } - - Ok(Self { - _d3d_device: d3d_device.clone(), - _media_device_manager: media_device_manager, - _device_manager_reset_token: device_manager_reset_token, - video_processor, + Ok(( + event_generator, + input_stream_id, + output_stream_id, + output_type, + )) + }; - transform, - event_generator, - input_stream_id, - output_stream_id, - bitrate, + let mut last_error = None; + for device in &devices { + let candidate = match device.create_transform() { + Ok(candidate) => candidate, + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder activation failed; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + continue; + } + }; + let attributes = match unsafe { candidate.GetAttributes() } { + Ok(attributes) => attributes, + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder attributes unavailable; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + continue; + } + }; + let configured = unsafe { + attributes + .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + .and_then(|_| { + attributes.SetUINT32( + &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, + enable_hardware_transforms as u32, + ) + }) + }; + if let Err(error) = configured { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder configuration failed; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + continue; + } + let attached = unsafe { + candidate.ProcessMessage( + MFT_MESSAGE_SET_D3D_MANAGER, + std::mem::transmute::( + media_device_manager.clone(), + ), + ) + }; + match attached { + Ok(()) => match configure(&candidate) { + Ok((event_generator, input_stream_id, output_stream_id, output_type)) => { + tracing::info!( + device = device.display_name(), + "Selected Media Foundation encoder MFT" + ); + return Ok(Self { + _d3d_device: d3d_device.clone(), + _media_device_manager: media_device_manager, + _device_manager_reset_token: device_manager_reset_token, + + video_processor, + + transform: candidate, + event_generator, + input_stream_id, + output_stream_id, + bitrate, + + output_type, + }); + } + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder type setup failed; trying the next MFT" + ); + last_error = Some(error); + } + }, + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder rejected the D3D manager; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + } + } + } - output_type, - }) + Err(last_error.unwrap_or(NewVideoEncoderError::NoVideoEncoderDevice)) } pub fn new_with_scaled_output( diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index c2753e339f9..179b02ec999 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -7,7 +7,7 @@ pub use windows_version::WindowsVersion; use std::{ sync::{ Arc, Condvar, Mutex, - atomic::{AtomicBool, AtomicU32, Ordering}, + atomic::{AtomicBool, Ordering}, mpsc::RecvError, }, time::{Duration, Instant}, @@ -27,7 +27,8 @@ use windows::{ Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, - D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_FLAG, + D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_BGRA_SUPPORT, + D3D11_CREATE_DEVICE_FLAG, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_FLAG_DO_NOT_WAIT, D3D11_MAP_READ, D3D11_MAP_WRITE, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, ID3D11Device, @@ -75,9 +76,8 @@ impl PixelFormat { const STAGING_POOL_SIZE: usize = 3; const STAGING_POOL_LIMIT: usize = STAGING_POOL_SIZE + 1; const STAGING_PADDING_ROWS: u32 = 1; -const READBACK_RETRY_INTERVAL: Duration = Duration::from_micros(250); -const READBACK_MISS_FALLBACK: u32 = 2; -const READBACK_BLOCKING_COOLDOWN_FRAMES: u32 = 150; +const READBACK_RETRY_INTERVAL: Duration = Duration::from_micros(100); +const READBACK_SPIN_BUDGET: Duration = Duration::from_millis(4); struct CallbackActivity { active: Mutex, @@ -186,8 +186,6 @@ pub struct StagingTexturePool { d3d_device: ID3D11Device, d3d_context: ID3D11DeviceContext, pixel_format: PixelFormat, - readback_misses: AtomicU32, - blocking_frames: AtomicU32, } impl StagingTexturePool { @@ -204,33 +202,6 @@ impl StagingTexturePool { d3d_device, d3d_context, pixel_format, - readback_misses: AtomicU32::new(0), - blocking_frames: AtomicU32::new(0), - } - } - - fn take_blocking_readback(&self) -> bool { - self.blocking_frames - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |remaining| { - remaining.checked_sub(1) - }) - .is_ok() - } - - fn reset_readback_misses(&self) { - self.readback_misses.store(0, Ordering::Relaxed); - } - - fn note_readback_miss(&self) { - let misses = self.readback_misses.fetch_add(1, Ordering::Relaxed) + 1; - if misses == READBACK_MISS_FALLBACK { - self.readback_misses.store(0, Ordering::Relaxed); - self.blocking_frames - .store(READBACK_BLOCKING_COOLDOWN_FRAMES, Ordering::Relaxed); - tracing::warn!( - cooldown_frames = READBACK_BLOCKING_COOLDOWN_FRAMES, - "GPU readback missed its deadline twice in a row; blocking for a cooldown instead of duplicating frames" - ); } } @@ -502,9 +473,27 @@ fn create_d3d_device_on_pinned_adapter( fn create_d3d_device_with_warp_fallback() -> windows::core::Result<(ID3D11Device, bool)> { let mut device = None; - let flags = D3D11_CREATE_DEVICE_FLAG::default(); + let video_flags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT; + let basic_flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + + match create_d3d_device_on_pinned_adapter(video_flags, &mut device) { + Ok(description) => { + tracing::info!( + adapter = %description, + "scap-direct3d: D3D11 device created on pinned hardware adapter with video support" + ); + let device = device.ok_or_else(windows::core::Error::from_win32)?; + return Ok((device, false)); + } + Err(e) => { + tracing::warn!( + error = %e, + "scap-direct3d: video-support device creation failed on pinned adapter, retrying without it" + ); + } + } - match create_d3d_device_on_pinned_adapter(flags, &mut device) { + match create_d3d_device_on_pinned_adapter(basic_flags, &mut device) { Ok(description) => { tracing::info!( adapter = %description, @@ -521,13 +510,13 @@ fn create_d3d_device_with_warp_fallback() -> windows::core::Result<(ID3D11Device } } - let result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, flags, &mut device); + let result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, basic_flags, &mut device); match result { Ok(()) => Ok((device.ok_or_else(windows::core::Error::from_win32)?, false)), Err(e) if e.code() == DXGI_ERROR_UNSUPPORTED => { tracing::info!("Hardware D3D11 device unavailable, attempting WARP fallback"); - create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device)?; + create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, basic_flags, &mut device)?; Ok((device.ok_or_else(windows::core::Error::from_win32)?, true)) } Err(e) => Err(e), @@ -988,35 +977,21 @@ impl Frame { }) } - /// Copies the frame into a staging texture and maps it without waiting for the GPU copy - /// (`D3D11_MAP_FLAG_DO_NOT_WAIT`). Returns [`ReadbackError::NotReady`] when the copy has not - /// completed within `budget`, so the caller can reuse the previous frame instead of stalling - /// its thread behind the GPU. + /// Copies the frame into a staging texture and maps it within `budget`. The default polls + /// `DO_NOT_WAIT` until the copy completes or the budget expires; the caller can reuse the + /// previous frame on `NotReady`. Set `CAP_READBACK_BLOCKING` to block in `Map` instead, which is + /// faster when the copy consistently finishes quickly but removes the deadline. pub fn map_readback(&self, budget: Duration) -> Result { + let blocking = std::env::var_os("CAP_READBACK_BLOCKING").is_some(); let lease = self.staging_pool.acquire(self.width, self.height)?; self.copy_into_staging(&lease); - // A GPU that cannot meet the deadline consistently would otherwise freeze the recording on - // repeated duplicates. After two consecutive misses, block for a cooldown so the encoder - // keeps receiving fresh frames at the slower readback rate, then probe the deadline again. - if self.staging_pool.take_blocking_readback() { - let mapped = self.map_staging(lease, None)?; - self.staging_pool.reset_readback_misses(); - return Ok(mapped); + if blocking { + return self.map_staging_hybrid(lease); } let deadline = Instant::now().checked_add(budget); - match self.map_staging(lease, deadline) { - Ok(mapped) => { - self.staging_pool.reset_readback_misses(); - Ok(mapped) - } - Err(ReadbackError::NotReady) => { - self.staging_pool.note_readback_miss(); - Err(ReadbackError::NotReady) - } - Err(error) => Err(error), - } + self.map_staging(lease, deadline) } fn copy_into_staging(&self, lease: &StagingLease) { @@ -1052,6 +1027,7 @@ impl Frame { let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); if let Some(deadline) = deadline { + let started = Instant::now(); loop { match unsafe { self.d3d_context.Map( @@ -1068,9 +1044,14 @@ impl Frame { if now >= deadline { return Err(ReadbackError::NotReady); } - std::thread::sleep( - READBACK_RETRY_INTERVAL.min(deadline.saturating_duration_since(now)), - ); + if now.duration_since(started) < READBACK_SPIN_BUDGET { + std::hint::spin_loop(); + } else { + std::thread::sleep( + READBACK_RETRY_INTERVAL + .min(deadline.saturating_duration_since(now)), + ); + } } Err(error) => return Err(error.into()), } @@ -1087,7 +1068,15 @@ impl Frame { } } - Ok(MappedFrame { + Ok(self.mapped_frame(lease, mapped_resource)) + } + + fn mapped_frame( + &self, + lease: StagingLease, + mapped_resource: D3D11_MAPPED_SUBRESOURCE, + ) -> MappedFrame { + MappedFrame { lease, d3d_context: self.d3d_context.clone(), data: mapped_resource.pData as *mut u8, @@ -1095,7 +1084,35 @@ impl Frame { width: self.width, height: self.height, pixel_format: self.pixel_format, - }) + } + } + + fn map_staging_hybrid(&self, lease: StagingLease) -> Result { + let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); + + match unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + D3D11_MAP_FLAG_DO_NOT_WAIT.0 as u32, + Some(&mut mapped_resource), + ) + } { + Ok(()) => {} + Err(error) if error.code() == DXGI_ERROR_WAS_STILL_DRAWING => unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + 0, + Some(&mut mapped_resource), + )?; + }, + Err(error) => return Err(error.into()), + } + + Ok(self.mapped_frame(lease, mapped_resource)) } } diff --git a/crates/scap-ffmpeg/examples/cli.rs b/crates/scap-ffmpeg/examples/cli.rs index c2e80034c38..9409f7e6a53 100644 --- a/crates/scap-ffmpeg/examples/cli.rs +++ b/crates/scap-ffmpeg/examples/cli.rs @@ -1,6 +1,6 @@ -//! Windows/macOS capture demo. On Windows, `--bench [--seconds N]` A/Bs the old two-transfer -//! readback against the staging ring + `DO_NOT_WAIT` + zero-copy path on live frames. WGC only -//! produces frames while the captured content changes, so keep something moving during the run. +//! Windows/macOS capture demo. On Windows, `--bench [--seconds N] [--path both|old|new|none]` +//! measures the old two-transfer readback against the zero-copy staging ring on live frames. WGC +//! only produces frames while the captured content changes, so keep something moving during the run. #[tokio::main] pub async fn main() { @@ -9,7 +9,11 @@ pub async fn main() { use scap_direct3d::*; use scap_targets::Display; use std::{ - sync::mpsc, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + mpsc, + }, time::{Duration, Instant}, }; @@ -22,26 +26,46 @@ pub async fn main() { .and_then(|value| value.parse::().ok()) .unwrap_or(15.0) .clamp(1.0, 120.0); + let path = args + .iter() + .position(|arg| arg == "--path") + .and_then(|index| args.get(index + 1)) + .cloned() + .unwrap_or_else(|| "both".to_string()); + if !matches!(path.as_str(), "both" | "old" | "new" | "none") { + eprintln!("invalid --path {path}; expected both, old, new, or none"); + std::process::exit(2); + } let display = Display::primary(); + let capture_fps = 60; + let min_update_interval = Settings::can_min_update_interval() + .ok() + .filter(|supported| *supported) + .map(|_| Duration::from_secs_f64(1.0 / capture_fps as f64)); let frame_sender = bench.then(|| { let (tx, rx) = mpsc::sync_channel::(4); (tx, rx) }); + let callback_drops = Arc::new(AtomicU64::new(0)); let frame_tx = frame_sender.as_ref().map(|(tx, _)| tx.clone()); + let callback_drops_tx = callback_drops.clone(); let mut capturer = Capturer::new( display.raw_handle().try_as_capture_item().unwrap(), Settings { is_border_required: Some(true), is_cursor_capture_enabled: Some(true), pixel_format: PixelFormat::R8G8B8A8Unorm, - fps: Some(60), + fps: Some(capture_fps), + min_update_interval, ..Default::default() }, move |frame| { if let Some(tx) = &frame_tx { - let _ = tx.try_send(frame); + if tx.try_send(frame).is_err() { + callback_drops_tx.fetch_add(1, Ordering::Relaxed); + } return Ok(()); } @@ -67,7 +91,7 @@ pub async fn main() { capturer.start().unwrap(); if let Some((_, frame_rx)) = frame_sender { - readback_benchmark(&display, frame_rx, seconds); + readback_benchmark(&display, frame_rx, seconds, &path, &callback_drops); } else { std::thread::sleep(Duration::from_secs(3)); } @@ -76,7 +100,13 @@ pub async fn main() { std::thread::sleep(Duration::from_secs(1)); - fn readback_benchmark(display: &Display, frame_rx: mpsc::Receiver, seconds: f64) { + fn readback_benchmark( + display: &Display, + frame_rx: mpsc::Receiver, + seconds: f64, + path: &str, + callback_drops: &Arc, + ) { use scap_ffmpeg::AsFFmpeg; let size = display.physical_size().unwrap(); @@ -87,12 +117,16 @@ pub async fn main() { let mut old = Samples::default(); let mut new = Samples::default(); let mut old_frame = None; + let mut frames_seen = 0u64; + let run_old_path = path != "new" && path != "none"; + let run_new_path = path != "old" && path != "none"; let started = Instant::now(); while started.elapsed() < Duration::from_secs_f64(seconds) { let Ok(frame) = frame_rx.recv_timeout(Duration::from_secs(1)) else { break; }; + frames_seen += 1; let mut run_old = || { let started = Instant::now(); @@ -135,17 +169,24 @@ pub async fn main() { } }; - let old_first = old.count.is_multiple_of(2); - let (old_elapsed, new_elapsed) = if old_first { - (run_old(), run_new()) - } else { - let new_elapsed = run_new(); - (run_old(), new_elapsed) + let (old_elapsed, new_elapsed) = match path { + "old" => (run_old(), None), + "new" => (None, run_new()), + "none" => (None, None), + _ => { + let old_first = old.count.is_multiple_of(2); + if old_first { + (run_old(), run_new()) + } else { + let new_elapsed = run_new(); + (run_old(), new_elapsed) + } + } }; if let Some(elapsed) = old_elapsed { old.record(elapsed); - } else { + } else if run_old_path { old.failures += 1; } @@ -157,20 +198,22 @@ pub async fn main() { new.record(elapsed); } } - None => new.failures += 1, + None if run_new_path => new.failures += 1, + None => {} } } println!( - "{}x{} RGBA ({:.1} MiB/frame), {seconds:.0}s", + "{}x{} RGBA ({:.1} MiB/frame), {seconds:.0}s, path={path}, callback_drops={}, source_frames={frames_seen}", size.width(), size.height(), - frame_bytes as f64 / (1024.0 * 1024.0) + frame_bytes as f64 / (1024.0 * 1024.0), + callback_drops.load(Ordering::Relaxed) ); println!("old (blocking two-transfer path): {old}"); - println!("new (ring + DO_NOT_WAIT + zero copy): {new}"); + println!("new (ring + probe/block zero copy): {new}"); println!( - "{{\"old\":{},\"new\":{},\"frame_bytes\":{frame_bytes}}}", + "{{\"old\":{},\"new\":{},\"source_frames\":{frames_seen},\"frame_bytes\":{frame_bytes}}}", old.json(), new.json() ); diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 5174ac03f44..1f4c7c07ca1 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -5,9 +5,8 @@ use std::time::Duration; pub type AsFFmpegError = ReadbackError; -/// Half a frame interval at 60 fps. A 4K GPU-to-CPU copy completes well inside it on PCIe, so a -/// frame that misses the deadline means the GPU is busy; the caller reuses the previous frame -/// rather than stalling the encoder thread behind `Map`. +/// Readback deadline. The default polls within it; `CAP_READBACK_BLOCKING` opts into an +/// unbounded `Map` when the copy is known to finish quickly. const READBACK_BUDGET: Duration = Duration::from_millis(8); impl super::AsFFmpeg for scap_direct3d::Frame { From 8a71a02c8404d624c5a0ec071606ca6489eaea29 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 16:05:47 +0200 Subject: [PATCH 03/16] fix(recording): address Windows readback and MF drain review feedback - Evict idle stale-size staging textures before reporting StagingBusy. - Size the benchmark copy from each mapped frame and rebuild old_frame on resize. - Require METransformDrainComplete and return errors on failed or timed-out drains. - Count drained samples in the encoder health output counters. - Keep benchmark runs alive across capture idle timeouts. --- crates/enc-mediafoundation/src/video/h264.rs | 21 ++++++++--- crates/scap-direct3d/src/lib.rs | 36 ++++++++++++++++++- crates/scap-ffmpeg/examples/cli.rs | 37 ++++++++++++++------ 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 9fb7f169ea2..9d34e047605 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -218,12 +218,14 @@ pub enum EncoderRuntimeError { process_failures: u32, frames_encoded: u64, }, + #[error("Media Foundation encoder did not confirm drain completion")] + DrainIncomplete, } impl EncoderRuntimeError { pub fn should_fallback(&self) -> bool { match self { - EncoderRuntimeError::Windows(_) => false, + EncoderRuntimeError::Windows(_) | EncoderRuntimeError::DrainIncomplete => false, EncoderRuntimeError::EncoderUnhealthy { .. } => true, } } @@ -786,6 +788,7 @@ impl H264Encoder { let _ = self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0); let drain_deadline = Instant::now() + Duration::from_millis(250); + let mut drain_complete = false; let mut drained = 0u64; while Instant::now() < drain_deadline { let event = match self.event_generator.GetEvent(MF_EVENT_FLAG_NO_WAIT) { @@ -815,24 +818,32 @@ impl H264Encoder { { Ok(()) => { if let Some(sample) = output_buffers[0].pSample.take() { + health_monitor.record_output(); on_sample(sample)?; drained += 1; } } - Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => break, + Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => {} Err(error) => { - debug!(%error, "Media Foundation drain output stopped early"); - break; + let _ = self.cleanup_encoder(); + return Err(error.into()); } } } - MediaFoundation::METransformDrainComplete => break, + MediaFoundation::METransformDrainComplete => { + drain_complete = true; + break; + } _ => {} } } if drained > 0 { debug!(drained, "Drained Media Foundation samples before shutdown"); } + if !drain_complete { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::DrainIncomplete); + } self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index 179b02ec999..5a87ead0ba0 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -226,7 +226,16 @@ impl StagingTexturePool { } if inner.live >= STAGING_POOL_LIMIT { - return Err(ReadbackError::StagingBusy); + // A cached texture at another size is idle, so it can be released. Without this a + // geometry change can wedge the pool at the limit while the previous frame is still + // mapped, and every frame at the new size is dropped. + let stale = inner + .free + .iter() + .position(|pooled| pooled.width != width || pooled.height != height) + .ok_or(ReadbackError::StagingBusy)?; + inner.free.swap_remove(stale); + inner.live -= 1; } let texture = create_staging_texture( @@ -1190,4 +1199,29 @@ mod tests { assert!(inner.live <= STAGING_POOL_LIMIT); } } + + #[test] + fn pool_evicts_stale_sizes_at_the_limit() { + let pool = warp_pool(); + let held = pool.acquire(320, 240).unwrap(); + for (width, height) in [(640, 480), (800, 600), (1024, 768)] { + drop(pool.acquire(width, height).unwrap()); + } + + { + let inner = pool.inner.lock().unwrap(); + assert_eq!(inner.live, STAGING_POOL_LIMIT); + assert_eq!(inner.free.len(), STAGING_POOL_SIZE); + } + + let lease = pool + .acquire(1280, 720) + .unwrap_or_else(|error| panic!("acquire after a size change: {error}")); + drop(lease); + drop(held); + + let inner = pool.inner.lock().unwrap(); + assert_eq!(inner.live, inner.free.len()); + assert!(inner.live <= STAGING_POOL_LIMIT); + } } diff --git a/crates/scap-ffmpeg/examples/cli.rs b/crates/scap-ffmpeg/examples/cli.rs index 9409f7e6a53..bfebeba715a 100644 --- a/crates/scap-ffmpeg/examples/cli.rs +++ b/crates/scap-ffmpeg/examples/cli.rs @@ -109,10 +109,8 @@ pub async fn main() { ) { use scap_ffmpeg::AsFFmpeg; - let size = display.physical_size().unwrap(); - let height = size.height() as usize; - let row_length = size.width() as usize * 4; - let frame_bytes = (height * row_length) as u64; + let initial_size = display.physical_size().unwrap(); + let mut last_frame_size = (initial_size.width() as u32, initial_size.height() as u32); let mut old = Samples::default(); let mut new = Samples::default(); @@ -123,22 +121,39 @@ pub async fn main() { let started = Instant::now(); while started.elapsed() < Duration::from_secs_f64(seconds) { - let Ok(frame) = frame_rx.recv_timeout(Duration::from_secs(1)) else { - break; + let frame = match frame_rx.recv_timeout(Duration::from_secs(1)) { + Ok(frame) => frame, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, }; frames_seen += 1; + last_frame_size = (frame.width(), frame.height()); let mut run_old = || { let started = Instant::now(); let buffer = frame.as_buffer().ok()?; + let width = buffer.width() as usize; + let height = buffer.height() as usize; let source_stride = buffer.stride() as usize; + let row_length = width * 4; let destination = old_frame.get_or_insert_with(|| { ffmpeg::frame::Video::new( ffmpeg::format::Pixel::RGBA, - size.width() as u32, - size.height() as u32, + width as u32, + height as u32, ) }); + + if destination.width() as usize != width + || destination.height() as usize != height + { + *destination = ffmpeg::frame::Video::new( + ffmpeg::format::Pixel::RGBA, + width as u32, + height as u32, + ); + } + let destination_stride = destination.stride(0); let destination_data = destination.data_mut(0); @@ -203,10 +218,12 @@ pub async fn main() { } } + let (last_width, last_height) = last_frame_size; + let frame_bytes = last_width as u64 * last_height as u64 * 4; println!( "{}x{} RGBA ({:.1} MiB/frame), {seconds:.0}s, path={path}, callback_drops={}, source_frames={frames_seen}", - size.width(), - size.height(), + last_width, + last_height, frame_bytes as f64 / (1024.0 * 1024.0), callback_drops.load(Ordering::Relaxed) ); From 68deb14f3e802a5e86ea4adc817df2d5e1c00f64 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 18:17:20 +0200 Subject: [PATCH 04/16] fix(scap-ffmpeg): mark mapped readback frames read-only --- crates/scap-ffmpeg/src/direct3d.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 1f4c7c07ca1..2b774fc80c5 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -38,8 +38,13 @@ fn wrap_mapped_frame( unsafe { let opaque = Box::into_raw(Box::new(mapped)).cast::(); - let buffer = - ffmpeg::ffi::av_buffer_create(data, size, Some(release_mapped_frame), opaque, 0); + let buffer = ffmpeg::ffi::av_buffer_create( + data, + size, + Some(release_mapped_frame), + opaque, + ffmpeg::ffi::AV_BUFFER_FLAG_READONLY, + ); if buffer.is_null() { drop(Box::from_raw(opaque.cast::())); From 0bae86a59c26b994b6522ff4484701d1546c2886 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 16:08:58 +0200 Subject: [PATCH 05/16] perf(frame-converter): reuse D3D11 views and staging across frames Cache the video processor input/output views at converter creation, replace the single staging pair with a reusable ring, and keep CPU upload/readback copies outside the GPU mutex. Share one D3D11 converter across pool workers so camera recording uses one device instead of one per worker. Collapse contiguous plane copies into a single memcpy and add a benchmark example that compares the D3D11 path with single-threaded swscale. --- .../examples/d3d11-benchmark.rs | 214 ++++++ crates/frame-converter/src/d3d11.rs | 651 ++++++++++-------- crates/frame-converter/src/pool.rs | 12 +- 3 files changed, 592 insertions(+), 285 deletions(-) create mode 100644 crates/frame-converter/examples/d3d11-benchmark.rs diff --git a/crates/frame-converter/examples/d3d11-benchmark.rs b/crates/frame-converter/examples/d3d11-benchmark.rs new file mode 100644 index 00000000000..5f9bebbe8b7 --- /dev/null +++ b/crates/frame-converter/examples/d3d11-benchmark.rs @@ -0,0 +1,214 @@ +#[cfg(target_os = "windows")] +mod benchmark { + use cap_frame_converter::{ + AsyncConverterPool, ConversionConfig, ConverterPoolConfig, DropStrategy, FrameConverter, + SwscaleConverter, + }; + use ffmpeg::{format::Pixel, frame}; + use std::{ + sync::Arc, + time::{Duration, Instant}, + }; + + const INPUT_FORMAT: Pixel = Pixel::YUYV422; + const OUTPUT_FORMAT: Pixel = Pixel::NV12; + const RESOLUTIONS: [(u32, u32); 2] = [(1280, 720), (1920, 1080)]; + const WARMUP_FRAMES: usize = 10; + const MEASURED_FRAMES: usize = 120; + const POOL_MEASURED_FRAMES: usize = 60; + const MAX_IN_FLIGHT: usize = 8; + + fn create_frame(format: Pixel, width: u32, height: u32) -> frame::Video { + let mut frame = frame::Video::new(format, width, height); + for plane in 0..frame.planes() { + for (index, byte) in frame.data_mut(plane).iter_mut().enumerate() { + *byte = ((index * 17 + plane * 31) % 256) as u8; + } + } + frame.set_pts(Some(0)); + frame + } + + fn conversion_config(width: u32, height: u32) -> ConversionConfig { + ConversionConfig::new(INPUT_FORMAT, width, height, OUTPUT_FORMAT, width, height) + } + + fn benchmark_single_threaded( + converter: &dyn FrameConverter, + config: &ConversionConfig, + ) -> Duration { + for _ in 0..WARMUP_FRAMES { + let frame = create_frame(config.input_format, config.input_width, config.input_height); + let _ = converter.convert(frame).expect("warmup conversion failed"); + } + + let mut total = Duration::ZERO; + for _ in 0..MEASURED_FRAMES { + let frame = create_frame(config.input_format, config.input_width, config.input_height); + let start = Instant::now(); + let _ = converter.convert(frame).expect("conversion failed"); + total += start.elapsed(); + } + total + } + + fn pool_config(worker_count: usize) -> ConverterPoolConfig { + ConverterPoolConfig { + worker_count, + input_capacity: MAX_IN_FLIGHT * 2, + output_capacity: MAX_IN_FLIGHT * 2, + drop_strategy: DropStrategy::DropOldest, + frame_pool_capacity: MAX_IN_FLIGHT * 2, + } + } + + fn measure_pool(pool: AsyncConverterPool, config: &ConversionConfig) -> (Duration, Duration) { + let mut frames = (0..WARMUP_FRAMES + POOL_MEASURED_FRAMES) + .map(|_| create_frame(config.input_format, config.input_width, config.input_height)) + .collect::>(); + + for sequence in 0..WARMUP_FRAMES { + let frame = frames.pop().expect("warmup frame"); + pool.submit(frame, sequence as u64) + .expect("warmup submit failed"); + } + for _ in 0..WARMUP_FRAMES { + pool.recv_timeout(Duration::from_secs(5)) + .expect("warmup conversion timed out"); + } + + let start = Instant::now(); + let mut next_sequence = 0usize; + let mut in_flight = 0usize; + let mut received = 0usize; + let mut conversion_time = Duration::ZERO; + + while received < POOL_MEASURED_FRAMES { + while in_flight < MAX_IN_FLIGHT && next_sequence < POOL_MEASURED_FRAMES { + let frame = frames.pop().expect("measurement frame"); + pool.submit(frame, next_sequence as u64) + .expect("submit failed"); + next_sequence += 1; + in_flight += 1; + } + + let converted = pool + .recv_timeout(Duration::from_secs(5)) + .expect("conversion timed out"); + conversion_time += converted.conversion_duration; + in_flight -= 1; + received += 1; + } + + ( + start.elapsed(), + conversion_time / POOL_MEASURED_FRAMES as u32, + ) + } + + fn benchmark_pool( + converter: Arc, + config: &ConversionConfig, + worker_count: usize, + ) -> (Duration, Duration) { + measure_pool( + AsyncConverterPool::new(converter, pool_config(worker_count)), + config, + ) + } + + fn benchmark_pool_from_config( + config: &ConversionConfig, + worker_count: usize, + ) -> (Duration, Duration) { + let pool = AsyncConverterPool::from_config(config.clone(), pool_config(worker_count)) + .expect("converter pool from config"); + measure_pool(pool, config) + } + + fn print_pool_result(label: &str, (elapsed, conversion_time): (Duration, Duration)) { + let per_frame = elapsed / POOL_MEASURED_FRAMES as u32; + println!( + "{label}: wall {:?}/frame ({:.1} fps), convert {:?}", + per_frame, + 1.0 / per_frame.as_secs_f64(), + conversion_time + ); + } + + pub fn run() { + ffmpeg::init().expect("failed to initialize ffmpeg"); + let single_threaded_only = std::env::var_os("BENCH_SINGLE_ONLY").is_some(); + + println!("D3D11 frame conversion benchmark"); + println!("Input: {INPUT_FORMAT:?}, output: {OUTPUT_FORMAT:?}\n"); + + for (width, height) in RESOLUTIONS { + let config = conversion_config(width, height); + println!("--- {width}x{height} ---"); + + let swscale = SwscaleConverter::new(config.clone()).expect("swscale converter"); + let swscale_time = benchmark_single_threaded(&swscale, &config); + let swscale_per_frame = swscale_time / MEASURED_FRAMES as u32; + println!( + "swscale single-thread: {:>10?}/frame ({:>6.1} fps)", + swscale_per_frame, + 1.0 / swscale_per_frame.as_secs_f64() + ); + + match cap_frame_converter::D3D11Converter::new(config.clone()) { + Ok(d3d11) => { + let d3d11_time = benchmark_single_threaded(&d3d11, &config); + let d3d11_per_frame = d3d11_time / MEASURED_FRAMES as u32; + println!( + "d3d11 single-thread: {:>10?}/frame ({:>6.1} fps, {:.1}x)", + d3d11_per_frame, + 1.0 / d3d11_per_frame.as_secs_f64(), + swscale_per_frame.as_secs_f64() / d3d11_per_frame.as_secs_f64() + ); + + if !single_threaded_only { + let d3d11: Arc = Arc::new(d3d11); + for worker_count in [1, 2, 4] { + print_pool_result( + &format!("d3d11 pool ({worker_count} workers, shared)"), + benchmark_pool(Arc::clone(&d3d11), &config, worker_count), + ); + } + drop(d3d11); + + for worker_count in [1, 2, 4] { + print_pool_result( + &format!("d3d11 pool ({worker_count} workers, config)"), + benchmark_pool_from_config(&config, worker_count), + ); + } + } + } + Err(error) => println!("d3d11 unavailable: {error}"), + } + + if !single_threaded_only { + let swscale: Arc = Arc::new(swscale); + for worker_count in [1, 2, 4] { + print_pool_result( + &format!("swscale pool ({worker_count} workers)"), + benchmark_pool(Arc::clone(&swscale), &config, worker_count), + ); + } + } + + println!(); + } + } +} + +#[cfg(target_os = "windows")] +fn main() { + benchmark::run(); +} + +#[cfg(not(target_os = "windows"))] +fn main() { + println!("D3D11 benchmark is only available on Windows"); +} diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 863d57fdcb9..704c4a78d43 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -1,6 +1,6 @@ use crate::{ConversionConfig, ConvertError, ConverterBackend, FrameConverter}; use ffmpeg::{format::Pixel, frame}; -use parking_lot::Mutex; +use parking_lot::{Condvar, Mutex}; use std::{ mem::ManuallyDrop, ptr, @@ -288,24 +288,55 @@ fn select_best_gpu(gpus: &[GpuInfo]) -> Option { Some((*best).clone()) } +const STAGING_RING_DEPTH: usize = 2; + struct D3D11Resources { #[allow(dead_code)] device: ID3D11Device, context: ID3D11DeviceContext, - video_device: ID3D11VideoDevice, video_context: ID3D11VideoContext, processor: ID3D11VideoProcessor, - enumerator: ID3D11VideoProcessorEnumerator, + input_view: ID3D11VideoProcessorInputView, + output_view: ID3D11VideoProcessorOutputView, input_texture: ID3D11Texture2D, output_texture: ID3D11Texture2D, input_shared_handle: Option, output_shared_handle: Option, - staging_input: ID3D11Texture2D, - staging_output: ID3D11Texture2D, +} + +struct StagingTextures { + input: ID3D11Texture2D, + output: ID3D11Texture2D, +} + +struct StagingLease<'a> { + converter: &'a D3D11Converter, + staging: Option, +} + +impl std::ops::Deref for StagingLease<'_> { + type Target = StagingTextures; + + fn deref(&self) -> &Self::Target { + self.staging + .as_ref() + .expect("staging textures are present until the lease is dropped") + } +} + +impl Drop for StagingLease<'_> { + fn drop(&mut self) { + if let Some(staging) = self.staging.take() { + self.converter.staging_free.lock().push(staging); + self.converter.staging_available.notify_one(); + } + } } pub struct D3D11Converter { resources: Mutex, + staging_free: Mutex>, + staging_available: Condvar, input_format: Pixel, output_format: Pixel, input_width: u32, @@ -355,6 +386,60 @@ fn get_gpu_info(device: &ID3D11Device) -> Result { } } +fn create_input_view( + video_device: &ID3D11VideoDevice, + enumerator: &ID3D11VideoProcessorEnumerator, + texture: &ID3D11Texture2D, +) -> Result { + let desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { + FourCC: 0, + ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D, + Anonymous: windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 { + Texture2D: windows::Win32::Graphics::Direct3D11::D3D11_TEX2D_VPIV { + MipSlice: 0, + ArraySlice: 0, + }, + }, + }; + + unsafe { + let mut view: Option = None; + video_device + .CreateVideoProcessorInputView(texture, enumerator, &desc, Some(&mut view)) + .map_err(|e| { + ConvertError::ConversionFailed(format!("CreateInputView failed: {e:?}")) + })?; + view.ok_or_else(|| { + ConvertError::ConversionFailed("CreateInputView returned null".to_string()) + }) + } +} + +fn create_output_view( + video_device: &ID3D11VideoDevice, + enumerator: &ID3D11VideoProcessorEnumerator, + texture: &ID3D11Texture2D, +) -> Result { + let desc = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC { + ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D, + Anonymous: windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 { + Texture2D: windows::Win32::Graphics::Direct3D11::D3D11_TEX2D_VPOV { MipSlice: 0 }, + }, + }; + + unsafe { + let mut view: Option = None; + video_device + .CreateVideoProcessorOutputView(texture, enumerator, &desc, Some(&mut view)) + .map_err(|e| { + ConvertError::ConversionFailed(format!("CreateOutputView failed: {e:?}")) + })?; + view.ok_or_else(|| { + ConvertError::ConversionFailed("CreateOutputView returned null".to_string()) + }) + } +} + impl D3D11Converter { fn create_device_on_best_adapter() -> Result<(ID3D11Device, ID3D11DeviceContext), ConvertError> { @@ -549,25 +634,32 @@ impl D3D11Converter { D3D11_BIND_RENDER_TARGET.0 as u32, )?; - let staging_input = create_texture( - &device, - config.input_width, - config.input_height, - input_dxgi, - D3D11_USAGE_STAGING, - 0, - D3D11_CPU_ACCESS_WRITE.0 as u32, - )?; - - let staging_output = create_texture( - &device, - config.output_width, - config.output_height, - output_dxgi, - D3D11_USAGE_STAGING, - 0, - D3D11_CPU_ACCESS_READ.0 as u32, - )?; + let input_view = create_input_view(&video_device, &enumerator, &input_texture)?; + let output_view = create_output_view(&video_device, &enumerator, &output_texture)?; + + let mut staging_free = Vec::with_capacity(STAGING_RING_DEPTH); + for _ in 0..STAGING_RING_DEPTH { + staging_free.push(StagingTextures { + input: create_texture( + &device, + config.input_width, + config.input_height, + input_dxgi, + D3D11_USAGE_STAGING, + 0, + D3D11_CPU_ACCESS_WRITE.0 as u32, + )?, + output: create_texture( + &device, + config.output_width, + config.output_height, + output_dxgi, + D3D11_USAGE_STAGING, + 0, + D3D11_CPU_ACCESS_READ.0 as u32, + )?, + }); + } if input_shared_handle.is_some() && output_shared_handle.is_some() { tracing::info!("D3D11 converter created with shared texture handles enabled"); @@ -582,16 +674,14 @@ impl D3D11Converter { let resources = D3D11Resources { device, context, - video_device, video_context, processor, - enumerator, + input_view, + output_view, input_texture, output_texture, input_shared_handle, output_shared_handle, - staging_input, - staging_output, }; tracing::debug!( @@ -607,6 +697,8 @@ impl D3D11Converter { Ok(Self { resources: Mutex::new(resources), + staging_free: Mutex::new(staging_free), + staging_available: Condvar::new(), input_format: config.input_format, output_format: config.output_format, input_width: config.input_width, @@ -620,6 +712,18 @@ impl D3D11Converter { }) } + fn acquire_staging(&self) -> StagingLease<'_> { + let mut free = self.staging_free.lock(); + while free.is_empty() { + self.staging_available.wait(&mut free); + } + + StagingLease { + converter: self, + staging: free.pop(), + } + } + pub fn gpu_info(&self) -> &GpuInfo { &self.gpu_info } @@ -707,153 +811,111 @@ impl FrameConverter for D3D11Converter { } let pts = input.pts(); - let resources = self.resources.lock(); + let staging = self.acquire_staging(); - unsafe { + let (input_ptr, input_pitch) = { + let resources = self.resources.lock(); let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); - resources - .context - .Map( - &resources.staging_input, - 0, - D3D11_MAP_WRITE, - 0, - Some(&mut mapped), - ) - .map_err(|e| ConvertError::ConversionFailed(format!("Map input failed: {e:?}")))?; - - copy_frame_to_mapped(&input, mapped.pData as *mut u8, mapped.RowPitch as usize); - - resources.context.Unmap(&resources.staging_input, 0); - - resources - .context - .CopyResource(&resources.input_texture, &resources.staging_input); - - let input_view_desc = D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC { - FourCC: 0, - ViewDimension: D3D11_VPIV_DIMENSION_TEXTURE2D, - Anonymous: - windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC_0 { - Texture2D: windows::Win32::Graphics::Direct3D11::D3D11_TEX2D_VPIV { - MipSlice: 0, - ArraySlice: 0, - }, - }, - }; - - let mut input_view: Option = None; - resources - .video_device - .CreateVideoProcessorInputView( - &resources.input_texture, - &resources.enumerator, - &input_view_desc, - Some(&mut input_view), - ) - .map_err(|e| { - ConvertError::ConversionFailed(format!("CreateInputView failed: {e:?}")) - })?; - let input_view = input_view.ok_or_else(|| { - ConvertError::ConversionFailed("CreateInputView returned null".to_string()) - })?; - - let output_view_desc = D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC { - ViewDimension: D3D11_VPOV_DIMENSION_TEXTURE2D, - Anonymous: - windows::Win32::Graphics::Direct3D11::D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC_0 { - Texture2D: windows::Win32::Graphics::Direct3D11::D3D11_TEX2D_VPOV { - MipSlice: 0, - }, - }, - }; - - let mut output_view: Option = None; - resources - .video_device - .CreateVideoProcessorOutputView( - &resources.output_texture, - &resources.enumerator, - &output_view_desc, - Some(&mut output_view), - ) - .map_err(|e| { - ConvertError::ConversionFailed(format!("CreateOutputView failed: {e:?}")) - })?; - let output_view = output_view.ok_or_else(|| { - ConvertError::ConversionFailed("CreateOutputView returned null".to_string()) - })?; + unsafe { + resources + .context + .Map(&staging.input, 0, D3D11_MAP_WRITE, 0, Some(&mut mapped)) + .map_err(|e| { + ConvertError::ConversionFailed(format!("Map input failed: {e:?}")) + })?; + } + (mapped.pData as *mut u8, mapped.RowPitch as usize) + }; - let stream = D3D11_VIDEO_PROCESSOR_STREAM { - Enable: true.into(), - OutputIndex: 0, - InputFrameOrField: 0, - PastFrames: 0, - FutureFrames: 0, - ppPastSurfaces: ptr::null_mut(), - pInputSurface: std::mem::transmute_copy(&input_view), - ppFutureSurfaces: ptr::null_mut(), - ppPastSurfacesRight: ptr::null_mut(), - pInputSurfaceRight: ManuallyDrop::new(None), - ppFutureSurfacesRight: ptr::null_mut(), - }; + unsafe { + copy_frame_to_mapped(&input, input_ptr, input_pitch); + } - resources - .video_context - .VideoProcessorBlt(&resources.processor, &output_view, 0, &[stream]) - .map_err(|e| { - ConvertError::ConversionFailed(format!("VideoProcessorBlt failed: {e:?}")) - })?; + let (output_ptr, output_pitch) = { + let resources = self.resources.lock(); + + unsafe { + resources.context.Unmap(&staging.input, 0); + resources + .context + .CopyResource(&resources.input_texture, &staging.input); + + let stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + OutputIndex: 0, + InputFrameOrField: 0, + PastFrames: 0, + FutureFrames: 0, + ppPastSurfaces: ptr::null_mut(), + pInputSurface: std::mem::transmute_copy(&resources.input_view), + ppFutureSurfaces: ptr::null_mut(), + ppPastSurfacesRight: ptr::null_mut(), + pInputSurfaceRight: ManuallyDrop::new(None), + ppFutureSurfacesRight: ptr::null_mut(), + }; - if !self.verified_gpu_usage.swap(true, Ordering::Relaxed) { - tracing::info!( - "D3D11 VideoProcessorBlt succeeded - confirmed GPU hardware acceleration on {}", - self.gpu_info.description - ); + resources + .video_context + .VideoProcessorBlt(&resources.processor, &resources.output_view, 0, &[stream]) + .map_err(|e| { + ConvertError::ConversionFailed(format!("VideoProcessorBlt failed: {e:?}")) + })?; + + resources + .context + .CopyResource(&staging.output, &resources.output_texture); + + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + resources + .context + .Map(&staging.output, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) + .map_err(|e| { + ConvertError::ConversionFailed(format!("Map output failed: {e:?}")) + })?; + + (mapped.pData as *const u8, mapped.RowPitch as usize) } + }; - resources - .context - .CopyResource(&resources.staging_output, &resources.output_texture); - - let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); - resources - .context - .Map( - &resources.staging_output, - 0, - D3D11_MAP_READ, - 0, - Some(&mut mapped), - ) - .map_err(|e| ConvertError::ConversionFailed(format!("Map output failed: {e:?}")))?; - - copy_mapped_to_frame(mapped.pData as *const u8, mapped.RowPitch as usize, output); + unsafe { + copy_mapped_to_frame(output_ptr, output_pitch, output); + } - resources.context.Unmap(&resources.staging_output, 0); + { + let resources = self.resources.lock(); + unsafe { + resources.context.Unmap(&staging.output, 0); + } + } - output.set_pts(pts); + if !self.verified_gpu_usage.swap(true, Ordering::Relaxed) { + tracing::info!( + "D3D11 VideoProcessorBlt succeeded - confirmed GPU hardware acceleration on {}", + self.gpu_info.description + ); + } - let elapsed_ns = start.elapsed().as_nanos() as u64; - self.total_conversion_time_ns - .fetch_add(elapsed_ns, Ordering::Relaxed); + output.set_pts(pts); - let frame_count = count + 1; - if frame_count.is_multiple_of(300) { - let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); - let avg_ms = (total_ns as f64 / frame_count as f64) / 1_000_000.0; - tracing::debug!( - "D3D11 converter: {:.2}ms avg over {} frames ({:?} -> {:?}) on {}", - avg_ms, - frame_count, - self.input_format, - self.output_format, - self.gpu_info.description - ); - } + let elapsed_ns = start.elapsed().as_nanos() as u64; + self.total_conversion_time_ns + .fetch_add(elapsed_ns, Ordering::Relaxed); - Ok(()) + let frame_count = count + 1; + if frame_count.is_multiple_of(300) { + let total_ns = self.total_conversion_time_ns.load(Ordering::Relaxed); + let avg_ms = (total_ns as f64 / frame_count as f64) / 1_000_000.0; + tracing::debug!( + "D3D11 converter: {:.2}ms avg over {} frames ({:?} -> {:?}) on {}", + avg_ms, + frame_count, + self.input_format, + self.output_format, + self.gpu_info.description + ); } + + Ok(()) } fn name(&self) -> &'static str { @@ -1009,79 +1071,90 @@ fn extract_shared_handle(texture: &ID3D11Texture2D) -> Option { } } +unsafe fn copy_plane_rows( + src: *const u8, + src_stride: usize, + dst: *mut u8, + dst_stride: usize, + row_bytes: usize, + rows: usize, +) { + if src_stride == dst_stride { + unsafe { + ptr::copy_nonoverlapping(src, dst, src_stride * rows); + } + return; + } + + for y in 0..rows { + unsafe { + ptr::copy_nonoverlapping(src.add(y * src_stride), dst.add(y * dst_stride), row_bytes); + } + } +} + unsafe fn copy_frame_to_mapped(frame: &frame::Video, dst: *mut u8, dst_stride: usize) { let height = frame.height() as usize; let width = frame.width() as usize; let format = frame.format(); match format { - Pixel::NV12 => { - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - frame.data(0).as_ptr().add(y * frame.stride(0)), - dst.add(y * dst_stride), - width, - ); - } - } - let uv_offset = height * dst_stride; - for y in 0..height / 2 { - unsafe { - ptr::copy_nonoverlapping( - frame.data(1).as_ptr().add(y * frame.stride(1)), - dst.add(uv_offset + y * dst_stride), - width, - ); - } - } - } - Pixel::YUYV422 => { - let row_bytes = width * 2; - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - frame.data(0).as_ptr().add(y * frame.stride(0)), - dst.add(y * dst_stride), - row_bytes, - ); - } - } - } - Pixel::BGRA | Pixel::RGBA => { - let row_bytes = width * 4; - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - frame.data(0).as_ptr().add(y * frame.stride(0)), - dst.add(y * dst_stride), - row_bytes, - ); - } - } - } - Pixel::P010LE => { - let row_bytes = width * 2; - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - frame.data(0).as_ptr().add(y * frame.stride(0)), - dst.add(y * dst_stride), - row_bytes, - ); - } - } - let uv_offset = height * dst_stride; - for y in 0..height / 2 { - unsafe { - ptr::copy_nonoverlapping( - frame.data(1).as_ptr().add(y * frame.stride(1)), - dst.add(uv_offset + y * dst_stride), - row_bytes, - ); - } - } - } + Pixel::NV12 => unsafe { + copy_plane_rows( + frame.data(0).as_ptr(), + frame.stride(0), + dst, + dst_stride, + width, + height, + ); + copy_plane_rows( + frame.data(1).as_ptr(), + frame.stride(1), + dst.add(height * dst_stride), + dst_stride, + width, + height / 2, + ); + }, + Pixel::YUYV422 => unsafe { + copy_plane_rows( + frame.data(0).as_ptr(), + frame.stride(0), + dst, + dst_stride, + width * 2, + height, + ); + }, + Pixel::BGRA | Pixel::RGBA => unsafe { + copy_plane_rows( + frame.data(0).as_ptr(), + frame.stride(0), + dst, + dst_stride, + width * 4, + height, + ); + }, + Pixel::P010LE => unsafe { + copy_plane_rows( + frame.data(0).as_ptr(), + frame.stride(0), + dst, + dst_stride, + width * 2, + height, + ); + copy_plane_rows( + frame.data(1).as_ptr(), + frame.stride(1), + dst.add(height * dst_stride), + dst_stride, + width * 2, + height / 2, + ); + }, _ => {} } } @@ -1093,70 +1166,51 @@ unsafe fn copy_mapped_to_frame(src: *const u8, src_stride: usize, frame: &mut fr match format { Pixel::NV12 => { - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - src.add(y * src_stride), - frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), - width, - ); - } - } - let uv_offset = height * src_stride; - for y in 0..height / 2 { - unsafe { - ptr::copy_nonoverlapping( - src.add(uv_offset + y * src_stride), - frame.data_mut(1).as_mut_ptr().add(y * frame.stride(1)), - width, - ); - } + let stride0 = frame.stride(0); + let stride1 = frame.stride(1); + let dst0 = frame.data_mut(0).as_mut_ptr(); + let dst1 = frame.data_mut(1).as_mut_ptr(); + unsafe { + copy_plane_rows(src, src_stride, dst0, stride0, width, height); + copy_plane_rows( + src.add(height * src_stride), + src_stride, + dst1, + stride1, + width, + height / 2, + ); } } Pixel::YUYV422 => { - let row_bytes = width * 2; - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - src.add(y * src_stride), - frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), - row_bytes, - ); - } + let stride0 = frame.stride(0); + let dst0 = frame.data_mut(0).as_mut_ptr(); + unsafe { + copy_plane_rows(src, src_stride, dst0, stride0, width * 2, height); } } Pixel::BGRA | Pixel::RGBA => { - let row_bytes = width * 4; - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - src.add(y * src_stride), - frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), - row_bytes, - ); - } + let stride0 = frame.stride(0); + let dst0 = frame.data_mut(0).as_mut_ptr(); + unsafe { + copy_plane_rows(src, src_stride, dst0, stride0, width * 4, height); } } Pixel::P010LE => { - let row_bytes = width * 2; - for y in 0..height { - unsafe { - ptr::copy_nonoverlapping( - src.add(y * src_stride), - frame.data_mut(0).as_mut_ptr().add(y * frame.stride(0)), - row_bytes, - ); - } - } - let uv_offset = height * src_stride; - for y in 0..height / 2 { - unsafe { - ptr::copy_nonoverlapping( - src.add(uv_offset + y * src_stride), - frame.data_mut(1).as_mut_ptr().add(y * frame.stride(1)), - row_bytes, - ); - } + let stride0 = frame.stride(0); + let stride1 = frame.stride(1); + let dst0 = frame.data_mut(0).as_mut_ptr(); + let dst1 = frame.data_mut(1).as_mut_ptr(); + unsafe { + copy_plane_rows(src, src_stride, dst0, stride0, width * 2, height); + copy_plane_rows( + src.add(height * src_stride), + src_stride, + dst1, + stride1, + width * 2, + height / 2, + ); } } _ => {} @@ -1184,3 +1238,32 @@ impl Drop for D3D11Resources { } } } + +#[cfg(test)] +mod tests { + use super::copy_plane_rows; + + #[test] + fn copies_a_contiguous_plane_in_one_pass() { + let source = [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let mut destination = [0u8; 12]; + + unsafe { + copy_plane_rows(source.as_ptr(), 6, destination.as_mut_ptr(), 6, 4, 2); + } + + assert_eq!(destination, source); + } + + #[test] + fn copies_rows_when_strides_differ() { + let source = [1u8, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + let mut destination = [0u8; 14]; + + unsafe { + copy_plane_rows(source.as_ptr(), 6, destination.as_mut_ptr(), 7, 4, 2); + } + + assert_eq!(destination, [1, 2, 3, 4, 0, 0, 0, 7, 8, 9, 10, 0, 0, 0]); + } +} diff --git a/crates/frame-converter/src/pool.rs b/crates/frame-converter/src/pool.rs index ee8ac5a499f..d24f81a353b 100644 --- a/crates/frame-converter/src/pool.rs +++ b/crates/frame-converter/src/pool.rs @@ -99,6 +99,16 @@ impl AsyncConverterPool { ) -> Result { let first_converter = create_converter(conversion_config.clone())?; let converter_name = first_converter.name().to_string(); + let share_converter = { + #[cfg(target_os = "windows")] + { + first_converter.backend() == crate::ConverterBackend::D3D11 + } + #[cfg(not(target_os = "windows"))] + { + false + } + }; let frame_pool = if pool_config.frame_pool_capacity > 0 { Some(VideoFramePool::new( @@ -113,7 +123,7 @@ impl AsyncConverterPool { Self::new_with_factory( move |worker_id| { - if worker_id == 0 { + if worker_id == 0 || share_converter { Ok(Arc::clone(&first_converter)) } else { create_converter(conversion_config.clone()) From b2b12c506bcfe4ff9fdc48bc8fd6b0abcf008c14 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 16:29:59 +0200 Subject: [PATCH 06/16] test(frame-converter): add D3D11 benchmark controls Allow selecting the input format, worker counts, and single-thread, pool-only, or D3D11-only runs so the benchmark can isolate transfer, conversion, and concurrency costs. --- .../examples/d3d11-benchmark.rs | 75 +++++++++++++------ 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/crates/frame-converter/examples/d3d11-benchmark.rs b/crates/frame-converter/examples/d3d11-benchmark.rs index 5f9bebbe8b7..d5b7335f8d8 100644 --- a/crates/frame-converter/examples/d3d11-benchmark.rs +++ b/crates/frame-converter/examples/d3d11-benchmark.rs @@ -10,7 +10,6 @@ mod benchmark { time::{Duration, Instant}, }; - const INPUT_FORMAT: Pixel = Pixel::YUYV422; const OUTPUT_FORMAT: Pixel = Pixel::NV12; const RESOLUTIONS: [(u32, u32); 2] = [(1280, 720), (1920, 1080)]; const WARMUP_FRAMES: usize = 10; @@ -29,8 +28,17 @@ mod benchmark { frame } + fn input_format() -> Pixel { + match std::env::var("BENCH_INPUT").as_deref() { + Ok("nv12") => Pixel::NV12, + Ok("bgra") => Pixel::BGRA, + Ok("rgba") => Pixel::RGBA, + _ => Pixel::YUYV422, + } + } + fn conversion_config(width: u32, height: u32) -> ConversionConfig { - ConversionConfig::new(INPUT_FORMAT, width, height, OUTPUT_FORMAT, width, height) + ConversionConfig::new(input_format(), width, height, OUTPUT_FORMAT, width, height) } fn benchmark_single_threaded( @@ -136,40 +144,63 @@ mod benchmark { ); } + fn worker_counts() -> Vec { + std::env::var("BENCH_WORKERS") + .ok() + .and_then(|raw| { + let counts = raw + .split(',') + .filter_map(|value| value.trim().parse().ok()) + .collect::>(); + (!counts.is_empty()).then_some(counts) + }) + .unwrap_or_else(|| vec![1, 2, 4]) + } + pub fn run() { ffmpeg::init().expect("failed to initialize ffmpeg"); let single_threaded_only = std::env::var_os("BENCH_SINGLE_ONLY").is_some(); + let pool_only = std::env::var_os("BENCH_POOL_ONLY").is_some(); + let d3d11_only = std::env::var_os("BENCH_D3D11_ONLY").is_some(); + let worker_counts = worker_counts(); println!("D3D11 frame conversion benchmark"); - println!("Input: {INPUT_FORMAT:?}, output: {OUTPUT_FORMAT:?}\n"); + println!("Input: {:?}, output: {OUTPUT_FORMAT:?}\n", input_format()); for (width, height) in RESOLUTIONS { let config = conversion_config(width, height); println!("--- {width}x{height} ---"); let swscale = SwscaleConverter::new(config.clone()).expect("swscale converter"); - let swscale_time = benchmark_single_threaded(&swscale, &config); - let swscale_per_frame = swscale_time / MEASURED_FRAMES as u32; - println!( - "swscale single-thread: {:>10?}/frame ({:>6.1} fps)", - swscale_per_frame, - 1.0 / swscale_per_frame.as_secs_f64() - ); + let swscale_per_frame = if pool_only { + None + } else { + let swscale_time = benchmark_single_threaded(&swscale, &config); + let per_frame = swscale_time / MEASURED_FRAMES as u32; + println!( + "swscale single-thread: {:>10?}/frame ({:>6.1} fps)", + per_frame, + 1.0 / per_frame.as_secs_f64() + ); + Some(per_frame) + }; match cap_frame_converter::D3D11Converter::new(config.clone()) { Ok(d3d11) => { - let d3d11_time = benchmark_single_threaded(&d3d11, &config); - let d3d11_per_frame = d3d11_time / MEASURED_FRAMES as u32; - println!( - "d3d11 single-thread: {:>10?}/frame ({:>6.1} fps, {:.1}x)", - d3d11_per_frame, - 1.0 / d3d11_per_frame.as_secs_f64(), - swscale_per_frame.as_secs_f64() / d3d11_per_frame.as_secs_f64() - ); + if let Some(swscale_per_frame) = swscale_per_frame { + let d3d11_time = benchmark_single_threaded(&d3d11, &config); + let d3d11_per_frame = d3d11_time / MEASURED_FRAMES as u32; + println!( + "d3d11 single-thread: {:>10?}/frame ({:>6.1} fps, {:.1}x)", + d3d11_per_frame, + 1.0 / d3d11_per_frame.as_secs_f64(), + swscale_per_frame.as_secs_f64() / d3d11_per_frame.as_secs_f64() + ); + } if !single_threaded_only { let d3d11: Arc = Arc::new(d3d11); - for worker_count in [1, 2, 4] { + for &worker_count in &worker_counts { print_pool_result( &format!("d3d11 pool ({worker_count} workers, shared)"), benchmark_pool(Arc::clone(&d3d11), &config, worker_count), @@ -177,7 +208,7 @@ mod benchmark { } drop(d3d11); - for worker_count in [1, 2, 4] { + for &worker_count in &worker_counts { print_pool_result( &format!("d3d11 pool ({worker_count} workers, config)"), benchmark_pool_from_config(&config, worker_count), @@ -188,9 +219,9 @@ mod benchmark { Err(error) => println!("d3d11 unavailable: {error}"), } - if !single_threaded_only { + if !single_threaded_only && !d3d11_only { let swscale: Arc = Arc::new(swscale); - for worker_count in [1, 2, 4] { + for &worker_count in &worker_counts { print_pool_result( &format!("swscale pool ({worker_count} workers)"), benchmark_pool(Arc::clone(&swscale), &config, worker_count), From 046e7c0e23724864819d8cd15ce1e140ad4526d8 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 16:37:34 +0200 Subject: [PATCH 07/16] perf(frame-converter): poll D3D11 readback without holding the context Map the staging output with DO_NOT_WAIT and release the shared GPU mutex between retries so other camera workers can submit their blits and copies while one readback completes. The 4-worker 1080p pool drops from about 3.6 ms/frame to 2.2 ms/frame on the GTX 950M. --- crates/frame-converter/src/d3d11.rs | 48 ++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 704c4a78d43..3b66be7a560 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -16,8 +16,8 @@ use windows::{ Graphics::{ Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, - D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_READ, D3D11_MAP_WRITE, - D3D11_MAPPED_SUBRESOURCE, D3D11_RESOURCE_MISC_SHARED, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_MAP_FLAG_DO_NOT_WAIT, D3D11_MAP_READ, + D3D11_MAP_WRITE, D3D11_MAPPED_SUBRESOURCE, D3D11_RESOURCE_MISC_SHARED, D3D11_RESOURCE_MISC_SHARED_NTHANDLE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11_VIDEO_PROCESSOR_CONTENT_DESC, D3D11_VIDEO_PROCESSOR_INPUT_VIEW_DESC, D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC, @@ -32,8 +32,9 @@ use windows::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_P010, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_FORMAT_YUY2, }, - CreateDXGIFactory1, DXGI_SHARED_RESOURCE_READ, DXGI_SHARED_RESOURCE_WRITE, - IDXGIAdapter, IDXGIDevice, IDXGIFactory1, IDXGIResource1, + CreateDXGIFactory1, DXGI_ERROR_WAS_STILL_DRAWING, DXGI_SHARED_RESOURCE_READ, + DXGI_SHARED_RESOURCE_WRITE, IDXGIAdapter, IDXGIDevice, IDXGIFactory1, + IDXGIResource1, }, }, }, @@ -832,7 +833,7 @@ impl FrameConverter for D3D11Converter { } let (output_ptr, output_pitch) = { - let resources = self.resources.lock(); + let mut resources = self.resources.lock(); unsafe { resources.context.Unmap(&staging.input, 0); @@ -865,13 +866,38 @@ impl FrameConverter for D3D11Converter { .context .CopyResource(&staging.output, &resources.output_texture); + // Polling releases the shared context between retries so other + // workers can submit GPU work while this readback completes. let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); - resources - .context - .Map(&staging.output, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) - .map_err(|e| { - ConvertError::ConversionFailed(format!("Map output failed: {e:?}")) - })?; + loop { + match resources.context.Map( + &staging.output, + 0, + D3D11_MAP_READ, + D3D11_MAP_FLAG_DO_NOT_WAIT.0 as u32, + Some(&mut mapped), + ) { + Ok(()) => break, + Err(e) if e.code() == DXGI_ERROR_WAS_STILL_DRAWING => { + drop(resources); + std::thread::yield_now(); + resources = self.resources.lock(); + } + Err(_) => { + drop(resources); + resources = self.resources.lock(); + resources + .context + .Map(&staging.output, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) + .map_err(|e| { + ConvertError::ConversionFailed(format!( + "Map output failed: {e:?}" + )) + })?; + break; + } + } + } (mapped.pData as *const u8, mapped.RowPitch as usize) } From 17f81e42bd37b202623ba41c9eab4e829d91d789 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 16:40:05 +0200 Subject: [PATCH 08/16] test(frame-converter): report the selected D3D11 adapter --- crates/frame-converter/examples/d3d11-benchmark.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/frame-converter/examples/d3d11-benchmark.rs b/crates/frame-converter/examples/d3d11-benchmark.rs index d5b7335f8d8..0f8567c03b7 100644 --- a/crates/frame-converter/examples/d3d11-benchmark.rs +++ b/crates/frame-converter/examples/d3d11-benchmark.rs @@ -187,6 +187,7 @@ mod benchmark { match cap_frame_converter::D3D11Converter::new(config.clone()) { Ok(d3d11) => { + println!("d3d11 gpu: {}", d3d11.gpu_info().description); if let Some(swscale_per_frame) = swscale_per_frame { let d3d11_time = benchmark_single_threaded(&d3d11, &config); let d3d11_per_frame = d3d11_time / MEASURED_FRAMES as u32; From be2b60bc36113e479acaee8ab3a19b8e665706f2 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 16:44:40 +0200 Subject: [PATCH 09/16] perf(frame-converter): size the D3D11 staging ring for four workers Four staging slots match the default camera pool so workers spend less time waiting for an upload/readback pair; deeper rings showed no additional gain. --- crates/frame-converter/src/d3d11.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 3b66be7a560..b68cf847671 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -289,7 +289,7 @@ fn select_best_gpu(gpus: &[GpuInfo]) -> Option { Some((*best).clone()) } -const STAGING_RING_DEPTH: usize = 2; +const STAGING_RING_DEPTH: usize = 4; struct D3D11Resources { #[allow(dead_code)] From ffb679cce6d2e92279f22831546d9bef6fc406bf Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 17:22:41 +0200 Subject: [PATCH 10/16] perf(recording): give capture D3D11 devices video support Media Foundation hardware encoders need a video-capable D3D11 device to attach their DXGI device manager. Without it the texture-native screen encoder falls back to FFmpeg and the full-frame readback path. Keep the display-notch tests below the platform helpers so clippy accepts the module layout. --- crates/recording/src/capture_pipeline.rs | 114 ++++++++++++----------- 1 file changed, 58 insertions(+), 56 deletions(-) diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 7c5da374acd..486a936ca3c 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -633,60 +633,6 @@ fn resolve_area_display_notch( }) } -#[cfg(test)] -mod display_notch_tests { - use super::*; - use scap_targets::bounds::{LogicalBounds, LogicalPosition, LogicalSize}; - - const NOTCH: scap_targets::NotchGeometry = scap_targets::NotchGeometry { - x: 0.4, - width: 0.2, - height: 0.1, - }; - - fn display_size() -> LogicalSize { - LogicalSize::new(1_000.0, 800.0) - } - - #[test] - fn area_containing_the_full_notch_rebases_it() { - let bounds = LogicalBounds::new( - LogicalPosition::new(300.0, 0.0), - LogicalSize::new(400.0, 200.0), - ); - - assert_eq!( - resolve_area_display_notch(NOTCH, display_size(), bounds), - Some(cap_project::DisplayNotch { - x: 0.25, - width: 0.5, - height: 0.4, - }) - ); - } - - #[test] - fn partially_intersected_notches_are_omitted() { - let horizontal = LogicalBounds::new( - LogicalPosition::new(500.0, 0.0), - LogicalSize::new(200.0, 200.0), - ); - let vertical = LogicalBounds::new( - LogicalPosition::new(300.0, 40.0), - LogicalSize::new(400.0, 200.0), - ); - - assert_eq!( - resolve_area_display_notch(NOTCH, display_size(), horizontal), - None - ); - assert_eq!( - resolve_area_display_notch(NOTCH, display_size(), vertical), - None - ); - } -} - #[cfg(windows)] pub fn create_d3d_device() -> windows::core::Result { @@ -697,9 +643,11 @@ pub fn create_d3d_device() let mut device = None; let flags = { - use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_BGRA_SUPPORT; + use windows::Win32::Graphics::Direct3D11::{ + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + }; - let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; if cfg!(feature = "d3ddebug") { use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_DEBUG; @@ -784,3 +732,57 @@ pub fn create_d3d_device() Ok(device.unwrap()) } + +#[cfg(test)] +mod display_notch_tests { + use super::*; + use scap_targets::bounds::{LogicalBounds, LogicalPosition, LogicalSize}; + + const NOTCH: scap_targets::NotchGeometry = scap_targets::NotchGeometry { + x: 0.4, + width: 0.2, + height: 0.1, + }; + + fn display_size() -> LogicalSize { + LogicalSize::new(1_000.0, 800.0) + } + + #[test] + fn area_containing_the_full_notch_rebases_it() { + let bounds = LogicalBounds::new( + LogicalPosition::new(300.0, 0.0), + LogicalSize::new(400.0, 200.0), + ); + + assert_eq!( + resolve_area_display_notch(NOTCH, display_size(), bounds), + Some(cap_project::DisplayNotch { + x: 0.25, + width: 0.5, + height: 0.4, + }) + ); + } + + #[test] + fn partially_intersected_notches_are_omitted() { + let horizontal = LogicalBounds::new( + LogicalPosition::new(500.0, 0.0), + LogicalSize::new(200.0, 200.0), + ); + let vertical = LogicalBounds::new( + LogicalPosition::new(300.0, 40.0), + LogicalSize::new(400.0, 200.0), + ); + + assert_eq!( + resolve_area_display_notch(NOTCH, display_size(), horizontal), + None + ); + assert_eq!( + resolve_area_display_notch(NOTCH, display_size(), vertical), + None + ); + } +} From 83d499134e046aabc88d18d4f938782cd4d08987 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 17:22:42 +0200 Subject: [PATCH 11/16] test(recording): instrument Windows capture encoder stages Time the Media Foundation encoder wait/input/process/output stages, the native camera upload/mux stages, and the screen D3D11 readback itself instead of including pacer waits in the conversion metric. Also make the camera benchmark run on Windows and honor an externally supplied RUST_LOG. --- crates/enc-mediafoundation/src/video/h264.rs | 26 +++ crates/recording/examples/camera-benchmark.rs | 6 +- .../recording/examples/recording-benchmark.rs | 4 +- crates/recording/src/output_pipeline/win.rs | 23 ++- .../src/output_pipeline/win_fragmented_m4s.rs | 151 ++++++++++-------- 5 files changed, 135 insertions(+), 75 deletions(-) diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 9d34e047605..6278e1c8a1c 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -664,6 +664,11 @@ impl H264Encoder { mut on_sample: impl FnMut(IMFSample) -> windows::core::Result<()>, ) -> Result { let mut health_monitor = EncoderHealthMonitor::new(); + let mut frames = 0u64; + let mut wait_ns = 0u64; + let mut input_ns = 0u64; + let mut process_ns = 0u64; + let mut output_ns = 0u64; unsafe { self.transform @@ -689,13 +694,17 @@ impl H264Encoder { }); } + let wait_started = Instant::now(); let event = self.event_generator.GetEvent(MF_EVENT_FLAG_NONE)?; + wait_ns += wait_started.elapsed().as_nanos() as u64; let event_type = MF_EVENT_TYPE(event.GetType()? as i32); match event_type { MediaFoundation::METransformNeedInput => { health_monitor.record_input(); + frames += 1; should_exit = true; + let input_started = Instant::now(); let get_frame_started = Instant::now(); if !should_stop.load(Ordering::SeqCst) && let Some((texture, timestamp)) = get_frame()? @@ -709,6 +718,7 @@ impl H264Encoder { health_monitor.notify_input_stall_recovered(); } + let process_started = Instant::now(); let process_result = (|| -> windows::core::Result<()> { self.video_processor.process_texture(&texture)?; let input_buffer = MFCreateDXGISurfaceBuffer( @@ -724,6 +734,7 @@ impl H264Encoder { .ProcessInput(self.input_stream_id, &mf_sample, 0)?; Ok(()) })(); + process_ns += process_started.elapsed().as_nanos() as u64; match process_result { Ok(()) => { @@ -750,8 +761,22 @@ impl H264Encoder { } } } + + input_ns += input_started.elapsed().as_nanos() as u64; + if frames.is_multiple_of(60) { + let frame_count = frames as f64; + tracing::debug!( + frames, + wait_ms = wait_ns as f64 / frame_count / 1_000_000.0, + input_ms = input_ns as f64 / frame_count / 1_000_000.0, + process_ms = process_ns as f64 / frame_count / 1_000_000.0, + output_ms = output_ns as f64 / frame_count / 1_000_000.0, + "MF H264 loop timings" + ); + } } MediaFoundation::METransformHaveOutput => { + let output_started = Instant::now(); let mut status = 0; let output_buffer = MFT_OUTPUT_DATA_BUFFER { dwStreamID: self.output_stream_id, @@ -778,6 +803,7 @@ impl H264Encoder { }); } } + output_ns += output_started.elapsed().as_nanos() as u64; } _ => { panic!("Unknown media event type: {}", event_type.0); diff --git a/crates/recording/examples/camera-benchmark.rs b/crates/recording/examples/camera-benchmark.rs index e4fa6080821..dfcd74c7249 100644 --- a/crates/recording/examples/camera-benchmark.rs +++ b/crates/recording/examples/camera-benchmark.rs @@ -155,8 +155,10 @@ async fn run_camera_encoding_benchmark( None }; + let output_path = + std::env::temp_dir().join(format!("cap-camera-benchmark-{}.mp4", std::process::id())); let mut output = - ffmpeg::format::output_as("/dev/null", "mp4").expect("Failed to create dummy output"); + ffmpeg::format::output_as(&output_path, "mp4").expect("Failed to create dummy output"); let video_info = cap_media_info::VideoInfo { width, height, @@ -315,6 +317,8 @@ async fn run_camera_encoding_benchmark( metrics.stop(); let _ = encoder.flush(&mut output); let _ = output.write_trailer(); + drop(output); + let _ = std::fs::remove_file(&output_path); metrics.snapshot() } diff --git a/crates/recording/examples/recording-benchmark.rs b/crates/recording/examples/recording-benchmark.rs index 55f7531df98..03a85a81967 100644 --- a/crates/recording/examples/recording-benchmark.rs +++ b/crates/recording/examples/recording-benchmark.rs @@ -219,7 +219,9 @@ async fn stress_test_recording( #[tokio::main] async fn main() -> Result<(), Box> { - unsafe { std::env::set_var("RUST_LOG", "info") }; + if std::env::var_os("RUST_LOG").is_none() { + unsafe { std::env::set_var("RUST_LOG", "info") }; + } tracing_subscriber::fmt::init(); println!("=== Cap Recording Benchmark ===\n"); diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 8d2fc226df9..36c6f1a69f8 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -13,13 +13,14 @@ use cap_media_info::{AudioInfo, VideoInfo}; use cap_timestamp::Timestamp; use futures::channel::oneshot; use std::{ + cell::Cell, path::PathBuf, sync::{ Arc, Mutex, atomic::AtomicBool, mpsc::{SyncSender, TrySendError, sync_channel}, }, - time::Duration, + time::{Duration, Instant}, }; use tracing::*; @@ -897,6 +898,8 @@ impl Muxer for WindowsCameraMuxer { let mut first_timestamp: Option = None; let mut frame_count = 0u64; let mut camera_buffers = CameraBuffers::new(); + let upload_ns = Cell::new(0u64); + let mux_ns = Cell::new(0u64); let result = encoder.run( Arc::new(AtomicBool::default()), @@ -918,11 +921,21 @@ impl Muxer for WindowsCameraMuxer { let normalized_ts = normalize_camera_timestamp(timestamp, &mut first_timestamp); frame_count += 1; - if frame_count.is_multiple_of(30) { - debug!("Windows camera encoder: processed {} frames", frame_count); - } + let upload_start = Instant::now(); let texture = upload_mf_buffer_to_texture(&d3d_device, frame, &mut camera_buffers)?; + upload_ns + .set(upload_ns.get() + upload_start.elapsed().as_nanos() as u64); + if frame_count.is_multiple_of(30) { + let upload_avg = + upload_ns.get() as f64 / frame_count as f64 / 1_000_000.0; + let mux_avg = + mux_ns.get() as f64 / frame_count as f64 / 1_000_000.0; + debug!( + "Windows camera encoder: processed {} frames (upload {:.3}ms, mux {:.3}ms)", + frame_count, upload_avg, mux_avg + ); + } return Ok(Some((texture, duration_to_timespan(normalized_ts)))); }, |output_sample| { @@ -936,10 +949,12 @@ impl Muxer for WindowsCameraMuxer { )); } }; + let mux_start = Instant::now(); if let Err(e) = muxer.write_sample(&output_sample, &mut output) { tracing::error!("Camera WriteSample failed: {e}"); } + mux_ns.set(mux_ns.get() + mux_start.elapsed().as_nanos() as u64); Ok(()) }, ); diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index 4e61d2300ed..52a5b651901 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -479,96 +479,109 @@ impl WindowsFragmentedM4SMuxer { break; } - let convert_start = std::time::Instant::now(); - - let (ffmpeg_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) { - Paced::Frame(frame, ts) => match frame.as_ffmpeg() { - Ok(f) => { - last_ffmpeg_frame = Some(reference_video_frame(&f)); - (Some(f), ts) - } - Err(e) => { - let readback_miss = matches!( - e, - scap_ffmpeg::AsFFmpegError::NotReady - | scap_ffmpeg::AsFFmpegError::StagingBusy - ); - if !readback_miss { - warn!("Failed to convert D3D11 frame: {e:?}"); - } else { - readback_misses += 1; - if readback_misses == 1 || readback_misses.is_multiple_of(90) { - warn!( - readback_misses, - "Screen readback is missing deadlines; the encoder is duplicating the last frame" + let (ffmpeg_frame, timestamp, readback_elapsed) = + match pacer.next(&video_rx, &pause_flag) { + Paced::Frame(frame, ts) => { + let readback_start = std::time::Instant::now(); + let ffmpeg_frame = frame.as_ffmpeg(); + let readback_elapsed = readback_start.elapsed(); + match ffmpeg_frame { + Ok(f) => { + last_ffmpeg_frame = Some(reference_video_frame(&f)); + (Some(f), ts, Some(readback_elapsed)) + } + Err(e) => { + let readback_miss = matches!( + e, + scap_ffmpeg::AsFFmpegError::NotReady + | scap_ffmpeg::AsFFmpegError::StagingBusy ); + if !readback_miss { + warn!("Failed to convert D3D11 frame: {e:?}"); + } else { + readback_misses += 1; + if readback_misses == 1 + || readback_misses.is_multiple_of(90) + { + warn!( + readback_misses, + "Screen readback is missing deadlines; the encoder is duplicating the last frame" + ); + } + } + duplicated_frames += 1; + ( + last_ffmpeg_frame.as_ref().map(reference_video_frame), + ts, + Some(readback_elapsed), + ) } } + } + Paced::Repeat(ts) => { + let Some(f) = &last_ffmpeg_frame else { + continue; + }; duplicated_frames += 1; - (last_ffmpeg_frame.as_ref().map(reference_video_frame), ts) + (Some(reference_video_frame(f)), ts, None) } - }, - Paced::Repeat(ts) => { - let Some(f) = &last_ffmpeg_frame else { - continue; - }; - duplicated_frames += 1; - (Some(reference_video_frame(f)), ts) - } - Paced::End => { - let mut drained = 0u64; - let drain_start = std::time::Instant::now(); - let drain_timeout = Duration::from_millis(500); - - loop { - match video_rx.recv_timeout(Duration::from_millis(10)) { - Ok(Some((frame, ts))) => { - if let Ok(f) = frame.as_ffmpeg() { - let normalized_ts = - normalize_timestamp(ts, &mut first_timestamp); - if let Err(e) = encode_frame_fn( - f, - normalized_ts, - &mut slow_encode_count, - &mut total_frames, - &encoder_clone, - ) { - warn!("Failed to encode drained frame: {e}"); - break; + Paced::End => { + let mut drained = 0u64; + let drain_start = std::time::Instant::now(); + let drain_timeout = Duration::from_millis(500); + + loop { + match video_rx.recv_timeout(Duration::from_millis(10)) { + Ok(Some((frame, ts))) => { + if let Ok(f) = frame.as_ffmpeg() { + let normalized_ts = + normalize_timestamp(ts, &mut first_timestamp); + if let Err(e) = encode_frame_fn( + f, + normalized_ts, + &mut slow_encode_count, + &mut total_frames, + &encoder_clone, + ) { + warn!("Failed to encode drained frame: {e}"); + break; + } + drained += 1; } - drained += 1; } - } - Ok(None) => break, - Err(RecvTimeoutError::Timeout) => { - if drain_start.elapsed() > drain_timeout { - break; + Ok(None) => break, + Err(RecvTimeoutError::Timeout) => { + if drain_start.elapsed() > drain_timeout { + break; + } } + Err(RecvTimeoutError::Disconnected) => break, } - Err(RecvTimeoutError::Disconnected) => break, } - } - if drained > 0 { - debug!(drained = drained, "Drained remaining frames before exit"); + if drained > 0 { + debug!( + drained = drained, + "Drained remaining frames before exit" + ); + } + break; } - break; - } - }; - - let convert_elapsed_ms = convert_start.elapsed().as_millis(); + }; let Some(ffmpeg_frame) = ffmpeg_frame else { continue; }; - if convert_elapsed_ms > SLOW_THRESHOLD_MS { + if let Some(readback_elapsed) = readback_elapsed + && readback_elapsed.as_millis() > SLOW_THRESHOLD_MS + { slow_convert_count += 1; if slow_convert_count <= 5 || slow_convert_count.is_multiple_of(100) { debug!( - elapsed_ms = convert_elapsed_ms, + elapsed_ms = readback_elapsed.as_millis(), count = slow_convert_count, - "D3D11 frame conversion exceeded {}ms threshold", + "D3D11 frame readback exceeded {}ms threshold", SLOW_THRESHOLD_MS ); } From 19a7c69bea3e510a94e73ee677478c39536948f7 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 17:54:52 +0200 Subject: [PATCH 12/16] fix(recording): correct encoder and readback timing metrics --- crates/enc-mediafoundation/src/video/h264.rs | 111 +++++++++--------- .../src/output_pipeline/win_fragmented_m4s.rs | 37 +++--- 2 files changed, 78 insertions(+), 70 deletions(-) diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 6278e1c8a1c..7f529b460b9 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -666,7 +666,7 @@ impl H264Encoder { let mut health_monitor = EncoderHealthMonitor::new(); let mut frames = 0u64; let mut wait_ns = 0u64; - let mut input_ns = 0u64; + let mut get_frame_ns = 0u64; let mut process_ns = 0u64; let mut output_ns = 0u64; @@ -704,71 +704,74 @@ impl H264Encoder { health_monitor.record_input(); frames += 1; should_exit = true; - let input_started = Instant::now(); let get_frame_started = Instant::now(); - if !should_stop.load(Ordering::SeqCst) - && let Some((texture, timestamp)) = get_frame()? - { - // `get_frame` blocks for the whole pause (frames are - // dropped upstream while paused), so a long park here - // is idle time, not a hardware stall. Rebase the - // timeout clock so the next `check_health` at the top - // of the loop doesn't false-trip `Timeout` on resume. - if get_frame_started.elapsed() > INPUT_WAIT_HEALTH_RESET { - health_monitor.notify_input_stall_recovered(); - } - - let process_started = Instant::now(); - let process_result = (|| -> windows::core::Result<()> { - self.video_processor.process_texture(&texture)?; - let input_buffer = MFCreateDXGISurfaceBuffer( - &ID3D11Texture2D::IID, - self.video_processor.output_texture(), - 0, - false, - )?; - let mf_sample = MFCreateSample()?; - mf_sample.AddBuffer(&input_buffer)?; - mf_sample.SetSampleTime(timestamp.Duration)?; - self.transform - .ProcessInput(self.input_stream_id, &mf_sample, 0)?; - Ok(()) - })(); - process_ns += process_started.elapsed().as_nanos() as u64; - - match process_result { - Ok(()) => { - health_monitor.reset_process_failures(); - should_exit = false; + if !should_stop.load(Ordering::SeqCst) { + let frame = get_frame()?; + get_frame_ns += get_frame_started.elapsed().as_nanos() as u64; + if let Some((texture, timestamp)) = frame { + // `get_frame` blocks for the whole pause (frames are + // dropped upstream while paused), so a long park here + // is idle time, not a hardware stall. Rebase the + // timeout clock so the next `check_health` at the top + // of the loop doesn't false-trip `Timeout` on resume. + if get_frame_started.elapsed() > INPUT_WAIT_HEALTH_RESET { + health_monitor.notify_input_stall_recovered(); } - Err(_) => { - health_monitor.record_process_failure(); - let health_status = health_monitor.check_health(); - if !health_status.is_healthy - && let Some(reason) = health_status.failure_reason - { - let _ = self.cleanup_encoder(); - return Err(EncoderRuntimeError::EncoderUnhealthy { - reason, - inputs_without_output: health_status - .inputs_without_output, - process_failures: health_status - .consecutive_process_failures, - frames_encoded: health_status.total_frames_encoded, - }); + + let process_started = Instant::now(); + let process_result = (|| -> windows::core::Result<()> { + self.video_processor.process_texture(&texture)?; + let input_buffer = MFCreateDXGISurfaceBuffer( + &ID3D11Texture2D::IID, + self.video_processor.output_texture(), + 0, + false, + )?; + let mf_sample = MFCreateSample()?; + mf_sample.AddBuffer(&input_buffer)?; + mf_sample.SetSampleTime(timestamp.Duration)?; + self.transform.ProcessInput( + self.input_stream_id, + &mf_sample, + 0, + )?; + Ok(()) + })(); + process_ns += process_started.elapsed().as_nanos() as u64; + + match process_result { + Ok(()) => { + health_monitor.reset_process_failures(); + should_exit = false; + } + Err(_) => { + health_monitor.record_process_failure(); + let health_status = health_monitor.check_health(); + if !health_status.is_healthy + && let Some(reason) = health_status.failure_reason + { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::EncoderUnhealthy { + reason, + inputs_without_output: health_status + .inputs_without_output, + process_failures: health_status + .consecutive_process_failures, + frames_encoded: health_status.total_frames_encoded, + }); + } + should_exit = false; } - should_exit = false; } } } - input_ns += input_started.elapsed().as_nanos() as u64; if frames.is_multiple_of(60) { let frame_count = frames as f64; tracing::debug!( frames, wait_ms = wait_ns as f64 / frame_count / 1_000_000.0, - input_ms = input_ns as f64 / frame_count / 1_000_000.0, + get_frame_ms = get_frame_ns as f64 / frame_count / 1_000_000.0, process_ms = process_ns as f64 / frame_count / 1_000_000.0, output_ms = output_ns as f64 / frame_count / 1_000_000.0, "MF H264 loop timings" diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index 52a5b651901..af6d1b00fde 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -423,6 +423,7 @@ impl WindowsFragmentedM4SMuxer { let mut slow_encode_count = 0u32; let mut total_frames = 0u64; let mut duplicated_frames = 0u64; + let mut readback_frames = 0u64; let mut readback_misses = 0u64; let mut disk_monitor = DiskSpaceMonitor::new(); let mut disk_exhausted = false; @@ -573,17 +574,18 @@ impl WindowsFragmentedM4SMuxer { continue; }; - if let Some(readback_elapsed) = readback_elapsed - && readback_elapsed.as_millis() > SLOW_THRESHOLD_MS - { - slow_convert_count += 1; - if slow_convert_count <= 5 || slow_convert_count.is_multiple_of(100) { - debug!( - elapsed_ms = readback_elapsed.as_millis(), - count = slow_convert_count, - "D3D11 frame readback exceeded {}ms threshold", - SLOW_THRESHOLD_MS - ); + if let Some(readback_elapsed) = readback_elapsed { + readback_frames += 1; + if readback_elapsed.as_millis() > SLOW_THRESHOLD_MS { + slow_convert_count += 1; + if slow_convert_count <= 5 || slow_convert_count.is_multiple_of(100) { + debug!( + elapsed_ms = readback_elapsed.as_millis(), + count = slow_convert_count, + "D3D11 frame readback exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); + } } } @@ -601,15 +603,18 @@ impl WindowsFragmentedM4SMuxer { } if total_frames > 0 { + let readback_slow_pct = if readback_frames > 0 { + 100.0 * slow_convert_count as f64 / readback_frames as f64 + } else { + 0.0 + }; debug!( total_frames = total_frames, duplicated_frames = duplicated_frames, - slow_converts = slow_convert_count, + slow_readbacks = slow_convert_count, + readback_frames = readback_frames, slow_encodes = slow_encode_count, - slow_convert_pct = format!( - "{:.1}%", - 100.0 * slow_convert_count as f64 / total_frames as f64 - ), + slow_readback_pct = format!("{readback_slow_pct:.1}%"), slow_encode_pct = format!( "{:.1}%", 100.0 * slow_encode_count as f64 / total_frames as f64 From e8333db6e8b57a776e3fc1f002af0f21d3be7ba8 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 17:54:53 +0200 Subject: [PATCH 13/16] perf(frame-converter): allocate D3D11 staging textures lazily --- crates/frame-converter/src/d3d11.rs | 102 ++++++++++++++++++---------- crates/frame-converter/src/pool.rs | 3 + 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index b68cf847671..8dbbfe302fd 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -310,6 +310,11 @@ struct StagingTextures { output: ID3D11Texture2D, } +struct StagingPool { + free: Vec, + created: usize, +} + struct StagingLease<'a> { converter: &'a D3D11Converter, staging: Option, @@ -328,15 +333,16 @@ impl std::ops::Deref for StagingLease<'_> { impl Drop for StagingLease<'_> { fn drop(&mut self) { if let Some(staging) = self.staging.take() { - self.converter.staging_free.lock().push(staging); + self.converter.staging_pool.lock().free.push(staging); self.converter.staging_available.notify_one(); } } } pub struct D3D11Converter { + device: ID3D11Device, resources: Mutex, - staging_free: Mutex>, + staging_pool: Mutex, staging_available: Condvar, input_format: Pixel, output_format: Pixel, @@ -638,29 +644,7 @@ impl D3D11Converter { let input_view = create_input_view(&video_device, &enumerator, &input_texture)?; let output_view = create_output_view(&video_device, &enumerator, &output_texture)?; - let mut staging_free = Vec::with_capacity(STAGING_RING_DEPTH); - for _ in 0..STAGING_RING_DEPTH { - staging_free.push(StagingTextures { - input: create_texture( - &device, - config.input_width, - config.input_height, - input_dxgi, - D3D11_USAGE_STAGING, - 0, - D3D11_CPU_ACCESS_WRITE.0 as u32, - )?, - output: create_texture( - &device, - config.output_width, - config.output_height, - output_dxgi, - D3D11_USAGE_STAGING, - 0, - D3D11_CPU_ACCESS_READ.0 as u32, - )?, - }); - } + let staging_device = device.clone(); if input_shared_handle.is_some() && output_shared_handle.is_some() { tracing::info!("D3D11 converter created with shared texture handles enabled"); @@ -697,8 +681,12 @@ impl D3D11Converter { ); Ok(Self { + device: staging_device, resources: Mutex::new(resources), - staging_free: Mutex::new(staging_free), + staging_pool: Mutex::new(StagingPool { + free: Vec::new(), + created: 0, + }), staging_available: Condvar::new(), input_format: config.input_format, output_format: config.output_format, @@ -713,18 +701,60 @@ impl D3D11Converter { }) } - fn acquire_staging(&self) -> StagingLease<'_> { - let mut free = self.staging_free.lock(); - while free.is_empty() { - self.staging_available.wait(&mut free); - } + fn acquire_staging(&self) -> Result, ConvertError> { + let mut pool = self.staging_pool.lock(); + loop { + if let Some(staging) = pool.free.pop() { + return Ok(StagingLease { + converter: self, + staging: Some(staging), + }); + } + + if pool.created < STAGING_RING_DEPTH { + pool.created += 1; + drop(pool); + return match self.create_staging_pair() { + Ok(staging) => Ok(StagingLease { + converter: self, + staging: Some(staging), + }), + Err(error) => { + let mut pool = self.staging_pool.lock(); + pool.created -= 1; + self.staging_available.notify_one(); + Err(error) + } + }; + } - StagingLease { - converter: self, - staging: free.pop(), + self.staging_available.wait(&mut pool); } } + fn create_staging_pair(&self) -> Result { + Ok(StagingTextures { + input: create_texture( + &self.device, + self.input_width, + self.input_height, + pixel_to_dxgi(self.input_format)?, + D3D11_USAGE_STAGING, + 0, + D3D11_CPU_ACCESS_WRITE.0 as u32, + )?, + output: create_texture( + &self.device, + self.output_width, + self.output_height, + pixel_to_dxgi(self.output_format)?, + D3D11_USAGE_STAGING, + 0, + D3D11_CPU_ACCESS_READ.0 as u32, + )?, + }) + } + pub fn gpu_info(&self) -> &GpuInfo { &self.gpu_info } @@ -737,6 +767,8 @@ impl D3D11Converter { self.resources.lock().output_shared_handle } + /// Returns the current output texture. When this converter is shared by a pool, every + /// conversion reuses the texture; external readers must synchronize with pool activity. pub fn output_texture(&self) -> ID3D11Texture2D { self.resources.lock().output_texture.clone() } @@ -812,7 +844,7 @@ impl FrameConverter for D3D11Converter { } let pts = input.pts(); - let staging = self.acquire_staging(); + let staging = self.acquire_staging()?; let (input_ptr, input_pitch) = { let resources = self.resources.lock(); diff --git a/crates/frame-converter/src/pool.rs b/crates/frame-converter/src/pool.rs index d24f81a353b..f17c0235c7c 100644 --- a/crates/frame-converter/src/pool.rs +++ b/crates/frame-converter/src/pool.rs @@ -93,6 +93,9 @@ impl AsyncConverterPool { .expect("Factory using pre-created converter should not fail") } + /// On Windows, D3D11 converters are shared across workers and their textures are reused + /// by every conversion. External texture consumers must not read them while the pool is + /// converting. pub fn from_config( conversion_config: ConversionConfig, pool_config: ConverterPoolConfig, From 44411da44d6fb5cdd261d92ec8f290c26e35a09b Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 18:17:17 +0200 Subject: [PATCH 14/16] fix(frame-converter): submit queued D3D11 work before polling readback --- crates/frame-converter/src/d3d11.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/frame-converter/src/d3d11.rs b/crates/frame-converter/src/d3d11.rs index 8dbbfe302fd..4b76bf51e63 100644 --- a/crates/frame-converter/src/d3d11.rs +++ b/crates/frame-converter/src/d3d11.rs @@ -898,6 +898,8 @@ impl FrameConverter for D3D11Converter { .context .CopyResource(&staging.output, &resources.output_texture); + resources.context.Flush(); + // Polling releases the shared context between retries so other // workers can submit GPU work while this readback completes. let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); From 29ea30e5f26898f59e49a5c14e4a42276c72e308 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 18:17:17 +0200 Subject: [PATCH 15/16] fix(recording): drop video support from the WARP fallback --- crates/recording/src/capture_pipeline.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index 486a936ca3c..d832766359b 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -674,8 +674,12 @@ pub fn create_d3d_device() if error.code() == DXGI_ERROR_UNSUPPORTED { use windows::Win32::Graphics::Direct3D::D3D_DRIVER_TYPE_WARP; + use windows::Win32::Graphics::Direct3D11::D3D11_CREATE_DEVICE_VIDEO_SUPPORT; - result = create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device); + // WARP rejects VIDEO_SUPPORT; it cannot hardware-encode anyway. + let warp_flags = + D3D11_CREATE_DEVICE_FLAG(flags.0 & !D3D11_CREATE_DEVICE_VIDEO_SUPPORT.0); + result = create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, warp_flags, &mut device); } } result?; From 6341fe675d4f5333ea69502efe115b06bb8d51ef Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Fri, 23 Oct 2026 18:29:27 +0200 Subject: [PATCH 16/16] fix(recording): report MF encoder failures after draining --- crates/recording/src/output_pipeline/win_fragmented_m4s.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index af6d1b00fde..c013899e2b0 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -844,7 +844,7 @@ impl WindowsFragmentedM4SMuxer { } } - Ok(()) + Err(error.into()) } } })?;