diff --git a/crates/enc-ffmpeg/src/mux/segmented_stream.rs b/crates/enc-ffmpeg/src/mux/segmented_stream.rs index 0ea15ad1566..aa7eb4d669a 100644 --- a/crates/enc-ffmpeg/src/mux/segmented_stream.rs +++ b/crates/enc-ffmpeg/src/mux/segmented_stream.rs @@ -1,6 +1,7 @@ use super::fragment_metadata::read_fragment_metadata; use super::write_behind::{MuxSink, MuxTarget}; -use cap_media_info::VideoInfo; +use crate::base::PacketSink; +use cap_media_info::{VideoInfo, ensure_even}; use ffmpeg::{format, frame}; use serde::Serialize; use std::{ @@ -16,16 +17,7 @@ use crate::video::h264::{ }; const INIT_SEGMENT_NAME: &str = "init.mp4"; - -#[derive(Debug, Clone)] -pub struct DiskSpaceWarning { - pub available_mb: u64, - pub threshold_mb: u64, - pub path: String, - pub is_critical: bool, -} - -pub type DiskSpaceCallback = Arc; +const STREAM_TIME_BASE: ffmpeg::Rational = ffmpeg::Rational(1, 10_000_000); /// Shared so a write-behind muxer can install the callback without waiting /// for the writer thread, which may be blocked on storage. @@ -95,10 +87,6 @@ pub struct SegmentedVideoEncoder { encoder: H264Encoder, mux: MuxSink, segment_tx: SegmentSender, - - segment_duration: Duration, - - disk_space_callback: Option, } struct SegmentMux { @@ -199,6 +187,10 @@ pub enum InitError { Encoder(#[from] H264EncoderError), #[error("IO: {0}")] Io(#[from] std::io::Error), + #[error("The encoder did not expose an H.264 sequence header for init.mp4")] + MissingExtradata, + #[error("Failed to allocate the H.264 extradata buffer")] + Extradata, } #[derive(thiserror::Error, Debug)] @@ -245,6 +237,92 @@ impl Default for SegmentedVideoEncoderConfig { } } +fn init_segment_path(base_path: &Path) -> PathBuf { + base_path.join(INIT_SEGMENT_NAME) +} + +fn validate_init_segment(init_path: &Path) -> Result<(), String> { + if !init_path.exists() { + return Err(format!( + "init.mp4 is missing at {}. M4S segments will be unplayable without it.", + init_path.display() + )); + } + + match std::fs::metadata(init_path) { + Ok(metadata) => { + let size = metadata.len(); + if size < 100 { + return Err(format!( + "init.mp4 at {} is too small ({} bytes). It may be corrupted.", + init_path.display(), + size + )); + } + Ok(()) + } + Err(e) => Err(format!( + "Cannot read init.mp4 metadata at {}: {e}", + init_path.display() + )), + } +} + +fn open_dash_output( + base_path: &Path, + segment_duration: Duration, + timed_fragments: bool, +) -> Result { + let manifest_path = base_path.join("dash_manifest.mpd"); + + #[cfg(target_os = "windows")] + let manifest_path_str = manifest_path.to_string_lossy().replace('\\', "/"); + #[cfg(not(target_os = "windows"))] + let manifest_path_str = manifest_path.to_string_lossy().to_string(); + + let mut output = super::dash_output::create(&manifest_path_str)?; + + let init_seg_str = INIT_SEGMENT_NAME; + let media_seg_str = "segment_$Number%03d$.m4s"; + + unsafe { + let opts = output.as_mut_ptr(); + + let set_opt = |key: &str, value: &str| { + let k = CString::new(key).unwrap(); + let v = CString::new(value).unwrap(); + ffmpeg::ffi::av_opt_set((*opts).priv_data, k.as_ptr(), v.as_ptr(), 0); + }; + + set_opt("init_seg_name", init_seg_str); + set_opt("media_seg_name", media_seg_str); + set_opt("seg_duration", &segment_duration.as_secs_f64().to_string()); + set_opt("use_timeline", "1"); + set_opt("use_template", "1"); + set_opt("single_file", "0"); + set_opt("hls_playlist", "1"); + if timed_fragments { + set_opt("frag_type", "duration"); + set_opt("frag_duration", "2"); + set_opt("streaming", "1"); + } + } + + Ok(output) +} + +fn dash_codec_info(video_config: VideoInfo, stream_size: (u32, u32)) -> CodecInfo { + CodecInfo { + width: stream_size.0, + height: stream_size.1, + frame_rate_num: video_config.frame_rate.0, + frame_rate_den: video_config.frame_rate.1, + time_base_num: video_config.time_base.0, + time_base_den: video_config.time_base.1, + pixel_format: format!("{:?}", video_config.pixel_format), + } +} + impl SegmentedVideoEncoder { pub fn init( base_path: PathBuf, @@ -287,43 +365,16 @@ impl SegmentedVideoEncoder { ) -> Result { std::fs::create_dir_all(&base_path)?; - let manifest_path = base_path.join("dash_manifest.mpd"); - - #[cfg(target_os = "windows")] - let manifest_path_str = manifest_path.to_string_lossy().replace('\\', "/"); - #[cfg(not(target_os = "windows"))] - let manifest_path_str = manifest_path.to_string_lossy().to_string(); - - let mut output = super::dash_output::create(&manifest_path_str)?; - - let init_seg_str = INIT_SEGMENT_NAME; - let media_seg_str = "segment_$Number%03d$.m4s"; - - unsafe { - let opts = output.as_mut_ptr(); - - let set_opt = |key: &str, value: &str| { - let k = CString::new(key).unwrap(); - let v = CString::new(value).unwrap(); - ffmpeg::ffi::av_opt_set((*opts).priv_data, k.as_ptr(), v.as_ptr(), 0); - }; + let stream_size = config + .output_size + .map(|(width, height)| (ensure_even(width), ensure_even(height))) + .unwrap_or(( + ensure_even(video_config.width), + ensure_even(video_config.height), + )); - set_opt("init_seg_name", init_seg_str); - set_opt("media_seg_name", media_seg_str); - set_opt( - "seg_duration", - &config.segment_duration.as_secs_f64().to_string(), - ); - set_opt("use_timeline", "1"); - set_opt("use_template", "1"); - set_opt("single_file", "0"); - set_opt("hls_playlist", "1"); - if timed_fragments { - set_opt("frag_type", "duration"); - set_opt("frag_duration", "2"); - set_opt("streaming", "1"); - } - } + let manifest_path = base_path.join("dash_manifest.mpd"); + let mut output = open_dash_output(&base_path, config.segment_duration, timed_fragments)?; let mut builder = H264EncoderBuilder::new(video_config) .with_preset(config.preset) @@ -369,15 +420,7 @@ impl SegmentedVideoEncoder { "FFmpeg DASH muxer state after write_header()" ); - let codec_info = CodecInfo { - width: video_config.width, - height: video_config.height, - frame_rate_num: video_config.frame_rate.0, - frame_rate_den: video_config.frame_rate.1, - time_base_num: video_config.time_base.0, - time_base_den: video_config.time_base.1, - pixel_format: format!("{:?}", video_config.pixel_format), - }; + let codec_info = dash_codec_info(video_config, stream_size); tracing::info!( path = %base_path.display(), @@ -405,8 +448,6 @@ impl SegmentedVideoEncoder { encoder, mux: MuxSink::Direct(mux), segment_tx, - segment_duration: config.segment_duration, - disk_space_callback: None, }; if write_behind { @@ -425,10 +466,6 @@ impl SegmentedVideoEncoder { Ok(instance) } - pub fn set_disk_space_callback(&mut self, callback: DiskSpaceCallback) { - self.disk_space_callback = Some(callback); - } - pub fn set_segment_callback(&mut self, tx: std::sync::mpsc::Sender) { *self .segment_tx @@ -512,50 +549,16 @@ impl SegmentedVideoEncoder { Some(&mut self.encoder) } - pub fn base_path(&self) -> &Path { - &self.base_path - } - - pub fn segment_duration(&self) -> Duration { - self.segment_duration - } - pub fn current_index(&self) -> u32 { self.mux.with_target_ref(|mux| mux.current_index) } pub fn init_segment_path(&self) -> PathBuf { - self.base_path.join(INIT_SEGMENT_NAME) + init_segment_path(&self.base_path) } pub fn validate_init_segment(&self) -> Result<(), String> { - let init_path = self.init_segment_path(); - - if !init_path.exists() { - return Err(format!( - "init.mp4 is missing at {}. M4S segments will be unplayable without it.", - init_path.display() - )); - } - - match std::fs::metadata(&init_path) { - Ok(metadata) => { - let size = metadata.len(); - if size < 100 { - return Err(format!( - "init.mp4 at {} is too small ({} bytes). It may be corrupted.", - init_path.display(), - size - )); - } - Ok(()) - } - Err(e) => Err(format!( - "Cannot read init.mp4 metadata at {}: {}", - init_path.display(), - e - )), - } + validate_init_segment(&self.init_segment_path()) } } @@ -872,6 +875,156 @@ impl SegmentMux { } } +/// Muxes externally encoded H.264 access units into the same DASH segments as +/// [`SegmentedVideoEncoder`], for encoders that cannot hand over raw frames (the Windows Media +/// Foundation encoder takes D3D11 textures and emits compressed samples). +pub struct SegmentedPacketMuxer { + mux: MuxSink, + stream_index: usize, + segment_tx: SegmentSender, + last_written_pts: Option, + consecutive_pts_bumps: u64, +} + +impl SegmentedPacketMuxer { + /// `extradata` must be the H.264 sequence header (SPS/PPS) in Annex B; `init.mp4` is written + /// during construction and cannot carry a codec configuration discovered later. + pub fn new( + base_path: PathBuf, + video_config: VideoInfo, + stream_size: (u32, u32), + segment_duration: Duration, + timed_fragments: bool, + bitrate: u32, + extradata: &[u8], + ) -> Result { + if extradata.is_empty() { + return Err(InitError::MissingExtradata); + } + + let stream_size = (ensure_even(stream_size.0), ensure_even(stream_size.1)); + + std::fs::create_dir_all(&base_path)?; + + let mut output = open_dash_output(&base_path, segment_duration, timed_fragments)?; + + let h264 = ffmpeg::codec::decoder::find(ffmpeg::codec::Id::H264) + .ok_or(ffmpeg::Error::DecoderNotFound)?; + let mut stream = output.add_stream(h264)?; + let stream_index = stream.index(); + stream.set_time_base(STREAM_TIME_BASE); + + unsafe { + let codecpar = (*stream.as_mut_ptr()).codecpar; + (*codecpar).codec_type = ffmpeg::ffi::AVMediaType::AVMEDIA_TYPE_VIDEO; + (*codecpar).codec_id = ffmpeg::ffi::AVCodecID::AV_CODEC_ID_H264; + (*codecpar).width = stream_size.0 as i32; + (*codecpar).height = stream_size.1 as i32; + (*codecpar).bit_rate = i64::from(bitrate); + (*codecpar).format = ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NV12 as i32; + + let stream_ptr = stream.as_mut_ptr(); + (*stream_ptr).avg_frame_rate = ffmpeg::ffi::AVRational { + num: video_config.frame_rate.0, + den: video_config.frame_rate.1, + }; + (*stream_ptr).r_frame_rate = ffmpeg::ffi::AVRational { + num: video_config.frame_rate.0, + den: video_config.frame_rate.1, + }; + + let size = extradata.len() + ffmpeg::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize; + let buffer = ffmpeg::ffi::av_mallocz(size).cast::(); + if buffer.is_null() { + return Err(InitError::Extradata); + } + std::ptr::copy_nonoverlapping(extradata.as_ptr(), buffer, extradata.len()); + (*codecpar).extradata = buffer; + (*codecpar).extradata_size = extradata.len() as i32; + } + + output.write_header()?; + + let mut mux = SegmentMux { + base_path: base_path.clone(), + output, + current_index: 1, + completed_segments: Vec::new(), + frames_since_segment_scan: 0, + codec_info: dash_codec_info(video_config, stream_size), + segment_tx: SegmentSender::default(), + init_notified: false, + initial_manifest_pending: true, + }; + mux.write_initial_manifest(); + + Ok(Self { + segment_tx: mux.segment_tx.clone(), + mux: MuxSink::Direct(mux), + stream_index, + last_written_pts: None, + consecutive_pts_bumps: 0, + }) + } + + pub fn set_segment_callback(&mut self, tx: std::sync::mpsc::Sender) { + *self + .segment_tx + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(tx); + self.mux.with_target(SegmentMux::try_notify_init_segment); + } + + pub fn write_packet( + &mut self, + mut packet: ffmpeg::Packet, + source_time_base: ffmpeg::Rational, + ) -> Result<(), ffmpeg::Error> { + packet.set_stream(self.stream_index); + + let destination_time_base = self.mux.stream_time_base(self.stream_index); + packet.rescale_ts(source_time_base, destination_time_base); + + // The mov/DASH muxer rejects duplicate pts/dts. MF timestamps normally advance in step with + // the stream time base, but a re-quantized or stuck source clock can repeat one; advance the + // tie by a tick so muxing never fails on it. + if let Some(pts) = packet.pts() { + let pts = match self.last_written_pts { + Some(last) if pts <= last => { + self.consecutive_pts_bumps += 1; + if self.consecutive_pts_bumps == 30 + || self.consecutive_pts_bumps.is_multiple_of(300) + { + tracing::warn!( + consecutive_bumps = self.consecutive_pts_bumps, + "Media Foundation sample times are not advancing; the muxer is tie-bumping every packet (stuck source clock?)" + ); + } + last + 1 + } + _ => { + self.consecutive_pts_bumps = 0; + pts + } + }; + self.last_written_pts = Some(pts); + packet.set_pts(Some(pts)); + packet.set_dts(Some(pts)); + } + + self.mux.write_packet(packet)?; + self.mux.frame_queued(); + + Ok(()) + } + + pub fn finish(&mut self) -> Result<(), FinishError> { + self.mux + .finish(|_| Ok(()), |mux, flush_result| mux.finish(flush_result)) + .map_err(std::io::Error::from)? + } +} + #[cfg(test)] mod tests { use super::*; @@ -899,6 +1052,95 @@ mod tests { frame } + #[test] + fn packet_muxer_writes_the_same_init_and_manifest_contract() { + ffmpeg::init().unwrap(); + let temp = tempfile::tempdir().unwrap(); + let base_path = temp.path().to_path_buf(); + let video_info = test_video_info(); + let capture_config = VideoInfo { + width: 640, + height: 480, + ..video_info + }; + + let mut encoder = H264EncoderBuilder::new(video_info) + .with_preset(H264Preset::Ultrafast) + .build_standalone() + .unwrap(); + let extradata = encoder.extradata(); + assert!(!extradata.is_empty(), "encoder must expose SPS/PPS"); + + let mut muxer = SegmentedPacketMuxer::new( + base_path.clone(), + capture_config, + (video_info.width, video_info.height), + Duration::from_secs(2), + false, + 300_000, + &extradata, + ) + .unwrap(); + + let mut packets = Vec::new(); + for index in 0..90u64 { + encoder + .encode_frame( + create_test_frame(320, 240), + Duration::from_micros(index * 33_333), + |encoded| { + packets.push(encoded); + Ok(()) + }, + ) + .unwrap(); + } + + let source_time_base = encoder.time_base(); + for encoded in packets { + let mut packet = ffmpeg::Packet::copy(&encoded.data); + packet.set_pts(Some(encoded.pts)); + packet.set_dts(Some(encoded.dts)); + packet.set_duration(encoded.duration); + if encoded.is_keyframe { + packet.set_flags(ffmpeg::packet::Flags::KEY); + } + muxer + .write_packet(packet, source_time_base) + .expect("write packet"); + } + muxer.finish().unwrap(); + + let init_path = base_path.join(INIT_SEGMENT_NAME); + validate_init_segment(&init_path).unwrap(); + let init = std::fs::read(&init_path).unwrap(); + assert!( + init.windows(4).any(|window| window == b"avcC"), + "init.mp4 must carry the H.264 decoder configuration" + ); + + let input = ffmpeg::format::input(&init_path).expect("open init.mp4"); + let stream = input + .streams() + .best(ffmpeg::media::Type::Video) + .expect("video stream"); + let parameters = stream.parameters(); + let (declared_width, declared_height) = + unsafe { ((*parameters.as_ptr()).width, (*parameters.as_ptr()).height) }; + assert_eq!((declared_width, declared_height), (320, 240)); + + let manifest: serde_json::Value = + serde_json::from_slice(&std::fs::read(base_path.join("manifest.json")).unwrap()) + .unwrap(); + assert_eq!(manifest["is_complete"].as_bool(), Some(true)); + assert!( + !manifest["segments"].as_array().unwrap().is_empty(), + "the DASH muxer must publish at least one media segment" + ); + assert_eq!(manifest["codec_info"]["width"].as_u64(), Some(320)); + assert_eq!(manifest["codec_info"]["height"].as_u64(), Some(240)); + } + #[test] fn video_manifest_and_events_match_muxer_playlist() { ffmpeg::init().unwrap(); diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index 0042b4610af..46299035428 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -1663,6 +1663,13 @@ fn get_bitrate(width: u32, height: u32, frame_rate: f32, bpp: f32) -> usize { ((pixels_per_second * bpp as f64) as usize).min(MAX_BITRATE) } +/// Bitrate this module's encoders target for `bpp`, so a hardware encoder fed straight from the +/// capture texture keeps the recording quality of the software path. +pub fn bitrate_for_bpp(width: u32, height: u32, frame_rate: ffmpeg::Rational, bpp: f32) -> u32 { + let fps = frame_rate.0 as f32 / frame_rate.1.max(1) as f32; + get_bitrate(width, height, fps, bpp) as u32 +} + const MAX_BITRATE: usize = 1_000_000_000; #[cfg(test)] diff --git a/crates/enc-mediafoundation/src/d3d.rs b/crates/enc-mediafoundation/src/d3d.rs index 77ae38543bd..7d9406c8dad 100644 --- a/crates/enc-mediafoundation/src/d3d.rs +++ b/crates/enc-mediafoundation/src/d3d.rs @@ -7,8 +7,8 @@ use windows::Win32::Graphics::{ D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_UNKNOWN, D3D_DRIVER_TYPE_WARP, }, Direct3D11::{ - D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG, D3D11_SDK_VERSION, - D3D11CreateDevice, ID3D11Device, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, D3D11CreateDevice, ID3D11Device, }, Dxgi::{DXGI_ERROR_UNSUPPORTED, IDXGIDevice}, }; @@ -70,7 +70,7 @@ fn create_d3d_device_on_pinned_adapter( pub fn create_d3d_device() -> Result { let mut device = None; let flags = { - let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; + let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT; if cfg!(feature = "d3ddebug") { flags |= D3D11_CREATE_DEVICE_DEBUG; } @@ -97,7 +97,12 @@ pub fn create_d3d_device() -> Result { if let Err(error) = &result && error.code() == DXGI_ERROR_UNSUPPORTED { - result = create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device); + // WARP rejects VIDEO_SUPPORT; a WARP device cannot hardware-encode anyway. + result = create_d3d_device_with_type( + D3D_DRIVER_TYPE_WARP, + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + &mut device, + ); } result?; device.ok_or_else(windows::core::Error::from_win32) diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 2e496e47f64..9d34e047605 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -10,6 +10,7 @@ use std::{ }, time::{Duration, Instant}, }; +use tracing::{debug, warn}; use windows::{ Foundation::TimeSpan, Graphics::SizeInt32, @@ -22,13 +23,14 @@ use windows::{ Media::MediaFoundation::{ self, IMFAttributes, IMFDXGIDeviceManager, IMFMediaEventGenerator, IMFMediaType, IMFSample, IMFTransform, MF_E_INVALIDMEDIATYPE, MF_E_NO_MORE_TYPES, - MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_FLAG_NONE, MF_EVENT_TYPE, - MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, - MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SUBTYPE, - MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, + MF_E_TRANSFORM_NEED_MORE_INPUT, MF_E_TRANSFORM_TYPE_NOT_SET, MF_EVENT_FLAG_NO_WAIT, + MF_EVENT_FLAG_NONE, MF_EVENT_TYPE, MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, + MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, + MF_MT_MAX_KEYFRAME_SPACING, MF_MT_MPEG_SEQUENCE_HEADER, MF_MT_PIXEL_ASPECT_RATIO, + MF_MT_SUBTYPE, MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, MF_TRANSFORM_ASYNC_UNLOCK, MFCreateDXGIDeviceManager, MFCreateDXGISurfaceBuffer, MFCreateMediaType, MFCreateSample, MFMediaType_Video, MFT_ENUM_FLAG, MFT_ENUM_FLAG_HARDWARE, - MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_FLUSH, + MFT_ENUM_FLAG_TRANSCODE_ONLY, MFT_MESSAGE_COMMAND_DRAIN, MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_OF_STREAM, MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_MESSAGE_SET_D3D_MANAGER, MFT_OUTPUT_DATA_BUFFER, MFT_SET_TYPE_TEST_ONLY, @@ -38,6 +40,8 @@ use windows::{ core::{Error, Interface}, }; +/// DASH segments are cut on keyframes; 2 s matches the software encoder's GOP. +const MUXED_KEYFRAME_INTERVAL_SECS: u32 = 2; const MAX_CONSECUTIVE_EMPTY_SAMPLES: u8 = 20; const MAX_INPUT_WITHOUT_OUTPUT: u32 = 30; const MAX_PROCESS_INPUT_FAILURES: u32 = 5; @@ -214,12 +218,14 @@ pub enum EncoderRuntimeError { process_failures: u32, frames_encoded: u64, }, + #[error("Media Foundation encoder did not confirm drain completion")] + DrainIncomplete, } impl EncoderRuntimeError { pub fn should_fallback(&self) -> bool { match self { - EncoderRuntimeError::Windows(_) => false, + EncoderRuntimeError::Windows(_) | EncoderRuntimeError::DrainIncomplete => false, EncoderRuntimeError::EncoderUnhealthy { .. } => true, } } @@ -241,25 +247,13 @@ impl H264Encoder { input_resolution: SizeInt32, output_resolution: SizeInt32, frame_rate: u32, - bitrate_multipler: f32, + bitrate: u32, flags: MFT_ENUM_FLAG, enable_hardware_transforms: bool, ) -> Result { - let bitrate = calculate_bitrate( - output_resolution.Width as u32, - output_resolution.Height as u32, - frame_rate, - bitrate_multipler, - ); - - let transform = + let devices = EncoderDevice::enumerate_with_flags(MFMediaType_Video, MFVideoFormat_H264, flags) - .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)? - .first() - .cloned() - .ok_or(NewVideoEncoderError::NoVideoEncoderDevice)? - .create_transform() - .map_err(NewVideoEncoderError::EncoderTransform)?; + .map_err(|_| NewVideoEncoderError::NoVideoEncoderDevice)?; let video_processor = VideoProcessor::new( d3d_device.clone(), @@ -289,103 +283,57 @@ impl H264Encoder { .map_err(NewVideoEncoderError::DeviceManager)? }; - let event_generator: IMFMediaEventGenerator = transform - .cast() - .map_err(NewVideoEncoderError::EventGenerator)?; - let attributes = unsafe { - transform - .GetAttributes() - .map_err(NewVideoEncoderError::EventGenerator)? - }; - unsafe { - attributes - .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + let configure = |transform: &IMFTransform| -> Result< + (IMFMediaEventGenerator, u32, u32, IMFMediaType), + NewVideoEncoderError, + > { + let event_generator: IMFMediaEventGenerator = transform + .cast() .map_err(NewVideoEncoderError::EventGenerator)?; - attributes - .SetUINT32( - &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, - enable_hardware_transforms as u32, - ) - .map_err(NewVideoEncoderError::EventGenerator)?; - }; - let mut number_of_input_streams = 0; - let mut number_of_output_streams = 0; - unsafe { - transform - .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams) - .map_err(NewVideoEncoderError::EventGenerator)? - }; - let (input_stream_ids, output_stream_ids) = { - let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; - let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; - let result = - unsafe { transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) }; - match result { - Ok(_) => {} - Err(error) => { - if error.code() == E_NOTIMPL { - for i in 0..number_of_input_streams { - input_stream_ids[i as usize] = i; - } - for i in 0..number_of_output_streams { - output_stream_ids[i as usize] = i; + let mut number_of_input_streams = 0; + let mut number_of_output_streams = 0; + unsafe { + transform + .GetStreamCount(&mut number_of_input_streams, &mut number_of_output_streams) + .map_err(NewVideoEncoderError::EventGenerator)? + }; + let (input_stream_ids, output_stream_ids) = { + let mut input_stream_ids = vec![0u32; number_of_input_streams as usize]; + let mut output_stream_ids = vec![0u32; number_of_output_streams as usize]; + let result = unsafe { + transform.GetStreamIDs(&mut input_stream_ids, &mut output_stream_ids) + }; + match result { + Ok(_) => {} + Err(error) => { + if error.code() == E_NOTIMPL { + for i in 0..number_of_input_streams { + input_stream_ids[i as usize] = i; + } + for i in 0..number_of_output_streams { + output_stream_ids[i as usize] = i; + } + } else { + return Err(NewVideoEncoderError::ConfigureStreams(error)); } - } else { - return Err(NewVideoEncoderError::ConfigureStreams(error)); } } - } - (input_stream_ids, output_stream_ids) - }; - let input_stream_id = input_stream_ids[0]; - let output_stream_id = output_stream_ids[0]; - - unsafe { - let temp = media_device_manager.clone(); - transform - .ProcessMessage( - MFT_MESSAGE_SET_D3D_MANAGER, - std::mem::transmute::(temp), - ) - .map_err(NewVideoEncoderError::EncoderTransform)?; - }; - - let output_type = (|| unsafe { - let output_type = MFCreateMediaType()?; - let attributes: IMFAttributes = output_type.cast()?; - output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; - output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; - output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; - MFSetAttributeSize( - &attributes, - &MF_MT_FRAME_SIZE, - output_resolution.Width as u32, - output_resolution.Height as u32, - )?; - MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; - MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; - output_type.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; - output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; - transform.SetOutputType(output_stream_id, &output_type, 0)?; - Ok(output_type) - })() - .map_err(NewVideoEncoderError::OutputType)?; - - let input_type: Option = (|| unsafe { - let mut count = 0; - loop { - let result = transform.GetInputAvailableType(input_stream_id, count); - if let Err(error) = &result - && error.code() == MF_E_NO_MORE_TYPES - { - break Ok(None); - } - - let input_type = result?; - let attributes: IMFAttributes = input_type.cast()?; - input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; - input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; + (input_stream_ids, output_stream_ids) + }; + let input_stream_id = input_stream_ids[0]; + let output_stream_id = output_stream_ids[0]; + + let output_type = (|| unsafe { + let output_type = MFCreateMediaType()?; + let attributes: IMFAttributes = output_type.cast()?; + output_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + output_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264)?; + output_type.SetUINT32(&MF_MT_AVG_BITRATE, bitrate)?; + output_type.SetUINT32( + &MF_MT_MAX_KEYFRAME_SPACING, + frame_rate.saturating_mul(MUXED_KEYFRAME_INTERVAL_SECS), + )?; MFSetAttributeSize( &attributes, &MF_MT_FRAME_SIZE, @@ -393,47 +341,167 @@ impl H264Encoder { output_resolution.Height as u32, )?; MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; - let result = transform.SetInputType( - input_stream_id, - &input_type, - MFT_SET_TYPE_TEST_ONLY.0 as u32, - ); - if let Err(error) = &result - && error.code() == MF_E_INVALIDMEDIATYPE - { - count += 1; - continue; + MFSetAttributeRatio(&attributes, &MF_MT_PIXEL_ASPECT_RATIO, 1, 1)?; + output_type + .SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32)?; + output_type.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1)?; + transform.SetOutputType(output_stream_id, &output_type, 0)?; + transform.GetOutputCurrentType(output_stream_id) + })() + .map_err(NewVideoEncoderError::OutputType)?; + + let input_type: Option = (|| unsafe { + let mut count = 0; + loop { + let result = transform.GetInputAvailableType(input_stream_id, count); + if let Err(error) = &result + && error.code() == MF_E_NO_MORE_TYPES + { + break Ok(None); + } + + let input_type = result?; + let attributes: IMFAttributes = input_type.cast()?; + input_type.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video)?; + input_type.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12)?; + MFSetAttributeSize( + &attributes, + &MF_MT_FRAME_SIZE, + output_resolution.Width as u32, + output_resolution.Height as u32, + )?; + MFSetAttributeRatio(&attributes, &MF_MT_FRAME_RATE, frame_rate, 1)?; + let result = transform.SetInputType( + input_stream_id, + &input_type, + MFT_SET_TYPE_TEST_ONLY.0 as u32, + ); + if let Err(error) = &result + && error.code() == MF_E_INVALIDMEDIATYPE + { + count += 1; + continue; + } + result?; + break Ok(Some(input_type)); } - result?; - break Ok(Some(input_type)); + })() + .map_err(NewVideoEncoderError::InputType)?; + if let Some(input_type) = input_type { + unsafe { transform.SetInputType(input_stream_id, &input_type, 0) } + .map_err(NewVideoEncoderError::InputType)?; + } else { + return Err(NewVideoEncoderError::InputType(Error::new( + MF_E_TRANSFORM_TYPE_NOT_SET, + "No suitable input type found! Try a different set of encoding settings.", + ))); } - })() - .map_err(NewVideoEncoderError::InputType)?; - if let Some(input_type) = input_type { - unsafe { transform.SetInputType(input_stream_id, &input_type, 0) } - .map_err(NewVideoEncoderError::InputType)?; - } else { - return Err(NewVideoEncoderError::InputType(Error::new( - MF_E_TRANSFORM_TYPE_NOT_SET, - "No suitable input type found! Try a different set of encoding settings.", - ))); - } - - Ok(Self { - _d3d_device: d3d_device.clone(), - _media_device_manager: media_device_manager, - _device_manager_reset_token: device_manager_reset_token, - video_processor, + Ok(( + event_generator, + input_stream_id, + output_stream_id, + output_type, + )) + }; - transform, - event_generator, - input_stream_id, - output_stream_id, - bitrate, + let mut last_error = None; + for device in &devices { + let candidate = match device.create_transform() { + Ok(candidate) => candidate, + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder activation failed; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + continue; + } + }; + let attributes = match unsafe { candidate.GetAttributes() } { + Ok(attributes) => attributes, + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder attributes unavailable; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + continue; + } + }; + let configured = unsafe { + attributes + .SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + .and_then(|_| { + attributes.SetUINT32( + &MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, + enable_hardware_transforms as u32, + ) + }) + }; + if let Err(error) = configured { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder configuration failed; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + continue; + } + let attached = unsafe { + candidate.ProcessMessage( + MFT_MESSAGE_SET_D3D_MANAGER, + std::mem::transmute::( + media_device_manager.clone(), + ), + ) + }; + match attached { + Ok(()) => match configure(&candidate) { + Ok((event_generator, input_stream_id, output_stream_id, output_type)) => { + tracing::info!( + device = device.display_name(), + "Selected Media Foundation encoder MFT" + ); + return Ok(Self { + _d3d_device: d3d_device.clone(), + _media_device_manager: media_device_manager, + _device_manager_reset_token: device_manager_reset_token, + + video_processor, + + transform: candidate, + event_generator, + input_stream_id, + output_stream_id, + bitrate, + + output_type, + }); + } + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder type setup failed; trying the next MFT" + ); + last_error = Some(error); + } + }, + Err(error) => { + warn!( + device = device.display_name(), + %error, + "Media Foundation encoder rejected the D3D manager; trying the next MFT" + ); + last_error = Some(NewVideoEncoderError::EncoderTransform(error)); + } + } + } - output_type, - }) + Err(last_error.unwrap_or(NewVideoEncoderError::NoVideoEncoderDevice)) } pub fn new_with_scaled_output( @@ -443,6 +511,29 @@ impl H264Encoder { output_resolution: SizeInt32, frame_rate: u32, bitrate_multipler: f32, + ) -> Result { + Self::new_with_scaled_output_and_bitrate( + d3d_device, + format, + input_resolution, + output_resolution, + frame_rate, + calculate_bitrate( + output_resolution.Width as u32, + output_resolution.Height as u32, + frame_rate, + bitrate_multipler, + ), + ) + } + + pub fn new_with_scaled_output_and_bitrate( + d3d_device: &ID3D11Device, + format: DXGI_FORMAT, + input_resolution: SizeInt32, + output_resolution: SizeInt32, + frame_rate: u32, + bitrate: u32, ) -> Result { Self::new_with_scaled_output_with_flags( d3d_device, @@ -450,7 +541,7 @@ impl H264Encoder { input_resolution, output_resolution, frame_rate, - bitrate_multipler, + bitrate, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_TRANSCODE_ONLY, true, ) @@ -470,7 +561,12 @@ impl H264Encoder { input_resolution, output_resolution, frame_rate, - bitrate_multipler, + calculate_bitrate( + output_resolution.Width as u32, + output_resolution.Height as u32, + frame_rate, + bitrate_multipler, + ), MFT_ENUM_FLAG_TRANSCODE_ONLY, false, ) @@ -518,6 +614,28 @@ impl H264Encoder { &self.output_type } + /// SPS/PPS in Annex B, as the encoder exposes them on its output type. The DASH `init.mp4` is + /// written before the first sample, so a caller muxing compressed samples must have this up + /// front. + pub fn sequence_header(&self) -> Option> { + let attributes: IMFAttributes = self.output_type.cast().ok()?; + let size = unsafe { attributes.GetBlobSize(&MF_MT_MPEG_SEQUENCE_HEADER) }.ok()?; + if size == 0 { + return None; + } + + let mut header = vec![0u8; size as usize]; + let mut written = 0u32; + unsafe { + attributes + .GetBlob(&MF_MT_MPEG_SEQUENCE_HEADER, &mut header, Some(&mut written)) + .ok()?; + } + header.truncate(written as usize); + + Some(header) + } + pub fn validate(&self) -> Result<(), NewVideoEncoderError> { unsafe { self.transform @@ -667,6 +785,66 @@ impl H264Encoder { } } + let _ = self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_DRAIN, 0); + + let drain_deadline = Instant::now() + Duration::from_millis(250); + let mut drain_complete = false; + let mut drained = 0u64; + while Instant::now() < drain_deadline { + let event = match self.event_generator.GetEvent(MF_EVENT_FLAG_NO_WAIT) { + Ok(event) => event, + Err(_) => { + std::thread::sleep(Duration::from_millis(1)); + continue; + } + }; + let event_type = match event.GetType() { + Ok(event_type) => event_type, + Err(_) => break, + }; + + match MF_EVENT_TYPE(event_type as i32) { + MediaFoundation::METransformHaveOutput => { + let mut status = 0; + let output_buffer = MFT_OUTPUT_DATA_BUFFER { + dwStreamID: self.output_stream_id, + ..Default::default() + }; + let mut output_buffers = [output_buffer]; + + match self + .transform + .ProcessOutput(0, &mut output_buffers, &mut status) + { + Ok(()) => { + if let Some(sample) = output_buffers[0].pSample.take() { + health_monitor.record_output(); + on_sample(sample)?; + drained += 1; + } + } + Err(error) if error.code() == MF_E_TRANSFORM_NEED_MORE_INPUT => {} + Err(error) => { + let _ = self.cleanup_encoder(); + return Err(error.into()); + } + } + } + MediaFoundation::METransformDrainComplete => { + drain_complete = true; + break; + } + _ => {} + } + } + if drained > 0 { + debug!(drained, "Drained Media Foundation samples before shutdown"); + } + if !drain_complete { + let _ = self.cleanup_encoder(); + return Err(EncoderRuntimeError::DrainIncomplete); + } + self.transform .ProcessMessage(MFT_MESSAGE_NOTIFY_END_OF_STREAM, 0)?; self.transform diff --git a/crates/mediafoundation-ffmpeg/src/h264.rs b/crates/mediafoundation-ffmpeg/src/h264.rs index f1077860e9d..199dccbe15b 100644 --- a/crates/mediafoundation-ffmpeg/src/h264.rs +++ b/crates/mediafoundation-ffmpeg/src/h264.rs @@ -27,6 +27,8 @@ impl Default for MuxerConfig { } } +/// Writes Media Foundation H.264 samples into an mp4/mov output. Production callers are the +/// Windows MP4 hardware paths in `recording::output_pipeline::win` (studio and camera). pub struct H264StreamMuxer { stream_index: usize, time_base: ffmpeg::Rational, @@ -99,7 +101,7 @@ impl H264StreamMuxer { return Err("Muxer is already finished".into()); } - let mut packet = self.mf_sample_to_avpacket(sample)?; + let mut packet = sample_to_packet(sample, self.time_base, self.stream_index)?; packet.rescale_ts( self.time_base, @@ -145,44 +147,9 @@ impl H264StreamMuxer { packet.write_interleaved(output)?; - Ok(()) - } - - fn mf_sample_to_avpacket(&self, sample: &IMFSample) -> windows::core::Result { - let len = unsafe { sample.GetTotalLength()? }; - let mut packet = ffmpeg::Packet::new(len as usize); - - { - let buffer = unsafe { sample.ConvertToContiguousBuffer()? }; - let data = buffer.lock()?; - - packet - .data_mut() - .unwrap() - .copy_from_slice(&data[0..len as usize]); - } - - let pts = unsafe { sample.GetSampleTime() } - .ok() - .map(|v| mf_from_mf_time(self.time_base, v)); - packet.set_pts(pts); - packet.set_dts(pts); + self.frame_count += 1; - let duration = unsafe { sample.GetSampleDuration() } - .ok() - .map(|v| mf_from_mf_time(self.time_base, v)) - .unwrap_or_default(); - packet.set_duration(duration); - - if let Ok(t) = unsafe { sample.GetUINT32(&MFSampleExtension_CleanPoint) } - && t != 0 - { - packet.set_flags(packet::Flags::KEY); - } - - packet.set_stream(self.stream_index); - - Ok(packet) + Ok(()) } pub fn finish(&mut self) -> Result<(), ffmpeg::Error> { @@ -206,7 +173,74 @@ impl H264StreamMuxer { } } -const MF_TIMEBASE: ffmpeg::Rational = ffmpeg::Rational(1, 10_000_000); +/// Media Foundation sample times are in 100 ns ticks. +pub const MF_TIMEBASE: ffmpeg::Rational = ffmpeg::Rational(1, 10_000_000); + +pub fn sample_to_packet( + sample: &IMFSample, + time_base: Rational, + stream_index: usize, +) -> windows::core::Result { + let len = unsafe { sample.GetTotalLength()? }; + let mut packet = ffmpeg::Packet::new(len as usize); + + { + let buffer = unsafe { sample.ConvertToContiguousBuffer()? }; + let data = buffer.lock()?; + + packet + .data_mut() + .unwrap() + .copy_from_slice(&data[0..len as usize]); + } + + let pts = unsafe { sample.GetSampleTime() } + .ok() + .map(|v| mf_from_mf_time(time_base, v)); + packet.set_pts(pts); + packet.set_dts(pts); + + let duration = unsafe { sample.GetSampleDuration() } + .ok() + .map(|v| mf_from_mf_time(time_base, v)) + .unwrap_or_default(); + packet.set_duration(duration); + + let clean_point = + unsafe { sample.GetUINT32(&MFSampleExtension_CleanPoint) }.is_ok_and(|value| value != 0); + if clean_point || contains_idr_nal(packet.data().unwrap_or_default()) { + packet.set_flags(packet::Flags::KEY); + } + + packet.set_stream(stream_index); + + Ok(packet) +} + +/// Some Media Foundation encoders leave `MFSampleExtension_CleanPoint` unset. The DASH muxer +/// splits segments on keyframes, so fall back to the bitstream: an IDR NAL (type 5) after an +/// Annex B start code. +fn contains_idr_nal(data: &[u8]) -> bool { + let mut index = 0; + while index + 3 < data.len() { + if data[index] == 0 + && data[index + 1] == 0 + && (data[index + 2] == 1 + || (data[index + 2] == 0 && index + 4 < data.len() && data[index + 3] == 1)) + { + let nal = if data[index + 2] == 1 { + index + 3 + } else { + index + 4 + }; + if data.get(nal).is_some_and(|byte| byte & 0x1F == 5) { + return true; + } + } + index += 1; + } + false +} fn mf_from_mf_time(tb: Rational, stime: i64) -> i64 { unsafe { av_rescale_q(stime, MF_TIMEBASE.into(), tb.into()) } @@ -253,3 +287,17 @@ pub fn set_fragmented_mp4_options( Ok(()) } + +#[cfg(test)] +mod tests { + use super::contains_idr_nal; + + #[test] + fn detects_idr_after_three_and_four_byte_start_codes() { + assert!(contains_idr_nal(&[0, 0, 1, 0x65])); + assert!(contains_idr_nal(&[0, 0, 0, 1, 0x65])); + assert!(!contains_idr_nal(&[0, 0, 1, 0x41])); + assert!(!contains_idr_nal(&[0, 0, 1])); + assert!(!contains_idr_nal(&[])); + } +} diff --git a/crates/mediafoundation-ffmpeg/src/lib.rs b/crates/mediafoundation-ffmpeg/src/lib.rs index 1080a2c25c0..ac7ac10b601 100644 --- a/crates/mediafoundation-ffmpeg/src/lib.rs +++ b/crates/mediafoundation-ffmpeg/src/lib.rs @@ -4,4 +4,6 @@ mod audio; mod h264; pub use audio::AudioExt; -pub use h264::{H264StreamMuxer, MuxerConfig, set_fragmented_mp4_options}; +pub use h264::{ + H264StreamMuxer, MF_TIMEBASE, MuxerConfig, sample_to_packet, set_fragmented_mp4_options, +}; diff --git a/crates/recording/src/capture_pipeline.rs b/crates/recording/src/capture_pipeline.rs index d935e0e30f2..7c5da374acd 100644 --- a/crates/recording/src/capture_pipeline.rs +++ b/crates/recording/src/capture_pipeline.rs @@ -251,6 +251,8 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { .parent() .map(|p| p.join("display")) .unwrap_or_else(|| output_path.with_file_name("display")); + let d3d_device = screen_capture.d3d_device.clone(); + let pixel_format = screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi(); let bpp = if ultra { H264EncoderBuilder::ULTRA_BPP @@ -301,7 +303,6 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { bpp, output_size, shared_pause_state, - disk_space_callback: None, segment_tx: None, ..Default::default() }) @@ -318,7 +319,9 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { bpp, output_size, shared_pause_state, - disk_space_callback: None, + d3d_device: Some(d3d_device), + pixel_format: Some(pixel_format), + encoder_preferences: encoder_preferences.clone(), segment_tx: None, }) .await @@ -356,6 +359,7 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { start_gate: Option, segment_tx: Option>, ) -> anyhow::Result { + let d3d_device = screen_capture.d3d_device.clone(); OutputPipeline::builder(segments_dir) .with_video::(screen_capture) .with_timestamps(start_time) @@ -367,7 +371,9 @@ impl MakeCapturePipeline for screen_capture::Direct3DCapture { bpp: H264EncoderBuilder::INSTANT_MODE_BPP, output_size: Some(output_size), shared_pause_state: None, - disk_space_callback: None, + d3d_device: Some(d3d_device), + pixel_format: Some(screen_capture::Direct3DCapture::PIXEL_FORMAT.as_dxgi()), + encoder_preferences: EncoderPreferences::new(), segment_tx, }) .await diff --git a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs index cd36e3bff3b..a4cb8cf9f23 100644 --- a/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/macos_fragmented_m4s.rs @@ -10,7 +10,7 @@ use crate::{ use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::segmented_stream::{ - DiskSpaceCallback, SegmentCompletedEvent, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, + SegmentCompletedEvent, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, }; use cap_media_info::{AudioInfo, VideoInfo}; use std::{ @@ -165,7 +165,6 @@ pub struct MacOSFragmentedM4SMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, } @@ -177,7 +176,6 @@ pub struct MacOSFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, } @@ -190,7 +188,6 @@ impl Default for MacOSFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, } } @@ -232,7 +229,6 @@ impl Muxer for MacOSFragmentedM4SMuxer { pause, frame_drops: FrameDropTracker::new(SharedHealthSender::new(), "muxer:macos-fragmented"), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), }) @@ -305,9 +301,6 @@ impl MacOSFragmentedM4SMuxer { encoder_config, self.timed_fragments, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } if let Some(tx) = &self.segment_tx { encoder.set_segment_callback(tx.clone()); } @@ -553,7 +546,6 @@ pub struct MacOSFragmentedM4SCameraMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, } @@ -564,7 +556,6 @@ pub struct MacOSFragmentedM4SCameraMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, } @@ -576,7 +567,6 @@ impl Default for MacOSFragmentedM4SCameraMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, } } @@ -621,7 +611,6 @@ impl Muxer for MacOSFragmentedM4SCameraMuxer { "muxer:macos-fragmented-camera", ), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), }) @@ -696,9 +685,6 @@ impl MacOSFragmentedM4SCameraMuxer { encoder_config, true, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } if let Some(tx) = &self.segment_tx { encoder.set_segment_callback(tx.clone()); } diff --git a/crates/recording/src/output_pipeline/mod.rs b/crates/recording/src/output_pipeline/mod.rs index 25ecb5c52aa..c93c84217e4 100644 --- a/crates/recording/src/output_pipeline/mod.rs +++ b/crates/recording/src/output_pipeline/mod.rs @@ -47,16 +47,6 @@ mod win; #[cfg(windows)] pub use win::*; -#[cfg(windows)] -mod win_segmented; -#[cfg(windows)] -pub use win_segmented::*; - -#[cfg(windows)] -mod win_segmented_camera; -#[cfg(windows)] -pub use win_segmented_camera::*; - #[cfg(windows)] mod win_fragmented_m4s; #[cfg(windows)] diff --git a/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs b/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs index 9afbb643412..0ee7c7991db 100644 --- a/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/oop_fragmented_m4s.rs @@ -16,7 +16,7 @@ use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::fragment_manifest::FragmentManifestTracker; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::h264_packet::EncodePacketError; -use cap_enc_ffmpeg::segmented_stream::{DiskSpaceCallback, SegmentCompletedEvent}; +use cap_enc_ffmpeg::segmented_stream::SegmentCompletedEvent; use cap_media_info::{AudioInfo, VideoInfo}; use std::{ path::PathBuf, @@ -117,7 +117,6 @@ pub struct OutOfProcessFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, pub max_respawns: u32, } @@ -130,7 +129,6 @@ impl Default for OutOfProcessFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, max_respawns: DEFAULT_MAX_RESPAWNS, } @@ -148,7 +146,6 @@ pub struct OutOfProcessFragmentedM4SMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, max_respawns: u32, @@ -189,7 +186,6 @@ impl Muxer for OutOfProcessFragmentedM4SMuxer { pause, frame_drops: FrameDropTracker::new(SharedHealthSender::new(), "muxer:macos-oop"), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), max_respawns: config.max_respawns, @@ -270,7 +266,6 @@ impl OutOfProcessFragmentedM4SMuxer { let preset = self.preset; let bpp = self.bpp; let output_size = self.output_size; - let disk_space_callback = self.disk_space_callback.clone(); let segment_tx = self.segment_tx.clone(); let health_tx = self.health_tx.clone(); let max_respawns = self.max_respawns; @@ -363,7 +358,6 @@ impl OutOfProcessFragmentedM4SMuxer { }; let mut disk_monitor = DiskSpaceMonitor::new(); - let _ = disk_space_callback; disk_monitor.poll(&base_path, &health_tx); if ready_tx.send(Ok(())).is_err() { diff --git a/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs b/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs index d90b19cf7c9..1867f248607 100644 --- a/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs +++ b/crates/recording/src/output_pipeline/oop_fragmented_m4s_win.rs @@ -15,7 +15,7 @@ use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::fragment_manifest::FragmentManifestTracker; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::h264_packet::EncodePacketError; -use cap_enc_ffmpeg::segmented_stream::{DiskSpaceCallback, SegmentCompletedEvent}; +use cap_enc_ffmpeg::segmented_stream::SegmentCompletedEvent; use cap_media_info::{AudioInfo, VideoInfo}; use std::{ path::PathBuf, @@ -122,7 +122,6 @@ pub struct WindowsOOPFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, pub segment_tx: Option>, pub max_respawns: u32, } @@ -135,7 +134,6 @@ impl Default for WindowsOOPFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, segment_tx: None, max_respawns: DEFAULT_MAX_RESPAWNS, } @@ -152,7 +150,6 @@ pub struct WindowsOOPFragmentedM4SMuxer { state: Option, pause: SharedPauseState, frame_drops: FrameDropTracker, - disk_space_callback: Option, segment_tx: Option>, health_tx: SharedHealthSender, max_respawns: u32, @@ -193,7 +190,6 @@ impl Muxer for WindowsOOPFragmentedM4SMuxer { state: None, pause, frame_drops: FrameDropTracker::new(SharedHealthSender::new(), "muxer:windows-oop"), - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), max_respawns: config.max_respawns, @@ -264,7 +260,6 @@ impl WindowsOOPFragmentedM4SMuxer { let preset = self.preset; let bpp = self.bpp; let output_size = self.output_size; - let disk_space_callback = self.disk_space_callback.clone(); let segment_tx = self.segment_tx.clone(); let health_tx = self.health_tx.clone(); let max_respawns = self.max_respawns; @@ -357,7 +352,6 @@ impl WindowsOOPFragmentedM4SMuxer { }; let mut disk_monitor = DiskSpaceMonitor::new(); - let _ = disk_space_callback; disk_monitor.poll(&base_path, &health_tx); if ready_tx.send(Ok(())).is_err() { @@ -370,6 +364,7 @@ impl WindowsOOPFragmentedM4SMuxer { let mut last_timestamp: Option = None; let mut first_timestamp: Option = None; let mut total_frames = 0u64; + let mut readback_misses = 0u64; let mut duplicated_frames = 0u64; let mut slow_convert_count = 0u32; let mut slow_encode_count = 0u32; @@ -471,7 +466,24 @@ impl WindowsOOPFragmentedM4SMuxer { (Some(f), ts) } Err(e) => { - warn!("Windows OOP failed to convert D3D11 frame: {e:?}"); + let readback_miss = matches!( + e, + scap_ffmpeg::AsFFmpegError::NotReady + | scap_ffmpeg::AsFFmpegError::StagingBusy + ); + if !readback_miss { + warn!("Windows OOP failed to convert D3D11 frame: {e:?}"); + } else { + readback_misses += 1; + if readback_misses == 1 + || readback_misses.is_multiple_of(90) + { + warn!( + readback_misses, + "Screen readback is missing deadlines; the encoder is duplicating the last frame" + ); + } + } duplicated_frames += 1; (last_ffmpeg_frame.as_ref().map(reference_video_frame), ts) } diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index faab33d6e76..8d2fc226df9 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -418,6 +418,7 @@ impl Muxer for WindowsMuxer { let mut pacer = FramePacer::new(config.frame_rate); let mut last_ffmpeg_frame: Option = None; let mut first_timestamp: Option = None; + let mut readback_misses = 0u64; loop { let (ffmpeg_frame, ts) = match pacer.next(&video_rx, &pause_flag) { @@ -427,7 +428,24 @@ impl Muxer for WindowsMuxer { (Some(f), timestamp) } Err(e) => { - warn!("Failed to convert frame: {e:?}"); + let readback_miss = matches!( + e, + scap_ffmpeg::AsFFmpegError::NotReady + | scap_ffmpeg::AsFFmpegError::StagingBusy + ); + if !readback_miss { + warn!("Failed to convert frame: {e:?}"); + } else { + readback_misses += 1; + if readback_misses == 1 + || readback_misses.is_multiple_of(90) + { + warn!( + readback_misses, + "Screen readback is missing deadlines; the encoder is duplicating the last frame" + ); + } + } ( last_ffmpeg_frame.as_ref().map(reference_video_frame), timestamp, @@ -587,7 +605,7 @@ impl AudioMuxer for WindowsMuxer { } } -fn duration_to_timespan(duration: Duration) -> TimeSpan { +pub(crate) fn duration_to_timespan(duration: Duration) -> TimeSpan { const TICKS_PER_SEC: u64 = 10_000_000; const NANOS_PER_TICK: u32 = 100; diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index b517b575903..4e61d2300ed 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -4,17 +4,19 @@ use super::core::{ }; use super::frame_pacing::{FramePacer, Paced}; use super::mjpeg::MjpegDecoder; -use super::win::reference_video_frame; +use super::win::{duration_to_timespan, reference_video_frame}; use crate::{ AudioFrame, AudioMuxer, Muxer, SharedPauseState, TaskPool, VideoMuxer, + capture_pipeline::EncoderPreferences, output_pipeline::{NativeCameraFrame, camera_frame_to_ffmpeg}, screen_capture, }; use anyhow::{Context, anyhow}; use cap_enc_ffmpeg::h264::{H264EncoderBuilder, H264Preset}; use cap_enc_ffmpeg::segmented_stream::{ - DiskSpaceCallback, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, + SegmentCompletedEvent, SegmentedPacketMuxer, SegmentedVideoEncoder, SegmentedVideoEncoderConfig, }; +use cap_enc_mediafoundation::H264Encoder; use cap_media_info::{AudioInfo, Pixel, VideoInfo}; use std::{ path::PathBuf, @@ -27,6 +29,10 @@ use std::{ time::Duration, }; use tracing::*; +use windows::{ + Graphics::SizeInt32, + Win32::Graphics::{Direct3D11::ID3D11Device, Dxgi::Common::DXGI_FORMAT}, +}; const DEFAULT_MUXER_BUFFER_SIZE: usize = 240; @@ -132,14 +138,9 @@ impl FinishableEncoderState for EncoderState { } fn finish_encoder(&self, timestamp: Duration, finish_label: &str) -> anyhow::Result<()> { - match self.encoder.lock() { - Ok(mut encoder) => encoder - .finish_with_timestamp(timestamp) - .map_err(|error| anyhow!("{finish_label}: {error:#}")), - Err(_) => Err(anyhow!( - "{finish_label}: encoder mutex poisoned - recording may be corrupt or incomplete" - )), - } + self.backend + .finish_with_timestamp(timestamp) + .map_err(|error| anyhow!("{finish_label}: {error:#}")) } } @@ -166,6 +167,16 @@ impl FinishableEncoderState for CameraEncoderState { } } +fn normalize_timestamp(ts: Duration, first: &mut Option) -> Duration { + match *first { + Some(first_ts) => ts.checked_sub(first_ts).unwrap_or(Duration::ZERO), + None => { + *first = Some(ts); + Duration::ZERO + } + } +} + fn finish_segmented_encoder_impl( state: &mut impl FinishableEncoderState, timestamp: Duration, @@ -196,10 +207,36 @@ fn finish_segmented_encoder_impl( struct EncoderState { video_tx: SyncSender>, - encoder: Arc>, + backend: EncoderBackend, encoder_handle: Option>>, } +enum EncoderBackend { + Ffmpeg(Arc>), + MediaFoundation(Arc>), +} + +impl EncoderBackend { + fn finish_with_timestamp(&self, timestamp: Duration) -> anyhow::Result<()> { + match self { + Self::Ffmpeg(encoder) => match encoder.lock() { + Ok(mut encoder) => encoder + .finish_with_timestamp(timestamp) + .map_err(|error| anyhow!("{error:#}")), + Err(_) => Err(anyhow!( + "encoder mutex poisoned - recording may be corrupt or incomplete" + )), + }, + Self::MediaFoundation(muxer) => match muxer.lock() { + Ok(mut muxer) => muxer.finish().map_err(|error| anyhow!("{error:#}")), + Err(_) => Err(anyhow!( + "muxer mutex poisoned - recording may be corrupt or incomplete" + )), + }, + } + } +} + pub struct WindowsFragmentedM4SMuxer { timed_fragments: bool, base_path: PathBuf, @@ -208,13 +245,14 @@ pub struct WindowsFragmentedM4SMuxer { preset: H264Preset, bpp: f32, output_size: Option<(u32, u32)>, + d3d_device: Option, + pixel_format: Option, + encoder_preferences: EncoderPreferences, state: Option, pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, - segment_tx: - Option>, + segment_tx: Option>, health_tx: SharedHealthSender, } @@ -225,9 +263,10 @@ pub struct WindowsFragmentedM4SMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, - pub segment_tx: - Option>, + pub d3d_device: Option, + pub pixel_format: Option, + pub encoder_preferences: EncoderPreferences, + pub segment_tx: Option>, } impl Default for WindowsFragmentedM4SMuxerConfig { @@ -239,7 +278,9 @@ impl Default for WindowsFragmentedM4SMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, + d3d_device: None, + pixel_format: None, + encoder_preferences: EncoderPreferences::new(), segment_tx: None, } } @@ -277,6 +318,9 @@ impl Muxer for WindowsFragmentedM4SMuxer { preset: config.preset, bpp: config.bpp, output_size: config.output_size, + d3d_device: config.d3d_device, + pixel_format: config.pixel_format, + encoder_preferences: config.encoder_preferences, state: None, pause, frame_drops: FrameDropTracker::new( @@ -284,7 +328,6 @@ impl Muxer for WindowsFragmentedM4SMuxer { "muxer:windows-fragmented", ), started: false, - disk_space_callback: config.disk_space_callback, segment_tx: config.segment_tx, health_tx: SharedHealthSender::new(), }; @@ -325,6 +368,10 @@ impl Muxer for WindowsFragmentedM4SMuxer { impl WindowsFragmentedM4SMuxer { fn start_encoder(&mut self) -> anyhow::Result<()> { + if self.start_media_foundation_encoder()? { + return Ok(()); + } + let buffer_size = get_muxer_buffer_size(); debug!( buffer_size = buffer_size, @@ -349,9 +396,6 @@ impl WindowsFragmentedM4SMuxer { encoder_config, self.timed_fragments, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } if let Some(tx) = self.segment_tx.take() { encoder.set_segment_callback(tx); } @@ -379,20 +423,11 @@ impl WindowsFragmentedM4SMuxer { let mut slow_encode_count = 0u32; let mut total_frames = 0u64; let mut duplicated_frames = 0u64; + let mut readback_misses = 0u64; let mut disk_monitor = DiskSpaceMonitor::new(); let mut disk_exhausted = false; const SLOW_THRESHOLD_MS: u128 = 5; - let normalize_timestamp = - |ts: Duration, first: &mut Option| -> Duration { - if let Some(first_ts) = *first { - ts.checked_sub(first_ts).unwrap_or(Duration::ZERO) - } else { - *first = Some(ts); - Duration::ZERO - } - }; - let encode_frame_fn = |ffmpeg_frame: ffmpeg::frame::Video, timestamp: Duration, slow_encode_count: &mut u32, @@ -453,7 +488,22 @@ impl WindowsFragmentedM4SMuxer { (Some(f), ts) } Err(e) => { - warn!("Failed to convert D3D11 frame: {e:?}"); + let readback_miss = matches!( + e, + scap_ffmpeg::AsFFmpegError::NotReady + | scap_ffmpeg::AsFFmpegError::StagingBusy + ); + if !readback_miss { + warn!("Failed to convert D3D11 frame: {e:?}"); + } else { + readback_misses += 1; + if readback_misses == 1 || readback_misses.is_multiple_of(90) { + warn!( + readback_misses, + "Screen readback is missing deadlines; the encoder is duplicating the last frame" + ); + } + } duplicated_frames += 1; (last_ffmpeg_frame.as_ref().map(reference_video_frame), ts) } @@ -564,7 +614,7 @@ impl WindowsFragmentedM4SMuxer { self.state = Some(EncoderState { video_tx, - encoder, + backend: EncoderBackend::Ffmpeg(encoder), encoder_handle: Some(encoder_handle), }); @@ -577,6 +627,241 @@ impl WindowsFragmentedM4SMuxer { Ok(()) } + + fn start_media_foundation_encoder(&mut self) -> anyhow::Result { + let (Some(d3d_device), Some(pixel_format)) = (self.d3d_device.clone(), self.pixel_format) + else { + return Ok(false); + }; + + if self.encoder_preferences.should_force_software() + || matches!( + cap_frame_converter::detect_primary_gpu().map(|gpu| gpu.vendor), + Some(cap_frame_converter::GpuVendor::Amd) + ) + { + return Ok(false); + } + + let (video_tx, video_rx) = sync_channel::>( + get_muxer_buffer_size(), + ); + let (ready_tx, ready_rx) = + sync_channel::>, String>>(1); + + let video_config = self.video_config; + let output_size = self.output_size; + let bpp = self.bpp; + let segment_duration = self.segment_duration; + let timed_fragments = self.timed_fragments; + let base_path = self.base_path.clone(); + let pause_flag = self.pause.flag(); + let segment_tx = self.segment_tx.clone(); + let encoder_preferences = self.encoder_preferences.clone(); + let health_tx = self.health_tx.clone(); + + let encoder_handle = std::thread::Builder::new() + .name("win-m4s-mf-encoder".to_string()) + .spawn(move || { + cap_mediafoundation_utils::thread_init(); + + let disk_base_path = base_path.clone(); + let setup = + (|| -> anyhow::Result<(H264Encoder, Arc>)> { + let input_size = SizeInt32 { + Width: video_config.width as i32, + Height: video_config.height as i32, + }; + let (bitrate_width, bitrate_height) = + output_size.unwrap_or((video_config.width, video_config.height)); + let output_size = output_size + .map(|(width, height)| SizeInt32 { + Width: width as i32, + Height: height as i32, + }) + .unwrap_or(input_size); + let frame_rate = video_config.fps(); + let bitrate = cap_enc_ffmpeg::h264::bitrate_for_bpp( + bitrate_width, + bitrate_height, + video_config.frame_rate, + bpp, + ); + + let encoder = H264Encoder::new_with_scaled_output_and_bitrate( + &d3d_device, + pixel_format, + input_size, + output_size, + frame_rate, + bitrate, + )?; + encoder.validate().map_err(|error| anyhow!("{error}"))?; + let extradata = encoder.sequence_header().ok_or_else(|| { + anyhow!("no H.264 sequence header before the first frame") + })?; + let mut muxer = SegmentedPacketMuxer::new( + base_path, + video_config, + (output_size.Width as u32, output_size.Height as u32), + segment_duration, + timed_fragments, + encoder.bitrate(), + &extradata, + )?; + if let Some(tx) = segment_tx { + muxer.set_segment_callback(tx); + } + + Ok((encoder, Arc::new(Mutex::new(muxer)))) + })(); + + let (mut encoder, muxer) = match setup { + Ok(setup) => setup, + Err(error) => { + let _ = ready_tx.send(Err(format!("{error:#}"))); + return Ok(()); + } + }; + + if ready_tx.send(Ok(muxer.clone())).is_err() { + return Ok(()); + } + + let mut pacer = FramePacer::new(video_config.fps()); + let mut last_frame: Option = None; + let mut first_timestamp: Option = None; + let mut failed_writes = 0u64; + let mut disk_monitor = DiskSpaceMonitor::new(); + + let result = encoder.run( + Arc::new(AtomicBool::default()), + || loop { + if matches!( + disk_monitor.poll(&disk_base_path, &health_tx), + super::core::DiskSpacePollResult::Exhausted { .. } + | super::core::DiskSpacePollResult::Stopped + ) { + return Ok(None); + } + + let (texture, timestamp) = match pacer.next(&video_rx, &pause_flag) { + Paced::Frame(frame, timestamp) => { + let texture = frame.texture().clone(); + // Keep the frame itself, not just its texture: the capture frame + // owns the surface, and dropping it lets WGC recycle the buffer + // while a repeat is still fed to the encoder. + last_frame = Some(frame); + (texture, timestamp) + } + Paced::Repeat(timestamp) => { + let Some(frame) = &last_frame else { + continue; + }; + (frame.texture().clone(), timestamp) + } + Paced::End => return Ok(None), + }; + let normalized = normalize_timestamp(timestamp, &mut first_timestamp); + + return Ok(Some((texture, duration_to_timespan(normalized)))); + }, + |sample| { + let packet = match cap_mediafoundation_ffmpeg::sample_to_packet( + &sample, + cap_mediafoundation_ffmpeg::MF_TIMEBASE, + 0, + ) { + Ok(packet) => packet, + Err(error) => { + warn!("Failed to convert Media Foundation sample: {error}"); + return Ok(()); + } + }; + + let Ok(mut muxer) = muxer.lock() else { + return Ok(()); + }; + if let Err(error) = + muxer.write_packet(packet, cap_mediafoundation_ffmpeg::MF_TIMEBASE) + { + failed_writes += 1; + if failed_writes == 1 || failed_writes.is_multiple_of(300) { + warn!( + failed_writes, + "Failed to write Media Foundation sample: {error}" + ); + } + } + Ok(()) + }, + ); + + match result { + Ok(status) => { + debug!( + frames = status.total_frames_encoded, + "Windows M4S Media Foundation encoder finished" + ); + Ok(()) + } + Err(error) => { + if error.should_fallback() { + encoder_preferences.force_software_only(); + } + error!( + "Media Foundation encoder stopped; the rest of the video stream is dropped: {error}" + ); + health_tx.emit(super::core::PipelineHealthEvent::MuxerCrashed { + reason: format!("Media Foundation encoder stopped: {error}"), + }); + + // Keep the channel connected while the recording runs: a closed channel + // makes send_video_frame fail and aborts the whole mux-video task, losing + // audio and the segments already written. Draining lets the pipeline stop + // normally and finish() finalize what exists. + while let Ok(message) = video_rx.recv() { + if message.is_none() { + break; + } + } + + Ok(()) + } + } + })?; + + match ready_rx.recv() { + Ok(Ok(muxer)) => { + self.state = Some(EncoderState { + video_tx, + backend: EncoderBackend::MediaFoundation(muxer), + encoder_handle: Some(encoder_handle), + }); + self.started = true; + info!( + path = %self.base_path.display(), + "Started Windows M4S fragmented video encoder (Media Foundation, texture input)" + ); + Ok(true) + } + Ok(Err(reason)) => { + warn!( + %reason, + "Media Foundation texture encoding unavailable; falling back to FFmpeg" + ); + let _ = encoder_handle.join(); + Ok(false) + } + Err(_) => { + warn!( + "Media Foundation encoder thread ended before signalling readiness; falling back to FFmpeg" + ); + let _ = encoder_handle.join(); + Ok(false) + } + } + } } impl VideoMuxer for WindowsFragmentedM4SMuxer { @@ -629,7 +914,6 @@ pub struct WindowsFragmentedM4SCameraMuxer { pause: SharedPauseState, frame_drops: FrameDropTracker, started: bool, - disk_space_callback: Option, health_tx: SharedHealthSender, } @@ -639,7 +923,6 @@ pub struct WindowsFragmentedM4SCameraMuxerConfig { pub bpp: f32, pub output_size: Option<(u32, u32)>, pub shared_pause_state: Option, - pub disk_space_callback: Option, } impl Default for WindowsFragmentedM4SCameraMuxerConfig { @@ -650,7 +933,6 @@ impl Default for WindowsFragmentedM4SCameraMuxerConfig { bpp: H264EncoderBuilder::QUALITY_BPP, output_size: None, shared_pause_state: None, - disk_space_callback: None, } } } @@ -694,7 +976,6 @@ impl Muxer for WindowsFragmentedM4SCameraMuxer { "muxer:windows-fragmented-camera", ), started: false, - disk_space_callback: config.disk_space_callback, health_tx: SharedHealthSender::new(), }; @@ -794,15 +1075,12 @@ impl WindowsFragmentedM4SCameraMuxer { prefer_videotoolbox_hw_input: false, }; - let mut encoder = SegmentedVideoEncoder::init_with_timed_fragments( + let encoder = SegmentedVideoEncoder::init_with_timed_fragments( self.base_path.clone(), self.video_config, encoder_config, true, )?; - if let Some(callback) = &self.disk_space_callback { - encoder.set_disk_space_callback(callback.clone()); - } let encoder = Arc::new(Mutex::new(encoder)); let encoder_clone = encoder.clone(); diff --git a/crates/recording/src/output_pipeline/win_segmented.rs b/crates/recording/src/output_pipeline/win_segmented.rs deleted file mode 100644 index 1edaf97daf1..00000000000 --- a/crates/recording/src/output_pipeline/win_segmented.rs +++ /dev/null @@ -1,822 +0,0 @@ -use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; -use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation, screen_capture}; -use anyhow::{Context, anyhow}; -use cap_media_info::{AudioInfo, VideoInfo}; -use serde::Serialize; -use std::{ - path::PathBuf, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc::{SyncSender, sync_channel}, - }, - thread::JoinHandle, - time::Duration, -}; -use tracing::*; -use windows::{ - Foundation::TimeSpan, - Graphics::SizeInt32, - Win32::Graphics::{Direct3D11::ID3D11Device, Dxgi::Common::DXGI_FORMAT}, -}; - -#[derive(Debug, Clone)] -pub struct SegmentInfo { - pub path: PathBuf, - pub index: u32, - pub duration: Duration, - pub file_size: Option, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -struct SegmentState { - video_tx: SyncSender>, - output: Arc>, - encoder_handle: Option>>, -} - -struct PauseTracker { - flag: Arc, - paused_at: Option, - offset: Duration, -} - -struct FrameDropTracker { - count: u32, - last_warning: std::time::Instant, -} - -impl FrameDropTracker { - fn new() -> Self { - Self { - count: 0, - last_warning: std::time::Instant::now(), - } - } - - fn record_drop(&mut self) { - self.count += 1; - if self.count >= 30 && self.last_warning.elapsed() > Duration::from_secs(5) { - warn!( - "Dropped {} screen frames due to encoder backpressure", - self.count - ); - self.count = 0; - self.last_warning = std::time::Instant::now(); - } - } - - fn reset(&mut self) { - if self.count > 0 { - trace!("Frame drop count at segment boundary: {}", self.count); - } - self.count = 0; - } -} - -impl PauseTracker { - fn new(flag: Arc) -> Self { - Self { - flag, - paused_at: None, - offset: Duration::ZERO, - } - } - - fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { - if self.flag.load(Ordering::Acquire) { - if self.paused_at.is_none() { - self.paused_at = Some(timestamp); - } - return Ok(None); - } - - if let Some(start) = self.paused_at.take() { - let delta = match timestamp.checked_sub(start) { - Some(d) => d, - None => { - warn!( - resume_at = ?start, - current = ?timestamp, - "Timestamp anomaly: frame timestamp went backward during unpause (clock skew?), treating as zero delta" - ); - Duration::ZERO - } - }; - - self.offset = match self.offset.checked_add(delta) { - Some(o) => o, - None => { - warn!( - offset = ?self.offset, - delta = ?delta, - "Timestamp anomaly: pause offset overflow, clamping to MAX" - ); - Duration::MAX - } - }; - } - - let adjusted = match timestamp.checked_sub(self.offset) { - Some(t) => t, - None => { - warn!( - timestamp = ?timestamp, - offset = ?self.offset, - "Timestamp anomaly: adjusted timestamp underflow (clock skew?), using zero" - ); - Duration::ZERO - } - }; - - Ok(Some(adjusted)) - } -} - -pub struct WindowsSegmentedMuxer { - base_path: PathBuf, - segment_duration: Duration, - current_index: u32, - segment_start_time: Option, - completed_segments: Vec, - - current_state: Option, - - video_config: VideoInfo, - pixel_format: DXGI_FORMAT, - d3d_device: ID3D11Device, - frame_rate: u32, - bitrate_multiplier: f32, - output_size: Option, - encoder_preferences: crate::capture_pipeline::EncoderPreferences, - - pause: PauseTracker, - frame_drops: FrameDropTracker, -} - -pub struct WindowsSegmentedMuxerConfig { - pub pixel_format: DXGI_FORMAT, - pub d3d_device: ID3D11Device, - pub frame_rate: u32, - pub bitrate_multiplier: f32, - pub output_size: Option, - pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, - pub segment_duration: Duration, -} - -impl Muxer for WindowsSegmentedMuxer { - type Config = WindowsSegmentedMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - _audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result - where - Self: Sized, - { - let video_config = - video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; - - std::fs::create_dir_all(&output_path) - .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; - - Ok(Self { - base_path: output_path, - segment_duration: config.segment_duration, - current_index: 0, - segment_start_time: None, - completed_segments: Vec::new(), - current_state: None, - video_config, - pixel_format: config.pixel_format, - d3d_device: config.d3d_device, - frame_rate: config.frame_rate, - bitrate_multiplier: config.bitrate_multiplier, - output_size: config.output_size, - encoder_preferences: config.encoder_preferences, - pause: PauseTracker::new(pause_flag), - frame_drops: FrameDropTracker::new(), - }) - } - - fn stop(&mut self) { - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.send(None) - { - trace!("Screen encoder channel already closed during stop: {e}"); - } - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let mut finish_error = None; - let mut final_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Screen encoder thread", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - finish_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write screen trailer: {error:#}"); - finish_error = Some(match finish_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&segment_path); - - if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - final_segment = Some(SegmentInfo { - path: segment_path, - index: self.current_index, - duration: final_duration, - file_size, - }); - } - } - - if let Some(error) = finish_error { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - if let Some(segment) = final_segment { - self.completed_segments.push(segment); - } - - if let Err(error) = self.finalize_manifest() { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - Ok(Ok(())) - } -} - -impl WindowsSegmentedMuxer { - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn finalize_manifest(&self) -> anyhow::Result<()> { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - fragmentation::atomic_write_json(&manifest_path, &manifest).map_err(|error| { - anyhow!( - "Failed to write final manifest to {}: {error}", - manifest_path.display() - ) - }) - } - - fn create_segment(&mut self) -> anyhow::Result<()> { - let segment_path = self.current_segment_path(); - let input_size = SizeInt32 { - Width: self.video_config.width as i32, - Height: self.video_config.height as i32, - }; - let output_size = self.output_size.unwrap_or(input_size); - - let queue_depth = ((self.frame_rate as f32 / 30.0 * 5.0).ceil() as usize).clamp(3, 12); - let (video_tx, video_rx) = - sync_channel::>(queue_depth); - let (ready_tx, ready_rx) = sync_channel::>(1); - let output = ffmpeg::format::output(&segment_path)?; - let output = Arc::new(Mutex::new(output)); - - let d3d_device = self.d3d_device.clone(); - let pixel_format = self.pixel_format; - let frame_rate = self.frame_rate; - let bitrate_multiplier = self.bitrate_multiplier; - let video_config = self.video_config; - let encoder_preferences = self.encoder_preferences.clone(); - let output_clone = output.clone(); - - let encoder_handle = std::thread::Builder::new() - .name(format!("segment-encoder-{}", self.current_index)) - .spawn(move || { - cap_mediafoundation_utils::thread_init(); - - let encoder = (|| { - let fallback = |reason: Option| { - encoder_preferences.force_software_only(); - if let Some(reason) = reason.as_ref() { - error!("Falling back to software H264 encoder: {reason}"); - } else { - info!("Falling back to software H264 encoder"); - } - - let fallback_width = if output_size.Width > 0 { - output_size.Width as u32 - } else { - video_config.width - }; - let fallback_height = if output_size.Height > 0 { - output_size.Height as u32 - } else { - video_config.height - }; - - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return Err(anyhow!( - "ScreenSoftwareEncoder: failed to lock output mutex: {}", - poisoned - )); - } - }; - - cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) - .with_output_size(fallback_width, fallback_height) - .and_then(|builder| builder.build(&mut output_guard)) - .map(either::Right) - .map_err(|e| anyhow!("ScreenSoftwareEncoder/{e}")) - }; - - if encoder_preferences.should_force_software() { - return fallback(None); - } - - match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - pixel_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ) { - Ok(encoder) => { - if let Err(e) = encoder.validate() { - return fallback(Some(format!( - "Hardware encoder validation failed: {e}" - ))); - } - - let width = match u32::try_from(output_size.Width) { - Ok(width) if width > 0 => width, - _ => { - return fallback(Some(format!( - "Invalid output width: {}", - output_size.Width - ))); - } - }; - - let height = match u32::try_from(output_size.Height) { - Ok(height) if height > 0 => height, - _ => { - return fallback(Some(format!( - "Invalid output height: {}", - output_size.Height - ))); - } - }; - - let muxer = { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return fallback(Some(format!( - "Failed to lock output mutex: {poisoned}" - ))); - } - }; - - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width, - height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented: false, - frag_duration_us: 0, - }, - ) - }; - - match muxer { - Ok(muxer) => Ok(either::Left((encoder, muxer))), - Err(err) => fallback(Some(err.to_string())), - } - } - Err(err) => fallback(Some(err.to_string())), - } - })(); - - let encoder = match encoder { - Ok(encoder) => { - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } - encoder - } - Err(e) => { - error!("Encoder setup failed: {:#}", e); - let _ = ready_tx.send(Err(anyhow!("{e}"))); - return Err(anyhow!("{e}")); - } - }; - - match encoder { - either::Left((mut encoder, mut muxer)) => { - trace!("Running native encoder for segment"); - let mut first_timestamp: Option = None; - let result = encoder.run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more frames available for segment"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - let frame_time = duration_to_timespan(relative); - - Ok(Some((frame.texture().clone(), frame_time))) - }, - |output_sample| { - let mut output = match output_clone.lock() { - Ok(guard) => guard, - Err(e) => { - error!("Failed to lock output mutex: {e}"); - return Err(windows::core::Error::new( - windows::core::HRESULT(0x80004005u32 as i32), - format!("Mutex poisoned: {e}"), - )); - } - }; - - if let Err(e) = muxer.write_sample(&output_sample, &mut output) { - warn!("WriteSample failed: {e}"); - } - - Ok(()) - }, - ); - - match result { - Ok(health_status) => { - debug!( - "Hardware encoder completed for segment: {} frames encoded", - health_status.total_frames_encoded - ); - Ok(()) - } - Err(e) => { - if e.should_fallback() { - error!( - "Hardware encoder failed with recoverable error, marking for software fallback: {}", - e - ); - encoder_preferences.force_software_only(); - } - Err(anyhow!("Hardware encoder error: {}", e)) - } - } - } - either::Right(mut encoder) => { - while let Ok(Some((frame, time))) = video_rx.recv() { - let Ok(mut output) = output_clone.lock() else { - continue; - }; - - frame - .as_ffmpeg() - .context("frame as_ffmpeg") - .and_then(|frame| { - encoder - .queue_frame(frame, time, &mut output) - .context("queue_frame") - })?; - } - - Ok(()) - } - } - })?; - - ready_rx - .recv() - .map_err(|_| anyhow!("Encoder thread ended unexpectedly"))??; - - output.lock().unwrap().write_header()?; - - self.current_state = Some(SegmentState { - video_tx, - output, - encoder_handle: Some(encoder_handle), - }); - - Ok(()) - } - - fn rotate_segment(&mut self, timestamp: Duration) -> anyhow::Result<()> { - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let mut rotation_error = None; - let mut rotated_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Screen encoder channel already closed during rotation: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Screen encoder thread during rotation", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - rotation_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Err(error); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write rotated screen trailer: {error:#}"); - rotation_error = Some(match rotation_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&completed_segment_path); - - let file_size = std::fs::metadata(&completed_segment_path) - .ok() - .map(|m| m.len()); - - rotated_segment = Some(SegmentInfo { - path: completed_segment_path, - index: self.current_index, - duration: segment_duration, - file_size, - }); - } - - if let Some(error) = rotation_error { - self.write_in_progress_manifest(); - return Err(error); - } - - if let Some(segment) = rotated_segment { - self.completed_segments.push(segment); - } - - self.write_manifest(); - self.frame_drops.reset(); - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - self.create_segment()?; - self.write_in_progress_manifest(); - - info!( - "Rotated to segment {} at {:?}", - self.current_index, timestamp - ); - - Ok(()) - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } -} - -impl VideoMuxer for WindowsSegmentedMuxer { - type VideoFrame = screen_capture::VideoFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { - return Ok(()); - }; - - if self.current_state.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - self.create_segment()?; - self.write_in_progress_manifest(); - } - - if self.segment_start_time.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - } - - let segment_elapsed = - adjusted_timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(adjusted_timestamp)?; - } - - if let Some(state) = &self.current_state - && let Err(e) = state - .video_tx - .try_send(Some((frame.frame, adjusted_timestamp))) - { - match e { - std::sync::mpsc::TrySendError::Full(_) => { - self.frame_drops.record_drop(); - } - std::sync::mpsc::TrySendError::Disconnected(_) => { - return Err(anyhow!("Screen encoder channel disconnected")); - } - } - } - - Ok(()) - } -} - -impl AudioMuxer for WindowsSegmentedMuxer { - fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { - Ok(()) - } -} - -fn duration_to_timespan(duration: Duration) -> TimeSpan { - const TICKS_PER_SEC: u64 = 10_000_000; - const NANOS_PER_TICK: u32 = 100; - - let secs_ticks = duration.as_secs().saturating_mul(TICKS_PER_SEC); - let nanos_ticks = (duration.subsec_nanos() / NANOS_PER_TICK) as u64; - let total_ticks = secs_ticks.saturating_add(nanos_ticks); - let clamped = total_ticks.min(i64::MAX as u64); - - TimeSpan { - Duration: clamped as i64, - } -} diff --git a/crates/recording/src/output_pipeline/win_segmented_camera.rs b/crates/recording/src/output_pipeline/win_segmented_camera.rs deleted file mode 100644 index c377d33888a..00000000000 --- a/crates/recording/src/output_pipeline/win_segmented_camera.rs +++ /dev/null @@ -1,849 +0,0 @@ -use super::core::{BlockingThreadFinish, combine_finish_errors, wait_for_blocking_thread_finish}; -use crate::output_pipeline::win::{CameraBuffers, NativeCameraFrame, upload_mf_buffer_to_texture}; -use crate::{AudioFrame, AudioMuxer, Muxer, TaskPool, VideoMuxer, fragmentation}; -use anyhow::{Context, anyhow}; -use cap_media_info::{AudioInfo, VideoInfo}; -use serde::Serialize; -use std::{ - path::PathBuf, - sync::{ - Arc, Mutex, - atomic::{AtomicBool, Ordering}, - mpsc::{SyncSender, sync_channel}, - }, - thread::JoinHandle, - time::Duration, -}; -use tracing::*; -use windows::{Foundation::TimeSpan, Graphics::SizeInt32}; - -#[derive(Debug, Clone)] -struct SegmentInfo { - path: PathBuf, - index: u32, - duration: Duration, - file_size: Option, -} - -#[derive(Serialize)] -struct FragmentEntry { - path: String, - index: u32, - duration: f64, - is_complete: bool, - #[serde(skip_serializing_if = "Option::is_none")] - file_size: Option, -} - -const MANIFEST_VERSION: u32 = 2; - -#[derive(Serialize)] -struct Manifest { - version: u32, - fragments: Vec, - #[serde(skip_serializing_if = "Option::is_none")] - total_duration: Option, - is_complete: bool, -} - -struct SegmentState { - video_tx: SyncSender>, - output: Arc>, - encoder_handle: Option>>, -} - -struct PauseTracker { - flag: Arc, - paused_at: Option, - offset: Duration, -} - -struct FrameDropTracker { - count: u32, - last_warning: std::time::Instant, -} - -impl FrameDropTracker { - fn new() -> Self { - Self { - count: 0, - last_warning: std::time::Instant::now(), - } - } - - fn record_drop(&mut self) { - self.count += 1; - if self.count >= 30 && self.last_warning.elapsed() > Duration::from_secs(5) { - warn!( - "Dropped {} camera frames due to encoder backpressure", - self.count - ); - self.count = 0; - self.last_warning = std::time::Instant::now(); - } - } - - fn reset(&mut self) { - if self.count > 0 { - trace!( - "Camera frame drop count at segment boundary: {}", - self.count - ); - } - self.count = 0; - } -} - -impl PauseTracker { - fn new(flag: Arc) -> Self { - Self { - flag, - paused_at: None, - offset: Duration::ZERO, - } - } - - fn adjust(&mut self, timestamp: Duration) -> anyhow::Result> { - if self.flag.load(Ordering::Acquire) { - if self.paused_at.is_none() { - self.paused_at = Some(timestamp); - } - return Ok(None); - } - - if let Some(start) = self.paused_at.take() { - let delta = match timestamp.checked_sub(start) { - Some(d) => d, - None => { - warn!( - resume_at = ?start, - current = ?timestamp, - "Timestamp anomaly: frame timestamp went backward during unpause (clock skew?), treating as zero delta" - ); - Duration::ZERO - } - }; - - self.offset = match self.offset.checked_add(delta) { - Some(o) => o, - None => { - warn!( - offset = ?self.offset, - delta = ?delta, - "Timestamp anomaly: pause offset overflow, clamping to MAX" - ); - Duration::MAX - } - }; - } - - let adjusted = match timestamp.checked_sub(self.offset) { - Some(t) => t, - None => { - warn!( - timestamp = ?timestamp, - offset = ?self.offset, - "Timestamp anomaly: adjusted timestamp underflow (clock skew?), using zero" - ); - Duration::ZERO - } - }; - - Ok(Some(adjusted)) - } -} - -pub struct WindowsSegmentedCameraMuxer { - base_path: PathBuf, - segment_duration: Duration, - current_index: u32, - segment_start_time: Option, - completed_segments: Vec, - - current_state: Option, - - video_config: VideoInfo, - output_height: Option, - encoder_preferences: crate::capture_pipeline::EncoderPreferences, - - pause: PauseTracker, - frame_drops: FrameDropTracker, -} - -pub struct WindowsSegmentedCameraMuxerConfig { - pub output_height: Option, - pub segment_duration: Duration, - pub encoder_preferences: crate::capture_pipeline::EncoderPreferences, -} - -impl Default for WindowsSegmentedCameraMuxerConfig { - fn default() -> Self { - Self { - output_height: None, - segment_duration: Duration::from_secs(3), - encoder_preferences: crate::capture_pipeline::EncoderPreferences::new(), - } - } -} - -impl Muxer for WindowsSegmentedCameraMuxer { - type Config = WindowsSegmentedCameraMuxerConfig; - - async fn setup( - config: Self::Config, - output_path: PathBuf, - video_config: Option, - _audio_config: Option, - pause_flag: Arc, - _tasks: &mut TaskPool, - ) -> anyhow::Result - where - Self: Sized, - { - let video_config = - video_config.ok_or_else(|| anyhow!("invariant: video config expected"))?; - - std::fs::create_dir_all(&output_path) - .with_context(|| format!("Failed to create segments directory: {output_path:?}"))?; - - Ok(Self { - base_path: output_path, - segment_duration: config.segment_duration, - current_index: 0, - segment_start_time: None, - completed_segments: Vec::new(), - current_state: None, - video_config, - output_height: config.output_height, - encoder_preferences: config.encoder_preferences, - pause: PauseTracker::new(pause_flag), - frame_drops: FrameDropTracker::new(), - }) - } - - fn stop(&mut self) { - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.send(None) - { - trace!("Camera encoder channel already closed during stop: {e}"); - } - } - - fn finish(&mut self, timestamp: Duration) -> anyhow::Result> { - let segment_path = self.current_segment_path(); - let segment_start = self.segment_start_time; - let mut finish_error = None; - let mut final_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Camera encoder channel already closed during finish: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Camera encoder thread", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - finish_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write camera trailer: {error:#}"); - finish_error = Some(match finish_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&segment_path); - - if let Some(start) = segment_start { - let final_duration = timestamp.saturating_sub(start); - let file_size = std::fs::metadata(&segment_path).ok().map(|m| m.len()); - - final_segment = Some(SegmentInfo { - path: segment_path, - index: self.current_index, - duration: final_duration, - file_size, - }); - } - } - - if let Some(error) = finish_error { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - if let Some(segment) = final_segment { - self.completed_segments.push(segment); - } - - if let Err(error) = self.finalize_manifest() { - self.write_in_progress_manifest(); - return Ok(Err(error)); - } - - Ok(Ok(())) - } -} - -impl WindowsSegmentedCameraMuxer { - fn current_segment_path(&self) -> PathBuf { - self.base_path - .join(format!("fragment_{:03}.mp4", self.current_index)) - } - - fn write_manifest(&self) { - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write manifest to {}: {e}", - manifest_path.display() - ); - } - } - - fn finalize_manifest(&self) -> anyhow::Result<()> { - let total_duration: Duration = self.completed_segments.iter().map(|s| s.duration).sum(); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments: self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(), - total_duration: Some(total_duration.as_secs_f64()), - is_complete: true, - }; - - let manifest_path = self.base_path.join("manifest.json"); - fragmentation::atomic_write_json(&manifest_path, &manifest).map_err(|error| { - anyhow!( - "Failed to write final manifest to {}: {error}", - manifest_path.display() - ) - }) - } - - fn create_segment(&mut self, first_frame: &NativeCameraFrame) -> anyhow::Result<()> { - use crate::output_pipeline::{mjpeg::MjpegDecoder, win::camera_frame_to_ffmpeg}; - - let segment_path = self.current_segment_path(); - - let input_size = SizeInt32 { - Width: self.video_config.width as i32, - Height: self.video_config.height as i32, - }; - - let output_height = self.output_height.unwrap_or(self.video_config.height); - let output_width = (self.video_config.width * output_height) / self.video_config.height; - let output_width = output_width & !1; - let output_height = output_height & !1; - - let output_size = SizeInt32 { - Width: output_width as i32, - Height: output_height as i32, - }; - - let frame_rate = self.video_config.fps(); - let bitrate_multiplier = 0.2f32; - let input_format = first_frame.dxgi_format(); - let video_config = self.video_config; - let encoder_preferences = self.encoder_preferences.clone(); - - let (video_tx, video_rx) = sync_channel::>(30); - let (ready_tx, ready_rx) = sync_channel::>(1); - let output = ffmpeg::format::output(&segment_path)?; - let output = Arc::new(Mutex::new(output)); - let output_clone = output.clone(); - - let encoder_handle = std::thread::Builder::new() - .name(format!("camera-segment-encoder-{}", self.current_index)) - .spawn(move || { - cap_mediafoundation_utils::thread_init(); - - let d3d_device = match crate::capture_pipeline::create_d3d_device() { - Ok(device) => device, - Err(e) => { - let _ = ready_tx.send(Err(anyhow!("Failed to create D3D device: {e}"))); - return Err(anyhow!("Failed to create D3D device: {e}")); - } - }; - - let encoder = (|| { - let fallback = |reason: Option| { - encoder_preferences.force_software_only(); - if let Some(reason) = reason.as_ref() { - error!( - "Falling back to software H264 encoder for camera segment: {reason}" - ); - } else { - info!("Using software H264 encoder for camera segment"); - } - - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return Err(anyhow!( - "CameraSegmentSoftwareEncoder: failed to lock output mutex: {}", - poisoned - )); - } - }; - - cap_enc_ffmpeg::h264::H264Encoder::builder(video_config) - .with_output_size(output_width, output_height) - .and_then(|builder| builder.build(&mut output_guard)) - .map(either::Right) - .map_err(|e| anyhow!("CameraSegmentSoftwareEncoder/{e}")) - }; - - if encoder_preferences.should_force_software() { - return fallback(None); - } - - match cap_enc_mediafoundation::H264Encoder::new_with_scaled_output( - &d3d_device, - input_format, - input_size, - output_size, - frame_rate, - bitrate_multiplier, - ) { - Ok(encoder) => { - if let Err(e) = encoder.validate() { - return fallback(Some(format!( - "Camera hardware encoder validation failed: {e}" - ))); - } - - let muxer = { - let mut output_guard = match output_clone.lock() { - Ok(guard) => guard, - Err(poisoned) => { - return fallback(Some(format!( - "Failed to lock output mutex: {poisoned}" - ))); - } - }; - - cap_mediafoundation_ffmpeg::H264StreamMuxer::new( - &mut output_guard, - cap_mediafoundation_ffmpeg::MuxerConfig { - width: output_width, - height: output_height, - fps: frame_rate, - bitrate: encoder.bitrate(), - fragmented: false, - frag_duration_us: 0, - }, - ) - }; - - match muxer { - Ok(muxer) => Ok(either::Left((encoder, muxer))), - Err(err) => fallback(Some(err.to_string())), - } - } - Err(err) => fallback(Some(err.to_string())), - } - })(); - - let encoder = match encoder { - Ok(encoder) => { - if ready_tx.send(Ok(())).is_err() { - error!("Failed to send ready signal - receiver dropped"); - return Ok(()); - } - encoder - } - Err(e) => { - error!("Camera segment encoder setup failed: {:#}", e); - let _ = ready_tx.send(Err(anyhow!("{e}"))); - return Err(anyhow!("{e}")); - } - }; - - match encoder { - either::Left((mut encoder, mut muxer)) => { - info!( - "Camera segment encoder started (hardware): {:?} {}x{} -> NV12 {}x{} @ {}fps", - input_format, - input_size.Width, - input_size.Height, - output_size.Width, - output_size.Height, - frame_rate - ); - - let mut first_timestamp: Option = None; - let mut camera_buffers = CameraBuffers::new(); - - let result = encoder.run( - Arc::new(AtomicBool::default()), - || { - let Ok(Some((frame, timestamp))) = video_rx.recv() else { - trace!("No more camera frames available for segment"); - return Ok(None); - }; - - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - - let texture = upload_mf_buffer_to_texture(&d3d_device, &frame, &mut camera_buffers)?; - Ok(Some((texture, duration_to_timespan(relative)))) - }, - |output_sample| { - let mut output = output_clone.lock().map_err(|e| { - windows::core::Error::new( - windows::core::HRESULT(-1), - format!("Mutex poisoned: {e}"), - ) - })?; - muxer - .write_sample(&output_sample, &mut output) - .map_err(|e| { - windows::core::Error::new( - windows::core::HRESULT(-1), - format!("WriteSample: {e}"), - ) - }) - }, - ); - - match result { - Ok(health_status) => { - info!( - "Camera segment encoder finished (hardware): {} frames encoded", - health_status.total_frames_encoded - ); - Ok(()) - } - Err(e) => { - if e.should_fallback() { - error!( - "Camera hardware encoder failed with recoverable error, marking for software fallback: {}", - e - ); - encoder_preferences.force_software_only(); - } - Err(anyhow!("Camera hardware encoder error: {}", e)) - } - } - } - either::Right(mut encoder) => { - info!( - "Camera segment encoder started (software): {}x{} -> {}x{} @ {}fps", - video_config.width, - video_config.height, - output_width, - output_height, - frame_rate - ); - - let mut first_timestamp: Option = None; - let mut mjpeg_decoder = MjpegDecoder::default(); - - while let Ok(Some((frame, timestamp))) = video_rx.recv() { - let relative = if let Some(first) = first_timestamp { - timestamp.checked_sub(first).unwrap_or(Duration::ZERO) - } else { - first_timestamp = Some(timestamp); - Duration::ZERO - }; - - let ffmpeg_frame = match camera_frame_to_ffmpeg(&frame, &mut mjpeg_decoder) { - Ok(f) => f, - Err(e) => { - warn!("Failed to convert camera frame: {e}"); - continue; - } - }; - - let Ok(mut output_guard) = output_clone.lock() else { - continue; - }; - - if let Err(e) = - encoder.queue_frame(ffmpeg_frame, relative, &mut output_guard) - { - warn!("Failed to queue camera frame: {e}"); - } - } - - if let Ok(mut output_guard) = output_clone.lock() - && let Err(e) = encoder.flush(&mut output_guard) - { - warn!("Failed to flush software encoder: {e}"); - } - - Ok(()) - } - } - })?; - - ready_rx - .recv() - .map_err(|_| anyhow!("Camera encoder thread ended unexpectedly"))??; - - output.lock().unwrap().write_header()?; - - self.current_state = Some(SegmentState { - video_tx, - output, - encoder_handle: Some(encoder_handle), - }); - - Ok(()) - } - - fn rotate_segment( - &mut self, - timestamp: Duration, - next_frame: &NativeCameraFrame, - ) -> anyhow::Result<()> { - let segment_start = self.segment_start_time.unwrap_or(Duration::ZERO); - let segment_duration = timestamp.saturating_sub(segment_start); - let completed_segment_path = self.current_segment_path(); - let mut rotation_error = None; - let mut rotated_segment = None; - - if let Some(mut state) = self.current_state.take() { - if let Err(e) = state.video_tx.send(None) { - trace!("Camera encoder channel already closed during rotation: {e}"); - } - - if let Some(handle) = state.encoder_handle.take() { - match wait_for_blocking_thread_finish( - handle, - Duration::from_secs(5), - "Camera encoder thread during rotation", - ) { - BlockingThreadFinish::Clean => {} - BlockingThreadFinish::Failed(error) => { - rotation_error = Some(error); - } - BlockingThreadFinish::TimedOut(error) => { - self.write_in_progress_manifest(); - return Err(error); - } - } - } - - let mut output = state - .output - .lock() - .map_err(|_| anyhow!("Failed to lock output"))?; - if let Err(error) = output.write_trailer() { - let error = anyhow!("Failed to write rotated camera trailer: {error:#}"); - rotation_error = Some(match rotation_error.take() { - Some(existing) => combine_finish_errors(existing, error), - None => error, - }); - } - - fragmentation::sync_file(&completed_segment_path); - - let file_size = std::fs::metadata(&completed_segment_path) - .ok() - .map(|m| m.len()); - - rotated_segment = Some(SegmentInfo { - path: completed_segment_path, - index: self.current_index, - duration: segment_duration, - file_size, - }); - } - - if let Some(error) = rotation_error { - self.write_in_progress_manifest(); - return Err(error); - } - - if let Some(segment) = rotated_segment { - self.completed_segments.push(segment); - } - - self.write_manifest(); - self.frame_drops.reset(); - self.current_index += 1; - self.segment_start_time = Some(timestamp); - - self.create_segment(next_frame)?; - self.write_in_progress_manifest(); - - info!( - "Camera rotated to segment {} at {:?}", - self.current_index, timestamp - ); - - Ok(()) - } - - fn write_in_progress_manifest(&self) { - let mut fragments: Vec = self - .completed_segments - .iter() - .map(|s| FragmentEntry { - path: s - .path - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: s.index, - duration: s.duration.as_secs_f64(), - is_complete: true, - file_size: s.file_size, - }) - .collect(); - - fragments.push(FragmentEntry { - path: self - .current_segment_path() - .file_name() - .unwrap_or_default() - .to_string_lossy() - .into_owned(), - index: self.current_index, - duration: 0.0, - is_complete: false, - file_size: None, - }); - - let manifest = Manifest { - version: MANIFEST_VERSION, - fragments, - total_duration: None, - is_complete: false, - }; - - let manifest_path = self.base_path.join("manifest.json"); - if let Err(e) = fragmentation::atomic_write_json(&manifest_path, &manifest) { - warn!( - "Failed to write in-progress manifest to {}: {e}", - manifest_path.display() - ); - } - } -} - -impl VideoMuxer for WindowsSegmentedCameraMuxer { - type VideoFrame = NativeCameraFrame; - - fn send_video_frame( - &mut self, - frame: Self::VideoFrame, - timestamp: Duration, - ) -> anyhow::Result<()> { - let Some(adjusted_timestamp) = self.pause.adjust(timestamp)? else { - return Ok(()); - }; - - if self.current_state.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - self.create_segment(&frame)?; - self.write_in_progress_manifest(); - } - - if self.segment_start_time.is_none() { - self.segment_start_time = Some(adjusted_timestamp); - } - - let segment_elapsed = - adjusted_timestamp.saturating_sub(self.segment_start_time.unwrap_or(Duration::ZERO)); - - if segment_elapsed >= self.segment_duration { - self.rotate_segment(adjusted_timestamp, &frame)?; - } - - if let Some(state) = &self.current_state - && let Err(e) = state.video_tx.try_send(Some((frame, adjusted_timestamp))) - { - match e { - std::sync::mpsc::TrySendError::Full(_) => { - self.frame_drops.record_drop(); - } - std::sync::mpsc::TrySendError::Disconnected(_) => { - return Err(anyhow!("Camera encoder channel disconnected")); - } - } - } - - Ok(()) - } -} - -impl AudioMuxer for WindowsSegmentedCameraMuxer { - fn send_audio_frame(&mut self, _frame: AudioFrame, _timestamp: Duration) -> anyhow::Result<()> { - Ok(()) - } -} - -fn duration_to_timespan(duration: Duration) -> TimeSpan { - const TICKS_PER_SEC: u64 = 10_000_000; - const NANOS_PER_TICK: u32 = 100; - - let secs_ticks = duration.as_secs().saturating_mul(TICKS_PER_SEC); - let nanos_ticks = (duration.subsec_nanos() / NANOS_PER_TICK) as u64; - let total_ticks = secs_ticks.saturating_add(nanos_ticks); - let clamped = total_ticks.min(i64::MAX as u64); - - TimeSpan { - Duration: clamped as i64, - } -} diff --git a/crates/recording/src/sources/screen_capture/windows.rs b/crates/recording/src/sources/screen_capture/windows.rs index 36d5d448386..2148b35253e 100644 --- a/crates/recording/src/sources/screen_capture/windows.rs +++ b/crates/recording/src/sources/screen_capture/windows.rs @@ -130,7 +130,7 @@ impl ScreenFrame { } } - pub fn as_ffmpeg(&self) -> Result { + pub fn as_ffmpeg(&self) -> Result { match self { ScreenFrame::Captured(frame) => { use scap_ffmpeg::AsFFmpeg; diff --git a/crates/scap-direct3d/src/lib.rs b/crates/scap-direct3d/src/lib.rs index dbe42566731..5a87ead0ba0 100644 --- a/crates/scap-direct3d/src/lib.rs +++ b/crates/scap-direct3d/src/lib.rs @@ -7,7 +7,7 @@ pub use windows_version::WindowsVersion; use std::{ sync::{ Arc, Condvar, Mutex, - atomic::{AtomicBool, AtomicUsize, Ordering}, + atomic::{AtomicBool, Ordering}, mpsc::RecvError, }, time::{Duration, Instant}, @@ -27,7 +27,9 @@ use windows::{ Direct3D::{D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_WARP}, Direct3D11::{ D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_BOX, - D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_FLAG, D3D11_MAP_READ, + D3D11_CPU_ACCESS_READ, D3D11_CPU_ACCESS_WRITE, D3D11_CREATE_DEVICE_BGRA_SUPPORT, + D3D11_CREATE_DEVICE_FLAG, D3D11_CREATE_DEVICE_VIDEO_SUPPORT, + D3D11_MAP_FLAG_DO_NOT_WAIT, D3D11_MAP_READ, D3D11_MAP_WRITE, D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, D3D11_USAGE_STAGING, D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Multithread, ID3D11Texture2D, @@ -37,7 +39,7 @@ use windows::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_SAMPLE_DESC, }, - DXGI_ERROR_UNSUPPORTED, IDXGIDevice, + DXGI_ERROR_UNSUPPORTED, DXGI_ERROR_WAS_STILL_DRAWING, IDXGIDevice, }, }, System::WinRT::Direct3D11::{ @@ -72,6 +74,10 @@ impl PixelFormat { } const STAGING_POOL_SIZE: usize = 3; +const STAGING_POOL_LIMIT: usize = STAGING_POOL_SIZE + 1; +const STAGING_PADDING_ROWS: u32 = 1; +const READBACK_RETRY_INTERVAL: Duration = Duration::from_micros(100); +const READBACK_SPIN_BUDGET: Duration = Duration::from_millis(4); struct CallbackActivity { active: Mutex, @@ -152,87 +158,267 @@ impl Drop for CallbackActivityGuard { } } +#[derive(Debug, thiserror::Error)] +pub enum ReadbackError { + #[error("D3D11: {0}")] + D3D11(#[from] windows::core::Error), + #[error("every staging texture is in flight")] + StagingBusy, + #[error("the GPU did not finish the readback before the deadline")] + NotReady, + #[error("failed to allocate the frame buffer reference")] + FrameBufferAllocation, +} + struct PooledStagingTexture { texture: ID3D11Texture2D, width: u32, height: u32, } +struct StagingPoolInner { + free: Vec, + live: usize, +} + pub struct StagingTexturePool { - textures: Mutex>, + inner: Mutex, d3d_device: ID3D11Device, + d3d_context: ID3D11DeviceContext, pixel_format: PixelFormat, - next_index: AtomicUsize, } impl StagingTexturePool { - fn new(d3d_device: ID3D11Device, pixel_format: PixelFormat) -> Self { + fn new( + d3d_device: ID3D11Device, + d3d_context: ID3D11DeviceContext, + pixel_format: PixelFormat, + ) -> Self { Self { - textures: Mutex::new(Vec::with_capacity(STAGING_POOL_SIZE)), + inner: Mutex::new(StagingPoolInner { + free: Vec::with_capacity(STAGING_POOL_SIZE), + live: 0, + }), d3d_device, + d3d_context, pixel_format, - next_index: AtomicUsize::new(0), } } - fn get_or_create_texture( - &self, - width: u32, - height: u32, - ) -> windows::core::Result { - let mut textures = self - .textures + fn acquire(self: &Arc, width: u32, height: u32) -> Result { + let mut inner = self + .inner .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); - let index = self.next_index.fetch_add(1, Ordering::Relaxed) % STAGING_POOL_SIZE; - - if let Some(pooled) = textures.get(index) - && pooled.width == width - && pooled.height == height + if let Some(index) = inner + .free + .iter() + .position(|pooled| pooled.width == width && pooled.height == height) { - return Ok(pooled.texture.clone()); - } - - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: width, - Height: height, - MipLevels: 1, - ArraySize: 1, - Format: self.pixel_format.as_dxgi(), - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_STAGING, - BindFlags: 0, - CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32, - MiscFlags: 0, - }; + let pooled = inner.free.swap_remove(index); + return Ok(StagingLease { + pool: self.clone(), + texture: pooled.texture, + width, + height, + }); + } - let mut texture = None; - unsafe { - self.d3d_device - .CreateTexture2D(&texture_desc, None, Some(&mut texture))?; - }; + if inner.live >= STAGING_POOL_LIMIT { + // A cached texture at another size is idle, so it can be released. Without this a + // geometry change can wedge the pool at the limit while the previous frame is still + // mapped, and every frame at the new size is dropped. + let stale = inner + .free + .iter() + .position(|pooled| pooled.width != width || pooled.height != height) + .ok_or(ReadbackError::StagingBusy)?; + inner.free.swap_remove(stale); + inner.live -= 1; + } - let texture = texture.ok_or_else(windows::core::Error::from_win32)?; + let texture = create_staging_texture( + &self.d3d_device, + &self.d3d_context, + self.pixel_format, + width, + height, + )?; + inner.live += 1; + + Ok(StagingLease { + pool: self.clone(), + texture, + width, + height, + }) + } - if index < textures.len() { - textures[index] = PooledStagingTexture { + fn recycle(&self, texture: &ID3D11Texture2D, width: u32, height: u32) { + let mut inner = self + .inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + + if inner.free.len() < STAGING_POOL_SIZE { + inner.free.push(PooledStagingTexture { texture: texture.clone(), width, height, - }; - } else { - textures.push(PooledStagingTexture { + }); + } else if let Some(slot) = inner + .free + .iter_mut() + .find(|pooled| pooled.width != width || pooled.height != height) + { + // Capture size changed: keep the cache on the current geometry instead of allocating a + // fresh texture per frame while the stale sizes sit unused. + *slot = PooledStagingTexture { texture: texture.clone(), width, height, - }); + }; + // The replaced texture is dropped; the recycled one was an outstanding lease, so the + // pool now owns one fewer texture. + inner.live -= 1; + } else { + inner.live -= 1; } + } +} - Ok(texture) +struct StagingLease { + pool: Arc, + texture: ID3D11Texture2D, + width: u32, + height: u32, +} + +impl Drop for StagingLease { + fn drop(&mut self) { + self.pool.recycle(&self.texture, self.width, self.height); + } +} + +fn create_staging_texture( + device: &ID3D11Device, + context: &ID3D11DeviceContext, + pixel_format: PixelFormat, + width: u32, + height: u32, +) -> Result { + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: width, + Height: height + STAGING_PADDING_ROWS, + MipLevels: 1, + ArraySize: 1, + Format: pixel_format.as_dxgi(), + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: D3D11_CPU_ACCESS_READ.0 as u32 | D3D11_CPU_ACCESS_WRITE.0 as u32, + MiscFlags: 0, + }; + + let mut texture = None; + unsafe { + device.CreateTexture2D(&texture_desc, None, Some(&mut texture))?; + }; + let texture = texture.ok_or_else(windows::core::Error::from_win32)?; + zero_padding_row(context, &texture, height)?; + + Ok(texture) +} + +// ffmpeg may read up to AV_INPUT_BUFFER_PADDING_SIZE bytes past the last pixel and requires them +// to be zero. The staging texture carries one extra row for that, zeroed once because copies only +// ever touch the image rows. +fn zero_padding_row( + context: &ID3D11DeviceContext, + texture: &ID3D11Texture2D, + image_height: u32, +) -> Result<(), ReadbackError> { + let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); + unsafe { + context.Map(texture, 0, D3D11_MAP_WRITE, 0, Some(&mut mapped_resource))?; + + let padding = (mapped_resource.pData as *mut u8) + .add(mapped_resource.RowPitch as usize * image_height as usize); + std::ptr::write_bytes(padding, 0, mapped_resource.RowPitch as usize); + + context.Unmap(texture, 0); + }; + + Ok(()) +} + +pub struct MappedFrame { + lease: StagingLease, + d3d_context: ID3D11DeviceContext, + data: *mut u8, + row_pitch: u32, + width: u32, + height: u32, + pixel_format: PixelFormat, +} + +// SAFETY: the mapped staging memory belongs to the GPU resource, not to a thread; the only thread +// that touches the mapping owns this value, and the driver serializes the context (the capturer +// enables multithread protection). The AVBuffer built from it may be released on another thread, +// which only runs `Unmap` and returns the texture to the pool. +unsafe impl Send for MappedFrame {} + +impl std::fmt::Debug for MappedFrame { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MappedFrame") + .field("width", &self.width) + .field("height", &self.height) + .field("row_pitch", &self.row_pitch) + .finish() + } +} + +impl Drop for MappedFrame { + fn drop(&mut self) { + unsafe { + self.d3d_context.Unmap(&self.lease.texture, 0); + } + } +} + +impl MappedFrame { + pub fn width(&self) -> u32 { + self.width + } + + pub fn height(&self) -> u32 { + self.height + } + + pub fn row_pitch(&self) -> u32 { + self.row_pitch + } + + pub fn pixel_format(&self) -> PixelFormat { + self.pixel_format + } + + pub fn data_ptr(&self) -> *mut u8 { + self.data + } + + pub fn data_len(&self) -> usize { + self.row_pitch as usize * (self.height as usize + STAGING_PADDING_ROWS as usize) + } + + pub fn as_slice(&self) -> &[u8] { + unsafe { + std::slice::from_raw_parts(self.data, self.height as usize * self.row_pitch as usize) + } } } @@ -296,9 +482,27 @@ fn create_d3d_device_on_pinned_adapter( fn create_d3d_device_with_warp_fallback() -> windows::core::Result<(ID3D11Device, bool)> { let mut device = None; - let flags = D3D11_CREATE_DEVICE_FLAG::default(); + let video_flags = D3D11_CREATE_DEVICE_VIDEO_SUPPORT | D3D11_CREATE_DEVICE_BGRA_SUPPORT; + let basic_flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT; - match create_d3d_device_on_pinned_adapter(flags, &mut device) { + match create_d3d_device_on_pinned_adapter(video_flags, &mut device) { + Ok(description) => { + tracing::info!( + adapter = %description, + "scap-direct3d: D3D11 device created on pinned hardware adapter with video support" + ); + let device = device.ok_or_else(windows::core::Error::from_win32)?; + return Ok((device, false)); + } + Err(e) => { + tracing::warn!( + error = %e, + "scap-direct3d: video-support device creation failed on pinned adapter, retrying without it" + ); + } + } + + match create_d3d_device_on_pinned_adapter(basic_flags, &mut device) { Ok(description) => { tracing::info!( adapter = %description, @@ -315,13 +519,13 @@ fn create_d3d_device_with_warp_fallback() -> windows::core::Result<(ID3D11Device } } - let result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, flags, &mut device); + let result = create_d3d_device_with_type(D3D_DRIVER_TYPE_HARDWARE, basic_flags, &mut device); match result { Ok(()) => Ok((device.ok_or_else(windows::core::Error::from_win32)?, false)), Err(e) if e.code() == DXGI_ERROR_UNSUPPORTED => { tracing::info!("Hardware D3D11 device unavailable, attempting WARP fallback"); - create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device)?; + create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, basic_flags, &mut device)?; Ok((device.ok_or_else(windows::core::Error::from_win32)?, true)) } Err(e) => Err(e), @@ -486,6 +690,7 @@ impl Capturer { let staging_pool = Arc::new(StagingTexturePool::new( d3d_device.clone(), + d3d_context.clone(), settings.pixel_format, )); @@ -768,82 +973,255 @@ impl Frame { &self.d3d_context } - pub fn as_buffer(&self) -> windows::core::Result> { - let staging_texture = self - .staging_pool - .get_or_create_texture(self.width, self.height)?; + /// Copies the frame into a staging texture and maps it, blocking until the GPU copy is done. + /// Use [`Self::map_readback`] on latency-sensitive paths. + pub fn as_buffer(&self) -> Result, ReadbackError> { + let lease = self.staging_pool.acquire(self.width, self.height)?; + self.copy_into_staging(&lease); + let mapped = self.map_staging(lease, None)?; - unsafe { - self.d3d_context - .CopyResource(&staging_texture, &self.texture); + Ok(FrameBuffer { + mapped, + frame: std::marker::PhantomData, + }) + } + + /// Copies the frame into a staging texture and maps it within `budget`. The default polls + /// `DO_NOT_WAIT` until the copy completes or the budget expires; the caller can reuse the + /// previous frame on `NotReady`. Set `CAP_READBACK_BLOCKING` to block in `Map` instead, which is + /// faster when the copy consistently finishes quickly but removes the deadline. + pub fn map_readback(&self, budget: Duration) -> Result { + let blocking = std::env::var_os("CAP_READBACK_BLOCKING").is_some(); + let lease = self.staging_pool.acquire(self.width, self.height)?; + self.copy_into_staging(&lease); + + if blocking { + return self.map_staging_hybrid(lease); + } + + let deadline = Instant::now().checked_add(budget); + self.map_staging(lease, deadline) + } + + fn copy_into_staging(&self, lease: &StagingLease) { + let source_box = D3D11_BOX { + left: 0, + top: 0, + front: 0, + right: self.width, + bottom: self.height, + back: 1, }; - let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); unsafe { - self.d3d_context.Map( - &staging_texture, + self.d3d_context.CopySubresourceRegion( + &lease.texture, 0, - D3D11_MAP_READ, 0, - Some(&mut mapped_resource), - )?; - }; + 0, + 0, + &self.texture, + 0, + Some(&source_box), + ); + self.d3d_context.Flush(); + } + } - let data = unsafe { - std::slice::from_raw_parts( - mapped_resource.pData.cast(), - (self.height * mapped_resource.RowPitch) as usize, - ) - }; + fn map_staging( + &self, + lease: StagingLease, + deadline: Option, + ) -> Result { + let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); - Ok(FrameBuffer { - data, + if let Some(deadline) = deadline { + let started = Instant::now(); + loop { + match unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + D3D11_MAP_FLAG_DO_NOT_WAIT.0 as u32, + Some(&mut mapped_resource), + ) + } { + Ok(()) => break, + Err(error) if error.code() == DXGI_ERROR_WAS_STILL_DRAWING => { + let now = Instant::now(); + if now >= deadline { + return Err(ReadbackError::NotReady); + } + if now.duration_since(started) < READBACK_SPIN_BUDGET { + std::hint::spin_loop(); + } else { + std::thread::sleep( + READBACK_RETRY_INTERVAL + .min(deadline.saturating_duration_since(now)), + ); + } + } + Err(error) => return Err(error.into()), + } + } + } else { + unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + 0, + Some(&mut mapped_resource), + )?; + } + } + + Ok(self.mapped_frame(lease, mapped_resource)) + } + + fn mapped_frame( + &self, + lease: StagingLease, + mapped_resource: D3D11_MAPPED_SUBRESOURCE, + ) -> MappedFrame { + MappedFrame { + lease, + d3d_context: self.d3d_context.clone(), + data: mapped_resource.pData as *mut u8, + row_pitch: mapped_resource.RowPitch, width: self.width, height: self.height, - stride: mapped_resource.RowPitch, pixel_format: self.pixel_format, - staging_texture, - d3d_context: self.d3d_context.clone(), - }) + } } -} -pub struct FrameBuffer<'a> { - data: &'a [u8], - width: u32, - height: u32, - stride: u32, - pixel_format: PixelFormat, - staging_texture: ID3D11Texture2D, - d3d_context: ID3D11DeviceContext, -} + fn map_staging_hybrid(&self, lease: StagingLease) -> Result { + let mut mapped_resource = D3D11_MAPPED_SUBRESOURCE::default(); -impl Drop for FrameBuffer<'_> { - fn drop(&mut self) { - unsafe { - self.d3d_context.Unmap(&self.staging_texture, 0); + match unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + D3D11_MAP_FLAG_DO_NOT_WAIT.0 as u32, + Some(&mut mapped_resource), + ) + } { + Ok(()) => {} + Err(error) if error.code() == DXGI_ERROR_WAS_STILL_DRAWING => unsafe { + self.d3d_context.Map( + &lease.texture, + 0, + D3D11_MAP_READ, + 0, + Some(&mut mapped_resource), + )?; + }, + Err(error) => return Err(error.into()), } + + Ok(self.mapped_frame(lease, mapped_resource)) } } +pub struct FrameBuffer<'a> { + mapped: MappedFrame, + frame: std::marker::PhantomData<&'a Frame>, +} + impl FrameBuffer<'_> { pub fn width(&self) -> u32 { - self.width + self.mapped.width() } pub fn height(&self) -> u32 { - self.height + self.mapped.height() } pub fn stride(&self) -> u32 { - self.stride + self.mapped.row_pitch() } pub fn data(&self) -> &[u8] { - self.data + self.mapped.as_slice() } pub fn pixel_format(&self) -> PixelFormat { - self.pixel_format + self.mapped.pixel_format() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn warp_pool() -> Arc { + let mut device = None; + unsafe { + D3D11CreateDevice( + None, + D3D_DRIVER_TYPE_WARP, + HMODULE::default(), + D3D11_CREATE_DEVICE_FLAG::default(), + None, + D3D11_SDK_VERSION, + Some(&mut device), + None, + None, + ) + .unwrap(); + } + + let device = device.unwrap(); + let context = unsafe { device.GetImmediateContext().unwrap() }; + + Arc::new(StagingTexturePool::new( + device, + context, + PixelFormat::R8G8B8A8Unorm, + )) + } + + #[test] + fn pool_survives_capture_size_changes() { + let pool = warp_pool(); + let sizes = [(320, 240), (640, 480), (800, 600), (1024, 768), (1280, 720)]; + + for (index, (width, height)) in sizes.into_iter().enumerate() { + let lease = pool + .acquire(width, height) + .unwrap_or_else(|error| panic!("acquire size {index}: {error}")); + drop(lease); + + let inner = pool.inner.lock().unwrap(); + assert_eq!(inner.live, inner.free.len(), "live after size {index}"); + assert!(inner.live <= STAGING_POOL_LIMIT); + } + } + + #[test] + fn pool_evicts_stale_sizes_at_the_limit() { + let pool = warp_pool(); + let held = pool.acquire(320, 240).unwrap(); + for (width, height) in [(640, 480), (800, 600), (1024, 768)] { + drop(pool.acquire(width, height).unwrap()); + } + + { + let inner = pool.inner.lock().unwrap(); + assert_eq!(inner.live, STAGING_POOL_LIMIT); + assert_eq!(inner.free.len(), STAGING_POOL_SIZE); + } + + let lease = pool + .acquire(1280, 720) + .unwrap_or_else(|error| panic!("acquire after a size change: {error}")); + drop(lease); + drop(held); + + let inner = pool.inner.lock().unwrap(); + assert_eq!(inner.live, inner.free.len()); + assert!(inner.live <= STAGING_POOL_LIMIT); } } diff --git a/crates/scap-ffmpeg/examples/cli.rs b/crates/scap-ffmpeg/examples/cli.rs index 12aa005c2df..bfebeba715a 100644 --- a/crates/scap-ffmpeg/examples/cli.rs +++ b/crates/scap-ffmpeg/examples/cli.rs @@ -1,32 +1,85 @@ +//! Windows/macOS capture demo. On Windows, `--bench [--seconds N] [--path both|old|new|none]` +//! measures the old two-transfer readback against the zero-copy staging ring on live frames. WGC +//! only produces frames while the captured content changes, so keep something moving during the run. + #[tokio::main] pub async fn main() { #[cfg(windows)] { use scap_direct3d::*; use scap_targets::Display; - use std::time::Duration; + use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + mpsc, + }, + time::{Duration, Instant}, + }; + + let args: Vec = std::env::args().collect(); + let bench = args.iter().any(|arg| arg == "--bench"); + let seconds = args + .iter() + .position(|arg| arg == "--seconds") + .and_then(|index| args.get(index + 1)) + .and_then(|value| value.parse::().ok()) + .unwrap_or(15.0) + .clamp(1.0, 120.0); + let path = args + .iter() + .position(|arg| arg == "--path") + .and_then(|index| args.get(index + 1)) + .cloned() + .unwrap_or_else(|| "both".to_string()); + if !matches!(path.as_str(), "both" | "old" | "new" | "none") { + eprintln!("invalid --path {path}; expected both, old, new, or none"); + std::process::exit(2); + } let display = Display::primary(); + let capture_fps = 60; + let min_update_interval = Settings::can_min_update_interval() + .ok() + .filter(|supported| *supported) + .map(|_| Duration::from_secs_f64(1.0 / capture_fps as f64)); + let frame_sender = bench.then(|| { + let (tx, rx) = mpsc::sync_channel::(4); + (tx, rx) + }); + let callback_drops = Arc::new(AtomicU64::new(0)); + let frame_tx = frame_sender.as_ref().map(|(tx, _)| tx.clone()); + let callback_drops_tx = callback_drops.clone(); let mut capturer = Capturer::new( display.raw_handle().try_as_capture_item().unwrap(), Settings { is_border_required: Some(true), is_cursor_capture_enabled: Some(true), pixel_format: PixelFormat::R8G8B8A8Unorm, + fps: Some(capture_fps), + min_update_interval, ..Default::default() }, - |frame| { - use scap_ffmpeg::AsFFmpeg; + move |frame| { + if let Some(tx) = &frame_tx { + if tx.try_send(frame).is_err() { + callback_drops_tx.fetch_add(1, Ordering::Relaxed); + } + return Ok(()); + } - let ff_frame = frame.as_ffmpeg()?; + use scap_ffmpeg::AsFFmpeg; - println!( - "Frame: {}x{} format={:?}", - ff_frame.width(), - ff_frame.height(), - ff_frame.format() - ); + match frame.as_ffmpeg() { + Ok(ff_frame) => println!( + "Frame: {}x{} format={:?}", + ff_frame.width(), + ff_frame.height(), + ff_frame.format() + ), + Err(error) => eprintln!("Frame conversion failed: {error}"), + } Ok(()) }, @@ -37,11 +90,219 @@ pub async fn main() { capturer.start().unwrap(); - std::thread::sleep(Duration::from_secs(3)); + if let Some((_, frame_rx)) = frame_sender { + readback_benchmark(&display, frame_rx, seconds, &path, &callback_drops); + } else { + std::thread::sleep(Duration::from_secs(3)); + } capturer.stop().unwrap(); - std::thread::sleep(Duration::from_secs(3)); + std::thread::sleep(Duration::from_secs(1)); + + fn readback_benchmark( + display: &Display, + frame_rx: mpsc::Receiver, + seconds: f64, + path: &str, + callback_drops: &Arc, + ) { + use scap_ffmpeg::AsFFmpeg; + + let initial_size = display.physical_size().unwrap(); + let mut last_frame_size = (initial_size.width() as u32, initial_size.height() as u32); + + let mut old = Samples::default(); + let mut new = Samples::default(); + let mut old_frame = None; + let mut frames_seen = 0u64; + let run_old_path = path != "new" && path != "none"; + let run_new_path = path != "old" && path != "none"; + let started = Instant::now(); + + while started.elapsed() < Duration::from_secs_f64(seconds) { + let frame = match frame_rx.recv_timeout(Duration::from_secs(1)) { + Ok(frame) => frame, + Err(mpsc::RecvTimeoutError::Timeout) => continue, + Err(mpsc::RecvTimeoutError::Disconnected) => break, + }; + frames_seen += 1; + last_frame_size = (frame.width(), frame.height()); + + let mut run_old = || { + let started = Instant::now(); + let buffer = frame.as_buffer().ok()?; + let width = buffer.width() as usize; + let height = buffer.height() as usize; + let source_stride = buffer.stride() as usize; + let row_length = width * 4; + let destination = old_frame.get_or_insert_with(|| { + ffmpeg::frame::Video::new( + ffmpeg::format::Pixel::RGBA, + width as u32, + height as u32, + ) + }); + + if destination.width() as usize != width + || destination.height() as usize != height + { + *destination = ffmpeg::frame::Video::new( + ffmpeg::format::Pixel::RGBA, + width as u32, + height as u32, + ); + } + + let destination_stride = destination.stride(0); + let destination_data = destination.data_mut(0); + + for row in 0..height { + unsafe { + std::ptr::copy_nonoverlapping( + buffer.data().as_ptr().add(row * source_stride), + destination_data.as_mut_ptr().add(row * destination_stride), + row_length, + ); + } + } + + Some(started.elapsed()) + }; + + let run_new = || { + let started = Instant::now(); + match frame.as_ffmpeg() { + Ok(_) => Some((started.elapsed(), false)), + Err(scap_ffmpeg::AsFFmpegError::NotReady) => { + Some((started.elapsed(), true)) + } + Err(error) => { + eprintln!("as_ffmpeg failed: {error}"); + None + } + } + }; + + let (old_elapsed, new_elapsed) = match path { + "old" => (run_old(), None), + "new" => (None, run_new()), + "none" => (None, None), + _ => { + let old_first = old.count.is_multiple_of(2); + if old_first { + (run_old(), run_new()) + } else { + let new_elapsed = run_new(); + (run_old(), new_elapsed) + } + } + }; + + if let Some(elapsed) = old_elapsed { + old.record(elapsed); + } else if run_old_path { + old.failures += 1; + } + + match new_elapsed { + Some((elapsed, dropped)) => { + if dropped { + new.dropped += 1; + } else { + new.record(elapsed); + } + } + None if run_new_path => new.failures += 1, + None => {} + } + } + + let (last_width, last_height) = last_frame_size; + let frame_bytes = last_width as u64 * last_height as u64 * 4; + println!( + "{}x{} RGBA ({:.1} MiB/frame), {seconds:.0}s, path={path}, callback_drops={}, source_frames={frames_seen}", + last_width, + last_height, + frame_bytes as f64 / (1024.0 * 1024.0), + callback_drops.load(Ordering::Relaxed) + ); + println!("old (blocking two-transfer path): {old}"); + println!("new (ring + probe/block zero copy): {new}"); + println!( + "{{\"old\":{},\"new\":{},\"source_frames\":{frames_seen},\"frame_bytes\":{frame_bytes}}}", + old.json(), + new.json() + ); + } + + #[derive(Default)] + struct Samples { + count: u64, + dropped: u64, + failures: u64, + elapsed: Vec, + } + + impl Samples { + fn record(&mut self, elapsed: Duration) { + self.count += 1; + self.elapsed.push(elapsed); + } + + fn mean_ms(&self) -> f64 { + if self.elapsed.is_empty() { + return 0.0; + } + self.elapsed + .iter() + .map(|elapsed| elapsed.as_secs_f64()) + .sum::() + * 1000.0 + / self.elapsed.len() as f64 + } + + fn percentile_ms(&self, percentile: f64) -> f64 { + if self.elapsed.is_empty() { + return 0.0; + } + let mut sorted = self.elapsed.clone(); + sorted.sort_unstable(); + let index = ((sorted.len() as f64 - 1.0) * percentile).round() as usize; + sorted[index].as_secs_f64() * 1000.0 + } + + fn json(&self) -> String { + format!( + "{{\"frames\":{},\"dropped\":{},\"failures\":{},\"mean_ms\":{:.3},\"p50_ms\":{:.3},\"p95_ms\":{:.3},\"p99_ms\":{:.3},\"max_ms\":{:.3}}}", + self.count, + self.dropped, + self.failures, + self.mean_ms(), + self.percentile_ms(0.50), + self.percentile_ms(0.95), + self.percentile_ms(0.99), + self.percentile_ms(1.0) + ) + } + } + + impl std::fmt::Display for Samples { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "frames={} dropped={} failures={} mean={:.2}ms p50={:.2}ms p95={:.2}ms p99={:.2}ms max={:.2}ms", + self.count, + self.dropped, + self.failures, + self.mean_ms(), + self.percentile_ms(0.50), + self.percentile_ms(0.95), + self.percentile_ms(0.99), + self.percentile_ms(1.0) + ) + } + } } #[cfg(target_os = "macos")] diff --git a/crates/scap-ffmpeg/src/direct3d.rs b/crates/scap-ffmpeg/src/direct3d.rs index 762af9c58b9..2b774fc80c5 100644 --- a/crates/scap-ffmpeg/src/direct3d.rs +++ b/crates/scap-ffmpeg/src/direct3d.rs @@ -1,158 +1,67 @@ use ffmpeg::format::Pixel; -use scap_direct3d::PixelFormat; +use scap_direct3d::{MappedFrame, PixelFormat, ReadbackError}; +use std::ffi::c_void; +use std::time::Duration; -pub type AsFFmpegError = windows::core::Error; +pub type AsFFmpegError = ReadbackError; -#[inline] -fn copy_frame_data( - src_bytes: &[u8], - src_stride: usize, - dest_bytes: &mut [u8], - dest_stride: usize, - row_length: usize, - height: usize, -) { - debug_assert!(height > 0, "height must be positive"); - debug_assert!( - src_bytes.len() - >= (height - 1) - .saturating_mul(src_stride) - .saturating_add(row_length), - "source buffer too small" - ); - debug_assert!( - dest_bytes.len() - >= (height - 1) - .saturating_mul(dest_stride) - .saturating_add(row_length), - "destination buffer too small" - ); +/// Readback deadline. The default polls within it; `CAP_READBACK_BLOCKING` opts into an +/// unbounded `Map` when the copy is known to finish quickly. +const READBACK_BUDGET: Duration = Duration::from_millis(8); - if src_stride == row_length && dest_stride == row_length { - let total_bytes = row_length.saturating_mul(height); - unsafe { - std::ptr::copy_nonoverlapping(src_bytes.as_ptr(), dest_bytes.as_mut_ptr(), total_bytes); - } - } else { - for i in 0..height { - unsafe { - std::ptr::copy_nonoverlapping( - src_bytes.as_ptr().add(i * src_stride), - dest_bytes.as_mut_ptr().add(i * dest_stride), - row_length, - ); - } - } +impl super::AsFFmpeg for scap_direct3d::Frame { + fn as_ffmpeg(&self) -> Result { + let mapped = self.map_readback(READBACK_BUDGET)?; + let format = mapped.pixel_format().as_ffmpeg(); + wrap_mapped_frame(mapped, format) } } -struct FramePool { - key: (Pixel, u32, u32), - linesize: usize, - pool: *mut ffmpeg::ffi::AVBufferPool, +unsafe extern "C" fn release_mapped_frame(opaque: *mut c_void, _data: *mut u8) { + drop(unsafe { Box::from_raw(opaque.cast::()) }); } -impl Drop for FramePool { - fn drop(&mut self) { - // Frees the pool once every buffer still referenced by an in-flight frame is returned. - unsafe { ffmpeg::ffi::av_buffer_pool_uninit(&mut self.pool) }; - } -} - -thread_local! { - static FRAME_POOL: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; -} +/// Wraps the mapped staging texture into an `AVFrame` whose backing `AVBufferRef` owns the mapping. +/// The pixels are never copied: the encoder reads the staging texture directly, and the last frame +/// reference unmaps it and returns it to the pool. +fn wrap_mapped_frame( + mapped: MappedFrame, + format: Pixel, +) -> Result { + let data = mapped.data_ptr(); + let size = mapped.data_len(); + let row_pitch = mapped.row_pitch() as i32; + let width = mapped.width(); + let height = mapped.height(); + + let mut frame = ffmpeg::frame::Video::empty(); + + unsafe { + let opaque = Box::into_raw(Box::new(mapped)).cast::(); + let buffer = ffmpeg::ffi::av_buffer_create( + data, + size, + Some(release_mapped_frame), + opaque, + ffmpeg::ffi::AV_BUFFER_FLAG_READONLY, + ); -// A fresh multi-megabyte allocation per frame is served by fresh OS pages on Windows, so every -// capture paid page-fault and zeroing costs several times the copy itself (~2ms per 1080p frame). -fn pooled_frame(format: Pixel, width: u32, height: u32) -> ffmpeg::frame::Video { - debug_assert!(matches!(format, Pixel::RGBA | Pixel::BGRA)); - FRAME_POOL.with_borrow_mut(|slot| { - if slot - .as_ref() - .is_none_or(|pool| pool.key != (format, width, height)) - { - *slot = None; - let linesize = (width as usize).next_multiple_of(32) * 4; - let size = - linesize * height as usize + ffmpeg::ffi::AV_INPUT_BUFFER_PADDING_SIZE as usize; - let pool = unsafe { ffmpeg::ffi::av_buffer_pool_init(size, None) }; - if pool.is_null() { - return ffmpeg::frame::Video::new(format, width, height); - } - *slot = Some(FramePool { - key: (format, width, height), - linesize, - pool, - }); - } - let pool = slot.as_ref().expect("frame pool initialized above"); - unsafe { - let buffer = ffmpeg::ffi::av_buffer_pool_get(pool.pool); - if buffer.is_null() { - return ffmpeg::frame::Video::new(format, width, height); - } - let mut frame = ffmpeg::frame::Video::empty(); - frame.set_format(format); - frame.set_width(width); - frame.set_height(height); - let raw = frame.as_mut_ptr(); - (*raw).buf[0] = buffer; - (*raw).data[0] = (*buffer).data; - (*raw).linesize[0] = pool.linesize as i32; - frame + if buffer.is_null() { + drop(Box::from_raw(opaque.cast::())); + return Err(ReadbackError::FrameBufferAllocation); } - }) -} - -impl super::AsFFmpeg for scap_direct3d::Frame { - fn as_ffmpeg(&self) -> Result { - let buffer = self.as_buffer()?; - - let width = self.width() as usize; - let height = self.height() as usize; - - let src_bytes = buffer.data(); - let src_stride = buffer.stride() as usize; - let row_length = width * 4; - match self.pixel_format() { - PixelFormat::R8G8B8A8Unorm => { - let mut ff_frame = pooled_frame(Pixel::RGBA, self.width(), self.height()); + frame.set_format(format); + frame.set_width(width); + frame.set_height(height); - let dest_stride = ff_frame.stride(0); - let dest_bytes = ff_frame.data_mut(0); - - copy_frame_data( - src_bytes, - src_stride, - dest_bytes, - dest_stride, - row_length, - height, - ); - - Ok(ff_frame) - } - PixelFormat::B8G8R8A8Unorm => { - let mut ff_frame = pooled_frame(Pixel::BGRA, self.width(), self.height()); - - let dest_stride = ff_frame.stride(0); - let dest_bytes = ff_frame.data_mut(0); - - copy_frame_data( - src_bytes, - src_stride, - dest_bytes, - dest_stride, - row_length, - height, - ); - - Ok(ff_frame) - } - } + let raw = frame.as_mut_ptr(); + (*raw).buf[0] = buffer; + (*raw).data[0] = data; + (*raw).linesize[0] = row_pitch; } + + Ok(frame) } pub trait PixelFormatExt { @@ -167,31 +76,3 @@ impl PixelFormatExt for PixelFormat { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn pooled_frames_recycle_released_buffers() { - let first = pooled_frame(Pixel::BGRA, 1001, 3); - assert_eq!( - (first.format(), first.width(), first.height()), - (Pixel::BGRA, 1001, 3) - ); - assert!(first.stride(0) >= 1001 * 4); - assert!(first.data(0).len() >= first.stride(0) * 3); - let held = first.data(0).as_ptr(); - - let second = pooled_frame(Pixel::BGRA, 1001, 3); - assert_ne!(second.data(0).as_ptr(), held); - - drop(first); - let third = pooled_frame(Pixel::BGRA, 1001, 3); - assert_eq!(third.data(0).as_ptr(), held); - - let resized = pooled_frame(Pixel::BGRA, 640, 2); - assert_eq!((resized.width(), resized.height()), (640, 2)); - drop((second, third)); - } -}