From 5dab6ae629d59927a6e91086225109c01787e0de Mon Sep 17 00:00:00 2001 From: Kyzer Phneh Date: Fri, 23 Oct 2026 10:51:19 +0000 Subject: [PATCH 1/3] perf: speed up packed yuv to nv12 conversion Same-size UYVY422 and YUYV422 frames copy to NV12 in scalar or AVX2 code. convert reuses output buffers after the last frame reference drops. --- crates/frame-converter/src/frame_pool.rs | 35 +- crates/frame-converter/src/lib.rs | 3 + crates/frame-converter/src/output_frame.rs | 269 +++++++++ crates/frame-converter/src/packed_yuv.rs | 665 +++++++++++++++++++++ crates/frame-converter/src/swscale.rs | 494 ++++++++++++++- 5 files changed, 1457 insertions(+), 9 deletions(-) create mode 100644 crates/frame-converter/src/output_frame.rs create mode 100644 crates/frame-converter/src/packed_yuv.rs diff --git a/crates/frame-converter/src/frame_pool.rs b/crates/frame-converter/src/frame_pool.rs index 51d880fbf3b..4bba2af3eb6 100644 --- a/crates/frame-converter/src/frame_pool.rs +++ b/crates/frame-converter/src/frame_pool.rs @@ -1,3 +1,4 @@ +use crate::output_frame::cached_output_frame; use ffmpeg::{format::Pixel, frame}; use parking_lot::Mutex; use std::sync::Arc; @@ -14,7 +15,7 @@ impl VideoFramePool { pub fn new(capacity: usize, format: Pixel, width: u32, height: u32) -> Arc { let mut frames = Vec::with_capacity(capacity); for _ in 0..capacity { - frames.push(frame::Video::new(format, width, height)); + frames.push(cached_output_frame(format, width, height)); } Arc::new(Self { @@ -30,7 +31,7 @@ impl VideoFramePool { self.frames .lock() .pop() - .unwrap_or_else(|| frame::Video::new(self.format, self.width, self.height)) + .unwrap_or_else(|| cached_output_frame(self.format, self.width, self.height)) } pub fn put(&self, frame: frame::Video) { @@ -56,3 +57,33 @@ impl VideoFramePool { (self.width, self.height) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Once; + + fn init_ffmpeg() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + ffmpeg::init().expect("ffmpeg init"); + }); + } + + #[test] + fn pooled_frames_match_ffmpeg_geometry() { + init_ffmpeg(); + let pool = VideoFramePool::new(2, Pixel::NV12, 64, 32); + assert_eq!(pool.available(), 2); + let frame = pool.get(); + let fresh = frame::Video::new(Pixel::NV12, 64, 32); + assert_eq!(frame.format(), fresh.format()); + assert_eq!(frame.width(), fresh.width()); + assert_eq!(frame.height(), fresh.height()); + assert_eq!(frame.stride(0), fresh.stride(0)); + assert_eq!(frame.stride(1), fresh.stride(1)); + assert_eq!(pool.available(), 1); + pool.put(frame); + assert_eq!(pool.available(), 2); + } +} diff --git a/crates/frame-converter/src/lib.rs b/crates/frame-converter/src/lib.rs index cbe4c581072..82811bcc509 100644 --- a/crates/frame-converter/src/lib.rs +++ b/crates/frame-converter/src/lib.rs @@ -1,3 +1,6 @@ +mod output_frame; +mod packed_yuv; + mod swscale; pub use swscale::*; diff --git a/crates/frame-converter/src/output_frame.rs b/crates/frame-converter/src/output_frame.rs new file mode 100644 index 00000000000..83195a9f96b --- /dev/null +++ b/crates/frame-converter/src/output_frame.rs @@ -0,0 +1,269 @@ +use ffmpeg::format::Pixel; +use ffmpeg::frame; +use parking_lot::Mutex; +use std::ffi::c_void; + +// Image bytes stay in this pool after the last AVFrame reference drops, so +// `convert` can hand out the same allocation again. Callers still receive an +// exclusively owned frame::Video. +const POOLED_PER_SIZE: usize = 16; + +struct PooledBuffer { + ptr: *mut u8, + size: usize, +} + +// Exclusive ownership of `ptr` moves with the value under OUTPUT_POOL. +unsafe impl Send for PooledBuffer {} + +static OUTPUT_POOL: Mutex> = Mutex::new(Vec::new()); +static LAYOUTS: Mutex> = Mutex::new(Vec::new()); + +#[derive(Clone, Copy)] +struct FrameLayout { + format: Pixel, + width: u32, + height: u32, + linesize: [i32; 8], + offset: [isize; 8], + size: usize, +} + +pub(crate) fn cached_output_frame(format: Pixel, width: u32, height: u32) -> frame::Video { + if width == 0 || height == 0 { + return frame::Video::new(format, width, height); + } + let Some(layout) = layout_for(format, width, height) else { + return frame::Video::new(format, width, height); + }; + assemble(&layout).unwrap_or_else(|| frame::Video::new(format, width, height)) +} + +fn layout_for(format: Pixel, width: u32, height: u32) -> Option { + if let Some(layout) = find_layout(format, width, height) { + return Some(layout); + } + let probe = frame::Video::new(format, width, height); + let layout = capture(&probe)?; + drop(probe); + let mut layouts = LAYOUTS.lock(); + if let Some(existing) = layouts + .iter() + .copied() + .find(|layout| layout.format == format && layout.width == width && layout.height == height) + { + return Some(existing); + } + layouts.push(layout); + Some(layout) +} + +fn find_layout(format: Pixel, width: u32, height: u32) -> Option { + LAYOUTS + .lock() + .iter() + .copied() + .find(|layout| layout.format == format && layout.width == width && layout.height == height) +} + +fn capture(frame: &frame::Video) -> Option { + unsafe { + let raw = frame.as_ptr(); + if raw.is_null() || (*raw).buf[0].is_null() { + return None; + } + for index in 1..8 { + if !(*raw).buf[index].is_null() { + return None; + } + } + let buf = (*raw).buf[0]; + let base = (*buf).data; + let size = (*buf).size; + if base.is_null() || size == 0 { + return None; + } + let mut linesize = [0i32; 8]; + let mut offset = [-1isize; 8]; + for index in 0..8 { + linesize[index] = (*raw).linesize[index]; + let plane = (*raw).data[index]; + if plane.is_null() { + continue; + } + let off = (plane as usize).wrapping_sub(base as usize); + if off >= size { + return None; + } + offset[index] = off as isize; + } + if offset[0] < 0 { + return None; + } + Some(FrameLayout { + format: frame.format(), + width: frame.width(), + height: frame.height(), + linesize, + offset, + size, + }) + } +} + +fn assemble(layout: &FrameLayout) -> Option { + let data = take_pooled(layout.size).or_else(|| malloc_zeroed(layout.size))?; + let mut frame = frame::Video::empty(); + if unsafe { frame.as_ptr() }.is_null() { + free_data(data); + return None; + } + frame.set_format(layout.format); + frame.set_width(layout.width); + frame.set_height(layout.height); + unsafe { + let raw = frame.as_mut_ptr(); + let buf = ffmpeg::ffi::av_buffer_create( + data, + layout.size, + Some(release_output_buffer), + layout.size as *mut c_void, + 0, + ); + if buf.is_null() { + free_data(data); + return None; + } + (*raw).buf[0] = buf; + for index in 0..8 { + (*raw).linesize[index] = layout.linesize[index]; + if layout.offset[index] >= 0 { + (*raw).data[index] = data.offset(layout.offset[index]); + } + } + (*raw).extended_data = (*raw).data.as_mut_ptr(); + } + Some(frame) +} + +fn take_pooled(size: usize) -> Option<*mut u8> { + let mut pool = OUTPUT_POOL.lock(); + let index = pool.iter().rposition(|buf| buf.size == size)?; + Some(pool.swap_remove(index).ptr) +} + +fn malloc_zeroed(size: usize) -> Option<*mut u8> { + if size == 0 { + return None; + } + unsafe { + let ptr = ffmpeg::ffi::av_malloc(size) as *mut u8; + if ptr.is_null() { + None + } else { + std::ptr::write_bytes(ptr, 0, size); + Some(ptr) + } + } +} + +fn free_data(ptr: *mut u8) { + unsafe { + ffmpeg::ffi::av_free(ptr.cast()); + } +} + +unsafe extern "C" fn release_output_buffer(opaque: *mut c_void, data: *mut u8) { + if data.is_null() { + return; + } + let size = opaque as usize; + if size == 0 { + free_data(data); + return; + } + let mut pool = OUTPUT_POOL.lock(); + let same_size = pool.iter().filter(|buf| buf.size == size).count(); + if same_size < POOLED_PER_SIZE { + pool.push(PooledBuffer { ptr: data, size }); + } else { + free_data(data); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Once; + + fn init_ffmpeg() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + ffmpeg::init().expect("ffmpeg init"); + }); + } + + fn plane_offset(frame: &frame::Video, plane: usize) -> usize { + unsafe { + let raw = frame.as_ptr(); + let base = (*(*raw).buf[0]).data as usize; + (*raw).data[plane] as usize - base + } + } + + #[test] + fn cached_frame_matches_ffmpeg_geometry() { + init_ffmpeg(); + let mut cached = cached_output_frame(Pixel::NV12, 94, 46); + let fresh = frame::Video::new(Pixel::NV12, 94, 46); + assert_eq!(cached.format(), fresh.format()); + assert_eq!(cached.width(), fresh.width()); + assert_eq!(cached.height(), fresh.height()); + assert_eq!(cached.stride(0), fresh.stride(0)); + assert_eq!(cached.stride(1), fresh.stride(1)); + assert_eq!(cached.planes(), fresh.planes()); + assert_eq!(plane_offset(&cached, 0), plane_offset(&fresh, 0)); + assert_eq!(plane_offset(&cached, 1), plane_offset(&fresh, 1)); + unsafe { + assert_eq!(ffmpeg::ffi::av_frame_is_writable(cached.as_mut_ptr()), 1); + } + } + + #[test] + fn dropped_frame_returns_its_buffer() { + init_ffmpeg(); + let first = cached_output_frame(Pixel::NV12, 95, 46); + let ptr = unsafe { (*first.as_ptr()).data[0] }; + drop(first); + let mut second = cached_output_frame(Pixel::NV12, 95, 46); + let reused = unsafe { (*second.as_ptr()).data[0] }; + assert_eq!(reused, ptr); + second.data_mut(0).fill(7); + assert!(second.data(0).iter().all(|byte| *byte == 7)); + } + + #[test] + fn live_reference_keeps_the_buffer_out_of_the_pool() { + init_ffmpeg(); + let mut held = frame::Video::empty(); + let ptr = { + let mut frame = cached_output_frame(Pixel::YUYV422, 96, 40); + frame.data_mut(0).fill(19); + unsafe { + let status = ffmpeg::ffi::av_frame_ref(held.as_mut_ptr(), frame.as_ptr()); + assert!(status >= 0); + (*frame.as_ptr()).data[0] + } + }; + let next = cached_output_frame(Pixel::YUYV422, 96, 40); + let next_ptr = unsafe { (*next.as_ptr()).data[0] }; + assert_ne!(next_ptr, ptr); + assert!(held.data(0).iter().all(|byte| *byte == 19)); + drop(held); + let recycled = cached_output_frame(Pixel::YUYV422, 96, 40); + let recycled_ptr = unsafe { (*recycled.as_ptr()).data[0] }; + assert_eq!(recycled_ptr, ptr); + assert_ne!(next_ptr, recycled_ptr); + drop(next); + } +} diff --git a/crates/frame-converter/src/packed_yuv.rs b/crates/frame-converter/src/packed_yuv.rs new file mode 100644 index 00000000000..d8fd8f61dbe --- /dev/null +++ b/crates/frame-converter/src/packed_yuv.rs @@ -0,0 +1,665 @@ +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Packed422 { + Uyvy, + Yuyv, +} + +pub(crate) struct Nv12Mut<'a> { + pub y: &'a mut [u8], + pub y_stride: usize, + pub uv: &'a mut [u8], + pub uv_stride: usize, +} + +struct PackedView<'a> { + kind: Packed422, + src: &'a [u8], + stride: usize, + width: usize, + height: usize, +} + +pub(crate) fn convert_packed422_to_nv12( + kind: Packed422, + src: &[u8], + src_stride: usize, + width: usize, + height: usize, + dst: &mut Nv12Mut<'_>, +) -> bool { + if width == 0 || height == 0 { + return true; + } + let view = PackedView { + kind, + src, + stride: src_stride, + width, + height, + }; + if !buffers_fit(&view, dst) { + return false; + } + if avx2_available() { + unsafe { + avx_convert(&view, dst); + } + } else { + scalar_rows(&view, dst); + } + true +} + +#[cfg(test)] +pub(crate) fn convert_packed422_scalar( + kind: Packed422, + src: &[u8], + src_stride: usize, + width: usize, + height: usize, + dst: &mut Nv12Mut<'_>, +) -> bool { + if width == 0 || height == 0 { + return true; + } + let view = PackedView { + kind, + src, + stride: src_stride, + width, + height, + }; + if !buffers_fit(&view, dst) { + return false; + } + scalar_rows(&view, dst); + true +} + +fn avx2_available() -> bool { + #[cfg(target_arch = "x86_64")] + { + is_x86_feature_detected!("avx2") + } + #[cfg(not(target_arch = "x86_64"))] + { + false + } +} + +fn chroma_row_bytes(width: usize) -> usize { + width.div_ceil(2) * 2 +} + +fn buffers_fit(src: &PackedView<'_>, dst: &Nv12Mut<'_>) -> bool { + let Some(packed) = src.width.checked_mul(2) else { + return false; + }; + let chroma = chroma_row_bytes(src.width); + if src.stride < packed || dst.y_stride < src.width || dst.uv_stride < chroma { + return false; + } + let Some(src_need) = src.stride.checked_mul(src.height) else { + return false; + }; + let Some(y_need) = dst.y_stride.checked_mul(src.height) else { + return false; + }; + let Some(uv_need) = dst.uv_stride.checked_mul(src.height.div_ceil(2)) else { + return false; + }; + src.src.len() >= src_need && dst.y.len() >= y_need && dst.uv.len() >= uv_need +} + +fn scalar_rows(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) { + let uv_rows = src.height.div_ceil(2); + for row in 0..uv_rows { + write_pixels(src, dst, row, 0); + } +} + +fn sample(row: &[u8], packed_len: usize, index: usize) -> u8 { + if index < packed_len { row[index] } else { 128 } +} + +fn avg_u8(a: u8, b: u8) -> u8 { + ((u16::from(a) + u16::from(b) + 1) >> 1) as u8 +} + +// (y_left, y_right, u, v). A short last macropixel reads missing bytes as 128. +fn load_macro(kind: Packed422, row: &[u8], pair: usize, packed_len: usize) -> (u8, u8, u8, u8) { + let off = pair * 4; + match kind { + Packed422::Uyvy => ( + sample(row, packed_len, off + 1), + sample(row, packed_len, off + 3), + sample(row, packed_len, off), + sample(row, packed_len, off + 2), + ), + Packed422::Yuyv => ( + sample(row, packed_len, off), + sample(row, packed_len, off + 2), + sample(row, packed_len, off + 1), + sample(row, packed_len, off + 3), + ), + } +} + +fn write_pixels(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>, row: usize, x_start: usize) { + debug_assert!(x_start.is_multiple_of(2)); + let width = src.width; + let height = src.height; + let src_stride = src.stride; + let y_stride = dst.y_stride; + let uv_stride = dst.uv_stride; + let top = row * 2; + let bot = top + 1; + let average = bot < height; + let packed_len = width * 2; + let src0 = top * src_stride; + let src1 = bot * src_stride; + let y0 = top * y_stride; + let y1 = bot * y_stride; + let uv_base = row * uv_stride; + let mut x = x_start; + while x < width { + let pair = x / 2; + let (left, right, u0, v0) = load_macro( + src.kind, + &src.src[src0..src0 + src_stride], + pair, + packed_len, + ); + dst.y[y0 + x] = left; + if x + 1 < width { + dst.y[y0 + x + 1] = right; + } + let (u, v) = if average { + let (left1, right1, u1, v1) = load_macro( + src.kind, + &src.src[src1..src1 + src_stride], + pair, + packed_len, + ); + dst.y[y1 + x] = left1; + if x + 1 < width { + dst.y[y1 + x + 1] = right1; + } + (avg_u8(u0, u1), avg_u8(v0, v1)) + } else { + (u0, v0) + }; + let uv_at = uv_base + pair * 2; + dst.uv[uv_at] = u; + dst.uv[uv_at + 1] = v; + x += 2; + } +} + +#[cfg(target_arch = "x86_64")] +struct RowPtrs { + src0: *const u8, + src1: *const u8, + y0: *mut u8, + y1: *mut u8, + uv: *mut u8, + average: bool, +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn avx_convert(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) { + let masks = match src.kind { + Packed422::Uyvy => Masks { + y: unsafe { uyvy_y_mask() }, + u: unsafe { uyvy_u_mask() }, + v: unsafe { uyvy_v_mask() }, + }, + Packed422::Yuyv => Masks { + y: unsafe { yuyv_y_mask() }, + u: unsafe { yuyv_u_mask() }, + v: unsafe { yuyv_v_mask() }, + }, + }; + let width = src.width; + let height = src.height; + let src_stride = src.stride; + let y_stride = dst.y_stride; + let uv_stride = dst.uv_stride; + let uv_rows = height.div_ceil(2); + for row in 0..uv_rows { + let top = row * 2; + let bot = top + 1; + let average = bot < height; + let src0 = src.src.as_ptr().wrapping_add(top * src_stride); + let src1 = if average { + src.src.as_ptr().wrapping_add(bot * src_stride) + } else { + std::ptr::null() + }; + let y0 = dst.y.as_mut_ptr().wrapping_add(top * y_stride); + let y1 = if average { + dst.y.as_mut_ptr().wrapping_add(bot * y_stride) + } else { + std::ptr::null_mut() + }; + let uv = dst.uv.as_mut_ptr().wrapping_add(row * uv_stride); + let ptrs = RowPtrs { + src0, + src1, + y0, + y1, + uv, + average, + }; + let mut x = 0usize; + while x + 32 <= width { + unsafe { + convert16(&masks, &ptrs, x); + convert16(&masks, &ptrs, x + 16); + } + x += 32; + } + while x + 16 <= width { + unsafe { + convert16(&masks, &ptrs, x); + } + x += 16; + } + if x < width { + write_pixels(src, dst, row, x); + } + } +} + +#[cfg(not(target_arch = "x86_64"))] +unsafe fn avx_convert(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) { + scalar_rows(src, dst); +} + +#[cfg(target_arch = "x86_64")] +struct Masks { + y: std::arch::x86_64::__m256i, + u: std::arch::x86_64::__m256i, + v: std::arch::x86_64::__m256i, +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn convert16(masks: &Masks, row: &RowPtrs, x: usize) { + use std::arch::x86_64::*; + + let packed0 = unsafe { _mm256_loadu_si256(row.src0.add(x * 2).cast()) }; + unsafe { store_luma(packed0, masks.y, row.y0.add(x)) }; + let u0 = unsafe { pack_chroma8(packed0, masks.u) }; + let v0 = unsafe { pack_chroma8(packed0, masks.v) }; + let (u, v) = if row.average { + let packed1 = unsafe { _mm256_loadu_si256(row.src1.add(x * 2).cast()) }; + unsafe { store_luma(packed1, masks.y, row.y1.add(x)) }; + let u1 = unsafe { pack_chroma8(packed1, masks.u) }; + let v1 = unsafe { pack_chroma8(packed1, masks.v) }; + // avg_epu8 is (a + b + 1) >> 1 for each byte. + (_mm_avg_epu8(u0, u1), _mm_avg_epu8(v0, v1)) + } else { + (u0, v0) + }; + let interleaved = _mm_unpacklo_epi8(u, v); + unsafe { _mm_storeu_si128(row.uv.add(x).cast(), interleaved) }; +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn store_luma( + packed: std::arch::x86_64::__m256i, + mask: std::arch::x86_64::__m256i, + dst: *mut u8, +) { + use std::arch::x86_64::*; + let shuffled = _mm256_shuffle_epi8(packed, mask); + let lo = _mm256_castsi256_si128(shuffled); + let hi = _mm256_extracti128_si256(shuffled, 1); + let y = _mm_unpacklo_epi64(lo, hi); + unsafe { _mm_storeu_si128(dst.cast(), y) }; +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn pack_chroma8( + packed: std::arch::x86_64::__m256i, + mask: std::arch::x86_64::__m256i, +) -> std::arch::x86_64::__m128i { + use std::arch::x86_64::*; + let shuffled = _mm256_shuffle_epi8(packed, mask); + let lo = _mm256_castsi256_si128(shuffled); + let hi = _mm256_extracti128_si256(shuffled, 1); + _mm_unpacklo_epi32(lo, hi) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn uyvy_y_mask() -> std::arch::x86_64::__m256i { + use std::arch::x86_64::*; + _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, 1, 3, 5, 7, 9, 11, 13, 15, -1, + -1, -1, -1, -1, -1, -1, -1, + ) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn uyvy_u_mask() -> std::arch::x86_64::__m256i { + use std::arch::x86_64::*; + _mm256_setr_epi8( + 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + ) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn uyvy_v_mask() -> std::arch::x86_64::__m256i { + use std::arch::x86_64::*; + _mm256_setr_epi8( + 2, 6, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 6, 10, 14, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + ) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn yuyv_y_mask() -> std::arch::x86_64::__m256i { + use std::arch::x86_64::*; + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14, -1, + -1, -1, -1, -1, -1, -1, -1, + ) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn yuyv_u_mask() -> std::arch::x86_64::__m256i { + use std::arch::x86_64::*; + _mm256_setr_epi8( + 1, 5, 9, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 5, 9, 13, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + ) +} + +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "avx2")] +unsafe fn yuyv_v_mask() -> std::arch::x86_64::__m256i { + use std::arch::x86_64::*; + _mm256_setr_epi8( + 3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 7, 11, 15, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fill(buf: &mut [u8], mut seed: u32) { + for byte in buf.iter_mut() { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + *byte = (seed >> 24) as u8; + } + } + + fn ref_byte(row: &[u8], packed_len: usize, index: usize) -> u8 { + if index < packed_len { row[index] } else { 128 } + } + + fn ref_sample(kind: Packed422, row: &[u8], packed_len: usize, x: usize) -> (u8, u8, u8) { + let off = (x / 2) * 4; + let (y, u, v) = match kind { + Packed422::Uyvy => { + let y = if x.is_multiple_of(2) { + ref_byte(row, packed_len, off + 1) + } else { + ref_byte(row, packed_len, off + 3) + }; + ( + y, + ref_byte(row, packed_len, off), + ref_byte(row, packed_len, off + 2), + ) + } + Packed422::Yuyv => { + let y = if x.is_multiple_of(2) { + ref_byte(row, packed_len, off) + } else { + ref_byte(row, packed_len, off + 2) + }; + ( + y, + ref_byte(row, packed_len, off + 1), + ref_byte(row, packed_len, off + 3), + ) + } + }; + (y, u, v) + } + + fn reference( + src: &PackedView<'_>, + y: &mut [u8], + y_stride: usize, + uv: &mut [u8], + uv_stride: usize, + ) { + let packed_len = src.width * 2; + for row in 0..src.height { + let src_row = &src.src[row * src.stride..row * src.stride + src.stride]; + for x in 0..src.width { + let (sample_y, _, _) = ref_sample(src.kind, src_row, packed_len, x); + y[row * y_stride + x] = sample_y; + } + } + let chroma_pairs = src.width.div_ceil(2); + for row in 0..src.height.div_ceil(2) { + let top = row * 2; + let src_top = &src.src[top * src.stride..top * src.stride + src.stride]; + let bot = top + 1; + let src_bot = (bot < src.height) + .then(|| &src.src[bot * src.stride..bot * src.stride + src.stride]); + for pair in 0..chroma_pairs { + let x = pair * 2; + let (_, u0, v0) = ref_sample(src.kind, src_top, packed_len, x); + let (u, v) = if let Some(src_bot) = src_bot { + let (_, u1, v1) = ref_sample(src.kind, src_bot, packed_len, x); + ( + ((u16::from(u0) + u16::from(u1) + 1) >> 1) as u8, + ((u16::from(v0) + u16::from(v1) + 1) >> 1) as u8, + ) + } else { + (u0, v0) + }; + uv[row * uv_stride + pair * 2] = u; + uv[row * uv_stride + pair * 2 + 1] = v; + } + } + } + + fn run_scalar( + kind: Packed422, + width: usize, + height: usize, + src_pad: usize, + y_pad: usize, + uv_pad: usize, + seed: u32, + ) -> (Vec, Vec, Vec, Vec) { + let src_stride = width * 2 + src_pad; + let y_stride = width + y_pad; + let uv_stride = chroma_row_bytes(width) + uv_pad; + let mut src = vec![0u8; src_stride * height]; + fill(&mut src, seed); + let mut y = vec![0x5Au8; y_stride * height]; + let mut uv = vec![0x5Au8; uv_stride * height.div_ceil(2)]; + let mut y_ref = y.clone(); + let mut uv_ref = uv.clone(); + let mut dst = Nv12Mut { + y: &mut y, + y_stride, + uv: &mut uv, + uv_stride, + }; + let view = PackedView { + kind, + src: &src, + stride: src_stride, + width, + height, + }; + assert!(convert_rows_scalar(&view, &mut dst)); + reference(&view, &mut y_ref, y_stride, &mut uv_ref, uv_stride); + (y, uv, y_ref, uv_ref) + } + + fn convert_rows_scalar(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) -> bool { + if !buffers_fit(src, dst) { + return false; + } + scalar_rows(src, dst); + true + } + + fn case( + kind: Packed422, + width: usize, + height: usize, + src_pad: usize, + y_pad: usize, + uv_pad: usize, + ) { + let seed = (width as u32) + .wrapping_mul(1000) + .wrapping_add(height as u32 * 17) + .wrapping_add(src_pad as u32 * 3) + .wrapping_add(y_pad as u32 * 5) + .wrapping_add(uv_pad as u32 * 9) + .wrapping_add(match kind { + Packed422::Uyvy => 1, + Packed422::Yuyv => 2, + }); + let (y, uv, y_ref, uv_ref) = run_scalar(kind, width, height, src_pad, y_pad, uv_pad, seed); + assert_eq!( + y, y_ref, + "{kind:?} {width}x{height} y pad {src_pad}/{y_pad}/{uv_pad}" + ); + assert_eq!( + uv, uv_ref, + "{kind:?} {width}x{height} uv pad {src_pad}/{y_pad}/{uv_pad}" + ); + } + + #[test] + fn scalar_matches_reference_across_shapes() { + let widths = [ + 1usize, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128, + ]; + let heights = [1usize, 2, 3, 4, 5, 27]; + let src_pads = [0usize, 7, 32]; + let y_pads = [0usize, 13]; + let uv_pads = [0usize, 15]; + for kind in [Packed422::Uyvy, Packed422::Yuyv] { + for &width in &widths { + for &height in &heights { + for &src_pad in &src_pads { + for &y_pad in &y_pads { + for &uv_pad in &uv_pads { + case(kind, width, height, src_pad, y_pad, uv_pad); + } + } + } + } + } + case(kind, 1920, 2, 16, 8, 24); + case(kind, 1919, 3, 1, 0, 5); + case(kind, 1280, 4, 0, 0, 0); + } + } + + #[cfg(target_arch = "x86_64")] + #[test] + fn avx2_matches_scalar_across_shapes() { + if !is_x86_feature_detected!("avx2") { + return; + } + let widths = [ + 1usize, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128, + ]; + let heights = [1usize, 2, 3, 5, 27]; + let src_pads = [0usize, 7, 32]; + let y_pads = [0usize, 13]; + let uv_pads = [0usize, 15]; + for kind in [Packed422::Uyvy, Packed422::Yuyv] { + for &width in &widths { + for &height in &heights { + for &src_pad in &src_pads { + for &y_pad in &y_pads { + for &uv_pad in &uv_pads { + compare_avx(kind, width, height, src_pad, y_pad, uv_pad); + } + } + } + } + } + compare_avx(kind, 1920, 2, 16, 8, 24); + compare_avx(kind, 1919, 3, 3, 1, 9); + compare_avx(kind, 3840, 2, 0, 0, 0); + } + } + + #[cfg(target_arch = "x86_64")] + fn compare_avx( + kind: Packed422, + width: usize, + height: usize, + src_pad: usize, + y_pad: usize, + uv_pad: usize, + ) { + let src_stride = width * 2 + src_pad; + let y_stride = width + y_pad; + let uv_stride = chroma_row_bytes(width) + uv_pad; + let mut src = vec![0u8; src_stride * height]; + fill(&mut src, width as u32 * 13 + height as u32); + let mut y_avx = vec![0x5Au8; y_stride * height]; + let mut uv_avx = vec![0x5Au8; uv_stride * height.div_ceil(2)]; + let mut y_scalar = y_avx.clone(); + let mut uv_scalar = uv_avx.clone(); + { + let mut dst = Nv12Mut { + y: &mut y_avx, + y_stride, + uv: &mut uv_avx, + uv_stride, + }; + assert!(convert_packed422_to_nv12( + kind, &src, src_stride, width, height, &mut dst + )); + } + { + let mut dst = Nv12Mut { + y: &mut y_scalar, + y_stride, + uv: &mut uv_scalar, + uv_stride, + }; + assert!(convert_rows_scalar( + &PackedView { + kind, + src: &src, + stride: src_stride, + width, + height, + }, + &mut dst + )); + } + assert_eq!(y_avx, y_scalar, "{kind:?} {width}x{height} avx y"); + assert_eq!(uv_avx, uv_scalar, "{kind:?} {width}x{height} avx uv"); + } +} diff --git a/crates/frame-converter/src/swscale.rs b/crates/frame-converter/src/swscale.rs index eafff292494..18ee4bd80e2 100644 --- a/crates/frame-converter/src/swscale.rs +++ b/crates/frame-converter/src/swscale.rs @@ -1,3 +1,5 @@ +use crate::output_frame::cached_output_frame; +use crate::packed_yuv::{self, Nv12Mut, Packed422}; use crate::{ConversionConfig, ConvertError, ConverterBackend, FrameConverter}; use ffmpeg::{format::Pixel, frame, software::scaling}; use std::cell::UnsafeCell; @@ -61,12 +63,124 @@ impl SwscaleConverter { self.contexts .get_or_try(|| Self::create_context(&self.config).map(SendableContext::new)) } + + #[cfg(test)] + pub(crate) fn convert_swscale_into( + &self, + input: &frame::Video, + output: &mut frame::Video, + ) -> Result<(), ConvertError> { + let context = self.get_or_create_context()?; + context + .run(input, output) + .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?; + Ok(()) + } +} + +fn positive_linesize(frame: &frame::Video, plane: usize) -> Option { + if plane >= frame.planes() { + return None; + } + let linesize = unsafe { (*frame.as_ptr()).linesize[plane] }; + if linesize <= 0 { + None + } else { + Some(linesize as usize) + } +} + +fn planes_disjoint(a: *const u8, a_len: usize, b: *const u8, b_len: usize) -> bool { + if a_len == 0 || b_len == 0 { + return true; + } + let Some(a_end) = (a as usize).checked_add(a_len) else { + return false; + }; + let Some(b_end) = (b as usize).checked_add(b_len) else { + return false; + }; + a_end <= b as usize || b_end <= a as usize +} + +pub(crate) fn try_packed_yuv_nv12(input: &frame::Video, output: &mut frame::Video) -> bool { + let kind = match input.format() { + Pixel::UYVY422 => Packed422::Uyvy, + Pixel::YUYV422 => Packed422::Yuyv, + _ => return false, + }; + if output.format() != Pixel::NV12 { + return false; + } + let width_px = input.width(); + let height_px = input.height(); + if output.width() != width_px || output.height() != height_px { + return false; + } + if width_px == 0 || height_px == 0 { + return true; + } + if input.planes() < 1 || output.planes() < 2 { + return false; + } + let Some(src_stride) = positive_linesize(input, 0) else { + return false; + }; + let Some(y_stride) = positive_linesize(output, 0) else { + return false; + }; + let Some(uv_stride) = positive_linesize(output, 1) else { + return false; + }; + let width = width_px as usize; + let height = height_px as usize; + let src_h = input.plane_height(0) as usize; + let y_h = output.plane_height(0) as usize; + let uv_h = output.plane_height(1) as usize; + if src_h < height || y_h < height || uv_h < height.div_ceil(2) { + return false; + } + let Some(src_len) = src_stride.checked_mul(src_h) else { + return false; + }; + let Some(y_len) = y_stride.checked_mul(y_h) else { + return false; + }; + let Some(uv_len) = uv_stride.checked_mul(uv_h) else { + return false; + }; + unsafe { + let in_raw = input.as_ptr(); + let out_raw = output.as_mut_ptr(); + if in_raw.is_null() || out_raw.is_null() { + return false; + } + let src_ptr = (*in_raw).data[0]; + let y_ptr = (*out_raw).data[0]; + let uv_ptr = (*out_raw).data[1]; + if src_ptr.is_null() || y_ptr.is_null() || uv_ptr.is_null() { + return false; + } + if !planes_disjoint(y_ptr, y_len, uv_ptr, uv_len) { + return false; + } + let src = std::slice::from_raw_parts(src_ptr, src_len); + let y = std::slice::from_raw_parts_mut(y_ptr, y_len); + let uv = std::slice::from_raw_parts_mut(uv_ptr, uv_len); + let mut dst = Nv12Mut { + y, + y_stride, + uv, + uv_stride, + }; + packed_yuv::convert_packed422_to_nv12(kind, src, src_stride, width, height, &mut dst) + } } impl FrameConverter for SwscaleConverter { fn convert(&self, input: frame::Video) -> Result { let mut output = - frame::Video::new(self.output_format, self.output_width, self.output_height); + cached_output_frame(self.output_format, self.output_width, self.output_height); self.convert_into(input, &mut output)?; Ok(output) } @@ -77,12 +191,12 @@ impl FrameConverter for SwscaleConverter { output: &mut frame::Video, ) -> Result<(), ConvertError> { let pts = input.pts(); - - let context = self.get_or_create_context()?; - context - .run(&input, output) - .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?; - + if !try_packed_yuv_nv12(&input, output) { + let context = self.get_or_create_context()?; + context + .run(&input, output) + .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?; + } output.set_pts(pts); Ok(()) } @@ -98,3 +212,369 @@ impl FrameConverter for SwscaleConverter { unsafe impl Send for SwscaleConverter {} unsafe impl Sync for SwscaleConverter {} + +#[cfg(test)] +mod tests { + use super::*; + use crate::packed_yuv::{Packed422, convert_packed422_scalar}; + use std::sync::Once; + use std::time::{Duration, Instant}; + + fn init_ffmpeg() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + ffmpeg::init().expect("ffmpeg init"); + }); + } + + fn fill(buf: &mut [u8], mut seed: u32) { + for byte in buf.iter_mut() { + seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); + *byte = (seed >> 24) as u8; + } + } + + fn fill_frame(frame: &mut frame::Video, seed: u32) { + for plane in 0..frame.planes() { + fill(frame.data_mut(plane), seed.wrapping_add(plane as u32 * 97)); + } + } + + fn active_rows(plane: &[u8], stride: usize, rows: usize, active: usize) -> Vec { + let mut out = Vec::with_capacity(rows * active); + for row in 0..rows { + let start = row * stride; + out.extend_from_slice(&plane[start..start + active]); + } + out + } + + fn kind_of(format: Pixel) -> Packed422 { + match format { + Pixel::UYVY422 => Packed422::Uyvy, + Pixel::YUYV422 => Packed422::Yuyv, + _ => panic!("packed 4:2:2 format"), + } + } + + #[test] + fn convert_into_matches_scalar_kernel() { + init_ffmpeg(); + let shapes = [ + (16u32, 8u32), + (17, 3), + (15, 2), + (31, 5), + (32, 1), + (33, 4), + (64, 27), + (47, 9), + ]; + for format in [Pixel::UYVY422, Pixel::YUYV422] { + for (width, height) in shapes { + let config = + ConversionConfig::new(format, width, height, Pixel::NV12, width, height); + let converter = SwscaleConverter::new(config).expect("converter"); + let mut input = frame::Video::new(format, width, height); + fill_frame(&mut input, width.wrapping_mul(13).wrapping_add(height)); + input.set_pts(Some(i64::from(width) * 1000 + i64::from(height))); + let mut output = frame::Video::new(Pixel::NV12, width, height); + converter + .convert_into(input.clone(), &mut output) + .expect("convert_into"); + assert_eq!( + output.pts(), + Some(i64::from(width) * 1000 + i64::from(height)) + ); + assert_eq!(output.format(), Pixel::NV12); + + let src_stride = input.stride(0); + let y_stride = output.stride(0); + let uv_stride = output.stride(1); + let mut y = vec![0u8; y_stride * output.plane_height(0) as usize]; + let mut uv = vec![0u8; uv_stride * output.plane_height(1) as usize]; + { + let mut dst = crate::packed_yuv::Nv12Mut { + y: &mut y, + y_stride, + uv: &mut uv, + uv_stride, + }; + assert!(convert_packed422_scalar( + kind_of(format), + input.data(0), + src_stride, + width as usize, + height as usize, + &mut dst, + )); + } + assert_eq!( + active_rows(output.data(0), y_stride, height as usize, width as usize), + active_rows(&y, y_stride, height as usize, width as usize), + "{format:?} {width}x{height} y" + ); + let chroma_bytes = (width as usize).div_ceil(2) * 2; + assert_eq!( + active_rows( + output.data(1), + uv_stride, + height.div_ceil(2) as usize, + chroma_bytes + ), + active_rows(&uv, uv_stride, height.div_ceil(2) as usize, chroma_bytes), + "{format:?} {width}x{height} uv" + ); + } + } + } + + // FAST_BILINEAR is not an identity luma copy and is not (a + b + 1) >> 1. + // On even frames through 1920x1080 the max absolute difference was 20 luma + // and 19 chroma. Odd heights use a different swscale vertical filter + // (about 120 levels on 64x33) and stay in the exact kernel tests. + const LUMA_TOLERANCE: i32 = 24; + const CHROMA_TOLERANCE: i32 = 24; + + fn max_plane_diff(left: &[u8], right: &[u8], stride: usize, rows: usize, active: usize) -> i32 { + let mut max_diff = 0i32; + for row in 0..rows { + let start = row * stride; + for offset in 0..active { + let delta = i32::from(left[start + offset]) - i32::from(right[start + offset]); + max_diff = max_diff.max(delta.abs()); + } + } + max_diff + } + + #[test] + fn packed_path_stays_close_to_swscale() { + init_ffmpeg(); + let shapes = [ + (64u32, 48u32), + (320, 180), + (640, 360), + (1280, 720), + (1920, 1080), + ]; + for format in [Pixel::UYVY422, Pixel::YUYV422] { + for (width, height) in shapes { + let config = + ConversionConfig::new(format, width, height, Pixel::NV12, width, height); + let converter = SwscaleConverter::new(config).expect("converter"); + let mut input = frame::Video::new(format, width, height); + fill_frame( + &mut input, + 0x51C0_u32.wrapping_add(width).wrapping_add(height), + ); + let mut fast = frame::Video::new(Pixel::NV12, width, height); + let mut sws = frame::Video::new(Pixel::NV12, width, height); + assert!(try_packed_yuv_nv12(&input, &mut fast)); + converter + .convert_swscale_into(&input, &mut sws) + .expect("swscale"); + let luma = max_plane_diff( + fast.data(0), + sws.data(0), + fast.stride(0), + height as usize, + width as usize, + ); + let chroma_bytes = (width as usize / 2) * 2; + let chroma = max_plane_diff( + fast.data(1), + sws.data(1), + fast.stride(1), + height.div_ceil(2) as usize, + chroma_bytes, + ); + assert!( + luma <= LUMA_TOLERANCE && chroma <= CHROMA_TOLERANCE, + "{format:?} {width}x{height} luma {luma} chroma {chroma}" + ); + } + } + } + + #[test] + fn other_formats_and_scaling_still_use_swscale() { + init_ffmpeg(); + let scaled = SwscaleConverter::new(ConversionConfig::new( + Pixel::UYVY422, + 32, + 16, + Pixel::NV12, + 16, + 8, + )) + .expect("scaled"); + let mut input = frame::Video::new(Pixel::UYVY422, 32, 16); + fill_frame(&mut input, 9); + let output = scaled.convert(input).expect("scale"); + assert_eq!( + (output.width(), output.height(), output.format()), + (16, 8, Pixel::NV12) + ); + + let bgra = SwscaleConverter::new(ConversionConfig::new( + Pixel::BGRA, + 16, + 16, + Pixel::NV12, + 16, + 16, + )) + .expect("bgra"); + let mut input = frame::Video::new(Pixel::BGRA, 16, 16); + fill_frame(&mut input, 4); + let output = bgra.convert(input).expect("bgra convert"); + assert_eq!(output.format(), Pixel::NV12); + assert_eq!((output.width(), output.height()), (16, 16)); + } + + #[test] + fn convert_keeps_live_frames_independent() { + init_ffmpeg(); + let converter = SwscaleConverter::new(ConversionConfig::new( + Pixel::UYVY422, + 82, + 18, + Pixel::NV12, + 82, + 18, + )) + .expect("converter"); + let mut first_src = frame::Video::new(Pixel::UYVY422, 82, 18); + first_src.data_mut(0).fill(30); + first_src.set_pts(Some(42)); + let mut first = converter.convert(first_src).expect("first"); + assert_eq!(first.pts(), Some(42)); + let first_y = first.data(0).to_vec(); + let first_uv = first.data(1).to_vec(); + unsafe { + assert_eq!(ffmpeg::ffi::av_frame_is_writable(first.as_mut_ptr()), 1); + } + let mut second_src = frame::Video::new(Pixel::UYVY422, 82, 18); + second_src.data_mut(0).fill(90); + let mut second = converter.convert(second_src).expect("second"); + assert_eq!(first.data(0), first_y.as_slice()); + assert_eq!(first.data(1), first_uv.as_slice()); + assert!(second.data(0).contains(&90)); + assert!(!second.data(0).iter().take(82).all(|byte| *byte == 30)); + unsafe { + assert_eq!(ffmpeg::ffi::av_frame_is_writable(first.as_mut_ptr()), 1); + assert_eq!(ffmpeg::ffi::av_frame_is_writable(second.as_mut_ptr()), 1); + } + drop(first); + drop(second); + let mut third_src = frame::Video::new(Pixel::UYVY422, 82, 18); + third_src.data_mut(0).fill(11); + let third = converter.convert(third_src).expect("third"); + assert!(third.data(0).iter().take(82).all(|byte| *byte == 11)); + } + + #[inline(never)] + fn time_swscale(converter: &SwscaleConverter, input: &frame::Video, output: &mut frame::Video) { + converter + .convert_swscale_into(input, output) + .expect("swscale"); + let index = output.stride(0); + std::hint::black_box(output.data(0)[index]); + } + + #[inline(never)] + fn time_packed(input: &frame::Video, output: &mut frame::Video) { + assert!(try_packed_yuv_nv12(input, output)); + let index = output.stride(0); + std::hint::black_box(output.data(0)[index]); + } + + fn median(samples: &mut [Duration]) -> Duration { + samples.sort_unstable(); + samples[samples.len() / 2] + } + + fn cpu_model() -> String { + let Ok(text) = std::fs::read_to_string("/proc/cpuinfo") else { + return "unknown".to_string(); + }; + for line in text.lines() { + let Some(rest) = line.strip_prefix("model name") else { + continue; + }; + return rest.trim_start_matches([' ', ':', '\t']).to_string(); + } + "unknown".to_string() + } + + #[test] + #[ignore] + fn bench_packed_yuv_nv12() { + init_ffmpeg(); + if cfg!(debug_assertions) { + eprintln!("debug build; rerun with --release for representative medians"); + } + #[cfg(target_arch = "x86_64")] + let avx2 = is_x86_feature_detected!("avx2"); + #[cfg(not(target_arch = "x86_64"))] + let avx2 = false; + let warmup = 16; + let runs = 51; + println!("cpu: {}", cpu_model()); + println!("avx2: {avx2}"); + println!("warmup: {warmup}"); + println!("timed runs: {runs}"); + println!( + "before: swscale FAST_BILINEAR, after: packed kernel, frames allocated outside the timer" + ); + let shapes = [(1280u32, 720u32), (1920, 1080), (3840, 2160)]; + for format in [Pixel::UYVY422, Pixel::YUYV422] { + for (width, height) in shapes { + let config = + ConversionConfig::new(format, width, height, Pixel::NV12, width, height); + let converter = SwscaleConverter::new(config).expect("converter"); + let mut input = frame::Video::new(format, width, height); + fill_frame(&mut input, width.wrapping_mul(height)); + let mut sws_out = frame::Video::new(Pixel::NV12, width, height); + let mut fast_out = frame::Video::new(Pixel::NV12, width, height); + for _ in 0..warmup { + time_swscale(&converter, &input, &mut sws_out); + } + let mut sws_samples = Vec::with_capacity(runs); + for _ in 0..runs { + let start = Instant::now(); + time_swscale(&converter, &input, &mut sws_out); + sws_samples.push(start.elapsed()); + } + for _ in 0..warmup { + time_packed(&input, &mut fast_out); + } + let mut fast_samples = Vec::with_capacity(runs); + for _ in 0..runs { + let start = Instant::now(); + time_packed(&input, &mut fast_out); + fast_samples.push(start.elapsed()); + } + let sws_min = sws_samples.iter().copied().min().unwrap_or_default(); + let fast_min = fast_samples.iter().copied().min().unwrap_or_default(); + let sws_max = sws_samples.iter().copied().max().unwrap_or_default(); + let fast_max = fast_samples.iter().copied().max().unwrap_or_default(); + let sws_med = median(&mut sws_samples); + let fast_med = median(&mut fast_samples); + let speedup = sws_med.as_secs_f64() / fast_med.as_secs_f64(); + println!( + "{width}x{height} {format:?} before {:.3} ms after {:.3} ms speedup {speedup:.2}x n {runs} before_min {:.3} before_max {:.3} after_min {:.3} after_max {:.3}", + sws_med.as_secs_f64() * 1000.0, + fast_med.as_secs_f64() * 1000.0, + sws_min.as_secs_f64() * 1000.0, + sws_max.as_secs_f64() * 1000.0, + fast_min.as_secs_f64() * 1000.0, + fast_max.as_secs_f64() * 1000.0, + ); + let _ = std::hint::black_box(sws_out.data(0)[0]); + let _ = std::hint::black_box(fast_out.data(0)[0]); + } + } + } +} From 8734f17bfacbfeda48be49ca8ecbda92e12b1f08 Mon Sep 17 00:00:00 2001 From: Kyzer Phneh Date: Fri, 23 Oct 2026 13:37:20 +0000 Subject: [PATCH 2/3] perf: convert packed camera yuv in the h264 encoder UYVY422 and YUYV422 frames become YUV420P or NV12 through AVX2 on the encoder queue path. Odd sizes, scaling, other formats, and machines without AVX2 stay on swscale. The unused frame-converter output pool is removed. --- crates/enc-ffmpeg/src/video/h264.rs | 6 +- crates/enc-ffmpeg/src/video/h264_packet.rs | 6 +- crates/enc-ffmpeg/src/video/mod.rs | 1 + crates/enc-ffmpeg/src/video/packed_yuv.rs | 903 +++++++++++++++++++++ crates/frame-converter/src/frame_pool.rs | 35 +- crates/frame-converter/src/lib.rs | 3 - crates/frame-converter/src/output_frame.rs | 269 ------ crates/frame-converter/src/packed_yuv.rs | 665 --------------- crates/frame-converter/src/swscale.rs | 494 +---------- 9 files changed, 917 insertions(+), 1465 deletions(-) create mode 100644 crates/enc-ffmpeg/src/video/packed_yuv.rs delete mode 100644 crates/frame-converter/src/output_frame.rs delete mode 100644 crates/frame-converter/src/packed_yuv.rs diff --git a/crates/enc-ffmpeg/src/video/h264.rs b/crates/enc-ffmpeg/src/video/h264.rs index 0042b4610af..90645af991d 100644 --- a/crates/enc-ffmpeg/src/video/h264.rs +++ b/crates/enc-ffmpeg/src/video/h264.rs @@ -915,8 +915,7 @@ impl H264Encoder { let frame_to_send = if let Some(converter) = &mut self.converter { let pts = frame.pts(); let converted = self.converted_frame_pool.as_mut().unwrap(); - converter - .run(&frame, converted) + super::packed_yuv::convert_or_scale(&frame, converted, converter) .map_err(QueueFrameError::Converter)?; converted.set_pts(pts); converted as &frame::Video @@ -945,8 +944,7 @@ impl H264Encoder { let converted = converted_frame.get_or_insert_with(|| { frame::Video::new(self.output_format, self.output_width, self.output_height) }); - converter - .run(frame, converted) + super::packed_yuv::convert_or_scale(frame, converted, converter) .map_err(QueueFrameError::Converter)?; converted.set_pts(pts); converted as &frame::Video diff --git a/crates/enc-ffmpeg/src/video/h264_packet.rs b/crates/enc-ffmpeg/src/video/h264_packet.rs index fb7596f7c21..e344a09e500 100644 --- a/crates/enc-ffmpeg/src/video/h264_packet.rs +++ b/crates/enc-ffmpeg/src/video/h264_packet.rs @@ -140,8 +140,7 @@ impl H264PacketEncoder { let converted = self.converted_frame_pool.as_mut().expect( "converted_frame_pool present whenever converter is Some (invariant preserved by from_opened)", ); - converter - .run(&frame, converted) + super::packed_yuv::convert_or_scale(&frame, converted, converter) .map_err(EncodePacketError::Converter)?; converted.set_pts(pts); converted as &frame::Video @@ -173,8 +172,7 @@ impl H264PacketEncoder { let converted = converted_frame.get_or_insert_with(|| { frame::Video::new(self.output_format, self.output_width, self.output_height) }); - converter - .run(frame, converted) + super::packed_yuv::convert_or_scale(frame, converted, converter) .map_err(EncodePacketError::Converter)?; converted.set_pts(pts); converted as &frame::Video diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs index 104261fc075..839c61e68c5 100644 --- a/crates/enc-ffmpeg/src/video/mod.rs +++ b/crates/enc-ffmpeg/src/video/mod.rs @@ -1,4 +1,5 @@ pub mod colour; +mod packed_yuv; pub mod h264; pub mod h264_packet; pub mod hevc; diff --git a/crates/enc-ffmpeg/src/video/packed_yuv.rs b/crates/enc-ffmpeg/src/video/packed_yuv.rs new file mode 100644 index 00000000000..71ab404255e --- /dev/null +++ b/crates/enc-ffmpeg/src/video/packed_yuv.rs @@ -0,0 +1,903 @@ +use ffmpeg::format::Pixel; +use ffmpeg::frame; + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Packed422 { + Uyvy, + Yuyv, +} + +struct Job { + src: *const u8, + src_stride: usize, + width: usize, + height: usize, + kind: Packed422, + y: *mut u8, + y_stride: usize, + u: *mut u8, + u_stride: usize, + v: *mut u8, + v_stride: usize, + uv: *mut u8, + uv_stride: usize, + nv12: bool, +} + +fn positive_linesize(frame: &frame::Video, plane: usize) -> Option { + if plane >= frame.planes() { + return None; + } + let raw = unsafe { (*frame.as_ptr()).linesize[plane] }; + if raw <= 0 { None } else { Some(raw as usize) } +} + +fn covers(stride: usize, row_bytes: usize, height: usize, len: usize) -> bool { + if row_bytes == 0 || height == 0 { + return true; + } + if stride < row_bytes { + return false; + } + stride + .checked_mul(height - 1) + .and_then(|last| last.checked_add(row_bytes)) + .is_some_and(|need| need <= len) +} + +/// Same-size UYVY422 or YUYV422 into `dst` when both dimensions are even, +/// AVX2 is available at runtime, and `dst` is YUV420P or NV12. Any other +/// input returns false so the caller keeps swscale. +pub(crate) fn try_convert(src: &frame::Video, dst: &mut frame::Video) -> bool { + if src.width() != dst.width() || src.height() != dst.height() { + return false; + } + let kind = match src.format() { + Pixel::UYVY422 => Packed422::Uyvy, + Pixel::YUYV422 => Packed422::Yuyv, + _ => return false, + }; + let nv12 = match dst.format() { + Pixel::YUV420P => false, + Pixel::NV12 => true, + _ => return false, + }; + let width = src.width() as usize; + let height = src.height() as usize; + if !width.is_multiple_of(2) || !height.is_multiple_of(2) { + return false; + } + if width == 0 || height == 0 { + return true; + } + #[cfg(not(target_arch = "x86_64"))] + { + let _ = (kind, nv12); + return false; + } + #[cfg(target_arch = "x86_64")] + { + if !std::arch::is_x86_feature_detected!("avx2") { + return false; + } + let Some(src_stride) = positive_linesize(src, 0) else { + return false; + }; + let Some(y_stride) = positive_linesize(dst, 0) else { + return false; + }; + if !covers(src_stride, width * 2, height, src.data(0).len()) + || !covers(y_stride, width, height, dst.data(0).len()) + { + return false; + } + let chroma_w = width / 2; + let chroma_h = height / 2; + let (u_stride, v_stride, uv_stride) = if nv12 { + let Some(uv_stride) = positive_linesize(dst, 1) else { + return false; + }; + if !covers(uv_stride, chroma_w * 2, chroma_h, dst.data(1).len()) { + return false; + } + (0, 0, uv_stride) + } else { + let Some(u_stride) = positive_linesize(dst, 1) else { + return false; + }; + let Some(v_stride) = positive_linesize(dst, 2) else { + return false; + }; + if !covers(u_stride, chroma_w, chroma_h, dst.data(1).len()) + || !covers(v_stride, chroma_w, chroma_h, dst.data(2).len()) + { + return false; + } + (u_stride, v_stride, 0) + }; + + let src_ptr = src.data(0).as_ptr(); + let y_ptr = dst.data_mut(0).as_mut_ptr(); + let (u_ptr, v_ptr, uv_ptr) = if nv12 { + ( + std::ptr::null_mut(), + std::ptr::null_mut(), + dst.data_mut(1).as_mut_ptr(), + ) + } else { + let u_ptr = dst.data_mut(1).as_mut_ptr(); + let v_ptr = dst.data_mut(2).as_mut_ptr(); + (u_ptr, v_ptr, std::ptr::null_mut()) + }; + + let job = Job { + src: src_ptr, + src_stride, + width, + height, + kind, + y: y_ptr, + y_stride, + u: u_ptr, + u_stride, + v: v_ptr, + v_stride, + uv: uv_ptr, + uv_stride, + nv12, + }; + // SAFETY: AVX2 was detected, and `covers` checked every active plane. + unsafe { avx2::execute(&job) }; + true + } +} + +pub(crate) fn convert_or_scale( + src: &frame::Video, + dst: &mut frame::Video, + scaler: &mut ffmpeg::software::scaling::Context, +) -> Result<(), ffmpeg::Error> { + if try_convert(src, dst) { + Ok(()) + } else { + scaler.run(src, dst) + } +} + +#[cfg(target_arch = "x86_64")] +mod avx2 { + use std::arch::x86_64::*; + + use super::{Job, Packed422}; + + #[target_feature(enable = "avx2")] + pub(super) unsafe fn execute(job: &Job) { + unsafe { + let masks = match job.kind { + Packed422::Uyvy => Masks { + y: uyvy_y_mask(), + u: uyvy_u_mask(), + v: uyvy_v_mask(), + }, + Packed422::Yuyv => Masks { + y: yuyv_y_mask(), + u: yuyv_u_mask(), + v: yuyv_v_mask(), + }, + }; + if job.nv12 { + convert_frame::(job, &masks); + } else { + convert_frame::(job, &masks); + } + } + } + + struct Masks { + y: __m256i, + u: __m256i, + v: __m256i, + } + + #[target_feature(enable = "avx2")] + unsafe fn convert_frame(job: &Job, masks: &Masks) { + unsafe { + let mut top = 0; + while top < job.height { + let row = Row16 { + src0: job.src.add(top * job.src_stride), + src1: job.src.add((top + 1) * job.src_stride), + y0: job.y.add(top * job.y_stride), + y1: job.y.add((top + 1) * job.y_stride), + top, + }; + let mut x = 0; + while x + 32 <= job.width { + convert16::(job, masks, &row, x); + convert16::(job, masks, &row, x + 16); + x += 32; + } + while x + 16 <= job.width { + convert16::(job, masks, &row, x); + x += 16; + } + if x < job.width { + scalar_tail::(job, top, x); + } + top += 2; + } + } + } + + struct Row16 { + src0: *const u8, + src1: *const u8, + y0: *mut u8, + y1: *mut u8, + top: usize, + } + + #[target_feature(enable = "avx2")] + unsafe fn convert16(job: &Job, masks: &Masks, row: &Row16, x: usize) { + unsafe { + let packed0 = _mm256_loadu_si256(row.src0.add(x * 2).cast()); + store_luma(packed0, masks.y, row.y0.add(x)); + let u0 = pack_chroma8(packed0, masks.u); + let v0 = pack_chroma8(packed0, masks.v); + let packed1 = _mm256_loadu_si256(row.src1.add(x * 2).cast()); + store_luma(packed1, masks.y, row.y1.add(x)); + let u1 = pack_chroma8(packed1, masks.u); + let v1 = pack_chroma8(packed1, masks.v); + let u = _mm_avg_epu8(u0, u1); + let v = _mm_avg_epu8(v0, v1); + store_chroma8::(job, row.top / 2, x / 2, u, v); + } + } + + #[target_feature(enable = "avx2")] + unsafe fn store_luma(packed: __m256i, mask: __m256i, dst: *mut u8) { + unsafe { + let shuffled = _mm256_shuffle_epi8(packed, mask); + let lo = _mm256_castsi256_si128(shuffled); + let hi = _mm256_extracti128_si256(shuffled, 1); + let y = _mm_unpacklo_epi64(lo, hi); + _mm_storeu_si128(dst.cast(), y); + } + } + + #[target_feature(enable = "avx2")] + unsafe fn pack_chroma8(packed: __m256i, mask: __m256i) -> __m128i { + let shuffled = _mm256_shuffle_epi8(packed, mask); + let lo = _mm256_castsi256_si128(shuffled); + let hi = _mm256_extracti128_si256(shuffled, 1); + _mm_unpacklo_epi32(lo, hi) + } + + #[target_feature(enable = "avx2")] + unsafe fn store_chroma8( + job: &Job, + row: usize, + x: usize, + u: __m128i, + v: __m128i, + ) { + unsafe { + if NV12 { + let interleaved = _mm_unpacklo_epi8(u, v); + _mm_storeu_si128(job.uv.add(row * job.uv_stride + x * 2).cast(), interleaved); + } else { + _mm_storel_epi64(job.u.add(row * job.u_stride + x).cast(), u); + _mm_storel_epi64(job.v.add(row * job.v_stride + x).cast(), v); + } + } + } + + #[target_feature(enable = "avx2")] + unsafe fn uyvy_y_mask() -> __m256i { + _mm256_setr_epi8( + 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, 1, 3, 5, 7, 9, 11, 13, 15, + -1, -1, -1, -1, -1, -1, -1, -1, + ) + } + + #[target_feature(enable = "avx2")] + unsafe fn uyvy_u_mask() -> __m256i { + _mm256_setr_epi8( + 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, + ) + } + + #[target_feature(enable = "avx2")] + unsafe fn uyvy_v_mask() -> __m256i { + _mm256_setr_epi8( + 2, 6, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 6, 10, 14, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, + ) + } + + #[target_feature(enable = "avx2")] + unsafe fn yuyv_y_mask() -> __m256i { + _mm256_setr_epi8( + 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14, + -1, -1, -1, -1, -1, -1, -1, -1, + ) + } + + #[target_feature(enable = "avx2")] + unsafe fn yuyv_u_mask() -> __m256i { + _mm256_setr_epi8( + 1, 5, 9, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 5, 9, 13, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, + ) + } + + #[target_feature(enable = "avx2")] + unsafe fn yuyv_v_mask() -> __m256i { + _mm256_setr_epi8( + 3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 7, 11, 15, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, + ) + } + + unsafe fn scalar_tail(job: &Job, top: usize, x_start: usize) { + let mut x = x_start; + while x < job.width { + unsafe { + let (y0, y1, u0, v0) = load_macro(job, top, x); + let (y2, y3, u1, v1) = load_macro(job, top + 1, x); + *job.y.add(top * job.y_stride + x) = y0; + *job.y.add(top * job.y_stride + x + 1) = y1; + *job.y.add((top + 1) * job.y_stride + x) = y2; + *job.y.add((top + 1) * job.y_stride + x + 1) = y3; + let cx = x / 2; + let cy = top / 2; + let u = avg(u0, u1); + let v = avg(v0, v1); + if NV12 { + let dst = job.uv.add(cy * job.uv_stride + cx * 2); + *dst = u; + *dst.add(1) = v; + } else { + *job.u.add(cy * job.u_stride + cx) = u; + *job.v.add(cy * job.v_stride + cx) = v; + } + } + x += 2; + } + } + + unsafe fn load_macro(job: &Job, row: usize, x: usize) -> (u8, u8, u8, u8) { + unsafe { + let pixel = job.src.add(row * job.src_stride + x * 2); + let b0 = *pixel; + let b1 = *pixel.add(1); + let b2 = *pixel.add(2); + let b3 = *pixel.add(3); + match job.kind { + Packed422::Uyvy => (b1, b3, b0, b2), + Packed422::Yuyv => (b0, b2, b1, b3), + } + } + } + + fn avg(a: u8, b: u8) -> u8 { + ((u16::from(a) + u16::from(b) + 1) >> 1) as u8 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use ffmpeg::software::scaling::{self, flag::Flags}; + + fn avx2_ready() -> bool { + #[cfg(target_arch = "x86_64")] + { + std::arch::is_x86_feature_detected!("avx2") + } + #[cfg(not(target_arch = "x86_64"))] + { + false + } + } + + fn fast_scaler(src: Pixel, dst: Pixel, width: u32, height: u32) -> scaling::Context { + scaling::Context::get(src, width, height, dst, width, height, Flags::FAST_BILINEAR).unwrap() + } + + fn zero_frame(frame: &mut frame::Video) { + for plane in 0..frame.planes() { + frame.data_mut(plane).fill(0); + } + } + + fn fill_packed(frame: &mut frame::Video) { + let uyvy = frame.format() == Pixel::UYVY422; + let width = frame.width() as usize; + let height = frame.height() as usize; + let stride = frame.stride(0); + let data = frame.data_mut(0); + for y in 0..height { + let mut x = 0; + while x + 1 < width { + let y0 = ((x * 17 + y * 3) % 256) as u8; + let y1 = ((x * 13 + y * 11 + 40) % 256) as u8; + let u = ((x * 5 + y * 7 + 16) % 256) as u8; + let v = ((x * 9 + y * 2 + 80) % 256) as u8; + let off = y * stride + x * 2; + if uyvy { + data[off..off + 4].copy_from_slice(&[u, y0, v, y1]); + } else { + data[off..off + 4].copy_from_slice(&[y0, u, y1, v]); + } + x += 2; + } + } + } + + fn reference(src: &frame::Video) -> (Vec, Vec, Vec) { + let width = src.width() as usize; + let height = src.height() as usize; + let stride = src.stride(0); + let data = src.data(0); + let uyvy = src.format() == Pixel::UYVY422; + let cw = width / 2; + let ch = height / 2; + let mut y_plane = vec![0u8; width * height]; + let mut u_plane = vec![0u8; cw * ch]; + let mut v_plane = vec![0u8; cw * ch]; + let mut row = 0; + while row < height { + let mut x = 0; + while x < width { + let (y0, y1, u0, v0) = macro_at(data, stride, row, x, uyvy); + let (y2, y3, u1, v1) = macro_at(data, stride, row + 1, x, uyvy); + y_plane[row * width + x] = y0; + y_plane[row * width + x + 1] = y1; + y_plane[(row + 1) * width + x] = y2; + y_plane[(row + 1) * width + x + 1] = y3; + let cx = x / 2; + let cy = row / 2; + u_plane[cy * cw + cx] = ((u16::from(u0) + u16::from(u1) + 1) >> 1) as u8; + v_plane[cy * cw + cx] = ((u16::from(v0) + u16::from(v1) + 1) >> 1) as u8; + x += 2; + } + row += 2; + } + (y_plane, u_plane, v_plane) + } + + fn macro_at(data: &[u8], stride: usize, row: usize, x: usize, uyvy: bool) -> (u8, u8, u8, u8) { + let off = row * stride + x * 2; + let b0 = data[off]; + let b1 = data[off + 1]; + let b2 = data[off + 2]; + let b3 = data[off + 3]; + if uyvy { + (b1, b3, b0, b2) + } else { + (b0, b2, b1, b3) + } + } + + fn assert_matches_reference(actual: &frame::Video, y: &[u8], u: &[u8], v: &[u8]) { + let width = actual.width() as usize; + let height = actual.height() as usize; + let cw = width / 2; + let ch = height / 2; + for row in 0..height { + let got = &actual.data(0)[row * actual.stride(0)..][..width]; + assert_eq!(got, &y[row * width..][..width], "y row {row}"); + } + if actual.format() == Pixel::NV12 { + for row in 0..ch { + let got = &actual.data(1)[row * actual.stride(1)..][..cw * 2]; + for cx in 0..cw { + assert_eq!(got[cx * 2], u[row * cw + cx], "nv12 u {cx},{row}"); + assert_eq!(got[cx * 2 + 1], v[row * cw + cx], "nv12 v {cx},{row}"); + } + } + } else { + for row in 0..ch { + let got_u = &actual.data(1)[row * actual.stride(1)..][..cw]; + let got_v = &actual.data(2)[row * actual.stride(2)..][..cw]; + assert_eq!(got_u, &u[row * cw..][..cw], "u row {row}"); + assert_eq!(got_v, &v[row * cw..][..cw], "v row {row}"); + } + } + } + + struct PlaneStats { + max_diff: u8, + mean: f64, + col: usize, + row: usize, + } + + fn plane_delta( + actual: &frame::Video, + expected: &frame::Video, + plane: usize, + width: usize, + height: usize, + ) -> PlaneStats { + let mut max_diff = 0u8; + let mut sum = 0u64; + let mut count = 0u64; + let mut col = 0; + let mut row_at = 0; + for row in 0..height { + let actual_row = &actual.data(plane)[row * actual.stride(plane)..]; + let expected_row = &expected.data(plane)[row * expected.stride(plane)..]; + for x in 0..width { + let diff = actual_row[x].abs_diff(expected_row[x]); + sum += u64::from(diff); + count += 1; + if diff > max_diff { + max_diff = diff; + col = x; + row_at = row; + } + } + } + PlaneStats { + max_diff, + mean: sum as f64 / count as f64, + col, + row: row_at, + } + } + + fn assert_within_swscale(actual: &frame::Video, expected: &frame::Video) { + let width = actual.width() as usize; + let height = actual.height() as usize; + let y_stats = plane_delta(actual, expected, 0, width, height); + let y_limit = if actual.format() == Pixel::NV12 { 4 } else { 2 }; + assert!( + y_stats.max_diff <= y_limit && y_stats.mean < 1.0, + "{width}x{height} luma max {} mean {:.3} at {},{}", + y_stats.max_diff, + y_stats.mean, + y_stats.col, + y_stats.row + ); + let cw = width / 2; + let ch = height / 2; + if actual.format() == Pixel::NV12 { + let uv_stats = plane_delta(actual, expected, 1, cw * 2, ch); + assert!( + uv_stats.max_diff <= 3 && uv_stats.mean < 1.5, + "{width}x{height} nv12 max {} mean {:.3} at {},{}", + uv_stats.max_diff, + uv_stats.mean, + uv_stats.col, + uv_stats.row + ); + } else { + let u_stats = plane_delta(actual, expected, 1, cw, ch); + let v_stats = plane_delta(actual, expected, 2, cw, ch); + assert!( + u_stats.max_diff <= 3 + && v_stats.max_diff <= 3 + && u_stats.mean < 1.5 + && v_stats.mean < 1.5, + "{width}x{height} chroma u max {} mean {:.3} at {},{} v max {} mean {:.3} at {},{}", + u_stats.max_diff, + u_stats.mean, + u_stats.col, + u_stats.row, + v_stats.max_diff, + v_stats.mean, + v_stats.col, + v_stats.row + ); + } + } + + fn assert_active_eq(actual: &frame::Video, expected: &frame::Video) { + for plane in 0..expected.planes() { + let row_bytes = if expected.format() == Pixel::NV12 && plane == 1 { + expected.width() as usize + } else { + expected.plane_width(plane) as usize + }; + for row in 0..expected.plane_height(plane) as usize { + let got = &actual.data(plane)[row * actual.stride(plane)..][..row_bytes]; + let want = &expected.data(plane)[row * expected.stride(plane)..][..row_bytes]; + assert_eq!(got, want, "plane {plane} row {row}"); + } + } + } + + fn check_even(format: Pixel, width: u32, height: u32, dst_format: Pixel) { + let mut src = frame::Video::new(format, width, height); + fill_packed(&mut src); + let (y, u, v) = reference(&src); + let mut kernel = frame::Video::new(dst_format, width, height); + let mut sws = frame::Video::new(dst_format, width, height); + let mut scaler = fast_scaler(format, dst_format, width, height); + scaler.run(&src, &mut sws).unwrap(); + if avx2_ready() { + assert!( + try_convert(&src, &mut kernel), + "{format:?} {width}x{height} -> {dst_format:?}" + ); + assert_matches_reference(&kernel, &y, &u, &v); + assert_within_swscale(&kernel, &sws); + let mut via = frame::Video::new(dst_format, width, height); + let mut via_scaler = fast_scaler(format, dst_format, width, height); + convert_or_scale(&src, &mut via, &mut via_scaler).unwrap(); + assert_active_eq(&via, &kernel); + } else { + assert!(!try_convert(&src, &mut kernel)); + let mut via = frame::Video::new(dst_format, width, height); + let mut via_scaler = fast_scaler(format, dst_format, width, height); + convert_or_scale(&src, &mut via, &mut via_scaler).unwrap(); + assert_active_eq(&via, &sws); + } + } + + #[test] + fn even_uyvy_and_yuyv_match_reference_and_swscale() { + ffmpeg::init().unwrap(); + let sizes = [ + (2u32, 2u32), + (16, 2), + (18, 4), + (32, 8), + (48, 6), + (64, 48), + (66, 40), + (320, 180), + ]; + for format in [Pixel::UYVY422, Pixel::YUYV422] { + for (width, height) in sizes { + check_even(format, width, height, Pixel::YUV420P); + } + } + } + + #[test] + fn even_nv12_matches_where_the_encoder_outputs_nv12() { + ffmpeg::init().unwrap(); + for format in [Pixel::UYVY422, Pixel::YUYV422] { + for (width, height) in [(16u32, 2u32), (18, 4), (64, 48), (66, 40)] { + check_even(format, width, height, Pixel::NV12); + } + } + } + + #[test] + fn odd_sizes_fall_back_to_swscale() { + ffmpeg::init().unwrap(); + let mut compared = 0u32; + for format in [Pixel::UYVY422, Pixel::YUYV422] { + for (width, height) in [ + (2u32, 1u32), + (64, 63), + (32, 15), + (1, 2), + (65, 64), + (31, 17), + (65, 63), + ] { + let mut src = frame::Video::new(format, width, height); + fill_packed(&mut src); + let mut probe = frame::Video::new(Pixel::YUV420P, width, height); + assert!( + !try_convert(&src, &mut probe), + "{format:?} {width}x{height} should stay on swscale" + ); + let Ok(mut via_scaler) = scaling::Context::get( + format, + width, + height, + Pixel::YUV420P, + width, + height, + Flags::FAST_BILINEAR, + ) else { + continue; + }; + let Ok(mut direct_scaler) = scaling::Context::get( + format, + width, + height, + Pixel::YUV420P, + width, + height, + Flags::FAST_BILINEAR, + ) else { + continue; + }; + let mut via = frame::Video::new(Pixel::YUV420P, width, height); + let mut direct = frame::Video::new(Pixel::YUV420P, width, height); + // av_frame_get_buffer does not clear memory, and swscale can + // leave the odd-height chroma tail untouched. + zero_frame(&mut via); + zero_frame(&mut direct); + convert_or_scale(&src, &mut via, &mut via_scaler).unwrap(); + direct_scaler.run(&src, &mut direct).unwrap(); + assert_active_eq(&via, &direct); + compared += 1; + } + } + assert!(compared > 0, "swscale rejected every odd size"); + } + + #[test] + fn scaling_and_other_formats_stay_on_swscale() { + ffmpeg::init().unwrap(); + let src = frame::Video::new(Pixel::UYVY422, 64, 48); + let mut scaled = frame::Video::new(Pixel::YUV420P, 32, 24); + assert!(!try_convert(&src, &mut scaled)); + let mut yuv422 = frame::Video::new(Pixel::YUV422P, 64, 48); + assert!(!try_convert(&src, &mut yuv422)); + for format in [Pixel::BGRA, Pixel::YUV422P, Pixel::NV12, Pixel::RGB24] { + let other = frame::Video::new(format, 32, 16); + let mut yuv = frame::Video::new(Pixel::YUV420P, 32, 16); + assert!(!try_convert(&other, &mut yuv), "{format:?}"); + } + + let mut packed = frame::Video::new(Pixel::YUYV422, 64, 48); + fill_packed(&mut packed); + let mut scaler = scaling::Context::get( + Pixel::YUYV422, + 64, + 48, + Pixel::YUV420P, + 32, + 24, + Flags::BICUBIC, + ) + .unwrap(); + let mut direct_scaler = scaling::Context::get( + Pixel::YUYV422, + 64, + 48, + Pixel::YUV420P, + 32, + 24, + Flags::BICUBIC, + ) + .unwrap(); + let mut via = frame::Video::new(Pixel::YUV420P, 32, 24); + let mut direct = frame::Video::new(Pixel::YUV420P, 32, 24); + convert_or_scale(&packed, &mut via, &mut scaler).unwrap(); + direct_scaler.run(&packed, &mut direct).unwrap(); + assert_active_eq(&via, &direct); + } + + #[test] + fn libx264_queue_accepts_packed_yuv() { + ffmpeg::init().unwrap(); + queue_one(Pixel::UYVY422); + encode_one(Pixel::YUYV422); + } + + fn queue_one(format: Pixel) { + if ffmpeg::codec::encoder::find_by_name("libx264").is_none() { + return; + } + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("packed.mp4"); + let mut output = ffmpeg::format::output_as(&path, "mp4").unwrap(); + let info = cap_media_info::VideoInfo::from_raw_ffmpeg(format, 64, 48, 30); + let mut encoder = super::super::h264::H264Encoder::builder(info) + .with_preset(super::super::h264::H264Preset::Ultrafast) + .with_crf(18) + .with_encoder_priority_override(&["libx264"]) + .build(&mut output) + .unwrap(); + let requirements = encoder.conversion_requirements(); + assert_eq!(requirements.output_format, Pixel::YUV420P); + assert!(requirements.needs_conversion); + let mut frame = frame::Video::new(format, 64, 48); + fill_packed(&mut frame); + let mut sink = crate::PacketQueue::new(ffmpeg::Rational(1, 90_000)); + encoder + .queue_frame_to(frame, std::time::Duration::ZERO, &mut sink) + .unwrap(); + encoder.flush_to(&mut sink).unwrap(); + assert!(!sink.take().is_empty()); + } + + fn encode_one(format: Pixel) { + if ffmpeg::codec::encoder::find_by_name("libx264").is_none() { + return; + } + let info = cap_media_info::VideoInfo::from_raw_ffmpeg(format, 64, 48, 30); + let mut encoder = super::super::h264::H264Encoder::builder(info) + .with_preset(super::super::h264::H264Preset::Ultrafast) + .with_crf(18) + .with_encoder_priority_override(&["libx264"]) + .build_standalone() + .unwrap(); + assert_eq!(encoder.output_format(), Pixel::YUV420P); + assert!(encoder.needs_conversion()); + let mut frame = frame::Video::new(format, 64, 48); + fill_packed(&mut frame); + let mut packets = Vec::new(); + encoder + .encode_frame(frame, std::time::Duration::ZERO, |packet| { + packets.push(packet.data); + Ok(()) + }) + .unwrap(); + encoder + .flush(|packet| { + packets.push(packet.data); + Ok(()) + }) + .unwrap(); + assert!(!packets.is_empty()); + } + + #[test] + #[ignore = "release timing bench, run with --ignored --nocapture"] + fn bench_packed_yuv_conversion() { + ffmpeg::init().unwrap(); + let model = std::fs::read_to_string("/proc/cpuinfo") + .ok() + .and_then(|text| { + text.lines() + .find(|line| line.starts_with("model name")) + .and_then(|line| line.split(':').nth(1)) + .map(|value| value.trim().to_string()) + }) + .unwrap_or_else(|| "unknown".into()); + println!("cpu: {model}"); + println!("avx2: {}", avx2_ready()); + println!("warmup 5 excluded; median is the middle of 31 sorted samples"); + for format in [Pixel::UYVY422, Pixel::YUYV422] { + for (width, height) in [(1280u32, 720u32), (1920, 1080), (3840, 2160)] { + bench_conversion(format, width, height); + } + } + } + + fn bench_conversion(format: Pixel, width: u32, height: u32) { + let mut src = frame::Video::new(format, width, height); + fill_packed(&mut src); + let mut kernel_dst = frame::Video::new(Pixel::YUV420P, width, height); + let mut sws_dst = frame::Video::new(Pixel::YUV420P, width, height); + let mut kernel_scaler = fast_scaler(format, Pixel::YUV420P, width, height); + let mut sws_scaler = fast_scaler(format, Pixel::YUV420P, width, height); + let sws = time_median(5, 31, || { + sws_scaler.run(&src, &mut sws_dst).unwrap(); + let _ = std::hint::black_box(sws_dst.data(0)); + let _ = std::hint::black_box(sws_dst.data(1)); + let _ = std::hint::black_box(sws_dst.data(2)); + }); + let kernel = time_median(5, 31, || { + convert_or_scale(&src, &mut kernel_dst, &mut kernel_scaler).unwrap(); + let _ = std::hint::black_box(kernel_dst.data(0)); + let _ = std::hint::black_box(kernel_dst.data(1)); + let _ = std::hint::black_box(kernel_dst.data(2)); + }); + println!( + "{format:?} {width}x{height} swscale_fast_bilinear median {:.3} ms n=31", + sws / 1_000_000.0 + ); + println!( + "{format:?} {width}x{height} convert_or_scale median {:.3} ms n=31 speedup {:.2}x", + kernel / 1_000_000.0, + sws / kernel + ); + } + + fn time_median(warmup: usize, runs: usize, mut body: impl FnMut()) -> f64 { + for _ in 0..warmup { + body(); + } + let mut samples = Vec::with_capacity(runs); + for _ in 0..runs { + let started = std::time::Instant::now(); + body(); + samples.push(started.elapsed().as_nanos()); + } + samples.sort_unstable(); + samples[runs / 2] as f64 + } +} diff --git a/crates/frame-converter/src/frame_pool.rs b/crates/frame-converter/src/frame_pool.rs index 4bba2af3eb6..51d880fbf3b 100644 --- a/crates/frame-converter/src/frame_pool.rs +++ b/crates/frame-converter/src/frame_pool.rs @@ -1,4 +1,3 @@ -use crate::output_frame::cached_output_frame; use ffmpeg::{format::Pixel, frame}; use parking_lot::Mutex; use std::sync::Arc; @@ -15,7 +14,7 @@ impl VideoFramePool { pub fn new(capacity: usize, format: Pixel, width: u32, height: u32) -> Arc { let mut frames = Vec::with_capacity(capacity); for _ in 0..capacity { - frames.push(cached_output_frame(format, width, height)); + frames.push(frame::Video::new(format, width, height)); } Arc::new(Self { @@ -31,7 +30,7 @@ impl VideoFramePool { self.frames .lock() .pop() - .unwrap_or_else(|| cached_output_frame(self.format, self.width, self.height)) + .unwrap_or_else(|| frame::Video::new(self.format, self.width, self.height)) } pub fn put(&self, frame: frame::Video) { @@ -57,33 +56,3 @@ impl VideoFramePool { (self.width, self.height) } } - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Once; - - fn init_ffmpeg() { - static ONCE: Once = Once::new(); - ONCE.call_once(|| { - ffmpeg::init().expect("ffmpeg init"); - }); - } - - #[test] - fn pooled_frames_match_ffmpeg_geometry() { - init_ffmpeg(); - let pool = VideoFramePool::new(2, Pixel::NV12, 64, 32); - assert_eq!(pool.available(), 2); - let frame = pool.get(); - let fresh = frame::Video::new(Pixel::NV12, 64, 32); - assert_eq!(frame.format(), fresh.format()); - assert_eq!(frame.width(), fresh.width()); - assert_eq!(frame.height(), fresh.height()); - assert_eq!(frame.stride(0), fresh.stride(0)); - assert_eq!(frame.stride(1), fresh.stride(1)); - assert_eq!(pool.available(), 1); - pool.put(frame); - assert_eq!(pool.available(), 2); - } -} diff --git a/crates/frame-converter/src/lib.rs b/crates/frame-converter/src/lib.rs index 82811bcc509..cbe4c581072 100644 --- a/crates/frame-converter/src/lib.rs +++ b/crates/frame-converter/src/lib.rs @@ -1,6 +1,3 @@ -mod output_frame; -mod packed_yuv; - mod swscale; pub use swscale::*; diff --git a/crates/frame-converter/src/output_frame.rs b/crates/frame-converter/src/output_frame.rs deleted file mode 100644 index 83195a9f96b..00000000000 --- a/crates/frame-converter/src/output_frame.rs +++ /dev/null @@ -1,269 +0,0 @@ -use ffmpeg::format::Pixel; -use ffmpeg::frame; -use parking_lot::Mutex; -use std::ffi::c_void; - -// Image bytes stay in this pool after the last AVFrame reference drops, so -// `convert` can hand out the same allocation again. Callers still receive an -// exclusively owned frame::Video. -const POOLED_PER_SIZE: usize = 16; - -struct PooledBuffer { - ptr: *mut u8, - size: usize, -} - -// Exclusive ownership of `ptr` moves with the value under OUTPUT_POOL. -unsafe impl Send for PooledBuffer {} - -static OUTPUT_POOL: Mutex> = Mutex::new(Vec::new()); -static LAYOUTS: Mutex> = Mutex::new(Vec::new()); - -#[derive(Clone, Copy)] -struct FrameLayout { - format: Pixel, - width: u32, - height: u32, - linesize: [i32; 8], - offset: [isize; 8], - size: usize, -} - -pub(crate) fn cached_output_frame(format: Pixel, width: u32, height: u32) -> frame::Video { - if width == 0 || height == 0 { - return frame::Video::new(format, width, height); - } - let Some(layout) = layout_for(format, width, height) else { - return frame::Video::new(format, width, height); - }; - assemble(&layout).unwrap_or_else(|| frame::Video::new(format, width, height)) -} - -fn layout_for(format: Pixel, width: u32, height: u32) -> Option { - if let Some(layout) = find_layout(format, width, height) { - return Some(layout); - } - let probe = frame::Video::new(format, width, height); - let layout = capture(&probe)?; - drop(probe); - let mut layouts = LAYOUTS.lock(); - if let Some(existing) = layouts - .iter() - .copied() - .find(|layout| layout.format == format && layout.width == width && layout.height == height) - { - return Some(existing); - } - layouts.push(layout); - Some(layout) -} - -fn find_layout(format: Pixel, width: u32, height: u32) -> Option { - LAYOUTS - .lock() - .iter() - .copied() - .find(|layout| layout.format == format && layout.width == width && layout.height == height) -} - -fn capture(frame: &frame::Video) -> Option { - unsafe { - let raw = frame.as_ptr(); - if raw.is_null() || (*raw).buf[0].is_null() { - return None; - } - for index in 1..8 { - if !(*raw).buf[index].is_null() { - return None; - } - } - let buf = (*raw).buf[0]; - let base = (*buf).data; - let size = (*buf).size; - if base.is_null() || size == 0 { - return None; - } - let mut linesize = [0i32; 8]; - let mut offset = [-1isize; 8]; - for index in 0..8 { - linesize[index] = (*raw).linesize[index]; - let plane = (*raw).data[index]; - if plane.is_null() { - continue; - } - let off = (plane as usize).wrapping_sub(base as usize); - if off >= size { - return None; - } - offset[index] = off as isize; - } - if offset[0] < 0 { - return None; - } - Some(FrameLayout { - format: frame.format(), - width: frame.width(), - height: frame.height(), - linesize, - offset, - size, - }) - } -} - -fn assemble(layout: &FrameLayout) -> Option { - let data = take_pooled(layout.size).or_else(|| malloc_zeroed(layout.size))?; - let mut frame = frame::Video::empty(); - if unsafe { frame.as_ptr() }.is_null() { - free_data(data); - return None; - } - frame.set_format(layout.format); - frame.set_width(layout.width); - frame.set_height(layout.height); - unsafe { - let raw = frame.as_mut_ptr(); - let buf = ffmpeg::ffi::av_buffer_create( - data, - layout.size, - Some(release_output_buffer), - layout.size as *mut c_void, - 0, - ); - if buf.is_null() { - free_data(data); - return None; - } - (*raw).buf[0] = buf; - for index in 0..8 { - (*raw).linesize[index] = layout.linesize[index]; - if layout.offset[index] >= 0 { - (*raw).data[index] = data.offset(layout.offset[index]); - } - } - (*raw).extended_data = (*raw).data.as_mut_ptr(); - } - Some(frame) -} - -fn take_pooled(size: usize) -> Option<*mut u8> { - let mut pool = OUTPUT_POOL.lock(); - let index = pool.iter().rposition(|buf| buf.size == size)?; - Some(pool.swap_remove(index).ptr) -} - -fn malloc_zeroed(size: usize) -> Option<*mut u8> { - if size == 0 { - return None; - } - unsafe { - let ptr = ffmpeg::ffi::av_malloc(size) as *mut u8; - if ptr.is_null() { - None - } else { - std::ptr::write_bytes(ptr, 0, size); - Some(ptr) - } - } -} - -fn free_data(ptr: *mut u8) { - unsafe { - ffmpeg::ffi::av_free(ptr.cast()); - } -} - -unsafe extern "C" fn release_output_buffer(opaque: *mut c_void, data: *mut u8) { - if data.is_null() { - return; - } - let size = opaque as usize; - if size == 0 { - free_data(data); - return; - } - let mut pool = OUTPUT_POOL.lock(); - let same_size = pool.iter().filter(|buf| buf.size == size).count(); - if same_size < POOLED_PER_SIZE { - pool.push(PooledBuffer { ptr: data, size }); - } else { - free_data(data); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::sync::Once; - - fn init_ffmpeg() { - static ONCE: Once = Once::new(); - ONCE.call_once(|| { - ffmpeg::init().expect("ffmpeg init"); - }); - } - - fn plane_offset(frame: &frame::Video, plane: usize) -> usize { - unsafe { - let raw = frame.as_ptr(); - let base = (*(*raw).buf[0]).data as usize; - (*raw).data[plane] as usize - base - } - } - - #[test] - fn cached_frame_matches_ffmpeg_geometry() { - init_ffmpeg(); - let mut cached = cached_output_frame(Pixel::NV12, 94, 46); - let fresh = frame::Video::new(Pixel::NV12, 94, 46); - assert_eq!(cached.format(), fresh.format()); - assert_eq!(cached.width(), fresh.width()); - assert_eq!(cached.height(), fresh.height()); - assert_eq!(cached.stride(0), fresh.stride(0)); - assert_eq!(cached.stride(1), fresh.stride(1)); - assert_eq!(cached.planes(), fresh.planes()); - assert_eq!(plane_offset(&cached, 0), plane_offset(&fresh, 0)); - assert_eq!(plane_offset(&cached, 1), plane_offset(&fresh, 1)); - unsafe { - assert_eq!(ffmpeg::ffi::av_frame_is_writable(cached.as_mut_ptr()), 1); - } - } - - #[test] - fn dropped_frame_returns_its_buffer() { - init_ffmpeg(); - let first = cached_output_frame(Pixel::NV12, 95, 46); - let ptr = unsafe { (*first.as_ptr()).data[0] }; - drop(first); - let mut second = cached_output_frame(Pixel::NV12, 95, 46); - let reused = unsafe { (*second.as_ptr()).data[0] }; - assert_eq!(reused, ptr); - second.data_mut(0).fill(7); - assert!(second.data(0).iter().all(|byte| *byte == 7)); - } - - #[test] - fn live_reference_keeps_the_buffer_out_of_the_pool() { - init_ffmpeg(); - let mut held = frame::Video::empty(); - let ptr = { - let mut frame = cached_output_frame(Pixel::YUYV422, 96, 40); - frame.data_mut(0).fill(19); - unsafe { - let status = ffmpeg::ffi::av_frame_ref(held.as_mut_ptr(), frame.as_ptr()); - assert!(status >= 0); - (*frame.as_ptr()).data[0] - } - }; - let next = cached_output_frame(Pixel::YUYV422, 96, 40); - let next_ptr = unsafe { (*next.as_ptr()).data[0] }; - assert_ne!(next_ptr, ptr); - assert!(held.data(0).iter().all(|byte| *byte == 19)); - drop(held); - let recycled = cached_output_frame(Pixel::YUYV422, 96, 40); - let recycled_ptr = unsafe { (*recycled.as_ptr()).data[0] }; - assert_eq!(recycled_ptr, ptr); - assert_ne!(next_ptr, recycled_ptr); - drop(next); - } -} diff --git a/crates/frame-converter/src/packed_yuv.rs b/crates/frame-converter/src/packed_yuv.rs deleted file mode 100644 index d8fd8f61dbe..00000000000 --- a/crates/frame-converter/src/packed_yuv.rs +++ /dev/null @@ -1,665 +0,0 @@ -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(crate) enum Packed422 { - Uyvy, - Yuyv, -} - -pub(crate) struct Nv12Mut<'a> { - pub y: &'a mut [u8], - pub y_stride: usize, - pub uv: &'a mut [u8], - pub uv_stride: usize, -} - -struct PackedView<'a> { - kind: Packed422, - src: &'a [u8], - stride: usize, - width: usize, - height: usize, -} - -pub(crate) fn convert_packed422_to_nv12( - kind: Packed422, - src: &[u8], - src_stride: usize, - width: usize, - height: usize, - dst: &mut Nv12Mut<'_>, -) -> bool { - if width == 0 || height == 0 { - return true; - } - let view = PackedView { - kind, - src, - stride: src_stride, - width, - height, - }; - if !buffers_fit(&view, dst) { - return false; - } - if avx2_available() { - unsafe { - avx_convert(&view, dst); - } - } else { - scalar_rows(&view, dst); - } - true -} - -#[cfg(test)] -pub(crate) fn convert_packed422_scalar( - kind: Packed422, - src: &[u8], - src_stride: usize, - width: usize, - height: usize, - dst: &mut Nv12Mut<'_>, -) -> bool { - if width == 0 || height == 0 { - return true; - } - let view = PackedView { - kind, - src, - stride: src_stride, - width, - height, - }; - if !buffers_fit(&view, dst) { - return false; - } - scalar_rows(&view, dst); - true -} - -fn avx2_available() -> bool { - #[cfg(target_arch = "x86_64")] - { - is_x86_feature_detected!("avx2") - } - #[cfg(not(target_arch = "x86_64"))] - { - false - } -} - -fn chroma_row_bytes(width: usize) -> usize { - width.div_ceil(2) * 2 -} - -fn buffers_fit(src: &PackedView<'_>, dst: &Nv12Mut<'_>) -> bool { - let Some(packed) = src.width.checked_mul(2) else { - return false; - }; - let chroma = chroma_row_bytes(src.width); - if src.stride < packed || dst.y_stride < src.width || dst.uv_stride < chroma { - return false; - } - let Some(src_need) = src.stride.checked_mul(src.height) else { - return false; - }; - let Some(y_need) = dst.y_stride.checked_mul(src.height) else { - return false; - }; - let Some(uv_need) = dst.uv_stride.checked_mul(src.height.div_ceil(2)) else { - return false; - }; - src.src.len() >= src_need && dst.y.len() >= y_need && dst.uv.len() >= uv_need -} - -fn scalar_rows(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) { - let uv_rows = src.height.div_ceil(2); - for row in 0..uv_rows { - write_pixels(src, dst, row, 0); - } -} - -fn sample(row: &[u8], packed_len: usize, index: usize) -> u8 { - if index < packed_len { row[index] } else { 128 } -} - -fn avg_u8(a: u8, b: u8) -> u8 { - ((u16::from(a) + u16::from(b) + 1) >> 1) as u8 -} - -// (y_left, y_right, u, v). A short last macropixel reads missing bytes as 128. -fn load_macro(kind: Packed422, row: &[u8], pair: usize, packed_len: usize) -> (u8, u8, u8, u8) { - let off = pair * 4; - match kind { - Packed422::Uyvy => ( - sample(row, packed_len, off + 1), - sample(row, packed_len, off + 3), - sample(row, packed_len, off), - sample(row, packed_len, off + 2), - ), - Packed422::Yuyv => ( - sample(row, packed_len, off), - sample(row, packed_len, off + 2), - sample(row, packed_len, off + 1), - sample(row, packed_len, off + 3), - ), - } -} - -fn write_pixels(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>, row: usize, x_start: usize) { - debug_assert!(x_start.is_multiple_of(2)); - let width = src.width; - let height = src.height; - let src_stride = src.stride; - let y_stride = dst.y_stride; - let uv_stride = dst.uv_stride; - let top = row * 2; - let bot = top + 1; - let average = bot < height; - let packed_len = width * 2; - let src0 = top * src_stride; - let src1 = bot * src_stride; - let y0 = top * y_stride; - let y1 = bot * y_stride; - let uv_base = row * uv_stride; - let mut x = x_start; - while x < width { - let pair = x / 2; - let (left, right, u0, v0) = load_macro( - src.kind, - &src.src[src0..src0 + src_stride], - pair, - packed_len, - ); - dst.y[y0 + x] = left; - if x + 1 < width { - dst.y[y0 + x + 1] = right; - } - let (u, v) = if average { - let (left1, right1, u1, v1) = load_macro( - src.kind, - &src.src[src1..src1 + src_stride], - pair, - packed_len, - ); - dst.y[y1 + x] = left1; - if x + 1 < width { - dst.y[y1 + x + 1] = right1; - } - (avg_u8(u0, u1), avg_u8(v0, v1)) - } else { - (u0, v0) - }; - let uv_at = uv_base + pair * 2; - dst.uv[uv_at] = u; - dst.uv[uv_at + 1] = v; - x += 2; - } -} - -#[cfg(target_arch = "x86_64")] -struct RowPtrs { - src0: *const u8, - src1: *const u8, - y0: *mut u8, - y1: *mut u8, - uv: *mut u8, - average: bool, -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn avx_convert(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) { - let masks = match src.kind { - Packed422::Uyvy => Masks { - y: unsafe { uyvy_y_mask() }, - u: unsafe { uyvy_u_mask() }, - v: unsafe { uyvy_v_mask() }, - }, - Packed422::Yuyv => Masks { - y: unsafe { yuyv_y_mask() }, - u: unsafe { yuyv_u_mask() }, - v: unsafe { yuyv_v_mask() }, - }, - }; - let width = src.width; - let height = src.height; - let src_stride = src.stride; - let y_stride = dst.y_stride; - let uv_stride = dst.uv_stride; - let uv_rows = height.div_ceil(2); - for row in 0..uv_rows { - let top = row * 2; - let bot = top + 1; - let average = bot < height; - let src0 = src.src.as_ptr().wrapping_add(top * src_stride); - let src1 = if average { - src.src.as_ptr().wrapping_add(bot * src_stride) - } else { - std::ptr::null() - }; - let y0 = dst.y.as_mut_ptr().wrapping_add(top * y_stride); - let y1 = if average { - dst.y.as_mut_ptr().wrapping_add(bot * y_stride) - } else { - std::ptr::null_mut() - }; - let uv = dst.uv.as_mut_ptr().wrapping_add(row * uv_stride); - let ptrs = RowPtrs { - src0, - src1, - y0, - y1, - uv, - average, - }; - let mut x = 0usize; - while x + 32 <= width { - unsafe { - convert16(&masks, &ptrs, x); - convert16(&masks, &ptrs, x + 16); - } - x += 32; - } - while x + 16 <= width { - unsafe { - convert16(&masks, &ptrs, x); - } - x += 16; - } - if x < width { - write_pixels(src, dst, row, x); - } - } -} - -#[cfg(not(target_arch = "x86_64"))] -unsafe fn avx_convert(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) { - scalar_rows(src, dst); -} - -#[cfg(target_arch = "x86_64")] -struct Masks { - y: std::arch::x86_64::__m256i, - u: std::arch::x86_64::__m256i, - v: std::arch::x86_64::__m256i, -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn convert16(masks: &Masks, row: &RowPtrs, x: usize) { - use std::arch::x86_64::*; - - let packed0 = unsafe { _mm256_loadu_si256(row.src0.add(x * 2).cast()) }; - unsafe { store_luma(packed0, masks.y, row.y0.add(x)) }; - let u0 = unsafe { pack_chroma8(packed0, masks.u) }; - let v0 = unsafe { pack_chroma8(packed0, masks.v) }; - let (u, v) = if row.average { - let packed1 = unsafe { _mm256_loadu_si256(row.src1.add(x * 2).cast()) }; - unsafe { store_luma(packed1, masks.y, row.y1.add(x)) }; - let u1 = unsafe { pack_chroma8(packed1, masks.u) }; - let v1 = unsafe { pack_chroma8(packed1, masks.v) }; - // avg_epu8 is (a + b + 1) >> 1 for each byte. - (_mm_avg_epu8(u0, u1), _mm_avg_epu8(v0, v1)) - } else { - (u0, v0) - }; - let interleaved = _mm_unpacklo_epi8(u, v); - unsafe { _mm_storeu_si128(row.uv.add(x).cast(), interleaved) }; -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn store_luma( - packed: std::arch::x86_64::__m256i, - mask: std::arch::x86_64::__m256i, - dst: *mut u8, -) { - use std::arch::x86_64::*; - let shuffled = _mm256_shuffle_epi8(packed, mask); - let lo = _mm256_castsi256_si128(shuffled); - let hi = _mm256_extracti128_si256(shuffled, 1); - let y = _mm_unpacklo_epi64(lo, hi); - unsafe { _mm_storeu_si128(dst.cast(), y) }; -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn pack_chroma8( - packed: std::arch::x86_64::__m256i, - mask: std::arch::x86_64::__m256i, -) -> std::arch::x86_64::__m128i { - use std::arch::x86_64::*; - let shuffled = _mm256_shuffle_epi8(packed, mask); - let lo = _mm256_castsi256_si128(shuffled); - let hi = _mm256_extracti128_si256(shuffled, 1); - _mm_unpacklo_epi32(lo, hi) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn uyvy_y_mask() -> std::arch::x86_64::__m256i { - use std::arch::x86_64::*; - _mm256_setr_epi8( - 1, 3, 5, 7, 9, 11, 13, 15, -1, -1, -1, -1, -1, -1, -1, -1, 1, 3, 5, 7, 9, 11, 13, 15, -1, - -1, -1, -1, -1, -1, -1, -1, - ) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn uyvy_u_mask() -> std::arch::x86_64::__m256i { - use std::arch::x86_64::*; - _mm256_setr_epi8( - 0, 4, 8, 12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 4, 8, 12, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, - ) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn uyvy_v_mask() -> std::arch::x86_64::__m256i { - use std::arch::x86_64::*; - _mm256_setr_epi8( - 2, 6, 10, 14, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 2, 6, 10, 14, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, - ) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn yuyv_y_mask() -> std::arch::x86_64::__m256i { - use std::arch::x86_64::*; - _mm256_setr_epi8( - 0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1, 0, 2, 4, 6, 8, 10, 12, 14, -1, - -1, -1, -1, -1, -1, -1, -1, - ) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn yuyv_u_mask() -> std::arch::x86_64::__m256i { - use std::arch::x86_64::*; - _mm256_setr_epi8( - 1, 5, 9, 13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 1, 5, 9, 13, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, - ) -} - -#[cfg(target_arch = "x86_64")] -#[target_feature(enable = "avx2")] -unsafe fn yuyv_v_mask() -> std::arch::x86_64::__m256i { - use std::arch::x86_64::*; - _mm256_setr_epi8( - 3, 7, 11, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 3, 7, 11, 15, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn fill(buf: &mut [u8], mut seed: u32) { - for byte in buf.iter_mut() { - seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); - *byte = (seed >> 24) as u8; - } - } - - fn ref_byte(row: &[u8], packed_len: usize, index: usize) -> u8 { - if index < packed_len { row[index] } else { 128 } - } - - fn ref_sample(kind: Packed422, row: &[u8], packed_len: usize, x: usize) -> (u8, u8, u8) { - let off = (x / 2) * 4; - let (y, u, v) = match kind { - Packed422::Uyvy => { - let y = if x.is_multiple_of(2) { - ref_byte(row, packed_len, off + 1) - } else { - ref_byte(row, packed_len, off + 3) - }; - ( - y, - ref_byte(row, packed_len, off), - ref_byte(row, packed_len, off + 2), - ) - } - Packed422::Yuyv => { - let y = if x.is_multiple_of(2) { - ref_byte(row, packed_len, off) - } else { - ref_byte(row, packed_len, off + 2) - }; - ( - y, - ref_byte(row, packed_len, off + 1), - ref_byte(row, packed_len, off + 3), - ) - } - }; - (y, u, v) - } - - fn reference( - src: &PackedView<'_>, - y: &mut [u8], - y_stride: usize, - uv: &mut [u8], - uv_stride: usize, - ) { - let packed_len = src.width * 2; - for row in 0..src.height { - let src_row = &src.src[row * src.stride..row * src.stride + src.stride]; - for x in 0..src.width { - let (sample_y, _, _) = ref_sample(src.kind, src_row, packed_len, x); - y[row * y_stride + x] = sample_y; - } - } - let chroma_pairs = src.width.div_ceil(2); - for row in 0..src.height.div_ceil(2) { - let top = row * 2; - let src_top = &src.src[top * src.stride..top * src.stride + src.stride]; - let bot = top + 1; - let src_bot = (bot < src.height) - .then(|| &src.src[bot * src.stride..bot * src.stride + src.stride]); - for pair in 0..chroma_pairs { - let x = pair * 2; - let (_, u0, v0) = ref_sample(src.kind, src_top, packed_len, x); - let (u, v) = if let Some(src_bot) = src_bot { - let (_, u1, v1) = ref_sample(src.kind, src_bot, packed_len, x); - ( - ((u16::from(u0) + u16::from(u1) + 1) >> 1) as u8, - ((u16::from(v0) + u16::from(v1) + 1) >> 1) as u8, - ) - } else { - (u0, v0) - }; - uv[row * uv_stride + pair * 2] = u; - uv[row * uv_stride + pair * 2 + 1] = v; - } - } - } - - fn run_scalar( - kind: Packed422, - width: usize, - height: usize, - src_pad: usize, - y_pad: usize, - uv_pad: usize, - seed: u32, - ) -> (Vec, Vec, Vec, Vec) { - let src_stride = width * 2 + src_pad; - let y_stride = width + y_pad; - let uv_stride = chroma_row_bytes(width) + uv_pad; - let mut src = vec![0u8; src_stride * height]; - fill(&mut src, seed); - let mut y = vec![0x5Au8; y_stride * height]; - let mut uv = vec![0x5Au8; uv_stride * height.div_ceil(2)]; - let mut y_ref = y.clone(); - let mut uv_ref = uv.clone(); - let mut dst = Nv12Mut { - y: &mut y, - y_stride, - uv: &mut uv, - uv_stride, - }; - let view = PackedView { - kind, - src: &src, - stride: src_stride, - width, - height, - }; - assert!(convert_rows_scalar(&view, &mut dst)); - reference(&view, &mut y_ref, y_stride, &mut uv_ref, uv_stride); - (y, uv, y_ref, uv_ref) - } - - fn convert_rows_scalar(src: &PackedView<'_>, dst: &mut Nv12Mut<'_>) -> bool { - if !buffers_fit(src, dst) { - return false; - } - scalar_rows(src, dst); - true - } - - fn case( - kind: Packed422, - width: usize, - height: usize, - src_pad: usize, - y_pad: usize, - uv_pad: usize, - ) { - let seed = (width as u32) - .wrapping_mul(1000) - .wrapping_add(height as u32 * 17) - .wrapping_add(src_pad as u32 * 3) - .wrapping_add(y_pad as u32 * 5) - .wrapping_add(uv_pad as u32 * 9) - .wrapping_add(match kind { - Packed422::Uyvy => 1, - Packed422::Yuyv => 2, - }); - let (y, uv, y_ref, uv_ref) = run_scalar(kind, width, height, src_pad, y_pad, uv_pad, seed); - assert_eq!( - y, y_ref, - "{kind:?} {width}x{height} y pad {src_pad}/{y_pad}/{uv_pad}" - ); - assert_eq!( - uv, uv_ref, - "{kind:?} {width}x{height} uv pad {src_pad}/{y_pad}/{uv_pad}" - ); - } - - #[test] - fn scalar_matches_reference_across_shapes() { - let widths = [ - 1usize, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128, - ]; - let heights = [1usize, 2, 3, 4, 5, 27]; - let src_pads = [0usize, 7, 32]; - let y_pads = [0usize, 13]; - let uv_pads = [0usize, 15]; - for kind in [Packed422::Uyvy, Packed422::Yuyv] { - for &width in &widths { - for &height in &heights { - for &src_pad in &src_pads { - for &y_pad in &y_pads { - for &uv_pad in &uv_pads { - case(kind, width, height, src_pad, y_pad, uv_pad); - } - } - } - } - } - case(kind, 1920, 2, 16, 8, 24); - case(kind, 1919, 3, 1, 0, 5); - case(kind, 1280, 4, 0, 0, 0); - } - } - - #[cfg(target_arch = "x86_64")] - #[test] - fn avx2_matches_scalar_across_shapes() { - if !is_x86_feature_detected!("avx2") { - return; - } - let widths = [ - 1usize, 2, 3, 7, 8, 15, 16, 17, 31, 32, 33, 47, 63, 64, 65, 100, 127, 128, - ]; - let heights = [1usize, 2, 3, 5, 27]; - let src_pads = [0usize, 7, 32]; - let y_pads = [0usize, 13]; - let uv_pads = [0usize, 15]; - for kind in [Packed422::Uyvy, Packed422::Yuyv] { - for &width in &widths { - for &height in &heights { - for &src_pad in &src_pads { - for &y_pad in &y_pads { - for &uv_pad in &uv_pads { - compare_avx(kind, width, height, src_pad, y_pad, uv_pad); - } - } - } - } - } - compare_avx(kind, 1920, 2, 16, 8, 24); - compare_avx(kind, 1919, 3, 3, 1, 9); - compare_avx(kind, 3840, 2, 0, 0, 0); - } - } - - #[cfg(target_arch = "x86_64")] - fn compare_avx( - kind: Packed422, - width: usize, - height: usize, - src_pad: usize, - y_pad: usize, - uv_pad: usize, - ) { - let src_stride = width * 2 + src_pad; - let y_stride = width + y_pad; - let uv_stride = chroma_row_bytes(width) + uv_pad; - let mut src = vec![0u8; src_stride * height]; - fill(&mut src, width as u32 * 13 + height as u32); - let mut y_avx = vec![0x5Au8; y_stride * height]; - let mut uv_avx = vec![0x5Au8; uv_stride * height.div_ceil(2)]; - let mut y_scalar = y_avx.clone(); - let mut uv_scalar = uv_avx.clone(); - { - let mut dst = Nv12Mut { - y: &mut y_avx, - y_stride, - uv: &mut uv_avx, - uv_stride, - }; - assert!(convert_packed422_to_nv12( - kind, &src, src_stride, width, height, &mut dst - )); - } - { - let mut dst = Nv12Mut { - y: &mut y_scalar, - y_stride, - uv: &mut uv_scalar, - uv_stride, - }; - assert!(convert_rows_scalar( - &PackedView { - kind, - src: &src, - stride: src_stride, - width, - height, - }, - &mut dst - )); - } - assert_eq!(y_avx, y_scalar, "{kind:?} {width}x{height} avx y"); - assert_eq!(uv_avx, uv_scalar, "{kind:?} {width}x{height} avx uv"); - } -} diff --git a/crates/frame-converter/src/swscale.rs b/crates/frame-converter/src/swscale.rs index 18ee4bd80e2..eafff292494 100644 --- a/crates/frame-converter/src/swscale.rs +++ b/crates/frame-converter/src/swscale.rs @@ -1,5 +1,3 @@ -use crate::output_frame::cached_output_frame; -use crate::packed_yuv::{self, Nv12Mut, Packed422}; use crate::{ConversionConfig, ConvertError, ConverterBackend, FrameConverter}; use ffmpeg::{format::Pixel, frame, software::scaling}; use std::cell::UnsafeCell; @@ -63,124 +61,12 @@ impl SwscaleConverter { self.contexts .get_or_try(|| Self::create_context(&self.config).map(SendableContext::new)) } - - #[cfg(test)] - pub(crate) fn convert_swscale_into( - &self, - input: &frame::Video, - output: &mut frame::Video, - ) -> Result<(), ConvertError> { - let context = self.get_or_create_context()?; - context - .run(input, output) - .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?; - Ok(()) - } -} - -fn positive_linesize(frame: &frame::Video, plane: usize) -> Option { - if plane >= frame.planes() { - return None; - } - let linesize = unsafe { (*frame.as_ptr()).linesize[plane] }; - if linesize <= 0 { - None - } else { - Some(linesize as usize) - } -} - -fn planes_disjoint(a: *const u8, a_len: usize, b: *const u8, b_len: usize) -> bool { - if a_len == 0 || b_len == 0 { - return true; - } - let Some(a_end) = (a as usize).checked_add(a_len) else { - return false; - }; - let Some(b_end) = (b as usize).checked_add(b_len) else { - return false; - }; - a_end <= b as usize || b_end <= a as usize -} - -pub(crate) fn try_packed_yuv_nv12(input: &frame::Video, output: &mut frame::Video) -> bool { - let kind = match input.format() { - Pixel::UYVY422 => Packed422::Uyvy, - Pixel::YUYV422 => Packed422::Yuyv, - _ => return false, - }; - if output.format() != Pixel::NV12 { - return false; - } - let width_px = input.width(); - let height_px = input.height(); - if output.width() != width_px || output.height() != height_px { - return false; - } - if width_px == 0 || height_px == 0 { - return true; - } - if input.planes() < 1 || output.planes() < 2 { - return false; - } - let Some(src_stride) = positive_linesize(input, 0) else { - return false; - }; - let Some(y_stride) = positive_linesize(output, 0) else { - return false; - }; - let Some(uv_stride) = positive_linesize(output, 1) else { - return false; - }; - let width = width_px as usize; - let height = height_px as usize; - let src_h = input.plane_height(0) as usize; - let y_h = output.plane_height(0) as usize; - let uv_h = output.plane_height(1) as usize; - if src_h < height || y_h < height || uv_h < height.div_ceil(2) { - return false; - } - let Some(src_len) = src_stride.checked_mul(src_h) else { - return false; - }; - let Some(y_len) = y_stride.checked_mul(y_h) else { - return false; - }; - let Some(uv_len) = uv_stride.checked_mul(uv_h) else { - return false; - }; - unsafe { - let in_raw = input.as_ptr(); - let out_raw = output.as_mut_ptr(); - if in_raw.is_null() || out_raw.is_null() { - return false; - } - let src_ptr = (*in_raw).data[0]; - let y_ptr = (*out_raw).data[0]; - let uv_ptr = (*out_raw).data[1]; - if src_ptr.is_null() || y_ptr.is_null() || uv_ptr.is_null() { - return false; - } - if !planes_disjoint(y_ptr, y_len, uv_ptr, uv_len) { - return false; - } - let src = std::slice::from_raw_parts(src_ptr, src_len); - let y = std::slice::from_raw_parts_mut(y_ptr, y_len); - let uv = std::slice::from_raw_parts_mut(uv_ptr, uv_len); - let mut dst = Nv12Mut { - y, - y_stride, - uv, - uv_stride, - }; - packed_yuv::convert_packed422_to_nv12(kind, src, src_stride, width, height, &mut dst) - } } impl FrameConverter for SwscaleConverter { fn convert(&self, input: frame::Video) -> Result { let mut output = - cached_output_frame(self.output_format, self.output_width, self.output_height); + frame::Video::new(self.output_format, self.output_width, self.output_height); self.convert_into(input, &mut output)?; Ok(output) } @@ -191,12 +77,12 @@ impl FrameConverter for SwscaleConverter { output: &mut frame::Video, ) -> Result<(), ConvertError> { let pts = input.pts(); - if !try_packed_yuv_nv12(&input, output) { - let context = self.get_or_create_context()?; - context - .run(&input, output) - .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?; - } + + let context = self.get_or_create_context()?; + context + .run(&input, output) + .map_err(|e| ConvertError::ConversionFailed(e.to_string()))?; + output.set_pts(pts); Ok(()) } @@ -212,369 +98,3 @@ impl FrameConverter for SwscaleConverter { unsafe impl Send for SwscaleConverter {} unsafe impl Sync for SwscaleConverter {} - -#[cfg(test)] -mod tests { - use super::*; - use crate::packed_yuv::{Packed422, convert_packed422_scalar}; - use std::sync::Once; - use std::time::{Duration, Instant}; - - fn init_ffmpeg() { - static ONCE: Once = Once::new(); - ONCE.call_once(|| { - ffmpeg::init().expect("ffmpeg init"); - }); - } - - fn fill(buf: &mut [u8], mut seed: u32) { - for byte in buf.iter_mut() { - seed = seed.wrapping_mul(1664525).wrapping_add(1013904223); - *byte = (seed >> 24) as u8; - } - } - - fn fill_frame(frame: &mut frame::Video, seed: u32) { - for plane in 0..frame.planes() { - fill(frame.data_mut(plane), seed.wrapping_add(plane as u32 * 97)); - } - } - - fn active_rows(plane: &[u8], stride: usize, rows: usize, active: usize) -> Vec { - let mut out = Vec::with_capacity(rows * active); - for row in 0..rows { - let start = row * stride; - out.extend_from_slice(&plane[start..start + active]); - } - out - } - - fn kind_of(format: Pixel) -> Packed422 { - match format { - Pixel::UYVY422 => Packed422::Uyvy, - Pixel::YUYV422 => Packed422::Yuyv, - _ => panic!("packed 4:2:2 format"), - } - } - - #[test] - fn convert_into_matches_scalar_kernel() { - init_ffmpeg(); - let shapes = [ - (16u32, 8u32), - (17, 3), - (15, 2), - (31, 5), - (32, 1), - (33, 4), - (64, 27), - (47, 9), - ]; - for format in [Pixel::UYVY422, Pixel::YUYV422] { - for (width, height) in shapes { - let config = - ConversionConfig::new(format, width, height, Pixel::NV12, width, height); - let converter = SwscaleConverter::new(config).expect("converter"); - let mut input = frame::Video::new(format, width, height); - fill_frame(&mut input, width.wrapping_mul(13).wrapping_add(height)); - input.set_pts(Some(i64::from(width) * 1000 + i64::from(height))); - let mut output = frame::Video::new(Pixel::NV12, width, height); - converter - .convert_into(input.clone(), &mut output) - .expect("convert_into"); - assert_eq!( - output.pts(), - Some(i64::from(width) * 1000 + i64::from(height)) - ); - assert_eq!(output.format(), Pixel::NV12); - - let src_stride = input.stride(0); - let y_stride = output.stride(0); - let uv_stride = output.stride(1); - let mut y = vec![0u8; y_stride * output.plane_height(0) as usize]; - let mut uv = vec![0u8; uv_stride * output.plane_height(1) as usize]; - { - let mut dst = crate::packed_yuv::Nv12Mut { - y: &mut y, - y_stride, - uv: &mut uv, - uv_stride, - }; - assert!(convert_packed422_scalar( - kind_of(format), - input.data(0), - src_stride, - width as usize, - height as usize, - &mut dst, - )); - } - assert_eq!( - active_rows(output.data(0), y_stride, height as usize, width as usize), - active_rows(&y, y_stride, height as usize, width as usize), - "{format:?} {width}x{height} y" - ); - let chroma_bytes = (width as usize).div_ceil(2) * 2; - assert_eq!( - active_rows( - output.data(1), - uv_stride, - height.div_ceil(2) as usize, - chroma_bytes - ), - active_rows(&uv, uv_stride, height.div_ceil(2) as usize, chroma_bytes), - "{format:?} {width}x{height} uv" - ); - } - } - } - - // FAST_BILINEAR is not an identity luma copy and is not (a + b + 1) >> 1. - // On even frames through 1920x1080 the max absolute difference was 20 luma - // and 19 chroma. Odd heights use a different swscale vertical filter - // (about 120 levels on 64x33) and stay in the exact kernel tests. - const LUMA_TOLERANCE: i32 = 24; - const CHROMA_TOLERANCE: i32 = 24; - - fn max_plane_diff(left: &[u8], right: &[u8], stride: usize, rows: usize, active: usize) -> i32 { - let mut max_diff = 0i32; - for row in 0..rows { - let start = row * stride; - for offset in 0..active { - let delta = i32::from(left[start + offset]) - i32::from(right[start + offset]); - max_diff = max_diff.max(delta.abs()); - } - } - max_diff - } - - #[test] - fn packed_path_stays_close_to_swscale() { - init_ffmpeg(); - let shapes = [ - (64u32, 48u32), - (320, 180), - (640, 360), - (1280, 720), - (1920, 1080), - ]; - for format in [Pixel::UYVY422, Pixel::YUYV422] { - for (width, height) in shapes { - let config = - ConversionConfig::new(format, width, height, Pixel::NV12, width, height); - let converter = SwscaleConverter::new(config).expect("converter"); - let mut input = frame::Video::new(format, width, height); - fill_frame( - &mut input, - 0x51C0_u32.wrapping_add(width).wrapping_add(height), - ); - let mut fast = frame::Video::new(Pixel::NV12, width, height); - let mut sws = frame::Video::new(Pixel::NV12, width, height); - assert!(try_packed_yuv_nv12(&input, &mut fast)); - converter - .convert_swscale_into(&input, &mut sws) - .expect("swscale"); - let luma = max_plane_diff( - fast.data(0), - sws.data(0), - fast.stride(0), - height as usize, - width as usize, - ); - let chroma_bytes = (width as usize / 2) * 2; - let chroma = max_plane_diff( - fast.data(1), - sws.data(1), - fast.stride(1), - height.div_ceil(2) as usize, - chroma_bytes, - ); - assert!( - luma <= LUMA_TOLERANCE && chroma <= CHROMA_TOLERANCE, - "{format:?} {width}x{height} luma {luma} chroma {chroma}" - ); - } - } - } - - #[test] - fn other_formats_and_scaling_still_use_swscale() { - init_ffmpeg(); - let scaled = SwscaleConverter::new(ConversionConfig::new( - Pixel::UYVY422, - 32, - 16, - Pixel::NV12, - 16, - 8, - )) - .expect("scaled"); - let mut input = frame::Video::new(Pixel::UYVY422, 32, 16); - fill_frame(&mut input, 9); - let output = scaled.convert(input).expect("scale"); - assert_eq!( - (output.width(), output.height(), output.format()), - (16, 8, Pixel::NV12) - ); - - let bgra = SwscaleConverter::new(ConversionConfig::new( - Pixel::BGRA, - 16, - 16, - Pixel::NV12, - 16, - 16, - )) - .expect("bgra"); - let mut input = frame::Video::new(Pixel::BGRA, 16, 16); - fill_frame(&mut input, 4); - let output = bgra.convert(input).expect("bgra convert"); - assert_eq!(output.format(), Pixel::NV12); - assert_eq!((output.width(), output.height()), (16, 16)); - } - - #[test] - fn convert_keeps_live_frames_independent() { - init_ffmpeg(); - let converter = SwscaleConverter::new(ConversionConfig::new( - Pixel::UYVY422, - 82, - 18, - Pixel::NV12, - 82, - 18, - )) - .expect("converter"); - let mut first_src = frame::Video::new(Pixel::UYVY422, 82, 18); - first_src.data_mut(0).fill(30); - first_src.set_pts(Some(42)); - let mut first = converter.convert(first_src).expect("first"); - assert_eq!(first.pts(), Some(42)); - let first_y = first.data(0).to_vec(); - let first_uv = first.data(1).to_vec(); - unsafe { - assert_eq!(ffmpeg::ffi::av_frame_is_writable(first.as_mut_ptr()), 1); - } - let mut second_src = frame::Video::new(Pixel::UYVY422, 82, 18); - second_src.data_mut(0).fill(90); - let mut second = converter.convert(second_src).expect("second"); - assert_eq!(first.data(0), first_y.as_slice()); - assert_eq!(first.data(1), first_uv.as_slice()); - assert!(second.data(0).contains(&90)); - assert!(!second.data(0).iter().take(82).all(|byte| *byte == 30)); - unsafe { - assert_eq!(ffmpeg::ffi::av_frame_is_writable(first.as_mut_ptr()), 1); - assert_eq!(ffmpeg::ffi::av_frame_is_writable(second.as_mut_ptr()), 1); - } - drop(first); - drop(second); - let mut third_src = frame::Video::new(Pixel::UYVY422, 82, 18); - third_src.data_mut(0).fill(11); - let third = converter.convert(third_src).expect("third"); - assert!(third.data(0).iter().take(82).all(|byte| *byte == 11)); - } - - #[inline(never)] - fn time_swscale(converter: &SwscaleConverter, input: &frame::Video, output: &mut frame::Video) { - converter - .convert_swscale_into(input, output) - .expect("swscale"); - let index = output.stride(0); - std::hint::black_box(output.data(0)[index]); - } - - #[inline(never)] - fn time_packed(input: &frame::Video, output: &mut frame::Video) { - assert!(try_packed_yuv_nv12(input, output)); - let index = output.stride(0); - std::hint::black_box(output.data(0)[index]); - } - - fn median(samples: &mut [Duration]) -> Duration { - samples.sort_unstable(); - samples[samples.len() / 2] - } - - fn cpu_model() -> String { - let Ok(text) = std::fs::read_to_string("/proc/cpuinfo") else { - return "unknown".to_string(); - }; - for line in text.lines() { - let Some(rest) = line.strip_prefix("model name") else { - continue; - }; - return rest.trim_start_matches([' ', ':', '\t']).to_string(); - } - "unknown".to_string() - } - - #[test] - #[ignore] - fn bench_packed_yuv_nv12() { - init_ffmpeg(); - if cfg!(debug_assertions) { - eprintln!("debug build; rerun with --release for representative medians"); - } - #[cfg(target_arch = "x86_64")] - let avx2 = is_x86_feature_detected!("avx2"); - #[cfg(not(target_arch = "x86_64"))] - let avx2 = false; - let warmup = 16; - let runs = 51; - println!("cpu: {}", cpu_model()); - println!("avx2: {avx2}"); - println!("warmup: {warmup}"); - println!("timed runs: {runs}"); - println!( - "before: swscale FAST_BILINEAR, after: packed kernel, frames allocated outside the timer" - ); - let shapes = [(1280u32, 720u32), (1920, 1080), (3840, 2160)]; - for format in [Pixel::UYVY422, Pixel::YUYV422] { - for (width, height) in shapes { - let config = - ConversionConfig::new(format, width, height, Pixel::NV12, width, height); - let converter = SwscaleConverter::new(config).expect("converter"); - let mut input = frame::Video::new(format, width, height); - fill_frame(&mut input, width.wrapping_mul(height)); - let mut sws_out = frame::Video::new(Pixel::NV12, width, height); - let mut fast_out = frame::Video::new(Pixel::NV12, width, height); - for _ in 0..warmup { - time_swscale(&converter, &input, &mut sws_out); - } - let mut sws_samples = Vec::with_capacity(runs); - for _ in 0..runs { - let start = Instant::now(); - time_swscale(&converter, &input, &mut sws_out); - sws_samples.push(start.elapsed()); - } - for _ in 0..warmup { - time_packed(&input, &mut fast_out); - } - let mut fast_samples = Vec::with_capacity(runs); - for _ in 0..runs { - let start = Instant::now(); - time_packed(&input, &mut fast_out); - fast_samples.push(start.elapsed()); - } - let sws_min = sws_samples.iter().copied().min().unwrap_or_default(); - let fast_min = fast_samples.iter().copied().min().unwrap_or_default(); - let sws_max = sws_samples.iter().copied().max().unwrap_or_default(); - let fast_max = fast_samples.iter().copied().max().unwrap_or_default(); - let sws_med = median(&mut sws_samples); - let fast_med = median(&mut fast_samples); - let speedup = sws_med.as_secs_f64() / fast_med.as_secs_f64(); - println!( - "{width}x{height} {format:?} before {:.3} ms after {:.3} ms speedup {speedup:.2}x n {runs} before_min {:.3} before_max {:.3} after_min {:.3} after_max {:.3}", - sws_med.as_secs_f64() * 1000.0, - fast_med.as_secs_f64() * 1000.0, - sws_min.as_secs_f64() * 1000.0, - sws_max.as_secs_f64() * 1000.0, - fast_min.as_secs_f64() * 1000.0, - fast_max.as_secs_f64() * 1000.0, - ); - let _ = std::hint::black_box(sws_out.data(0)[0]); - let _ = std::hint::black_box(fast_out.data(0)[0]); - } - } - } -} From 333451c47db6f5cf95afdfc327690d4f7c137973 Mon Sep 17 00:00:00 2001 From: Kyzer Phneh Date: Fri, 23 Oct 2026 14:30:51 +0000 Subject: [PATCH 3/3] Gate x86 only helpers and fix formatting --- crates/enc-ffmpeg/src/video/mod.rs | 2 +- crates/enc-ffmpeg/src/video/packed_yuv.rs | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/crates/enc-ffmpeg/src/video/mod.rs b/crates/enc-ffmpeg/src/video/mod.rs index 839c61e68c5..040480d0f28 100644 --- a/crates/enc-ffmpeg/src/video/mod.rs +++ b/crates/enc-ffmpeg/src/video/mod.rs @@ -1,8 +1,8 @@ pub mod colour; -mod packed_yuv; pub mod h264; pub mod h264_packet; pub mod hevc; +mod packed_yuv; pub mod prores; #[cfg(target_os = "macos")] pub mod videotoolbox_hw; diff --git a/crates/enc-ffmpeg/src/video/packed_yuv.rs b/crates/enc-ffmpeg/src/video/packed_yuv.rs index 71ab404255e..2bc80f474a9 100644 --- a/crates/enc-ffmpeg/src/video/packed_yuv.rs +++ b/crates/enc-ffmpeg/src/video/packed_yuv.rs @@ -7,6 +7,7 @@ enum Packed422 { Yuyv, } +#[cfg(target_arch = "x86_64")] struct Job { src: *const u8, src_stride: usize, @@ -24,6 +25,7 @@ struct Job { nv12: bool, } +#[cfg(target_arch = "x86_64")] fn positive_linesize(frame: &frame::Video, plane: usize) -> Option { if plane >= frame.planes() { return None; @@ -32,6 +34,7 @@ fn positive_linesize(frame: &frame::Video, plane: usize) -> Option { if raw <= 0 { None } else { Some(raw as usize) } } +#[cfg(target_arch = "x86_64")] fn covers(stride: usize, row_bytes: usize, height: usize, len: usize) -> bool { if row_bytes == 0 || height == 0 { return true; @@ -73,7 +76,7 @@ pub(crate) fn try_convert(src: &frame::Video, dst: &mut frame::Video) -> bool { #[cfg(not(target_arch = "x86_64"))] { let _ = (kind, nv12); - return false; + false } #[cfg(target_arch = "x86_64")] {