perf(gpui): stream the editor preview into one texture (default on Linux)

apps/desktop-gpui/src/app_windows.rs

@@ -5261,30 +5261,43 @@ fn load_editor_project_candidate(
             return;
         }
 
-        // The pump: convert on the background executor (un-padding plus the
-        // BGRA swap is a few megabytes per frame), deliver on the main one.
+        // The pump: RGBA frames go into the preview's streaming image as-is on
+        // the main thread. With streaming off they are converted on the
+        // background executor instead (un-padding plus the BGRA swap is a few
+        // megabytes per frame) and delivered on the main one.
         cx.spawn({
             let stats = stats.clone();
             let pump_instance = Arc::downgrade(&instance);
+            let stream = editor_window::streaming_preview_enabled()
+                .then(|| gpui::StreamingImage::new(gpui::StreamingImageFormat::Rgba8));
             async move |cx| {
                 while let Ok((output, layout)) = frame_rx.recv_async().await {
+                    let mut streamed = None;
                     let (frame, number) = match output {
                         cap_editor::EditorFrameOutput::Rgba(rgba) => {
-                            let stats = stats.clone();
                             let number = rgba.frame_number;
-                            let image = cx
-                                .background_executor()
-                                .spawn(
-                                    async move { editor_window::frame_image_timed(&rgba, &stats) },
+                            if let Some(stream) = &stream {
+                                streamed = Some(rgba);
+                                (
+                                    editor_window::EditorPreviewFrame::Stream(stream.clone()),
+                                    number,
                                 )
-                                .await;
-                            let Some(image) = image else {
-                                tracing::warn!(
-                                    "a rendered frame could not be converted for display"
-                                );
-                                continue;
-                            };
-                            (editor_window::EditorPreviewFrame::Image(image), number)
+                            } else {
+                                let stats = stats.clone();
+                                let image = cx
+                                    .background_executor()
+                                    .spawn(async move {
+                                        editor_window::frame_image_timed(&rgba, &stats)
+                                    })
+                                    .await;
+                                let Some(image) = image else {
+                                    tracing::warn!(
+                                        "a rendered frame could not be converted for display"
+                                    );
+                                    continue;
+                                };
+                                (editor_window::EditorPreviewFrame::Image(image), number)
+                            }
                         }
                         cap_editor::EditorFrameOutput::Nv12(frame) => {
                             tracing::warn!(
@@ -5304,6 +5317,11 @@ fn load_editor_project_candidate(
                             if !view.holds_instance(&pump_instance) {
                                 return;
                             }
+                            if let (Some(stream), Some(rgba)) = (&stream, streamed)
+                                && !editor_window::stream_frame_timed(stream, rgba, &stats)
+                            {
+                                return;
+                            }
                             view.frame_arrived(
                                 editor_window::EditorFrame {
                                     frame,

apps/desktop-gpui/src/editor_window.rs

@@ -62,8 +62,8 @@ use gpui::{
     Animation, AnimationExt as _, AppContext as _, Bounds, Context, Entity, FocusHandle,
     FontWeight, Hsla, InteractiveElement, IntoElement, MouseButton, MouseDownEvent, MouseMoveEvent,
     MouseUpEvent, ParentElement, Pixels, Point, Render, RenderImage, SharedString,
-    StatefulInteractiveElement as _, StyleRefinement, Styled, Subscription, WeakEntity, Window,
-    div, point, prelude::FluentBuilder, px, svg,
+    StatefulInteractiveElement as _, StreamingImage, StyleRefinement, Styled, Subscription,
+    WeakEntity, Window, div, point, prelude::FluentBuilder, px, svg,
 };
 
 use crate::{
@@ -582,6 +582,9 @@ pub fn frame_image(frame: &RenderedFrame) -> Option<Arc<RenderImage>> {
 #[derive(Clone)]
 pub(crate) enum EditorPreviewFrame {
     Image(Arc<RenderImage>),
+    /// The pump's one streaming image, which every RGBA frame replaces in place
+    /// so each window keeps a single texture for it.
+    Stream(StreamingImage),
     #[cfg(target_os = "macos")]
     Surface(CVPixelBuffer),
 }
@@ -593,6 +596,9 @@ impl EditorPreviewFrame {
             Self::Image(image) => {
                 let _ = window.paint_image(bounds, corners, Arc::clone(image), 0, false);
             }
+            Self::Stream(image) => {
+                let _ = window.paint_streaming_image(bounds, corners, image, false);
+            }
             #[cfg(target_os = "macos")]
             Self::Surface(surface) => {
                 window.paint_surface_fitted(bounds, bounds, corners, surface.clone())
@@ -11871,6 +11877,53 @@ pub(crate) fn surface_preview_frame(frame: cap_rendering::SurfaceFrame) -> Edito
     EditorPreviewFrame::Surface(pixel_buffer)
 }
 
+/// Whether the pump streams RGBA frames into one [`StreamingImage`] instead of
+/// converting each into a [`RenderImage`]. On by default on Linux, where the
+/// in-app A/B measured it; `CAP_GPUI_STREAMING_IMAGE=0` turns it off and `=1`
+/// turns it on for Windows and for macOS's RGBA fallback.
+pub(crate) fn streaming_preview_enabled() -> bool {
+    static ENABLED: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
+    *ENABLED.get_or_init(|| {
+        streaming_preview_setting(std::env::var_os("CAP_GPUI_STREAMING_IMAGE").as_deref())
+    })
+}
+
+fn streaming_preview_setting(value: Option<&std::ffi::OsStr>) -> bool {
+    match value {
+        Some(value) => value != "0",
+        None => cfg!(target_os = "linux"),
+    }
+}
+
+/// Makes `frame` the stream's current frame exactly as the renderer produced
+/// it: the stream uploads RGBA rows at the padded stride, so nothing is
+/// un-padded or swapped. Takes the renderer's buffer without copying when the
+/// frame is its only owner. Timed into the same counters as [`frame_image`].
+pub fn stream_frame_timed(
+    stream: &StreamingImage,
+    frame: RenderedFrame,
+    stats: &PumpStats,
+) -> bool {
+    let started = Instant::now();
+    let size = gpui::size(
+        gpui::DevicePixels(frame.width as i32),
+        gpui::DevicePixels(frame.height as i32),
+    );
+    let stride = frame.padded_bytes_per_row as usize;
+    let result = match Arc::try_unwrap(frame.data) {
+        Ok(data) => stream.set_frame(size, stride, data).map(drop),
+        Err(data) => stream.copy_frame(size, stride, &data),
+    };
+    stats
+        .convert_nanos
+        .fetch_add(started.elapsed().as_nanos() as u64, Ordering::Relaxed);
+    stats.convert_samples.fetch_add(1, Ordering::Relaxed);
+    if let Err(error) = &result {
+        tracing::warn!(%error, "a rendered frame could not be streamed to the preview");
+    }
+    result.is_ok()
+}
+
 /// [`frame_image`], timed. The pump calls this so the CPU conversion's cost is
 /// a number rather than a hunch.
 pub fn frame_image_timed(frame: &RenderedFrame, stats: &PumpStats) -> Option<Arc<RenderImage>> {
@@ -12817,4 +12870,63 @@ mod tests {
         assert_eq!(&bytes[0..4], &[30, 20, 10, 255]);
         assert!(!bytes.contains(&99));
     }
+
+    #[test]
+    fn rendered_frames_stream_with_their_padded_rgba_rows() {
+        let width = 3u32;
+        let height = 2u32;
+        let padded = 16u32;
+        let data: Vec<u8> = (0..padded * height).map(|byte| byte as u8).collect();
+        let frame = RenderedFrame {
+            data: Arc::new(data.clone()),
+            width,
+            height,
+            padded_bytes_per_row: padded,
+            frame_number: 0,
+            target_time_ns: 0,
+        };
+        let stream = StreamingImage::new(gpui::StreamingImageFormat::Rgba8);
+        let stats = PumpStats::default();
+
+        let owned_ptr = frame.data.as_ptr();
+        let shared = frame.clone();
+        assert!(stream_frame_timed(&stream, shared, &stats));
+        let copied = stream.current().expect("shared frame streams");
+        assert_eq!(copied.bytes(), data.as_slice());
+        assert_ne!(copied.bytes().as_ptr(), owned_ptr);
+
+        assert!(stream_frame_timed(&stream, frame, &stats));
+        let moved = stream.current().expect("owned frame streams");
+        assert_eq!(
+            moved.size(),
+            gpui::size(gpui::DevicePixels(3), gpui::DevicePixels(2))
+        );
+        assert_eq!(moved.stride(), padded as usize);
+        assert_eq!(moved.bytes(), data.as_slice());
+        assert_eq!(moved.bytes().as_ptr(), owned_ptr);
+        assert_eq!(stats.convert_samples.load(Ordering::Relaxed), 2);
+    }
+
+    #[test]
+    fn misshapen_rendered_frames_are_not_streamed() {
+        let frame = RenderedFrame {
+            data: Arc::new(vec![0; 16]),
+            width: 3,
+            height: 2,
+            padded_bytes_per_row: 8,
+            frame_number: 0,
+            target_time_ns: 0,
+        };
+        let stream = StreamingImage::new(gpui::StreamingImageFormat::Rgba8);
+        assert!(!stream_frame_timed(&stream, frame, &PumpStats::default()));
+        assert!(stream.current().is_none());
+    }
+
+    #[test]
+    fn streaming_preview_follows_the_kill_switch() {
+        use std::ffi::OsStr;
+        assert!(!streaming_preview_setting(Some(OsStr::new("0"))));
+        assert!(streaming_preview_setting(Some(OsStr::new("1"))));
+        assert_eq!(streaming_preview_setting(None), cfg!(target_os = "linux"));
+    }
 }