From aa54ab1b2b6d960083657414b9920beda181f557 Mon Sep 17 00:00:00 2001 From: Samuel Onoja Date: Fri, 2 Oct 2026 07:44:13 +0100 Subject: [PATCH] Reuse window capture targets and offload pixel conversion --- compositor/src/handlers/screencopy.rs | 7 +- compositor/src/handlers/window_capture.rs | 115 +++++++++++++++--- compositor/src/render/capture.rs | 75 ++++++++++-- compositor/src/render/mod.rs | 2 +- compositor/src/state.rs | 2 +- scripts/tests/test_window_capture_isolated.py | 39 +++++- 6 files changed, 204 insertions(+), 36 deletions(-) diff --git a/compositor/src/handlers/screencopy.rs b/compositor/src/handlers/screencopy.rs index 15cedf7..6067f87 100644 --- a/compositor/src/handlers/screencopy.rs +++ b/compositor/src/handlers/screencopy.rs @@ -151,10 +151,13 @@ fn capture_opted_in(value: Option<&std::ffi::OsStr>) -> bool { value.is_some_and(|value| value == "1") } -pub(crate) fn init_global(display: &DisplayHandle) { +pub(crate) fn init_global( + display: &DisplayHandle, + loop_handle: &smithay::reexports::calloop::LoopHandle<'static, Ferese>, +) { if capture_allowed() { display.create_global::(3, ()); - super::window_capture::init_global(display); + super::window_capture::init_global(display, loop_handle); tracing::info!("authorized screencopy is enabled for this session"); } } diff --git a/compositor/src/handlers/window_capture.rs b/compositor/src/handlers/window_capture.rs index afc5e99..67885ce 100644 --- a/compositor/src/handlers/window_capture.rs +++ b/compositor/src/handlers/window_capture.rs @@ -62,8 +62,16 @@ pub(super) struct Snapshot { _permit: Permit, } +#[derive(Clone)] pub(crate) struct Global { budget: Arc>, + worker: std::sync::mpsc::SyncSender, +} + +struct Conversion { + snapshot: Snapshot, + frame: ZwlrScreencopyFrameV1, + manager: FereseWindowCaptureManagerV1, } pub(crate) fn is_portal(stream: &std::os::unix::net::UnixStream) -> bool { @@ -92,8 +100,79 @@ pub(crate) fn is_portal(stream: &std::os::unix::net::UnixStream) -> bool { matches!((expected, actual), (Some(expected), Some(actual)) if expected.dev() == actual.dev() && expected.ino() == actual.ino()) } -pub(super) fn init_global(display: &DisplayHandle) { - display.create_global::(1, Global { budget: Arc::default() }); +pub(super) fn init_global( + display: &DisplayHandle, + loop_handle: &smithay::reexports::calloop::LoopHandle<'static, Ferese>, +) { + use smithay::reexports::calloop::channel; + let (worker, requests) = std::sync::mpsc::sync_channel::(MAX_FRAMES); + let (results, source) = channel::sync_channel::(MAX_FRAMES); + let Ok(token) = loop_handle.insert_source(source, |event, _, state| { + if let channel::Event::Msg(result) = event { + finish_conversion(state, result); + } + }) else { + tracing::warn!("window capture result source could not be registered"); + return; + }; + + if let Err(error) = std::thread::Builder::new() + .name("ferese-capture".into()) + .spawn(move || { + while let Ok(mut job) = requests.recv() { + if job.frame.is_alive() { + crate::render::convert_window_pixels(&mut job.snapshot.pixels.pixels); + } + // Permits cap the total number of queued, working and completed + // frames at MAX_FRAMES, so the result queue cannot fill here. + if results.send(job).is_err() { + break; + } + } + }) + { + loop_handle.remove(token); + tracing::warn!(%error, "window capture worker could not be started"); + return; + } + + display.create_global::( + 1, + Global { + budget: Arc::default(), + worker, + }, + ); +} + +fn finish_conversion(state: &Ferese, result: Conversion) { + let Conversion { + snapshot, + frame, + manager, + } = result; + let Some(data) = frame.data::>() else { + return; + }; + if !frame.is_alive() || *data.used.lock().unwrap() { + return; + } + if state.session_lock.active + || !super::screencopy::capture_allowed() + || !state.windows.ids().values().any(|id| *id == snapshot.id) + { + *data.used.lock().unwrap() = true; + frame.failed(); + return; + } + + let size = (snapshot.pixels.width as u32, snapshot.pixels.height as u32); + let logical_size = snapshot.logical_size; + *data.snapshot.lock().unwrap() = Some(snapshot); + if manager.is_alive() { + manager.geometry(&frame, logical_size.0, logical_size.1); + } + frame.buffer(wl_shm::Format::Argb8888, size.0, size.1, size.0 * 4); } impl GlobalDispatch for Ferese { @@ -113,17 +192,17 @@ impl GlobalDispatch for Ferese { global: &Global, data_init: &mut DataInit<'_, Self>, ) { - data_init.init(resource, global.budget.clone()); + data_init.init(resource, global.clone()); } } -impl Dispatch>> for Ferese { +impl Dispatch for Ferese { fn request( state: &mut Self, _: &Client, manager: &FereseWindowCaptureManagerV1, request: manager::Request, - budget: &Arc>, + global: &Global, _: &DisplayHandle, data_init: &mut DataInit<'_, Self>, ) { @@ -137,13 +216,12 @@ impl Dispatch>> for Ferese { let id = WindowId((u64::from(window_hi) << 32) | u64::from(window_lo)); let mut busy = false; let snapshot = (overlay_cursor <= 1) - .then(|| state.window_snapshot(id, overlay_cursor == 1, budget, &mut busy)) + .then(|| state.window_snapshot(id, overlay_cursor == 1, &global.budget, &mut busy)) .flatten(); let size = snapshot .as_ref() .map(|snapshot| (snapshot.pixels.width, snapshot.pixels.height)) .unwrap_or((1, 1)); - let logical_size = snapshot.as_ref().map(|snapshot| snapshot.logical_size); let failed = snapshot.is_none(); let resource = data_init.init( frame, @@ -152,7 +230,7 @@ impl Dispatch>> for Ferese { region: Rectangle::from_size(size.into()), overlay_cursor: false, used: Mutex::new(failed), - snapshot: Mutex::new(snapshot), + snapshot: Mutex::new(None), }), ); if failed { @@ -160,15 +238,18 @@ impl Dispatch>> for Ferese { manager.busy(&resource); } resource.failed(); - } else { - let logical_size = logical_size.unwrap(); - manager.geometry(&resource, logical_size.0, logical_size.1); - resource.buffer( - wl_shm::Format::Argb8888, - size.0 as u32, - size.1 as u32, - size.0 as u32 * 4, - ); + } else if global + .worker + .try_send(Conversion { + snapshot: snapshot.unwrap(), + frame: resource.clone(), + manager: manager.clone(), + }) + .is_err() + { + *resource.data::>().unwrap().used.lock().unwrap() = true; + manager.busy(&resource); + resource.failed(); } } } diff --git a/compositor/src/render/capture.rs b/compositor/src/render/capture.rs index daf4ca7..d99ed4b 100644 --- a/compositor/src/render/capture.rs +++ b/compositor/src/render/capture.rs @@ -1,11 +1,27 @@ use super::*; +#[derive(Default)] +struct CaptureTarget(std::sync::Mutex>); + pub(crate) fn capture_resize_snapshot( renderer: &mut GlesRenderer, window: &smithay::desktop::Window, geometry: Rectangle, scale: f64, remaining: usize, +) -> Result, GlesError> { + // Called before on_commit_buffer_handler: renderer surface state still + // owns the previous buffer during an actual resize handoff. + render_window_texture(renderer, window, geometry, scale, remaining, None) +} + +fn render_window_texture( + renderer: &mut GlesRenderer, + window: &smithay::desktop::Window, + geometry: Rectangle, + scale: f64, + remaining: usize, + reusable: Option, ) -> Result, GlesError> { let Some(toplevel) = window.toplevel() else { return Ok(None); @@ -17,8 +33,6 @@ pub(crate) fn capture_resize_snapshot( if bytes == 0 || bytes > remaining { return Ok(None); } - // Called before on_commit_buffer_handler: renderer surface state still - // owns the previous buffer. Copy only during an actual resize handoff. let content = render_elements_from_surface_tree::>( renderer, toplevel.wl_surface(), @@ -30,7 +44,10 @@ pub(crate) fn capture_resize_snapshot( if content.is_empty() { return Ok(None); } - let mut texture = Offscreen::::create_buffer(renderer, Fourcc::Abgr8888, (size.w, size.h).into())?; + let mut texture = match reusable.filter(|texture| texture.size() == (size.w, size.h).into()) { + Some(texture) => texture, + None => Offscreen::::create_buffer(renderer, Fourcc::Abgr8888, (size.w, size.h).into())?, + }; { let mut target = renderer.bind(&mut texture)?; let mut frame = renderer.render(&mut target, size, Transform::Normal)?; @@ -56,7 +73,9 @@ pub(crate) fn capture_window_buffer( geometry: Rectangle, scale: f64, ) -> Result { - capture_window_frame(renderer, window, geometry, scale, None) + let mut buffer = capture_window_frame(renderer, window, geometry, scale, None)?; + convert_window_pixels(&mut buffer.pixels); + Ok(buffer) } pub(crate) fn capture_window_frame( @@ -66,7 +85,15 @@ pub(crate) fn capture_window_frame( scale: f64, cursor: Option<(&Ferese, Rectangle)>, ) -> Result { - let mut snapshot = capture_resize_snapshot(renderer, window, geometry, scale, 128 * 1024 * 1024) + // One reusable render target per EGL context. Resize handoff snapshots keep + // their own textures because they outlive the capture call. + let cache = renderer + .egl_context() + .user_data() + .get_or_insert_threadsafe(|| Arc::new(CaptureTarget::default())) + .clone(); + let reusable = cache.0.lock().unwrap().take(); + let mut snapshot = render_window_texture(renderer, window, geometry, scale, 128 * 1024 * 1024, reusable) .map_err(|error| error.to_string())? .ok_or("Window has no capturable content")?; let size = snapshot.texture.size(); @@ -95,11 +122,26 @@ pub(crate) fn capture_window_frame( .checked_mul(size.h as usize) .and_then(|pixels| pixels.checked_mul(4)) .ok_or("Window is too large")?; + // Smithay maps its PBO here and can wait for the GPU. Nonblocking readback + // needs a retained mapping plus an EGL-context-owned fence, polled before + // mapping; moving this call to the conversion worker is not context-safe. let source = renderer.map_texture(&mapping).map_err(|error| error.to_string())?; if source.len() < bytes { return Err("Incomplete window readback".into()); } - let mut pixels = source[..bytes].to_vec(); + let pixels = source[..bytes].to_vec(); + *cache.0.lock().unwrap() = Some(snapshot.texture); + + Ok(crate::handlers::screenshot::CaptureBuffer { + width: size.w, + height: size.h, + stride: size.w as usize * 4, + pixels, + }) +} + +/// Convert premultiplied ABGR readback into the capture protocol's straight ARGB. +pub(crate) fn convert_window_pixels(pixels: &mut [u8]) { for pixel in pixels.as_chunks_mut::<4>().0 { pixel.swap(0, 2); let alpha = pixel[3] as u16; @@ -110,10 +152,19 @@ pub(crate) fn capture_window_frame( .min(255) as u8; } } - Ok(crate::handlers::screenshot::CaptureBuffer { - width: size.w, - height: size.h, - stride: size.w as usize * 4, - pixels, - }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn readback_conversion_preserves_color_and_handles_zero_alpha() { + let mut pixels = vec![10, 20, 30, 255, 32, 64, 128, 128, 9, 8, 7, 0, 1, 1, 1, 1]; + convert_window_pixels(&mut pixels); + assert_eq!( + pixels, + [30, 20, 10, 255, 255, 128, 64, 128, 0, 0, 0, 0, 255, 255, 255, 1] + ); + } } diff --git a/compositor/src/render/mod.rs b/compositor/src/render/mod.rs index ac3d47a..6dbe22a 100644 --- a/compositor/src/render/mod.rs +++ b/compositor/src/render/mod.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; -pub(crate) use capture::{capture_resize_snapshot, capture_window_buffer, capture_window_frame}; +pub(crate) use capture::{capture_resize_snapshot, capture_window_buffer, capture_window_frame, convert_window_pixels}; use decorations::*; use materials::*; use overview::*; diff --git a/compositor/src/state.rs b/compositor/src/state.rs index 82e0636..cf8a4d8 100644 --- a/compositor/src/state.rs +++ b/compositor/src/state.rs @@ -463,7 +463,7 @@ impl Ferese { ) -> Result> { let display_handle = display.handle(); - crate::handlers::screencopy::init_global(&display_handle); + crate::handlers::screencopy::init_global(&display_handle, &event_loop.handle()); crate::effects::init_global(&display_handle); crate::shell_control::init_global(&display_handle); diff --git a/scripts/tests/test_window_capture_isolated.py b/scripts/tests/test_window_capture_isolated.py index 120f6e2..9d61909 100644 --- a/scripts/tests/test_window_capture_isolated.py +++ b/scripts/tests/test_window_capture_isolated.py @@ -1,13 +1,15 @@ """Verify isolated window capture without exposing host desktop content. FERESE_TEST_WINDOW_CAPTURE=1 python3 scripts/tests/test_window_capture_isolated.py -Requires built debug binaries, a Wayland host, cc, wayland-scanner and Pillow. +Requires built debug binaries, a Wayland host, PipeWire, gst-launch-1.0, +cc, wayland-scanner and Pillow. """ import io import json import os from pathlib import Path import signal +import selectors import subprocess import tempfile import time @@ -89,10 +91,41 @@ class WindowCapture(unittest.TestCase): self.assertEqual(image.convert("RGBA").getpixel((round(40 * scale), round(80 * scale))), (255, 0, 0, 128)) with Image.open(io.BytesIO(call("screenshot-window", str(cover["id"])))) as image: self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 2)), (0, 255, 0)) + # Exercise the asynchronous conversion worker and reuse its render + # target across a stream, not just the synchronous screenshot path. + stream_env = dict(env, PIPEWIRE_REMOTE=str(Path(os.environ["XDG_RUNTIME_DIR"]) / "pipewire-0")) + worker = subprocess.Popen([str(REPO / "target/debug/xdg-desktop-portal-ferese"), + "--stream-window", str(target["id"]), "hidden"], + env=stream_env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=log, start_new_session=True) + processes.append(worker) + with selectors.DefaultSelector() as selector: + selector.register(worker.stdout, selectors.EVENT_READ) + self.assertTrue(selector.select(10), "window stream did not publish a node") + ready = json.loads(worker.stdout.readline()) + self.assertEqual((ready["width"], ready["height"]), (round(640 * scale), round(480 * scale))) + raw = root / "frames.bgrx" + subprocess.run(["gst-launch-1.0", "-q", "pipewiresrc", f'path={ready["node"]}', + "num-buffers=15", "!", "video/x-raw,format=BGRx", "!", "filesink", + f"location={raw}"], env=stream_env, check=True, timeout=20) + pixels = raw.read_bytes() + width, height = ready["width"], ready["height"] + frame_bytes = width * height * 4 + self.assertEqual(len(pixels), frame_bytes * 15) + for frame_index in (0, 14): + start = frame_index * frame_bytes + red = start + ((height // 4) * width + width // 2) * 4 + blue = start + ((height * 3 // 4) * width + width // 2) * 4 + self.assertEqual(pixels[red:red + 3], bytes([0, 0, 255])) + self.assertEqual(pixels[blue:blue + 3], bytes([255, 0, 0])) + worker.stdin.close() + self.assertEqual(worker.wait(timeout=7), 0) + worker.stdout.close() finally: for process in reversed(processes): - os.killpg(process.pid, signal.SIGTERM) - process.wait(timeout=5) + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + process.wait(timeout=5) log.close()