diff --git a/.github/workflows/workspace-checks.yml b/.github/workflows/workspace-checks.yml index 0af98af..2707f3a 100644 --- a/.github/workflows/workspace-checks.yml +++ b/.github/workflows/workspace-checks.yml @@ -24,7 +24,7 @@ jobs: sudo apt-get install -y build-essential pkg-config dbus-daemon libwayland-dev \ libxkbcommon-dev xkb-data libinput-dev libudev-dev libseat-dev libgbm-dev \ python3-gi python3-pil libegl-dev libegl-mesa0 libgl1-mesa-dri mesa-vulkan-drivers libdrm-dev libfontconfig1-dev libfreetype-dev libexpat1-dev xauth \ - clang libclang-dev libpipewire-0.3-dev libspa-0.2-dev libglib2.0-dev libgstreamer1.0-dev libpam-modules + clang libclang-dev libpipewire-0.3-dev libspa-0.2-dev libglib2.0-dev libgstreamer1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good libpam-modules - name: Create private test runtime directory run: install -d -m 0700 "$XDG_RUNTIME_DIR" - name: Install Rust @@ -60,6 +60,9 @@ jobs: cargo +1.95.0 test --locked -p ferese state::capture_privacy::tests -- --ignored --nocapture cargo +1.95.0 test --locked -p ferese backends::direct::capture::tests::capture_recomposes_cursor_without_touching_the_display_target -- --ignored --nocapture cargo +1.95.0 test --locked -p ferese winit::tests::capture_failure_restores_display_target_and_subsequent_frames_render -- --ignored --nocapture + cargo +1.95.0 test --locked -p ferese render::capture::stream_gpu_tests -- --ignored --nocapture + - name: Preserve static recording duration + run: cargo +1.95.0 test --locked -p xdg-desktop-portal-ferese --bin ferese-record recorder::tests::encoder_preserves_the_duration_of_a_static_capture -- --ignored --nocapture - name: Share desktop binaries with native checks uses: actions/upload-artifact@v7 with: diff --git a/Cargo.lock b/Cargo.lock index ac8fcf3..178a88e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1903,6 +1903,7 @@ dependencies = [ "bitflags 2.13.2", "wayland-backend", "wayland-client", + "wayland-protocols", "wayland-protocols-wlr", "wayland-scanner", "wayland-server", @@ -8259,6 +8260,7 @@ dependencies = [ "ferese-theme", "ferese-theme-client", "futures-util", + "gbm", "gstreamer", "image", "jiff", diff --git a/compositor/src/backends/direct.rs b/compositor/src/backends/direct.rs index f35f0e1..5530b92 100644 --- a/compositor/src/backends/direct.rs +++ b/compositor/src/backends/direct.rs @@ -177,6 +177,24 @@ impl DirectBackendState { ) } + pub(crate) fn clear_stream_capture_targets(&mut self) { + for device in self.devices.values_mut() { + crate::render::clear_stream_capture_targets(&mut device.renderer); + } + } + + pub(crate) fn capture_renderer(&mut self, output: &Output) -> Option<&mut GlesRenderer> { + self.devices + .values_mut() + .find(|device| { + device + .outputs + .values() + .any(|candidate| &candidate.output == output) + }) + .map(|device| &mut device.renderer) + } + pub(crate) fn capture_window_buffer( &mut self, window: &smithay::desktop::Window, @@ -295,13 +313,13 @@ struct DirectOutput { identity: String, mirror_source: Option, mirror_texture: Option, - mirror_capture_texture: Option, + mirror_capture_texture: [Option; 2], mirror_canvas: Option, mirror_id: smithay::backend::renderer::element::Id, mirror_commit: CommitCounter, surface: OutputCompositor, primary_commit: Option, - capture_texture: Option, + capture_texture: [Option; 2], render_metrics: RenderMetrics, diagnostics: Diagnostics, dmabuf_feedback: feedback::OutputFeedback, @@ -452,7 +470,7 @@ pub fn init(event_loop: &mut EventLoop, state: &mut Ferese) -> Result<() output.surface.reset_buffers(); output.dmabuf_feedback.invalidate(); output.primary_commit = None; - output.capture_texture = None; + output.capture_texture = Default::default(); output.frame_pending = false; output.power.suspended(); } @@ -1226,16 +1244,33 @@ pub(crate) fn wake_locked_outputs(state: &mut Ferese) { #[track_caller] pub fn render_all(state: &mut Ferese) { + for output in state.space.outputs() { + capture::changes::content_changed(output); + } redraw(state, None, std::panic::Location::caller()); } #[track_caller] pub(crate) fn render_on(state: &mut Ferese, outputs: &[Output]) { + for output in outputs { + capture::changes::content_changed(output); + } + redraw(state, Some(outputs), std::panic::Location::caller()); +} + +/// Consumer demand schedules a frame without invalidating retained scene pixels. +#[track_caller] +pub(crate) fn schedule_capture(state: &mut Ferese, outputs: &[Output]) { redraw(state, Some(outputs), std::panic::Location::caller()); } #[track_caller] pub(crate) fn render_cursor(state: &mut Ferese) { + // Include nested outputs and both sides of an output crossing. This does not + // invalidate the content revision used by cursor-hidden captures. + for output in state.space.outputs() { + capture::changes::cursor_changed(output); + } let outputs = cursor_affected_outputs(state); redraw(state, Some(&outputs), std::panic::Location::caller()); } @@ -2603,13 +2638,13 @@ fn create_direct_output( identity, mirror_source, mirror_texture: None, - mirror_capture_texture: None, + mirror_capture_texture: Default::default(), mirror_canvas: None, mirror_id: smithay::backend::renderer::element::Id::new(), mirror_commit: CommitCounter::default(), surface, primary_commit: None, - capture_texture: None, + capture_texture: Default::default(), render_metrics, diagnostics, dmabuf_feedback, diff --git a/compositor/src/backends/direct/capture.rs b/compositor/src/backends/direct/capture.rs index 170a983..5838090 100644 --- a/compositor/src/backends/direct/capture.rs +++ b/compositor/src/backends/direct/capture.rs @@ -1,4 +1,5 @@ use smithay::backend::allocator::Fourcc; +use smithay::backend::renderer::damage::{Error as DamageError, OutputDamageTracker}; use smithay::backend::renderer::element::RenderElement; use smithay::backend::renderer::gles::{GlesError, GlesRenderer, GlesTarget, GlesTexture}; use smithay::backend::renderer::{Bind, Offscreen, Texture}; @@ -13,6 +14,42 @@ use crate::state::FrameScene; const CACHE_IDLE: Duration = Duration::from_secs(2); +pub(crate) mod changes; + +pub(crate) struct CaptureTarget { + texture: GlesTexture, + damage: OutputDamageTracker, + output: Output, + revision: u64, + id: smithay::backend::renderer::element::Id, + commit: smithay::backend::renderer::utils::CommitCounter, + stamp: Option, + preparations: u64, + reuses: u64, + compositions: u64, + trace: bool, + reported: Instant, +} + +impl CaptureTarget { + pub(crate) fn new(texture: GlesTexture, output: &Output) -> Self { + Self { + texture, + damage: OutputDamageTracker::from_output(output), + output: output.clone(), + revision: 0, + id: smithay::backend::renderer::element::Id::new(), + commit: Default::default(), + stamp: None, + preparations: 0, + reuses: 0, + compositions: 0, + trace: std::env::var_os("FERESE_TRACE_PERFORMANCE").is_some(), + reported: Instant::now(), + } + } +} + pub(crate) fn touch_cache(state: &mut Ferese) { if state.capture_last_used.replace(Instant::now()).is_some() { return; @@ -28,16 +65,28 @@ pub(crate) fn touch_cache(state: &mut Ferese) { if idle < CACHE_IDLE { return TimeoutAction::ToDuration(CACHE_IDLE - idle); } + if let Some(backend) = state.nested_backend.clone() + && let Ok(mut backend) = backend.try_borrow_mut() + { + crate::render::clear_stream_capture_targets(backend.renderer()); + } + if let Some(backend) = state.direct_backend.as_mut() { + backend.clear_stream_capture_targets(); + } + if !state.pending_screencopies.is_empty() || state.image_capture.has_sessions() { + // A damage-waiting stream must retain the comparison image. + return TimeoutAction::ToDuration(CACHE_IDLE); + } // A stream can pause or fail without another repaint. Release its // offscreen targets and filtered-scene caches from the event loop. state.capture_last_used = None; state.capture_render = Default::default(); - state.nested_capture_texture = None; + state.nested_capture_texture = Default::default(); if let Some(backend) = state.direct_backend.as_mut() { for device in backend.devices.values_mut() { for output in device.outputs.values_mut() { - output.capture_texture = None; - output.mirror_capture_texture = None; + output.capture_texture = Default::default(); + output.mirror_capture_texture = Default::default(); } } } @@ -52,7 +101,7 @@ pub(crate) fn touch_cache(state: &mut Ferese) { pub(crate) fn capture_output( state: &mut Ferese, renderer: &mut GlesRenderer, - texture: &mut Option, + textures: &mut [Option; 2], output: &Output, scene: &FrameScene, ) { @@ -61,15 +110,35 @@ pub(crate) fn capture_output( continue; } - // Sample the same scene into an independent target: the displayed - // primary buffer may omit both a scanned-out client and hardware cursor. - let elements = capture_output_elements(state, renderer, output, include_cursor, scene); - let Some(target) = capture_request(state, output, include_cursor, || { - render_capture(renderer, texture, output, &elements) + let texture = &mut textures[usize::from(include_cursor)]; + if !prepare_capture( + state, + renderer, + texture, + output, + output, + include_cursor, + scene, + ) { + continue; + } + let texture = texture.as_mut().expect("capture target allocated"); + let Some((target, revision)) = capture_request(state, output, include_cursor, || { + renderer + .bind(&mut texture.texture) + .map(|target| (target, texture.revision)) }) else { continue; }; - state.process_screencopies(renderer, &target, output, include_cursor); + state.process_screencopies_with_revision( + renderer, + &target, + output, + include_cursor, + Some(revision), + ); + drop(target); + state.process_output_streams(renderer, &texture.texture, output, include_cursor, revision); } } @@ -84,47 +153,121 @@ pub(super) fn capture_mirror( if !state.has_pending_screencopy(&output.output, include_cursor) { continue; } - let elements = capture_output_elements(state, renderer, source, include_cursor, scene); // Never sample mirror_texture: it contains the unfiltered display. let canvas = output .mirror_canvas .as_ref() .expect("mirror display composed"); - let Some(()) = capture_request(state, &output.output, include_cursor, || { - super::mirror::compose( - renderer, - &mut output.mirror_capture_texture, - canvas, - source, - &elements, - ) - }) else { + super::mirror::configure_canvas(canvas, source); + let source_target = &mut output.mirror_capture_texture[usize::from(include_cursor)]; + if !prepare_capture( + state, + renderer, + source_target, + source, + canvas, + include_cursor, + scene, + ) { + continue; + } + let source_target = source_target + .as_ref() + .expect("mirror capture target allocated"); + let element = super::mirror::fitted_texture( + source_target.texture.clone(), + &output.output, + source_target.id.clone(), + source_target.commit, + ); + let Some((target, revision)) = + capture_request(state, &output.output, include_cursor, || { + render_capture_with_revision( + renderer, + &mut output.capture_texture[usize::from(include_cursor)], + &output.output, + &[element], + ) + }) + else { continue; }; - let element = super::mirror::fitted_texture( - output - .mirror_capture_texture + state.process_screencopies_with_revision( + renderer, + &target, + &output.output, + include_cursor, + Some(revision), + ); + drop(target); + state.process_output_streams( + renderer, + &output.capture_texture[usize::from(include_cursor)] .as_ref() - .expect("successful mirror composition populated its capture texture") + .expect("capture target allocated") + .texture .clone(), &output.output, - smithay::backend::renderer::element::Id::new(), - Default::default(), + include_cursor, + revision, ); - let Some(target) = capture_request(state, &output.output, include_cursor, || { - render_capture( - renderer, - &mut output.capture_texture, - &output.output, - &[element], - ) - }) else { - continue; - }; - state.process_screencopies(renderer, &target, &output.output, include_cursor); } } +/// Display damage cannot prove a filtered scene is unchanged: an opaque private +/// window may hide updates that must remain visible to capture underneath it. +/// Retain a separate source revision and only prepare the filtered scene on a +/// real mutation. Delivery buffers still receive a complete image. +fn prepare_capture( + state: &mut Ferese, + renderer: &mut GlesRenderer, + target: &mut Option, + source: &Output, + destination: &Output, + include_cursor: bool, + scene: &FrameScene, +) -> bool { + touch_cache(state); + let stamp = changes::Stamp::new(state, source, include_cursor); + let reusable = target.as_ref().is_some_and(|target| { + target.stamp.as_ref() == Some(&stamp) + && target.output == *destination + && destination + .current_mode() + .is_some_and(|mode| target.texture.size() == (mode.size.w, mode.size.h).into()) + }); + if reusable { + target.as_mut().expect("cached capture target").reuses += 1; + } else { + // Keep the capture image separate from the display swapchain and planes. + let elements = capture_output_elements(state, renderer, source, include_cursor, scene); + let Some(framebuffer) = capture_request(state, destination, include_cursor, || { + render_capture_with_revision(renderer, target, destination, &elements) + }) else { + // A failed render can have changed the retained pixels already. + if let Some(target) = target.as_mut() { + target.stamp = None; + } + return false; + }; + drop(framebuffer); + let target = target.as_mut().expect("capture target allocated"); + target.stamp = Some(stamp); + target.preparations += 1; + } + let target = target.as_mut().expect("capture target allocated"); + if target.trace && target.reported.elapsed() >= Duration::from_secs(5) { + tracing::info!(target: "ferese::render", output = %source.name(), include_cursor, + preparations = target.preparations, scene_reuses = target.reuses, + compositions = target.compositions, "capture scene outcomes"); + target.reported = Instant::now(); + target.preparations = 0; + target.reuses = 0; + target.compositions = 0; + } + true +} + pub(crate) fn capture_request( state: &mut Ferese, output: &Output, @@ -147,7 +290,69 @@ pub(super) fn after_queue(queued: Result, capture: impl FnOnce()) -> Ok(queued) } +#[cfg(test)] pub(super) fn render_capture<'a, E: RenderElement>( + renderer: &mut GlesRenderer, + target: &'a mut Option, + output: &Output, + elements: &[E], +) -> Result, DamageError> { + render_capture_with_revision(renderer, target, output, elements).map(|(target, _)| target) +} + +pub(super) fn render_capture_with_revision<'a, E: RenderElement>( + renderer: &mut GlesRenderer, + target: &'a mut Option, + output: &Output, + elements: &[E], +) -> Result<(GlesTarget<'a>, u64), DamageError> { + let size = output.current_mode().expect("output has a mode").size; + if target + .as_ref() + .is_none_or(|target| target.texture.size() != (size.w, size.h).into()) + { + *target = Some(CaptureTarget::new( + Offscreen::::create_buffer( + renderer, + Fourcc::Abgr8888, + (size.w, size.h).into(), + ) + .map_err(DamageError::Rendering)?, + output, + )); + } + let target = target.as_mut().expect("capture target allocated"); + if target.output != *output { + target.damage = OutputDamageTracker::from_output(output); + target.output = output.clone(); + } + // This texture retains the previous capture, independently of display + // scanout and damage. Age one lets unchanged pixels survive; a new target + // has no history and is repainted in full. Rebind after every readback. + let mut framebuffer = renderer + .bind(&mut target.texture) + .map_err(DamageError::Rendering)?; + let result = target.damage.render_output( + renderer, + &mut framebuffer, + 1, + elements, + [0.035, 0.04, 0.055, 1.0], + )?; + if result + .damage + .as_ref() + .is_some_and(|damage| !damage.is_empty()) + { + static NEXT_REVISION: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); + target.commit.increment(); + target.compositions += 1; + target.revision = NEXT_REVISION.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + } + Ok((framebuffer, target.revision)) +} + +pub(super) fn render_texture<'a, E: RenderElement>( renderer: &mut GlesRenderer, texture: &'a mut Option, output: &Output, @@ -184,6 +389,326 @@ mod tests { use super::*; + #[test] + #[ignore = "requires an EGL rendering device"] + fn unchanged_capture_skips_preparation_without_losing_source_updates() { + if !crate::startup_tests::private_runtime( + "backends::direct::capture::tests::unchanged_capture_skips_preparation_without_losing_source_updates", + ) { + return; + } + let mut events = smithay::reexports::calloop::EventLoop::try_new().unwrap(); + let mut config = crate::config::Config::default().runtime_config().unwrap(); + config.wallpaper.path = None; + config.animations_enabled = false; + let mut state = Ferese::new( + &mut events, + smithay::reexports::wayland_server::Display::new().unwrap(), + config, + ) + .unwrap(); + let output = Output::new( + "capture-cache".into(), + PhysicalProperties { + serial_number: String::new(), + size: (0, 0).into(), + subpixel: Subpixel::Unknown, + make: "test".into(), + model: "test".into(), + }, + ); + output.change_current_state( + Some(Mode { + size: (320, 240).into(), + refresh: 60_000, + }), + Some(Transform::Normal), + None, + None, + ); + state.space.map_output(&output, (0, 0)); + state.register_output(&output, "capture-cache".into()); + let (_window, _client) = crate::state::test_window(&mut state, &mut events, 0xffff0000); + state.advance_animations(Instant::now()); + let device = EGLDevice::enumerate().unwrap().last().expect("EGL device"); + // SAFETY: the device and display outlive this exclusively owned context. + let display = unsafe { EGLDisplay::new(device).unwrap() }; + let context = EGLContext::new(&display).unwrap(); + // SAFETY: this test thread exclusively owns the fresh context. + let mut renderer = unsafe { GlesRenderer::new(context).unwrap() }; + let mut target = None; + let prepare = |state: &mut Ferese, + renderer: &mut GlesRenderer, + target: &mut Option| { + let scene = state.sample_frame(&output, Duration::ZERO); + assert!(prepare_capture( + state, renderer, target, &output, &output, false, &scene + )); + }; + let pixels = |renderer: &mut GlesRenderer, target: &mut Option| { + let target = target.as_mut().unwrap(); + let framebuffer = renderer.bind(&mut target.texture).unwrap(); + let mapping = renderer + .copy_framebuffer( + &framebuffer, + Rectangle::from_size((320, 240).into()), + Fourcc::Abgr8888, + ) + .unwrap(); + drop(framebuffer); + renderer.map_texture(&mapping).unwrap().to_vec() + }; + prepare(&mut state, &mut renderer, &mut target); + // Keep assertions independent of the opt-in periodic reporting clock. + target.as_mut().unwrap().trace = false; + let original = pixels(&mut renderer, &mut target); + for _ in 0..100 { + super::super::schedule_capture(&mut state, std::slice::from_ref(&output)); + prepare(&mut state, &mut renderer, &mut target); + } + assert_eq!(target.as_ref().unwrap().preparations, 1); + assert_eq!(target.as_ref().unwrap().reuses, 100); + assert_eq!(pixels(&mut renderer, &mut target), original); + super::super::render_cursor(&mut state); + prepare(&mut state, &mut renderer, &mut target); + assert_eq!(target.as_ref().unwrap().preparations, 1); + state.capture_epoch += 1; + prepare(&mut state, &mut renderer, &mut target); + assert_eq!(target.as_ref().unwrap().preparations, 2); + let (_cover, _cover_client) = + crate::state::test_window(&mut state, &mut events, 0xff00ff00); + state.advance_animations(Instant::now()); + prepare(&mut state, &mut renderer, &mut target); + assert_eq!(target.as_ref().unwrap().preparations, 3); + assert_ne!(pixels(&mut renderer, &mut target), original); + } + + #[test] + #[ignore = "requires an EGL rendering device"] + fn incremental_capture_matches_full_redraw_after_scene_and_output_changes() { + let device = EGLDevice::enumerate().unwrap().last().expect("EGL device"); + // SAFETY: the enumerated device remains alive through its EGL display. + let display = unsafe { EGLDisplay::new(device).unwrap() }; + let context = EGLContext::new(&display).unwrap(); + // SAFETY: this thread exclusively owns the renderer and its fresh context. + let mut renderer = unsafe { GlesRenderer::new(context).unwrap() }; + let output = |name: &str| { + Output::new( + name.into(), + PhysicalProperties { + serial_number: String::new(), + size: (0, 0).into(), + subpixel: Subpixel::Unknown, + make: "test".into(), + model: "test".into(), + }, + ) + }; + let outputs = [output("first"), output("second")]; + let mut capture = None; + let mut reference = None; + let background = SolidColorRenderElement::new( + Id::new(), + Rectangle::from_size((96, 96).into()), + CommitCounter::default(), + Color32F::new(0., 0., 1., 1.), + Kind::Unspecified, + ); + let item_id = Id::new(); + for output in outputs.iter().chain(std::iter::once(&outputs[0])) { + for (width, height) in [(64, 48), (48, 64)] { + for scale in [1., 1.25, 1.5] { + for transform in [ + Transform::Normal, + Transform::_90, + Transform::_180, + Transform::Flipped, + ] { + output.change_current_state( + Some(Mode { + size: (width, height).into(), + refresh: 60_000, + }), + Some(transform), + Some(smithay::output::Scale::Fractional(scale)), + None, + ); + let mut commit = CommitCounter::default(); + // Include moves, color/alpha changes, removal, cursor + // inclusion/exclusion and an entirely empty scene. + for step in [6, 0, 1, 2, 3, 4, 5, 6] { + if step == 2 { + commit.increment(); + } + let mut elements = Vec::new(); + if matches!(step, 0..=2 | 4) { + elements.push(SolidColorRenderElement::new( + item_id.clone(), + Rectangle::new( + (if step == 0 { 4 } else { 20 }, 4).into(), + (12, 12).into(), + ), + commit, + if step < 2 { + Color32F::new(0., 1., 0., 1.) + } else { + Color32F::new(0.5, 0., 0., 0.5) + }, + Kind::Cursor, + )); + } + if step != 6 { + elements.push(background.clone()); + } + drop( + render_capture(&mut renderer, &mut capture, output, &elements) + .unwrap(), + ); + // The repeated scene must preserve the same pixels when + // damage tracking skips all drawing on the next capture. + let actual = + render_capture(&mut renderer, &mut capture, output, &elements) + .unwrap(); + let mapping = renderer + .copy_framebuffer( + &actual, + Rectangle::from_size((width, height).into()), + Fourcc::Abgr8888, + ) + .unwrap(); + let actual_pixels = renderer.map_texture(&mapping).unwrap().to_vec(); + drop(actual); + let expected = + render_texture(&mut renderer, &mut reference, output, &elements) + .unwrap(); + let mapping = renderer + .copy_framebuffer( + &expected, + Rectangle::from_size((width, height).into()), + Fourcc::Abgr8888, + ) + .unwrap(); + let expected_pixels = renderer.map_texture(&mapping).unwrap(); + assert_eq!( + actual_pixels, + expected_pixels, + "{} {width}x{height} scale={scale} {transform:?} step={step}", + output.name() + ); + } + } + } + } + } + } + + #[test] + #[ignore = "manual 4K capture benchmark; requires a hardware EGL device"] + fn profile_output_capture() { + let device = EGLDevice::enumerate() + .unwrap() + .find(|device| device.render_device_path().is_ok()) + .expect("a hardware EGL rendering device"); + eprintln!("capture_benchmark device={:?}", device.render_device_path()); + // SAFETY: the enumerated device remains alive through its EGL display. + let display = unsafe { EGLDisplay::new(device).unwrap() }; + let context = EGLContext::new(&display).unwrap(); + // SAFETY: this thread exclusively owns the renderer and its fresh context. + let mut renderer = unsafe { GlesRenderer::new(context).unwrap() }; + let output = Output::new( + "capture-benchmark".into(), + PhysicalProperties { + serial_number: String::new(), + size: (0, 0).into(), + subpixel: Subpixel::Unknown, + make: "test".into(), + model: "test".into(), + }, + ); + output.change_current_state( + Some(Mode { + size: (3840, 2160).into(), + refresh: 60_000, + }), + Some(Transform::Normal), + None, + None, + ); + let mut texture = None; + let background_id = Id::new(); + let cursor_id = Id::new(); + let mut commit = CommitCounter::default(); + let panes: Vec<_> = (0..20) + .map(|i| { + SolidColorRenderElement::new( + Id::new(), + Rectangle::new((i * 150, i * 75).into(), (800, 600).into()), + CommitCounter::default(), + Color32F::new(0.1, 0.2, 0.3, 0.5), + Kind::Unspecified, + ) + }) + .collect(); + for readback in [false, true] { + let size = if readback { (3840, 2160) } else { (1, 1) }; + let mut pixels = vec![0u8; size.0 as usize * size.1 as usize * 4]; + for workload in ["static", "cursor", "full"] { + for run in 0..=3 { + let started = Instant::now(); + commit.increment(); + for tick in 0..60 { + let color = if workload == "full" { + commit.increment(); + tick as f32 / 60. + } else { + 0.1 + }; + let mut elements = vec![SolidColorRenderElement::new( + cursor_id.clone(), + Rectangle::new( + (if workload == "cursor" { tick * 30 } else { 0 }, 0).into(), + (24, 24).into(), + ), + CommitCounter::default(), + Color32F::new(1., 1., 1., 1.), + Kind::Cursor, + )]; + elements.extend_from_slice(&panes); + elements.push(SolidColorRenderElement::new( + background_id.clone(), + Rectangle::from_size((3840, 2160).into()), + commit, + Color32F::new(color, 0.1, 0.2, 1.), + Kind::Unspecified, + )); + let target = + render_capture(&mut renderer, &mut texture, &output, &elements) + .unwrap(); + // Mapping waits for GPU completion in both workloads. The full + // readback also includes the copy into the client's SHM buffer. + let mapping = renderer + .copy_framebuffer( + &target, + Rectangle::from_size(size.into()), + Fourcc::Argb8888, + ) + .unwrap(); + let source = renderer.map_texture(&mapping).unwrap(); + let bytes = pixels.len(); + pixels.copy_from_slice(&source[..bytes]); + std::hint::black_box(&pixels); + } + if run != 0 { + eprintln!( + "output_capture_benchmark readback={readback} workload={workload} run={run} frames=60 elapsed_us={}", + started.elapsed().as_micros() + ); + } + } + } + } + } + #[test] #[ignore = "requires an EGL rendering device"] fn capture_recomposes_cursor_without_touching_the_display_target() { @@ -234,7 +759,7 @@ mod tests { let mut displayed = None; let mut capture = None; { - let _ = render_capture( + let _ = render_texture( &mut renderer, &mut displayed, &output, @@ -289,7 +814,7 @@ mod tests { let _ = render_capture::(&mut renderer, &mut capture, &output, &[]) .unwrap(); - assert_eq!(capture.as_ref().unwrap().size(), (48, 16).into()); + assert_eq!(capture.as_ref().unwrap().texture.size(), (48, 16).into()); } } diff --git a/compositor/src/backends/direct/capture/changes.rs b/compositor/src/backends/direct/capture/changes.rs new file mode 100644 index 0000000..7228de4 --- /dev/null +++ b/compositor/src/backends/direct/capture/changes.rs @@ -0,0 +1,185 @@ +//! Capture invalidation is independent of display damage and consumer requests. +use std::sync::atomic::{AtomicU64, Ordering}; + +use smithay::output::Output; +use smithay::utils::{Logical, Physical, Rectangle, Size, Transform}; + +#[derive(Default)] +struct Changes { + content: AtomicU64, + cursor: AtomicU64, +} + +pub(crate) fn content_changed(output: &Output) { + if let Some(changes) = output.user_data().get::() { + changes.content.fetch_add(1, Ordering::Relaxed); + } +} + +pub(crate) fn cursor_changed(output: &Output) { + if let Some(changes) = output.user_data().get::() { + changes.cursor.fetch_add(1, Ordering::Relaxed); + } +} + +pub(crate) fn tracked(output: &Output) -> bool { + output.user_data().get::().is_some() +} + +/// Also compare output configuration: mode changes can precede a redraw request. +/// Keeping the source identity prevents a remapped mirror from reusing its old view. +#[derive(Debug, PartialEq)] +pub(super) struct Stamp { + output: Output, + content: u64, + cursor: Option, + epoch: u64, + cursor_protected: bool, + size: Option>, + scale: f64, + transform: Transform, + geometry: Option>, +} + +impl Stamp { + pub fn new(state: &crate::Ferese, output: &Output, include_cursor: bool) -> Self { + let changes = output + .user_data() + .get_or_insert_threadsafe(Changes::default); + Self { + output: output.clone(), + content: changes.content.load(Ordering::Relaxed), + cursor: include_cursor.then(|| changes.cursor.load(Ordering::Relaxed)), + epoch: state.capture_epoch, + cursor_protected: include_cursor && state.capture_cursor_protected(), + size: output.current_mode().map(|mode| mode.size), + scale: output.current_scale().fractional_scale(), + transform: output.current_transform(), + geometry: state.space.output_geometry(output), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use smithay::output::{Mode, PhysicalProperties, Scale, Subpixel}; + + fn output() -> Output { + let output = Output::new( + "capture-test".into(), + PhysicalProperties { + size: (0, 0).into(), + subpixel: Subpixel::Unknown, + make: String::new(), + model: String::new(), + serial_number: String::new(), + }, + ); + output.change_current_state( + Some(Mode { + size: (640, 480).into(), + refresh: 60_000, + }), + Some(Transform::Normal), + Some(Scale::Fractional(1.25)), + Some((0, 0).into()), + ); + output + } + + #[test] + fn consumer_demand_does_not_invalidate_but_scene_redraws_do() { + if !crate::startup_tests::private_runtime( + "backends::direct::capture::changes::tests::consumer_demand_does_not_invalidate_but_scene_redraws_do", + ) { + return; + } + let mut events = smithay::reexports::calloop::EventLoop::try_new().unwrap(); + let mut state = crate::startup_tests::state(&mut events); + let first = output(); + let second = output(); + state.space.map_output(&first, (0, 0)); + state.space.map_output(&second, (640, 0)); + let before = Stamp::new(&state, &first, false); + let other = Stamp::new(&state, &second, false); + crate::backends::direct::schedule_capture(&mut state, std::slice::from_ref(&first)); + assert_eq!(before, Stamp::new(&state, &first, false)); + crate::backends::direct::render_on(&mut state, std::slice::from_ref(&first)); + assert_ne!(before, Stamp::new(&state, &first, false)); + assert_eq!(other, Stamp::new(&state, &second, false)); + let before = Stamp::new(&state, &first, false); + state.defer_output_redraw(first.clone()); + assert_ne!(before, Stamp::new(&state, &first, false)); + crate::backends::direct::render_all(&mut state); + assert_ne!(other, Stamp::new(&state, &second, false)); + } + + #[test] + fn cursor_privacy_and_output_configuration_change_the_appropriate_view() { + if !crate::startup_tests::private_runtime( + "backends::direct::capture::changes::tests::cursor_privacy_and_output_configuration_change_the_appropriate_view", + ) { + return; + } + let mut events = smithay::reexports::calloop::EventLoop::try_new().unwrap(); + let mut state = crate::startup_tests::state(&mut events); + let output = output(); + state.space.map_output(&output, (0, 0)); + let hidden = Stamp::new(&state, &output, false); + let embedded = Stamp::new(&state, &output, true); + crate::backends::direct::render_cursor(&mut state); + assert_eq!(hidden, Stamp::new(&state, &output, false)); + assert_ne!(embedded, Stamp::new(&state, &output, true)); + state.capture_epoch += 1; + assert_ne!(hidden, Stamp::new(&state, &output, false)); + let before = Stamp::new(&state, &output, false); + output.change_current_state(None, Some(Transform::_180), None, None); + assert_ne!(before, Stamp::new(&state, &output, false)); + let before = Stamp::new(&state, &output, false); + output.change_current_state(None, None, Some(Scale::Fractional(1.5)), None); + assert_ne!(before, Stamp::new(&state, &output, false)); + let before = Stamp::new(&state, &output, false); + state.space.map_output(&output, (100, 0)); + assert_ne!(before, Stamp::new(&state, &output, false)); + } + + #[test] + fn commits_and_the_final_animation_frame_invalidate_capture() { + if !crate::startup_tests::private_runtime( + "backends::direct::capture::changes::tests::commits_and_the_final_animation_frame_invalidate_capture", + ) { + return; + } + let mut events = smithay::reexports::calloop::EventLoop::try_new().unwrap(); + let mut state = crate::startup_tests::state(&mut events); + let output = output(); + state.space.map_output(&output, (0, 0)); + state.register_output(&output, "capture-animation".into()); + let (_window, mut client) = crate::state::test_window(&mut state, &mut events, 0xffff0000); + let before = Stamp::new(&state, &output, false); + // Even a callback-only commit goes through the mutation path. Damage + // tracking can subsequently decide that the retained pixels still match. + crate::state::test_client_request(&mut client, 2, 6, &[], None); + crate::state::dispatch_test_client(&mut events, &mut state); + assert_ne!(before, Stamp::new(&state, &output, false)); + let mut now = std::time::Instant::now(); + state.advance_animations(now); + assert!(state.output_has_animations(&output)); + let mut settled = false; + for _ in 0..500 { + let before = Stamp::new(&state, &output, false); + now += std::time::Duration::from_millis(16); + state.advance_animations(now); + assert_ne!(before, Stamp::new(&state, &output, false)); + if !state.output_has_animations(&output) { + settled = true; + break; + } + } + assert!( + settled, + "the opening/focus animation must reach its final frame" + ); + } +} diff --git a/compositor/src/backends/direct/mirror.rs b/compositor/src/backends/direct/mirror.rs index 6a6f283..7a0d55c 100644 --- a/compositor/src/backends/direct/mirror.rs +++ b/compositor/src/backends/direct/mirror.rs @@ -79,6 +79,13 @@ pub(super) fn compose( source: &Output, elements: &[crate::render::AnimatedWindowRenderElement], ) -> Result<(), smithay::backend::renderer::gles::GlesError> { + configure_canvas(canvas, source); + let framebuffer = capture::render_texture(renderer, texture, canvas, elements)?; + drop(framebuffer); + Ok(()) +} + +pub(super) fn configure_canvas(canvas: &Output, source: &Output) { let source_mode = source.current_mode().expect("source has a mode"); canvas.change_current_state( Some(OutputMode { @@ -89,9 +96,6 @@ pub(super) fn compose( Some(source.current_scale()), None, ); - let framebuffer = capture::render_capture(renderer, texture, canvas, elements)?; - drop(framebuffer); - Ok(()) } pub(super) fn fitted_texture( diff --git a/compositor/src/backends/direct/topology.rs b/compositor/src/backends/direct/topology.rs index ec70f55..7ace9d7 100644 --- a/compositor/src/backends/direct/topology.rs +++ b/compositor/src/backends/direct/topology.rs @@ -324,7 +324,7 @@ pub(super) fn reconcile_outputs(state: &mut Ferese, reactivate: bool) { output.surface.reset_buffers(); output.dmabuf_feedback.invalidate(); output.primary_commit = None; - output.capture_texture = None; + output.capture_texture = Default::default(); output.frame_pending = false; output.lock_frame_pending = false; output.power.suspended(); diff --git a/compositor/src/backends/direct/transaction.rs b/compositor/src/backends/direct/transaction.rs index 759fdc2..8994966 100644 --- a/compositor/src/backends/direct/transaction.rs +++ b/compositor/src/backends/direct/transaction.rs @@ -557,7 +557,7 @@ pub(super) fn apply_device( cancel_output_timers(&state.loop_handle, &mut output); output.frame_pending = false; output.primary_commit = None; - output.capture_texture = None; + output.capture_texture = Default::default(); output.power.suspended(); let clock = state .direct_backend diff --git a/compositor/src/cursor.rs b/compositor/src/cursor.rs index 62a7151..19192f3 100644 --- a/compositor/src/cursor.rs +++ b/compositor/src/cursor.rs @@ -208,6 +208,12 @@ pub(crate) struct Animation { pub(crate) buffer_scale: i32, } +impl Animation { + pub(crate) fn capture_revision(&self) -> (Option, Instant, usize) { + (self.icon, self.epoch, self.frame) + } +} + impl Default for Animation { fn default() -> Self { Self { diff --git a/compositor/src/handlers/dispatch.rs b/compositor/src/handlers/dispatch.rs index ef9136e..d449d67 100644 --- a/compositor/src/handlers/dispatch.rs +++ b/compositor/src/handlers/dispatch.rs @@ -53,6 +53,12 @@ macro_rules! delegate_protocols { } delegate_protocols!( + protocols::ext::image_capture_source::v1::server::ext_image_capture_source_v1::ExtImageCaptureSourceV1, + protocols::ext::image_capture_source::v1::server::ext_output_image_capture_source_manager_v1::ExtOutputImageCaptureSourceManagerV1, + protocols::ext::image_copy_capture::v1::server::ext_image_copy_capture_manager_v1::ExtImageCopyCaptureManagerV1, + protocols::ext::image_copy_capture::v1::server::ext_image_copy_capture_session_v1::ExtImageCopyCaptureSessionV1, + protocols::ext::image_copy_capture::v1::server::ext_image_copy_capture_cursor_session_v1::ExtImageCopyCaptureCursorSessionV1, + protocols::ext::image_copy_capture::v1::server::ext_image_copy_capture_frame_v1::ExtImageCopyCaptureFrameV1, protocols::wp::idle_inhibit::zv1::server::zwp_idle_inhibit_manager_v1::ZwpIdleInhibitManagerV1, protocols::wp::idle_inhibit::zv1::server::zwp_idle_inhibitor_v1::ZwpIdleInhibitorV1, protocols::ext::data_control::v1::server::ext_data_control_device_v1::ExtDataControlDeviceV1, diff --git a/compositor/src/handlers/image_capture.rs b/compositor/src/handlers/image_capture.rs new file mode 100644 index 0000000..2b4727f --- /dev/null +++ b/compositor/src/handlers/image_capture.rs @@ -0,0 +1,742 @@ +//! Persistent, damage-driven capture for the native portal. One frame may be +//! outstanding per session; client buffers remain owned until GPU completion. +use smithay::reexports::calloop::RegistrationToken; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use ferese_layout::WindowId; +use smithay::backend::allocator::{Fourcc, Modifier}; +use smithay::backend::egl::EGLDevice; +use smithay::backend::renderer::gles::{Capability, GlesRenderer, GlesTexture}; +use smithay::backend::renderer::Texture; +use smithay::backend::renderer::utils::{CommitCounter, RendererSurfaceStateUserData, SurfaceView}; +use smithay::output::{Output, WeakOutput}; +use smithay::reexports::wayland_protocols::ext::image_copy_capture::v1::server::ext_image_copy_capture_frame_v1::FailureReason; +use smithay::reexports::wayland_server::backend::ObjectId; +use smithay::reexports::wayland_server::{Client, DisplayHandle, Resource}; +use smithay::reexports::wayland_server::protocol::wl_shm; +use smithay::utils::{Buffer, Logical, Point, Rectangle, Size, Transform}; +use smithay::wayland::compositor::{TraversalAction, with_surface_tree_downward}; +use smithay::wayland::image_capture_source::{ImageCaptureSource, ImageCaptureSourceHandler, OutputCaptureSourceHandler, OutputCaptureSourceState}; +use smithay::wayland::image_copy_capture::{BufferConstraints, DmabufConstraints, Frame, FrameRef, ImageCopyCaptureHandler, ImageCopyCaptureState, Session, SessionRef}; + +use crate::Ferese; + +const MAX_SESSIONS: usize = 16; + +pub(super) enum Source { + Output(WeakOutput), + Window(WindowId), +} + +#[derive(PartialEq)] +struct SurfaceStamp { + id: ObjectId, + commit: CommitCounter, + view: SurfaceView, + scale: i32, + transform: Transform, + alpha: Option, +} + +#[derive(PartialEq)] +struct WindowStamp { + surfaces: Vec, + geometry: Rectangle, + scale: f64, + epoch: u64, + cursor: Option, +} + +#[derive(PartialEq)] +struct CursorStamp { + location: Point, + geometry: Rectangle, + image: CursorImageStamp, +} + +#[derive(PartialEq)] +enum CursorImageStamp { + Named(Option, Instant, usize), + Surface(Vec, Point), + Hidden, +} + +fn surface_stamp( + root: &smithay::reexports::wayland_server::protocol::wl_surface::WlSurface, + crop: Option>, +) -> Vec { + let mut stamp = Vec::new(); + with_surface_tree_downward( + root, + Point::::default(), + |surface, states, location| { + let Some(data) = states.data_map.get::() else { + return TraversalAction::SkipChildren; + }; + use smithay::wayland::alpha_modifier::AlphaModifierSurfaceCachedState; + let alpha = states + .cached_state + .get::() + .current() + .multiplier_f32(); + let data = data.lock().expect("surface metadata mutex poisoned"); + let Some(view) = data.view() else { + return TraversalAction::SkipChildren; + }; + let location = *location + view.offset.to_f64(); + if crop.is_none_or(|crop| { + Rectangle::new(location, view.dst.to_f64()).overlaps(crop.to_f64()) + }) { + stamp.push(SurfaceStamp { + id: surface.id(), + commit: data.current_commit(), + view, + scale: data.buffer_scale(), + transform: data.buffer_transform(), + alpha, + }); + } + // An off-crop parent can have a child offset back into the captured area. + TraversalAction::DoChildren(location) + }, + |_, _, _| {}, + |_, _, _| true, + ); + stamp +} + +fn pending_frame_callbacks( + root: &smithay::reexports::wayland_server::protocol::wl_surface::WlSurface, +) -> bool { + let mut pending = false; + with_surface_tree_downward( + root, + (), + |_, states, _| { + if states + .data_map + .get::() + .is_some_and(|data| { + data.lock() + .expect("surface metadata mutex poisoned") + .view() + .is_none() + }) + { + return TraversalAction::SkipChildren; + } + pending |= !states + .cached_state + .get::() + .current() + .frame_callbacks + .is_empty(); + TraversalAction::DoChildren(()) + }, + |_, _, _| {}, + |_, _, _| true, + ); + pending +} + +struct Active { + session: Session, + pending: Option, + revision: Option, + window_stamp: Option, + size: Size, + dma_node: Option, +} + +struct InFlight { + frame: Frame, + session: SessionRef, + epoch: u64, + fence: RegistrationToken, + timeout: RegistrationToken, +} + +pub(crate) struct CaptureSessions { + copy: ImageCopyCaptureState, + outputs: OutputCaptureSourceState, + active: Vec, + in_flight: HashMap, + next_fence: u64, + callback_timer: Option, +} + +fn authorized(client: &Client) -> bool { + super::screencopy::capture_allowed() + && client + .get_data::() + .is_some_and(|state| { + state + .capabilities + .contains(crate::private_client::ClientCapabilities::WINDOW_CAPTURE) + }) +} + +impl CaptureSessions { + pub(crate) fn new(display: &DisplayHandle) -> Self { + Self { + copy: ImageCopyCaptureState::new_with_filter::(display, authorized), + outputs: OutputCaptureSourceState::new_with_filter::(display, authorized), + active: Vec::new(), + in_flight: HashMap::new(), + next_fence: 0, + callback_timer: None, + } + } + + pub(crate) fn has_sessions(&self) -> bool { + self.active.iter().any(|active| { + active.pending.is_some() + && matches!( + active.session.source().user_data().get::(), + Some(Source::Output(_)) + ) + }) + } + + pub(super) fn pending_output(&self, output: &Output, cursor: bool) -> bool { + self.active.iter().any(|active| { + active.pending.is_some() + && active.session.draw_cursor() == cursor + && matches!(active.session.source().user_data().get::(), + Some(Source::Output(weak)) if weak.upgrade().as_ref() == Some(output)) + }) + } +} + +pub(super) fn dma_constraints(renderer: &GlesRenderer) -> Option> { + struct Cached(Option>); + renderer + .egl_context() + .user_data() + .get_or_insert_threadsafe(|| Cached(query_dma_constraints(renderer).map(Arc::new))) + .0 + .clone() +} + +fn query_dma_constraints(renderer: &GlesRenderer) -> Option { + if !renderer.capabilities().contains(&Capability::ExportFence) { + return None; + } + let display = renderer.egl_context().display(); + if !display + .extensions() + .iter() + .any(|extension| extension == "EGL_ANDROID_native_fence_sync") + { + return None; + } + let node = EGLDevice::device_for_display(display) + .ok()? + .try_get_render_node() + .ok()??; + let format = smithay::backend::allocator::Format { + code: Fourcc::Xrgb8888, + modifier: Modifier::Linear, + }; + display + .dmabuf_render_formats() + .contains(&format) + .then_some(DmabufConstraints { + node, + formats: vec![(Fourcc::Xrgb8888, vec![Modifier::Linear])], + }) +} + +impl ImageCaptureSourceHandler for Ferese {} +impl OutputCaptureSourceHandler for Ferese { + fn output_capture_source_state(&mut self) -> &mut OutputCaptureSourceState { + &mut self.image_capture.outputs + } + fn output_source_created(&mut self, source: ImageCaptureSource, output: &Output) { + source + .user_data() + .insert_if_missing(|| Source::Output(output.downgrade())); + } +} + +impl ImageCopyCaptureHandler for Ferese { + fn image_copy_capture_state(&mut self) -> &mut ImageCopyCaptureState { + &mut self.image_capture.copy + } + fn capture_constraints(&mut self, source: &ImageCaptureSource) -> Option { + let (size, output) = self.capture_source_geometry(source)?; + Some(BufferConstraints { + size, + shm: vec![wl_shm::Format::Xrgb8888], + dma: self.capture_dma_constraints(&output).as_deref().cloned(), + }) + } + fn new_session(&mut self, session: Session) { + if self.image_capture.active.len() >= MAX_SESSIONS { + return; + } + let Some(constraints) = session.current_constraints() else { + return; + }; + self.image_capture.active.push(Active { + session, + pending: None, + revision: None, + window_stamp: None, + size: constraints.size, + dma_node: constraints.dma.as_ref().map(|dma| dma.node), + }); + } + fn frame(&mut self, session: &SessionRef, frame: Frame) { + let Some(active) = self + .image_capture + .active + .iter_mut() + .find(|active| active.session.as_ref() == *session) + else { + return; + }; + if active.pending.is_some() { + frame.fail(FailureReason::Unknown); + return; + } + active.pending = Some(frame); + if let Some(Source::Output(weak)) = session.source().user_data().get::() + && let Some(output) = weak.upgrade() + { + crate::backends::direct::schedule_capture(self, &[output]); + } + } + + fn session_destroyed(&mut self, session: SessionRef) { + self.image_capture + .active + .retain(|active| active.session.as_ref() != session); + let fences: Vec<_> = self + .image_capture + .in_flight + .iter() + .filter_map(|(id, pending)| (pending.session == session).then_some(*id)) + .collect(); + for id in fences { + self.cancel_capture_fence(id); + } + } + fn frame_aborted(&mut self, frame: FrameRef) { + let fences: Vec<_> = self + .image_capture + .in_flight + .iter() + .filter_map(|(id, pending)| (pending.frame == frame).then_some(*id)) + .collect(); + for id in fences { + self.cancel_capture_fence(id); + } + for active in &mut self.image_capture.active { + if active + .pending + .as_ref() + .is_some_and(|pending| *pending == frame) + { + active.pending.take(); + } + } + } +} + +impl Ferese { + fn capture_source_geometry( + &self, + source: &ImageCaptureSource, + ) -> Option<(Size, Output)> { + if self.session_lock.active() || !source.alive() || !super::screencopy::capture_allowed() { + return None; + } + let (size, output) = match source.user_data().get::()? { + Source::Window(id) => { + let info = self.window_capture_info(*id)?; + let size = info.geometry.size.to_physical_precise_round(info.scale); + (Size::from((size.w, size.h)), info.output) + } + Source::Output(weak) => { + let output = weak.upgrade()?; + if !self.output_ids.contains_key(&output) { + return None; + } + let size = output + .current_transform() + .transform_size(output.current_mode()?.size); + (Size::from((size.w, size.h)), output) + } + }; + let bytes = u64::try_from(size.w) + .ok()? + .checked_mul(u64::try_from(size.h).ok()?)? + .checked_mul(4)?; + if bytes == 0 || bytes > 128 * 1024 * 1024 { + return None; + } + Some((size, output)) + } + + fn capture_dma_constraints(&mut self, output: &Output) -> Option> { + if let Some(backend) = self.nested_backend.clone() { + backend + .try_borrow_mut() + .ok() + .and_then(|mut backend| dma_constraints(backend.renderer())) + } else { + self.direct_backend + .as_mut() + .and_then(|backend| backend.capture_renderer(output)) + .and_then(|renderer| dma_constraints(renderer)) + } + } + + pub(super) fn refresh_capture_source(&mut self, source: &ImageCaptureSource) { + let mut outputs = Vec::new(); + for active in &mut self.image_capture.active { + if active.session.source() != *source { + continue; + } + active.revision = None; + active.window_stamp = None; + if active.pending.is_some() + && let Some(Source::Output(weak)) = source.user_data().get::() + && let Some(output) = weak.upgrade() + && !outputs.contains(&output) + { + outputs.push(output); + } + } + crate::backends::direct::schedule_capture(self, &outputs); + } + + pub(crate) fn refresh_capture_sessions(&mut self) { + self.image_capture.copy.cleanup(); + let mut callback_delay: Option = None; + let mut sessions = std::mem::take(&mut self.image_capture.active); + sessions.retain_mut(|active| { + let Some((size, output)) = self.capture_source_geometry(&active.session.source()) + else { + let session = active.session.as_ref(); + let fences: Vec<_> = self + .image_capture + .in_flight + .iter() + .filter_map(|(id, pending)| (pending.session == session).then_some(*id)) + .collect(); + for id in fences { + self.cancel_capture_fence(id); + } + return false; + }; + let dma = self.capture_dma_constraints(&output); + let dma_node = dma.as_ref().map(|dma| dma.node); + if size != active.size || dma_node != active.dma_node { + active.size = size; + active.dma_node = dma_node; + active.revision = None; + active.window_stamp = None; + active.session.update_constraints(BufferConstraints { + size, + shm: vec![wl_shm::Format::Xrgb8888], + dma: dma.as_deref().cloned(), + }); + if let Some(frame) = active.pending.take() { + frame.fail(FailureReason::BufferConstraints); + } + } + if let Some(Source::Window(id)) = active.session.source().user_data().get::() { + let id = *id; + if let Some(frame) = active.pending.take() { + let Some(info) = self.window_capture_info(id) else { + return false; + }; + let callback_interval = Duration::from_nanos( + 1_000_000_000_000 + / info + .output + .current_mode() + .map(|mode| mode.refresh.max(1000) as u64) + .unwrap_or(60_000), + ); + if let Some(toplevel) = info.window.toplevel() + && pending_frame_callbacks(toplevel.wl_surface()) + { + smithay::desktop::utils::send_frames_surface_tree( + toplevel.wl_surface(), + &info.output, + self.start_time.elapsed(), + Some(callback_interval), + |_, _| None, + ); + } + let mut stamp = WindowStamp { + surfaces: Vec::new(), + geometry: info.geometry, + scale: info.scale, + epoch: self.capture_epoch, + cursor: None, + }; + if let Some(toplevel) = info.window.toplevel() { + stamp.surfaces = surface_stamp(toplevel.wl_surface(), Some(info.geometry)); + } + let cursor = active + .session + .draw_cursor() + .then(|| self.window_capture_cursor(id, &info.window, info.geometry)) + .flatten(); + if let Some(geometry) = cursor + && let Some(pointer) = self.seat.get_pointer() + { + use smithay::input::pointer::CursorImageStatus; + let image = match &self.cursor_status { + CursorImageStatus::Named(_) => { + let (icon, epoch, frame) = self.cursor_animation.capture_revision(); + CursorImageStamp::Named(icon, epoch, frame) + } + CursorImageStatus::Surface(surface) => { + let hotspot = + smithay::wayland::compositor::with_states(surface, |states| { + states + .data_map + .get::( + ) + .map(|data| { + data.lock() + .expect("cursor metadata mutex poisoned") + .hotspot + }) + .unwrap_or_default() + }); + CursorImageStamp::Surface(surface_stamp(surface, None), hotspot) + } + CursorImageStatus::Hidden => CursorImageStamp::Hidden, + }; + stamp.cursor = Some(CursorStamp { + location: pointer.current_location(), + geometry, + image, + }); + } + if active.window_stamp.as_ref() == Some(&stamp) { + active.pending = Some(frame); + if info + .window + .toplevel() + .is_some_and(|toplevel| pending_frame_callbacks(toplevel.wl_surface())) + { + callback_delay = + Some(callback_delay.map_or(callback_interval, |delay| { + delay.min(callback_interval) + })); + } + } else { + crate::backends::direct::capture::touch_cache(self); + let result = if let Some(backend) = self.nested_backend.clone() { + backend + .try_borrow_mut() + .map_err(|error| error.to_string()) + .and_then(|mut backend| { + crate::render::capture_stream_window( + backend.renderer(), + self, + &info, + cursor, + &frame.buffer(), + ) + }) + } else if let Some(mut backend) = self.direct_backend.take() { + let result = backend + .capture_renderer(&info.output) + .ok_or_else(|| "Capture output is unavailable".to_string()) + .and_then(|renderer| { + crate::render::capture_stream_window( + renderer, + self, + &info, + cursor, + &frame.buffer(), + ) + }); + self.direct_backend = Some(backend); + result + } else { + Err("Capture renderer is unavailable".into()) + }; + active.window_stamp = Some(stamp); + self.finish_stream_frame(frame, active.session.as_ref(), result); + } + } + } + true + }); + self.image_capture.active = sessions; + if let Some(delay) = callback_delay { + if self.image_capture.callback_timer.is_none() { + use smithay::reexports::calloop::timer::{TimeoutAction, Timer}; + match self + .loop_handle + .insert_source(Timer::from_duration(delay), |_, _, state| { + state.image_capture.callback_timer = None; + TimeoutAction::Drop + }) { + Ok(token) => self.image_capture.callback_timer = Some(token), + Err(error) => { + tracing::warn!(%error, "could not schedule captured window callbacks") + } + } + } + } else if let Some(timer) = self.image_capture.callback_timer.take() { + self.loop_handle.remove(timer); + } + } + + pub(crate) fn process_output_streams( + &mut self, + renderer: &mut GlesRenderer, + texture: &GlesTexture, + output: &Output, + cursor: bool, + revision: u64, + ) { + let mut sessions = std::mem::take(&mut self.image_capture.active); + for active in &mut sessions { + let matches = matches!(active.session.source().user_data().get::(), Some(Source::Output(weak)) if weak.upgrade().as_ref() == Some(output)); + if matches { + let size = output.current_transform().transform_size(texture.size()); + if active.size != Size::from((size.w, size.h)) { + if let Some(frame) = active.pending.take() { + frame.fail(FailureReason::BufferConstraints); + } + active.revision = None; + continue; + } + } + if matches + && active.session.draw_cursor() == cursor + && active.revision != Some(revision) + && let Some(frame) = active.pending.take() + { + let result = crate::render::capture_stream_texture( + renderer, + texture, + &frame.buffer(), + output.current_transform(), + false, + ); + active.revision = Some(revision); + self.finish_stream_frame(frame, active.session.as_ref(), result); + } + } + self.image_capture.active = sessions; + } + + fn cancel_capture_fence(&mut self, id: u64) { + if let Some(pending) = self.image_capture.in_flight.remove(&id) { + self.loop_handle.remove(pending.fence); + self.loop_handle.remove(pending.timeout); + } + } + + fn finish_stream_frame( + &mut self, + frame: Frame, + session: SessionRef, + result: Result, + ) { + match result { + Ok(sync) if sync.is_reached() => frame.success( + Transform::Normal, + None, + smithay::utils::Clock::::new().now(), + ), + Ok(sync) => { + let Some(fd) = sync.export() else { + frame.fail(FailureReason::Unknown); + return; + }; + use smithay::reexports::calloop::timer::{TimeoutAction, Timer}; + use smithay::reexports::calloop::{Interest, Mode, PostAction, generic::Generic}; + self.image_capture.next_fence = self.image_capture.next_fence.wrapping_add(1); + let id = self.image_capture.next_fence; + let source = Generic::new(fd, Interest::READ, Mode::OneShot); + let fence = match self.loop_handle.insert_source(source, move |_, _, state| { + if let Some(pending) = state.image_capture.in_flight.remove(&id) { + state.loop_handle.remove(pending.timeout); + if pending.epoch == state.capture_epoch + && state + .capture_source_geometry(&pending.session.source()) + .is_some() + && !pending.frame.has_failed() + { + pending.frame.success( + Transform::Normal, + None, + smithay::utils::Clock::::new().now(), + ); + } else { + pending.frame.fail( + if state + .capture_source_geometry(&pending.session.source()) + .is_some() + { + FailureReason::BufferConstraints + } else { + FailureReason::Stopped + }, + ); + } + } + Ok(PostAction::Remove) + }) { + Ok(token) => token, + Err(error) => { + tracing::warn!(%error, "could not register capture fence"); + frame.fail(FailureReason::Unknown); + return; + } + }; + let timeout = match self.loop_handle.insert_source( + Timer::from_duration(Duration::from_secs(5)), + move |_, _, state| { + if let Some(pending) = state.image_capture.in_flight.remove(&id) { + state.loop_handle.remove(pending.fence); + pending.frame.fail(FailureReason::Stopped); + state + .image_capture + .active + .retain(|active| active.session.as_ref() != pending.session); + } + TimeoutAction::Drop + }, + ) { + Ok(token) => token, + Err(error) => { + self.loop_handle.remove(fence); + tracing::warn!(%error, "could not register capture deadline"); + frame.fail(FailureReason::Unknown); + return; + } + }; + self.image_capture.in_flight.insert( + id, + InFlight { + frame, + session, + epoch: self.capture_epoch, + fence, + timeout, + }, + ); + } + Err(error) => { + tracing::warn!(%error, "stream capture failed"); + frame.fail(FailureReason::Unknown); + } + } + } +} diff --git a/compositor/src/handlers/mod.rs b/compositor/src/handlers/mod.rs index fc1fb04..2cc1af4 100644 --- a/compositor/src/handlers/mod.rs +++ b/compositor/src/handlers/mod.rs @@ -2,6 +2,7 @@ pub(crate) mod activation; mod compositor; mod dispatch; pub(crate) mod gamma_control; +pub(crate) mod image_capture; mod input_method; mod layer_shell; pub(crate) mod screencopy; diff --git a/compositor/src/handlers/screencopy.rs b/compositor/src/handlers/screencopy.rs index 4dacd34..9201f97 100644 --- a/compositor/src/handlers/screencopy.rs +++ b/compositor/src/handlers/screencopy.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -22,6 +23,8 @@ use crate::Ferese; const BYTES_PER_PIXEL: usize = 4; +type DamageHistory = Arc>>; + #[derive(Debug)] pub(crate) struct FrameData { pub(super) output: Option, @@ -29,6 +32,7 @@ pub(crate) struct FrameData { pub(super) overlay_cursor: bool, pub(super) used: Mutex, pub(super) snapshot: Mutex>, + pub(super) history: Option, } #[derive(Debug)] @@ -38,6 +42,7 @@ pub(crate) struct PendingScreencopy { region: Rectangle, overlay_cursor: bool, with_damage: bool, + history: Option, } #[derive(Debug)] @@ -140,6 +145,7 @@ impl PendingScreencopy { // The compositor-owned path deliberately omits the cursor. overlay_cursor: false, with_damage: false, + history: None, } } @@ -187,17 +193,17 @@ impl GlobalDispatch for Ferese { _global_data: &(), data_init: &mut DataInit<'_, Self>, ) { - data_init.init(resource, ()); + data_init.init(resource, DamageHistory::default()); } } -impl Dispatch for Ferese { +impl Dispatch for Ferese { fn request( _state: &mut Self, _client: &Client, _manager: &ZwlrScreencopyManagerV1, request: zwlr_screencopy_manager_v1::Request, - _data: &(), + history: &DamageHistory, _display: &DisplayHandle, data_init: &mut DataInit<'_, Self>, ) { @@ -207,7 +213,14 @@ impl Dispatch for Ferese { overlay_cursor, output, } => { - create_frame(data_init, frame, output, None, overlay_cursor != 0); + create_frame( + data_init, + frame, + output, + None, + overlay_cursor != 0, + history.clone(), + ); } zwlr_screencopy_manager_v1::Request::CaptureOutputRegion { frame, @@ -224,6 +237,7 @@ impl Dispatch for Ferese { output, Some(Rectangle::new((x, y).into(), (width, height).into())), overlay_cursor != 0, + history.clone(), ); } zwlr_screencopy_manager_v1::Request::Destroy => {} @@ -307,6 +321,12 @@ impl Ferese { return; }; + if let Some(history) = &data.history { + history + .lock() + .expect("capture damage history mutex poisoned") + .retain(|(output, _), _| self.output_ids.contains_key(output)); + } self.pending_screencopies.push(PendingScreencopy { sink: CaptureSink::Shm { frame: frame.clone(), @@ -316,8 +336,24 @@ impl Ferese { region: data.region, overlay_cursor: data.overlay_cursor, with_damage, + history: data.history.clone(), + }); + crate::backends::direct::schedule_capture(self, &[output]); + } + + pub(crate) fn prune_screencopies(&mut self) { + self.pending_screencopies.retain(|capture| { + if let CaptureSink::Shm { frame, buffer } = &capture.sink { + if !frame.is_alive() { + return false; + } + if !buffer.is_alive() { + frame.failed(); + return false; + } + } + true }); - crate::backends::direct::render_on(self, &[output]); } pub(crate) fn process_screencopies( @@ -330,6 +366,17 @@ impl Ferese { where R: ExportMem, { + self.process_screencopies_with_revision(renderer, framebuffer, output, overlay_cursor, None) + } + + pub(crate) fn process_screencopies_with_revision( + &mut self, + renderer: &mut R, + framebuffer: &R::Framebuffer<'_>, + output: &Output, + overlay_cursor: bool, + revision: Option, + ) -> bool { let mut remaining = Vec::new(); let mut framebuffer_binding_changed = false; let captures = std::mem::take(&mut self.pending_screencopies); @@ -347,10 +394,33 @@ impl Ferese { continue; } + let key = (output.clone(), overlay_cursor); + if capture.history.is_some() && revision.is_none() { + remaining.push(capture); + continue; + } + if capture.with_damage { + let previous = capture.history.as_ref().and_then(|history| { + history + .lock() + .expect("capture damage history mutex poisoned") + .get(&key) + .copied() + }); + if revision.is_none() || previous == revision { + remaining.push(capture); + continue; + } + } framebuffer_binding_changed = true; if complete_capture(renderer, framebuffer, &capture, self.start_time.elapsed()).is_err() { capture.fail(); + } else if let (Some(history), Some(revision)) = (&capture.history, revision) { + history + .lock() + .expect("capture damage history mutex poisoned") + .insert(key, revision); } } @@ -359,10 +429,11 @@ impl Ferese { } pub(crate) fn has_pending_screencopy(&self, output: &Output, overlay_cursor: bool) -> bool { - self.pending_screencopies.iter().any(|capture| { - capture.output == *output - && cursor_overlay_matches(capture.overlay_cursor, overlay_cursor) - }) + self.image_capture.pending_output(output, overlay_cursor) + || self.pending_screencopies.iter().any(|capture| { + capture.output == *output + && cursor_overlay_matches(capture.overlay_cursor, overlay_cursor) + }) } pub(crate) fn fail_screencopies(&mut self, output: &Output, overlay_cursor: bool) { @@ -391,6 +462,7 @@ fn create_frame( wl_output: smithay::reexports::wayland_server::protocol::wl_output::WlOutput, requested: Option>, overlay_cursor: bool, + history: DamageHistory, ) { let Some(output) = Output::from_resource(&wl_output) else { let frame = data_init.init( @@ -401,6 +473,7 @@ fn create_frame( overlay_cursor, used: Mutex::new(true), snapshot: Mutex::new(None), + history: Some(history.clone()), }), ); frame.failed(); @@ -415,6 +488,7 @@ fn create_frame( overlay_cursor, used: Mutex::new(true), snapshot: Mutex::new(None), + history: Some(history.clone()), }), ); frame.failed(); @@ -428,6 +502,7 @@ fn create_frame( overlay_cursor, used: Mutex::new(false), snapshot: Mutex::new(None), + history: Some(history.clone()), }); let frame = data_init.init(frame, data); diff --git a/compositor/src/handlers/window_capture.rs b/compositor/src/handlers/window_capture.rs index e1ac0ee..9397cc6 100644 --- a/compositor/src/handlers/window_capture.rs +++ b/compositor/src/handlers/window_capture.rs @@ -4,6 +4,7 @@ use std::sync::{Arc, Mutex}; use ferese_layout::WindowId; use ferese_protocols::window_capture::v1::server::ferese_window_capture_manager_v1 as manager; use ferese_protocols::window_capture::v1::server::ferese_window_capture_manager_v1::FereseWindowCaptureManagerV1; +use smithay::backend::renderer::utils::{RendererSurfaceStateUserData, SurfaceView}; use smithay::reexports::wayland_protocols_wlr::screencopy::v1::server::zwlr_screencopy_frame_v1 as frame; use smithay::reexports::wayland_protocols_wlr::screencopy::v1::server::zwlr_screencopy_frame_v1::ZwlrScreencopyFrameV1; use smithay::reexports::wayland_server::protocol::wl_buffer::WlBuffer; @@ -11,7 +12,8 @@ use smithay::reexports::wayland_server::protocol::wl_shm; use smithay::reexports::wayland_server::{ Client, DataInit, Dispatch, DisplayHandle, GlobalDispatch, New, Resource, }; -use smithay::utils::Rectangle; +use smithay::utils::{Logical, Point, Rectangle}; +use smithay::wayland::compositor::{TraversalAction, with_surface_tree_downward}; use smithay::wayland::shm::with_buffer_contents_mut; use super::screencopy::FrameData; @@ -21,6 +23,15 @@ use crate::Ferese; const MAX_BYTES: usize = 128 * 1024 * 1024; const MAX_FRAMES: usize = 8; +fn capture_scale(output_scale: f64, buffer_scale: i32, view: SurfaceView) -> f64 { + if view.dst.w <= 0 || view.dst.h <= 0 { + return output_scale; + } + let native_x = view.src.size.w * f64::from(buffer_scale) / f64::from(view.dst.w); + let native_y = view.src.size.h * f64::from(buffer_scale) / f64::from(view.dst.h); + output_scale.max(native_x).max(native_y) +} + #[derive(Debug, Default)] pub(crate) struct Budget { bytes: usize, @@ -155,7 +166,7 @@ pub(super) fn init_global( } display.create_global::( - 1, + 2, Global { budget: Arc::default(), worker, @@ -237,6 +248,44 @@ impl Dispatch for Ferese { _: &DisplayHandle, data_init: &mut DataInit<'_, Self>, ) { + if let manager::Request::RefreshSource { source } = request { + if let Some(source) = + smithay::wayland::image_capture_source::ImageCaptureSource::from_resource(&source) + { + state.refresh_capture_source(&source); + } + return; + } + if let manager::Request::CreateSource { + window_hi, + window_lo, + source, + } = request + { + use smithay::wayland::image_capture_source::{ + ImageCaptureSource, ImageCaptureSourceData, + }; + let id = WindowId((u64::from(window_hi) << 32) | u64::from(window_lo)); + let capture = ImageCaptureSource::new(); + capture + .user_data() + .insert_if_missing(|| super::image_capture::Source::Window(id)); + let resource = data_init.init( + source, + ImageCaptureSourceData { + source: capture.clone(), + }, + ); + capture.add_instance(&resource); + if let Some(info) = state.window_capture_info(id) { + manager.source_geometry( + &resource, + info.geometry.size.w as u32, + info.geometry.size.h as u32, + ); + } + return; + } if let manager::Request::CaptureWindow { window_hi, window_lo, @@ -262,6 +311,7 @@ impl Dispatch for Ferese { overlay_cursor: false, used: Mutex::new(failed), snapshot: Mutex::new(None), + history: None, }), ); if failed { @@ -293,14 +343,16 @@ impl Dispatch for Ferese { } } +pub(crate) struct WindowInfo { + pub window: smithay::desktop::Window, + pub output: smithay::output::Output, + pub geometry: Rectangle, + pub scale: f64, + pub bytes: usize, +} + impl Ferese { - fn window_snapshot( - &mut self, - id: WindowId, - overlay_cursor: bool, - budget: &Arc>, - busy: &mut bool, - ) -> Option { + pub(super) fn window_capture_info(&self, id: WindowId) -> Option { if self.session_lock.active() || !super::screencopy::capture_allowed() { return None; } @@ -323,7 +375,34 @@ impl Ferese { .0 .clone(); let geometry = window.geometry(); - let scale = output.current_scale().fractional_scale(); + let output_scale = output.current_scale().fractional_scale(); + // A low-density parent can contain higher-density content. Match the + // rendered surface tree, excluding buffers outside the captured geometry. + let mut scale = output_scale; + if let Some(toplevel) = window.toplevel() { + with_surface_tree_downward( + toplevel.wl_surface(), + Point::::default(), + |_, states, location| { + let Some(data) = states.data_map.get::() else { + return TraversalAction::SkipChildren; + }; + let data = data + .lock() + .expect("Smithay surface metadata must remain consistent across callbacks"); + let Some(view) = data.view() else { + return TraversalAction::SkipChildren; + }; + let location = *location + view.offset.to_f64(); + if Rectangle::new(location, view.dst.to_f64()).overlaps(geometry.to_f64()) { + scale = capture_scale(scale, data.buffer_scale(), view); + } + TraversalAction::DoChildren(location) + }, + |_, _, _| {}, + |_, _, _| true, + ); + } let size: smithay::utils::Size = geometry.size.to_physical_precise_round(scale); let bytes = usize::try_from(size.w) @@ -333,33 +412,65 @@ impl Ferese { if bytes == 0 || bytes > MAX_BYTES { return None; } + Some(WindowInfo { + window, + output, + geometry, + scale, + bytes, + }) + } + + pub(super) fn window_capture_cursor( + &self, + id: WindowId, + window: &smithay::desktop::Window, + geometry: Rectangle, + ) -> Option> { + if !self.overview.is_presenting() && !self.capture_cursor_protected() { + self.seat.get_pointer().and_then(|pointer| { + let location = pointer.current_location(); + if self.window_under_visual(location).as_ref() != Some(window) { + return None; + } + let (x, y) = self.inverse_presented_window_point(id, location.x, location.y)?; + let surface_point = smithay::utils::Point::from((x, y)) + geometry.loc.to_f64(); + let (surface, _) = window + .surface_under(surface_point, smithay::desktop::WindowSurfaceType::ALL)?; + if pointer.current_focus().as_ref() != Some(&surface) { + return None; + } + Some(Rectangle::new( + (location - smithay::utils::Point::from((x, y))).to_i32_round(), + geometry.size, + )) + }) + } else { + None + } + } + + fn window_snapshot( + &mut self, + id: WindowId, + overlay_cursor: bool, + budget: &Arc>, + busy: &mut bool, + ) -> Option { + let WindowInfo { + window, + output, + geometry, + scale, + bytes, + } = self.window_capture_info(id)?; let Some(permit) = Permit::reserve(budget, bytes) else { *busy = true; return None; }; - let cursor = - if overlay_cursor && !self.overview.is_presenting() && !self.capture_cursor_protected() - { - self.seat.get_pointer().and_then(|pointer| { - let location = pointer.current_location(); - if self.window_under_visual(location).as_ref() != Some(&window) { - return None; - } - let (x, y) = self.inverse_presented_window_point(id, location.x, location.y)?; - let surface_point = smithay::utils::Point::from((x, y)) + geometry.loc.to_f64(); - let (surface, _) = window - .surface_under(surface_point, smithay::desktop::WindowSurfaceType::ALL)?; - if pointer.current_focus().as_ref() != Some(&surface) { - return None; - } - Some(Rectangle::new( - (location - smithay::utils::Point::from((x, y))).to_i32_round(), - geometry.size, - )) - }) - } else { - None - }; + let cursor = overlay_cursor + .then(|| self.window_capture_cursor(id, &window, geometry)) + .flatten(); let result = if let Some(backend) = self.nested_backend.clone() { let mut backend = backend.try_borrow_mut().ok()?; crate::render::capture_window_frame( @@ -476,6 +587,40 @@ pub(super) fn copy_snapshot( pub(crate) mod tests { use super::*; + #[test] + fn sharing_preserves_backing_density_at_fractional_display_scales() { + let integer = SurfaceView { + src: Rectangle::from_size((640., 480.).into()), + dst: (640, 480).into(), + offset: (0, 0).into(), + }; + let viewport = SurfaceView { + src: Rectangle::from_size((1280., 960.).into()), + ..integer + }; + for output_scale in [1., 1.25, 1.5, 1.75, 2.] { + assert_eq!(capture_scale(output_scale, 2, integer), 2.); + assert_eq!(capture_scale(output_scale, 1, viewport), 2.); + } + assert_eq!(capture_scale(2.5, 2, integer), 2.5); + let fractional = SurfaceView { + src: Rectangle::from_size((1120., 840.).into()), + ..integer + }; + assert_eq!(capture_scale(1.25, 1, fractional), 1.75); + assert_eq!(capture_scale(1.75, 1, fractional), 1.75); + } + + #[test] + fn sharing_uses_the_viewport_crop_instead_of_the_whole_buffer() { + let cropped = SurfaceView { + src: Rectangle::new((320., 240.).into(), (640., 480.).into()), + dst: (640, 480).into(), + offset: (0, 0).into(), + }; + assert_eq!(capture_scale(1.75, 1, cropped), 1.75); + } + /// Exercise both asynchronous publication boundaries using real resources. pub(crate) fn check_deferred_snapshot( state: &Ferese, @@ -505,6 +650,7 @@ pub(crate) mod tests { overlay_cursor: false, used: Mutex::new(false), snapshot: Mutex::new(None), + history: None, }) }; let (worker, _requests) = std::sync::mpsc::sync_channel(1); diff --git a/compositor/src/main.rs b/compositor/src/main.rs index 45de6d1..f60ac69 100644 --- a/compositor/src/main.rs +++ b/compositor/src/main.rs @@ -209,6 +209,8 @@ fn after_dispatch_with_redraw( ) { crate::backends::direct::gamma::sync_night_light(state); state.process_pending_dmabuf_imports(); + state.prune_screencopies(); + state.refresh_capture_sessions(); state.update_named_cursor(std::time::Instant::now()); state.poll_theme(); if state.session_lock.active() || state.input_capture.active() { diff --git a/compositor/src/render/capture.rs b/compositor/src/render/capture.rs index 4290b12..1564960 100644 --- a/compositor/src/render/capture.rs +++ b/compositor/src/render/capture.rs @@ -179,14 +179,37 @@ pub(crate) fn capture_window_frame( /// Convert premultiplied ABGR readback into the capture protocol's straight ARGB. pub(crate) fn convert_window_pixels(pixels: &mut [u8]) { + // Compile the rounded, saturated conversion into 64 KiB of read-only data. + // Capture threads share this table without allocating or dividing per pixel. + static STRAIGHT_CHANNELS: [[u8; 256]; 256] = { + let mut values = [[0; 256]; 256]; + let mut alpha = 1; + while alpha < 256 { + let mut channel = 0; + while channel < 256 { + let straight = (channel * 255 + alpha / 2) / alpha; + values[alpha][channel] = if straight > 255 { 255 } else { straight as u8 }; + channel += 1; + } + alpha += 1; + } + values + }; for pixel in pixels.as_chunks_mut::<4>().0 { pixel.swap(0, 2); - let alpha = pixel[3] as u16; + let alpha = pixel[3]; + // Opaque content needs only the channel swap; avoid three divisions + // per pixel for the usual window-sharing frame. + if alpha == 255 { + continue; + } + if alpha == 0 { + pixel[..3].fill(0); + continue; + } + let channels = &STRAIGHT_CHANNELS[usize::from(alpha)]; for channel in &mut pixel[..3] { - *channel = (*channel as u16 * 255 + alpha / 2) - .checked_div(alpha) - .unwrap_or(0) - .min(255) as u8; + *channel = channels[usize::from(*channel)]; } } } @@ -195,6 +218,93 @@ pub(crate) fn convert_window_pixels(pixels: &mut [u8]) { mod tests { use super::*; + #[test] + #[ignore = "manual release benchmark for window-capture pixel conversion"] + fn profile_window_conversion() { + use std::{ + hint::black_box, + time::{Duration, Instant}, + }; + + for (width, height) in [(1920usize, 1080usize), (3840, 2160)] { + for workload in ["opaque", "edges", "translucent", "mixed"] { + let mut source = vec![0u8; width * height * 4]; + for (i, pixel) in source.chunks_exact_mut(4).enumerate() { + let alpha: u8 = if workload == "mixed" { + i.wrapping_mul(37) as u8 + } else if workload == "translucent" + || (workload == "edges" + && (i % width < 16 + || i % width >= width - 16 + || i / width < 16 + || i / width >= height - 16)) + { + 128 + } else { + 255 + }; + let color = (i as u32).wrapping_mul(1664525).wrapping_add(1013904223); + let premultiplied = + |shift: u32| (((color >> shift) & 255) * u32::from(alpha) / 255) as u8; + pixel.copy_from_slice(&[ + premultiplied(0), + premultiplied(8), + premultiplied(16), + alpha, + ]); + } + let mut pixels = source.clone(); + for run in 0..=3 { + let mut elapsed = Duration::ZERO; + for _ in 0..3 { + pixels.copy_from_slice(&source); + let started = Instant::now(); + convert_window_pixels(black_box(&mut pixels)); + elapsed += started.elapsed(); + black_box(&pixels); + } + if run != 0 { + eprintln!( + "window_conversion_benchmark width={width} workload={workload} run={run} frames=3 elapsed_us={}", + elapsed.as_micros() + ); + } + } + } + } + } + + #[test] + fn conversion_matches_straight_argb_for_every_alpha_and_channel_value() { + let mut pixels = Vec::new(); + let mut expected = Vec::new(); + for alpha in 0..=255u8 { + for channel in 0..=255u8 { + let source = [channel, 255 - channel, channel ^ alpha, alpha]; + pixels.extend_from_slice(&source); + let straight = |value: u8| { + if alpha == 0 { + 0 + } else { + ((u32::from(value) * 255 + u32::from(alpha) / 2) / u32::from(alpha)) + .min(255) as u8 + } + }; + expected.extend_from_slice(&[ + straight(source[2]), + straight(source[1]), + straight(source[0]), + alpha, + ]); + } + } + // Incomplete trailing pixels are outside the conversion's input domain. + pixels.extend_from_slice(&[17, 29, 43]); + expected.extend_from_slice(&[17, 29, 43]); + convert_window_pixels(&mut pixels); + assert_eq!(pixels, expected); + } + #[test] fn readback_conversion_preserves_color_and_handles_zero_alpha() { let mut pixels = vec![10, 20, 30, 255, 32, 64, 128, 128, 9, 8, 7, 0, 1, 1, 1, 1]; @@ -207,3 +317,481 @@ mod tests { ); } } + +#[derive(Default)] +struct StreamResources { + textures: std::collections::VecDeque, + bytes: usize, + straight: Option, + copy: Option, +} + +impl StreamResources { + fn take_texture(&mut self, size: Size) -> Option { + let index = self + .textures + .iter() + .position(|texture| texture.size() == size)?; + let texture = self.textures.remove(index)?; + self.bytes -= texture.size().w as usize * texture.size().h as usize * 4; + Some(texture) + } + + fn put_texture(&mut self, texture: GlesTexture) { + let bytes = texture.size().w as usize * texture.size().h as usize * 4; + // Scratch targets are shared by size across sources. Keep alternating + // window sizes from reallocating every frame without retaining an + // unbounded collection of old source sizes. + const MAX_BYTES: usize = 128 * 1024 * 1024; + if bytes > MAX_BYTES { + return; + } + while self.textures.len() >= 8 || self.bytes + bytes > MAX_BYTES { + let Some(old) = self.textures.pop_front() else { + break; + }; + self.bytes -= old.size().w as usize * old.size().h as usize * 4; + } + self.bytes += bytes; + self.textures.push_back(texture); + } +} + +#[derive(Default)] +struct StreamCache(std::sync::Mutex); + +const STREAM_CAPTURE_SHADER: &str = r#" +//_DEFINES +#ifdef EXTERNAL +#extension GL_OES_EGL_image_external : require +#endif +precision highp float; +#ifdef EXTERNAL +uniform samplerExternalOES tex; +#else +uniform sampler2D tex; +#endif +varying vec2 v_coords; +uniform float alpha; +uniform float y_offset; +void main() { + // The renderer negates Y for inverted textures; translate that origin + // back into [0, 1] so full-buffer captures sample both rows correctly. + vec4 pixel = texture2D(tex, v_coords + vec2(0.0, y_offset)); +#ifdef NO_ALPHA + pixel.a = 1.0; +#endif +#ifdef STRAIGHT + float a = floor(pixel.a * 255.0 + 0.5); + vec3 numerator = floor(pixel.rgb * 255.0 + 0.5) * 255.0 + floor(a * 0.5); + vec3 quotient = floor(numerator / max(a, 1.0)); + vec3 remainder = numerator - quotient * max(a, 1.0); + quotient += step(vec3(max(a, 1.0)), remainder); + quotient -= vec3(lessThan(remainder, vec3(0.0))); + gl_FragColor = vec4(a == 0.0 ? vec3(0.0) : clamp(quotient, 0.0, 255.0) / 255.0, 1.0); +#else + gl_FragColor = vec4(pixel.rgb, 1.0); +#endif +} +"#; + +pub(crate) fn capture_stream_window( + renderer: &mut GlesRenderer, + state: &Ferese, + info: &crate::handlers::window_capture::WindowInfo, + cursor: Option>, + buffer: &smithay::reexports::wayland_server::protocol::wl_buffer::WlBuffer, +) -> Result { + let cache = renderer + .egl_context() + .user_data() + .get_or_insert_threadsafe(|| Arc::new(StreamCache::default())) + .clone(); + let reusable = cache + .0 + .lock() + .expect("stream texture cache mutex poisoned") + .take_texture({ + let size = info.geometry.size.to_physical_precise_round(info.scale); + (size.w, size.h).into() + }); + let mut snapshot = render_window_texture( + renderer, + &info.window, + info.geometry, + info.scale, + 128 * 1024 * 1024, + reusable, + ) + .map_err(|error| error.to_string())? + .ok_or("Window has no capturable content")?; + let size = snapshot.texture.size(); + if let Some(cursor) = cursor { + let elements = cursor_elements(state, renderer, cursor, info.scale); + let mut target = renderer + .bind(&mut snapshot.texture) + .map_err(|error| error.to_string())?; + let mut frame = renderer + .render(&mut target, (size.w, size.h).into(), Transform::Normal) + .map_err(|error| error.to_string())?; + draw_render_elements( + &mut frame, + info.scale, + &elements, + &[Rectangle::from_size((size.w, size.h).into())], + ) + .map_err(|error| error.to_string())?; + let _ = frame.finish().map_err(|error| error.to_string())?; + } + let result = + capture_stream_texture(renderer, &snapshot.texture, buffer, Transform::Normal, true); + cache + .0 + .lock() + .expect("stream texture cache mutex poisoned") + .put_texture(snapshot.texture); + result +} + +pub(crate) fn capture_stream_texture( + renderer: &mut GlesRenderer, + texture: &GlesTexture, + buffer: &smithay::reexports::wayland_server::protocol::wl_buffer::WlBuffer, + transform: Transform, + straight: bool, +) -> Result { + use smithay::backend::renderer::sync::SyncPoint; + use smithay::wayland::dmabuf::get_dmabuf; + use smithay::wayland::shm::with_buffer_contents_mut; + let cache = renderer + .egl_context() + .user_data() + .get_or_insert_threadsafe(|| Arc::new(StreamCache::default())) + .clone(); + let mut cache = cache.0.lock().expect("stream texture cache mutex poisoned"); + let program = if straight { + &mut cache.straight + } else { + &mut cache.copy + }; + if program.is_none() { + *program = + Some(compile_stream_program(renderer, straight).map_err(|error| error.to_string())?); + } + let program = program.as_ref().expect("stream shader compiled").clone(); + let source_size = texture.size(); + let size: Size = transform.transform_size((source_size.w, source_size.h).into()); + let draw = |renderer: &mut GlesRenderer, + target: &mut smithay::backend::renderer::gles::GlesTarget<'_>| { + draw_stream_texture(renderer, texture, target, transform, &program) + }; + if let Ok(dmabuf) = get_dmabuf(buffer) { + let mut dmabuf = dmabuf.clone(); + let mut target = renderer + .bind(&mut dmabuf) + .map_err(|error| error.to_string())?; + return draw(renderer, &mut target); + } + let valid = with_buffer_contents_mut(buffer, |_, length, data| { + data.width == size.w + && data.height == size.h + && data.format == smithay::reexports::wayland_server::protocol::wl_shm::Format::Xrgb8888 + && data.stride >= size.w * 4 + && data.offset >= 0 + && usize::try_from( + i64::from(data.offset) + i64::from(data.stride) * i64::from(data.height), + ) + .is_ok_and(|required| required <= length) + }) + .unwrap_or(false); + if !valid { + return Err("Invalid stream SHM buffer".into()); + } + let mut texture = match cache.take_texture((size.w, size.h).into()) { + Some(texture) => texture, + None => Offscreen::::create_buffer( + renderer, + Fourcc::Abgr8888, + (size.w, size.h).into(), + ) + .map_err(|error| error.to_string())?, + }; + { + let mut target = renderer + .bind(&mut texture) + .map_err(|error| error.to_string())?; + let _ = draw(renderer, &mut target)?; + } + let mapping = renderer + .copy_texture( + &texture, + Rectangle::from_size((size.w, size.h).into()), + Fourcc::Argb8888, + ) + .map_err(|error| error.to_string())?; + let source = renderer + .map_texture(&mapping) + .map_err(|error| error.to_string())?; + let row = size.w as usize * 4; + let bytes = row * size.h as usize; + if source.len() < bytes { + return Err("Incomplete stream readback".into()); + } + with_buffer_contents_mut(buffer, |destination, _, data| { + for (y, pixels) in source[..bytes].chunks_exact(row).enumerate() { + // SAFETY: the buffer bounds and stride were checked above. Source + // memory belongs to the renderer and cannot overlap the client SHM. + unsafe { + std::ptr::copy_nonoverlapping( + pixels.as_ptr(), + destination.add(data.offset as usize + y * data.stride as usize), + row, + ); + } + } + }) + .map_err(|error| format!("{error:?}"))?; + cache.put_texture(texture); + Ok(SyncPoint::signaled()) +} + +fn compile_stream_program( + renderer: &mut GlesRenderer, + straight: bool, +) -> Result { + use smithay::backend::renderer::gles::{UniformName, UniformType}; + let source = if straight { + format!("#define STRAIGHT\n{STREAM_CAPTURE_SHADER}") + } else { + STREAM_CAPTURE_SHADER.to_string() + }; + renderer + .compile_custom_texture_shader(source, &[UniformName::new("y_offset", UniformType::_1f)]) +} + +fn draw_stream_texture( + renderer: &mut GlesRenderer, + texture: &GlesTexture, + target: &mut smithay::backend::renderer::gles::GlesTarget<'_>, + transform: Transform, + program: &smithay::backend::renderer::gles::GlesTexProgram, +) -> Result { + let source_size = texture.size(); + let size: Size = transform.transform_size((source_size.w, source_size.h).into()); + let mut frame = renderer + .render(target, size, Transform::Normal) + .map_err(|error| error.to_string())?; + let rect = Rectangle::from_size(size); + frame + .clear(Color32F::new(0., 0., 0., 1.), &[rect]) + .map_err(|error| error.to_string())?; + frame + .render_texture_from_to( + texture, + Rectangle::from_size(source_size).to_f64(), + rect, + &[rect], + &[], + transform, + 1.0, + Some(program), + &[smithay::backend::renderer::gles::Uniform::new( + "y_offset", + if texture.is_y_inverted() { + 1.0f32 + } else { + 0.0f32 + }, + )], + ) + .map_err(|error| error.to_string())?; + frame.finish().map_err(|error| error.to_string()) +} + +pub(crate) fn clear_stream_capture_targets(renderer: &mut GlesRenderer) { + if let Some(cache) = renderer.egl_context().user_data().get::>() { + let mut cache = cache.0.lock().expect("stream texture cache mutex poisoned"); + cache.textures.clear(); + cache.bytes = 0; + } +} + +#[cfg(test)] +mod stream_gpu_tests { + use super::*; + use smithay::backend::egl::{EGLContext, EGLDevice, EGLDisplay}; + use smithay::backend::renderer::ImportMem; + + #[test] + #[ignore = "requires an EGL rendering device"] + fn straight_capture_shader_matches_every_alpha_and_channel_value() { + let device = EGLDevice::enumerate() + .unwrap() + .last() + .expect("an EGL rendering device"); + // SAFETY: the display retains the enumerated device for this test. + let display = unsafe { EGLDisplay::new(device).unwrap() }; + let context = EGLContext::new(&display).unwrap(); + // SAFETY: this thread exclusively owns the fresh context. + let mut renderer = unsafe { GlesRenderer::new(context).unwrap() }; + let mut pixels = Vec::with_capacity(256 * 256 * 4); + for alpha in 0..=255u8 { + for channel in 0..=255u8 { + pixels.extend_from_slice(&[ + channel, + channel.wrapping_add(85), + channel.wrapping_add(170), + alpha, + ]); + } + } + let source = renderer + .import_memory(&pixels, Fourcc::Abgr8888, (256, 256).into(), false) + .unwrap(); + let program = compile_stream_program(&mut renderer, true).unwrap(); + let mut destination = Offscreen::::create_buffer( + &mut renderer, + Fourcc::Abgr8888, + (256, 256).into(), + ) + .unwrap(); + { + let mut target = renderer.bind(&mut destination).unwrap(); + let _ = draw_stream_texture( + &mut renderer, + &source, + &mut target, + Transform::Normal, + &program, + ) + .unwrap(); + } + let mapping = renderer + .copy_texture( + &destination, + Rectangle::from_size((256, 256).into()), + Fourcc::Argb8888, + ) + .unwrap(); + let actual = renderer.map_texture(&mapping).unwrap(); + convert_window_pixels(&mut pixels); + for (index, (actual, expected)) in actual + .chunks_exact(4) + .zip(pixels.chunks_exact(4)) + .enumerate() + { + assert_eq!( + &actual[..3], + &expected[..3], + "alpha={} channel={}", + index / 256, + index % 256 + ); + assert_eq!(actual[3], 255, "BGRx capture must be opaque"); + } + } + #[test] + #[ignore = "requires an EGL rendering device"] + fn stream_transfer_matches_output_orientation_for_all_transforms() { + let device = EGLDevice::enumerate() + .unwrap() + .last() + .expect("an EGL rendering device"); + // SAFETY: the renderer retains the enumerated device's display. + let display = unsafe { EGLDisplay::new(device).unwrap() }; + let context = EGLContext::new(&display).unwrap(); + // SAFETY: this test exclusively owns the renderer's context. + let mut renderer = unsafe { GlesRenderer::new(context).unwrap() }; + use Transform::*; + let cases = [ + (Normal, "abcdef"), + (_90, "daebfc"), + (_180, "fedcba"), + (_270, "cfbead"), + (Flipped, "cbafed"), + (Flipped90, "adbecf"), + (Flipped180, "defabc"), + (Flipped270, "fcebda"), + ]; + let program = compile_stream_program(&mut renderer, false).unwrap(); + let pixels: Vec<_> = b"abcdef" + .iter() + .flat_map(|value| [*value, 0, 0, 255]) + .collect(); + for flipped in [false, true] { + let source = renderer + .import_memory(&pixels, Fourcc::Abgr8888, (3, 2).into(), flipped) + .unwrap(); + for (transform, expected) in cases { + let size = + transform.transform_size(Size::::from((3, 2))); + let mut target = + Offscreen::::create_buffer(&mut renderer, Fourcc::Abgr8888, size) + .unwrap(); + { + let mut target = renderer.bind(&mut target).unwrap(); + let _ = draw_stream_texture( + &mut renderer, + &source, + &mut target, + transform, + &program, + ) + .unwrap(); + } + let mapping = renderer + .copy_texture(&target, Rectangle::from_size(size), Fourcc::Abgr8888) + .unwrap(); + let actual = renderer.map_texture(&mapping).unwrap(); + let actual: Vec<_> = actual.chunks_exact(4).map(|pixel| pixel[0]).collect(); + // A flipped texture describes the same image with reversed source + // rows. Derive that reference before applying the output transform. + let expected = if flipped { + expected + .bytes() + .map(|value| if value < b'd' { value + 3 } else { value - 3 }) + .collect::>() + } else { + expected.as_bytes().to_vec() + }; + assert_eq!(actual, expected, "{transform:?}, texture flipped={flipped}"); + } + } + } + #[test] + #[ignore = "requires an EGL rendering device"] + fn alternating_capture_sizes_reuse_their_gpu_allocations() { + let device = EGLDevice::enumerate() + .unwrap() + .last() + .expect("an EGL rendering device"); + // SAFETY: the renderer retains the enumerated device's display. + let display = unsafe { EGLDisplay::new(device).unwrap() }; + let context = EGLContext::new(&display).unwrap(); + // SAFETY: this test exclusively owns the renderer's context. + let mut renderer = unsafe { GlesRenderer::new(context).unwrap() }; + let mut cache = StreamResources::default(); + let sizes = [Size::from((64, 48)), Size::from((96, 64))]; + let mut allocations = [None, None]; + for _ in 0..100 { + for (index, size) in sizes.into_iter().enumerate() { + let texture = cache.take_texture(size).unwrap_or_else(|| { + Offscreen::::create_buffer(&mut renderer, Fourcc::Abgr8888, size) + .unwrap() + }); + if let Some(previous) = allocations[index] { + assert_eq!( + texture.tex_id(), + previous, + "switching source sizes allocated another GPU target" + ); + } else { + allocations[index] = Some(texture.tex_id()); + } + cache.put_texture(texture); + } + } + assert_eq!(cache.textures.len(), 2); + assert_eq!(cache.bytes, (64 * 48 + 96 * 64) * 4); + } +} diff --git a/compositor/src/render/mod.rs b/compositor/src/render/mod.rs index e3f7f0d..a493ea0 100644 --- a/compositor/src/render/mod.rs +++ b/compositor/src/render/mod.rs @@ -20,7 +20,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::time::Duration; pub(crate) use capture::{ - capture_resize_snapshot, capture_window_buffer, capture_window_frame, convert_window_pixels, + capture_resize_snapshot, capture_stream_texture, capture_stream_window, capture_window_buffer, + capture_window_frame, clear_stream_capture_targets, convert_window_pixels, }; use decorations::{ border_uniforms, color_with_alpha, corner_program, framebuffer_clip_rect, @@ -37,8 +38,7 @@ use overview::{overview_chrome_element, overview_strip_elements}; pub(crate) use resources::RenderResources; use scene::cursor_elements; pub(crate) use scene::{ - animated_window_elements, capture_output_elements, frame_effect_metrics, layer_surfaces, - sampled_output_elements, + capture_output_elements, frame_effect_metrics, layer_surfaces, sampled_output_elements, }; pub(crate) use shader::SharedPixelShaderElement; use smithay::backend::allocator::Fourcc; diff --git a/compositor/src/render/scene.rs b/compositor/src/render/scene.rs index 45e8df5..c7a78d2 100644 --- a/compositor/src/render/scene.rs +++ b/compositor/src/render/scene.rs @@ -7,26 +7,17 @@ use smithay::backend::renderer::utils::CommitCounter; use super::{ AnimatedWindowRenderElement, ConstrainAlign, ConstrainBehavior, ConstrainReference, - CropRenderElement, CursorImageStatus, CursorImageSurfaceData, Duration, Ferese, - FrameEffectMetrics, GlesRenderer, Layer, LayerSurface, Logical, MaterialSurface, - MemoryRenderBufferRenderElement, Output, Physical, Point, PopupManager, Rectangle, Relocate, - RelocateRenderElement, RenderElementKind, Renderer, RescaleRenderElement, Resource, - RoundedRect, Size, SolidColorRenderElement, Transform, WaylandSurfaceRenderElement, - append_material_surface, backdrop, constrain_space_element, corner_program, - layer_map_for_output, material_element, overview_strip_elements, - render_elements_from_surface_tree, resize_content_behavior, rounded_visual_rect, - rounded_window_elements, window_border_element, window_corner_shape, window_shadow_element, - window_tint_element, with_states, + CropRenderElement, CursorImageStatus, CursorImageSurfaceData, Ferese, FrameEffectMetrics, + GlesRenderer, Layer, LayerSurface, Logical, MaterialSurface, MemoryRenderBufferRenderElement, + Output, Physical, Point, PopupManager, Rectangle, Relocate, RelocateRenderElement, + RenderElementKind, Renderer, RescaleRenderElement, Resource, RoundedRect, Size, + SolidColorRenderElement, Transform, WaylandSurfaceRenderElement, append_material_surface, + backdrop, constrain_space_element, corner_program, layer_map_for_output, material_element, + overview_strip_elements, render_elements_from_surface_tree, resize_content_behavior, + rounded_visual_rect, rounded_window_elements, window_border_element, window_corner_shape, + window_shadow_element, window_tint_element, with_states, }; -pub(crate) fn animated_window_elements( - state: &mut Ferese, - renderer: &mut GlesRenderer, - output: &Output, -) -> Vec { - output_elements(state, renderer, output, true) -} - pub(crate) fn frame_effect_metrics( _elements: &[AnimatedWindowRenderElement], _scale: f64, @@ -34,17 +25,6 @@ pub(crate) fn frame_effect_metrics( FrameEffectMetrics } -pub(crate) fn output_elements( - state: &mut Ferese, - renderer: &mut GlesRenderer, - output: &Output, - include_cursor: bool, -) -> Vec { - let frame = state.sample_frame(output, Duration::ZERO); - - sampled_output_elements(state, renderer, output, include_cursor, &frame) -} - pub(crate) fn sampled_output_elements( state: &mut Ferese, renderer: &mut GlesRenderer, diff --git a/compositor/src/session_lock/tests.rs b/compositor/src/session_lock/tests.rs index 7b41fa8..96230d5 100644 --- a/compositor/src/session_lock/tests.rs +++ b/compositor/src/session_lock/tests.rs @@ -894,7 +894,7 @@ fn ipc_events_follow_lock_lifecycle_without_exposing_desktop_objects() { fixture.state.publish_ipc_events(); let snapshot = fixture.state.ipc_event_snapshot(); assert!(snapshot.workspaces.as_array().unwrap().is_empty()); - assert!(snapshot.windows.as_array().unwrap().is_empty()); + assert!(snapshot.windows.is_empty()); assert!(snapshot.focus.is_null()); assert_eq!(snapshot.lock.phase, "acquiring"); assert!(receive.try_iter().any( diff --git a/compositor/src/state.rs b/compositor/src/state.rs index cec2580..b359569 100644 --- a/compositor/src/state.rs +++ b/compositor/src/state.rs @@ -458,8 +458,9 @@ pub struct Ferese { pub(crate) render: crate::render::RenderResources, pub(crate) capture_render: crate::render::RenderResources, pub(crate) capture_last_used: Option, - pub(crate) nested_capture_texture: Option, + pub(crate) nested_capture_texture: [Option; 2], pub(crate) capture_epoch: u64, + pub(crate) image_capture: crate::handlers::image_capture::CaptureSessions, capture_protected_windows: HashSet, capture_protected_cursor: Option, pub(crate) daemons: Option>>, @@ -733,6 +734,7 @@ impl Ferese { let mut activities = crate::activities::Engine::default(); activities.configure(&config.activity_settings, config.activity_matcher.clone()); + let image_capture = crate::handlers::image_capture::CaptureSessions::new(&display_handle); let mut state = Self { session_lock_state, session_lock: crate::session_lock::Lock::default(), @@ -778,8 +780,9 @@ impl Ferese { render: Default::default(), capture_render: Default::default(), capture_last_used: None, - nested_capture_texture: None, + nested_capture_texture: Default::default(), capture_epoch: 0, + image_capture, capture_protected_windows: HashSet::new(), capture_protected_cursor: None, display_presentation: Default::default(), diff --git a/compositor/src/state/animation.rs b/compositor/src/state/animation.rs index e587e27..ea94cf1 100644 --- a/compositor/src/state/animation.rs +++ b/compositor/src/state/animation.rs @@ -191,6 +191,17 @@ impl Ferese { return false; } + // Mark before advancing so the final settling frame invalidates capture + // too, even if output_has_animations becomes false during this tick. + for output in self.space.outputs() { + if crate::backends::direct::capture::changes::tracked(output) + && (self.output_has_animations(output) + || self.output_has_pending_visual_changes(output)) + { + crate::backends::direct::capture::changes::content_changed(output); + } + } + let delta = delta.mul_f64(self.animation_speed); if let Some(workspace) = self .focus_swipe() diff --git a/compositor/src/state/capture_privacy_tests.rs b/compositor/src/state/capture_privacy_tests.rs index c22a93e..a5dd5ee 100644 --- a/compositor/src/state/capture_privacy_tests.rs +++ b/compositor/src/state/capture_privacy_tests.rs @@ -296,7 +296,12 @@ fn pixels( output.clone(), Rectangle::from_size((320, 240).into()), )); - let mut target = Some(texture.clone()); + // Retain damage history as well as pixels: changing privacy policy + // must erase content already present in the reused capture target. + let mut target = std::mem::take(&mut state.nested_capture_texture); + target[0].get_or_insert_with(|| { + crate::backends::direct::capture::CaptureTarget::new(texture.clone(), output) + }); crate::backends::direct::capture::capture_output( state, renderer, @@ -304,6 +309,7 @@ fn pixels( output, &scene, ); + state.nested_capture_texture = target; let mut result = receiver.try_recv().unwrap().result.unwrap().pixels; for pixel in result.chunks_exact_mut(4) { pixel.swap(0, 2); @@ -749,13 +755,15 @@ fn capture_privacy_pixels_and_policy_transitions() { Rectangle::new((32, 24).into(), (160, 120).into()), )); let scene = state.sample_frame(&output, Duration::ZERO); + let mut target = std::mem::take(&mut state.nested_capture_texture); crate::backends::direct::capture::capture_output( &mut state, &mut renderer, - &mut Some(capture_texture.clone()), + &mut target, &output, &scene, ); + state.nested_capture_texture = target; let region = receiver.try_recv().unwrap().result.unwrap(); assert_eq!((region.width, region.height), (160, 120)); assert!( diff --git a/compositor/src/state/outputs.rs b/compositor/src/state/outputs.rs index eb4ee6b..46dc343 100644 --- a/compositor/src/state/outputs.rs +++ b/compositor/src/state/outputs.rs @@ -2,6 +2,7 @@ use super::{DesktopOutput, Ferese, Logical, Output, OutputId, Point}; impl Ferese { pub(crate) fn defer_output_redraw(&mut self, output: Output) { + crate::backends::direct::capture::changes::content_changed(&output); if !self.output_redraw_pending.contains(&output) { self.output_redraw_pending.push(output); } diff --git a/compositor/src/winit.rs b/compositor/src/winit.rs index 00e4a8e..7985c50 100644 --- a/compositor/src/winit.rs +++ b/compositor/src/winit.rs @@ -16,7 +16,7 @@ use smithay::utils::{Physical, Rectangle, Size, Transform}; use crate::Ferese; use crate::metrics::{FrameEffectMetrics, RenderMetrics}; -use crate::render::{animated_window_elements, frame_effect_metrics, layer_surfaces}; +use crate::render::{frame_effect_metrics, layer_surfaces, sampled_output_elements}; pub(crate) type NestedBackend = Rc>>; type DamageRenderResult = Result< @@ -144,7 +144,8 @@ pub fn init(event_loop: &mut EventLoop, state: &mut Ferese) -> Result<() let rendered = (|| -> DamageRenderResult { { let (renderer, mut framebuffer) = backend.bind()?; - let elements = animated_window_elements(state, renderer, &output); + let scene = state.sample_frame(&output, Duration::ZERO); + let elements = sampled_output_elements(state, renderer, &output, true, &scene); let effects = frame_effect_metrics(&elements, output.current_scale().fractional_scale()); let result = damage_tracker.render_output( renderer, @@ -164,8 +165,7 @@ pub fn init(event_loop: &mut EventLoop, state: &mut Ferese) -> Result<() if state.has_pending_screencopy(&output, false) || state.has_pending_screencopy(&output, true) { - let scene = state.sample_frame(&output, Duration::ZERO); - let mut capture_texture = state.nested_capture_texture.take(); + let mut capture_texture = std::mem::take(&mut state.nested_capture_texture); crate::backends::direct::capture::capture_output( state, renderer, diff --git a/crates/ferese-protocols/Cargo.toml b/crates/ferese-protocols/Cargo.toml index c35f5fc..df75960 100644 --- a/crates/ferese-protocols/Cargo.toml +++ b/crates/ferese-protocols/Cargo.toml @@ -10,8 +10,8 @@ workspace = true [features] default = ["client", "server"] -client = ["dep:wayland-client", "wayland-protocols-wlr/client"] -server = ["dep:wayland-server", "wayland-protocols-wlr/server"] +client = ["dep:wayland-client", "wayland-protocols-wlr/client", "wayland-protocols/client"] +server = ["dep:wayland-server", "wayland-protocols-wlr/server", "wayland-protocols/server"] [dependencies] bitflags.workspace = true @@ -20,3 +20,5 @@ wayland-client = { workspace = true, optional = true } wayland-scanner.workspace = true wayland-server = { workspace = true, optional = true } wayland-protocols-wlr.workspace = true + +wayland-protocols = { version = "0.32", features = ["staging"] } diff --git a/crates/ferese-protocols/src/lib.rs b/crates/ferese-protocols/src/lib.rs index 74afbcd..de122dd 100644 --- a/crates/ferese-protocols/src/lib.rs +++ b/crates/ferese-protocols/src/lib.rs @@ -195,10 +195,12 @@ pub mod window_capture { pub mod client { use wayland_client; use wayland_client::protocol::*; + use wayland_protocols::ext::image_capture_source::v1::client::*; use wayland_protocols_wlr::screencopy::v1::client::*; pub mod __interfaces { use wayland_client::protocol::__interfaces::*; + use wayland_protocols::ext::image_capture_source::v1::client::__interfaces::*; use wayland_protocols_wlr::screencopy::v1::client::__interfaces::*; wayland_scanner::generate_interfaces!( @@ -214,11 +216,13 @@ pub mod window_capture { #[cfg(feature = "server")] pub mod server { + use wayland_protocols::ext::image_capture_source::v1::server::*; use wayland_protocols_wlr::screencopy::v1::server::*; use wayland_server; use wayland_server::protocol::*; pub mod __interfaces { + use wayland_protocols::ext::image_capture_source::v1::server::__interfaces::*; use wayland_protocols_wlr::screencopy::v1::server::__interfaces::*; use wayland_server::protocol::__interfaces::*; diff --git a/portal/Cargo.toml b/portal/Cargo.toml index 2885ded..1ebdf1b 100644 --- a/portal/Cargo.toml +++ b/portal/Cargo.toml @@ -40,6 +40,8 @@ serde.workspace = true serde_json.workspace = true tokio = { version = "1", features = ["rt-multi-thread", "macros", "time", "sync", "process", "io-util", "net"] } wayland-client.workspace = true -wayland-protocols = { version = "0.32", features = ["client", "unstable"] } +wayland-protocols = { version = "0.32", features = ["client", "unstable", "staging"] } wayland-protocols-wlr.workspace = true zbus = { version = "5.19.0", default-features = false, features = ["tokio"] } + +gbm = { version = "0.18", default-features = false } diff --git a/portal/src/capture.rs b/portal/src/capture.rs index cdb9620..268e874 100644 --- a/portal/src/capture.rs +++ b/portal/src/capture.rs @@ -1,19 +1,13 @@ -use std::fs::File; -use std::os::fd::{AsFd, AsRawFd, FromRawFd}; +use std::os::fd::AsRawFd; use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use std::time::{Duration, Instant}; use ferese_protocols::window_capture::v1::client::ferese_window_capture_manager_v1 as window_manager; -use memmap2::MmapMut; use serde::{Deserialize, Serialize}; -use wayland_client::protocol::{wl_buffer, wl_output, wl_registry, wl_shm, wl_shm_pool}; +use wayland_client::protocol::{wl_output, wl_registry, wl_shm}; use wayland_client::{Connection, Dispatch, EventQueue, QueueHandle, WEnum}; -use wayland_protocols_wlr::screencopy::v1::client::{ - zwlr_screencopy_frame_v1 as frame, zwlr_screencopy_manager_v1 as manager, -}; - -pub const BUSY: &str = "Window capture snapshot pool is busy"; +use wayland_protocols_wlr::screencopy::v1::client::zwlr_screencopy_manager_v1 as manager; #[derive(Clone, Debug, Serialize, Deserialize)] pub struct Source { @@ -34,53 +28,23 @@ impl Source { struct Output { global: u32, - proxy: wl_output::WlOutput, + _proxy: wl_output::WlOutput, source: Source, transform: wl_output::Transform, } -pub struct Frame { - pub logical_size: Option<(u32, u32)>, - pub width: u32, - pub height: u32, - pub stride: u32, - pub pixels: Vec, -} - -struct Buffer { - _file: File, - map: MmapMut, - proxy: wl_buffer::WlBuffer, - width: u32, - height: u32, - stride: u32, - format: wl_shm::Format, -} - -impl Drop for Buffer { - fn drop(&mut self) { - self.proxy.destroy(); - } -} - #[derive(Default)] struct State { outputs: Vec, manager: Option, window_manager: Option, shm: Option, - buffer: Option, - result: Option>, - flipped: bool, - logical_size: Option<(u32, u32)>, - busy: bool, } pub struct Capture { connection: Connection, queue: EventQueue, state: State, - selected_output: Option, } impl Capture { @@ -96,7 +60,6 @@ impl Capture { connection, queue, state: State::default(), - selected_output: None, }; // Two syncs deliver registry bindings and output metadata without an // uninterruptible roundtrip on a stalled compositor. @@ -134,14 +97,6 @@ impl Capture { .map(|output| output.global) } - pub fn pin_output(&mut self, name: &str, generation: u32) -> Result<(), String> { - if self.generation(name) != Some(generation) { - return Err("Shared monitor was replaced".into()); - } - self.selected_output = Some(generation); - Ok(()) - } - pub fn validate_sources( &mut self, selected: &[(String, u32)], @@ -170,99 +125,6 @@ impl Capture { .collect() } - pub fn frame( - &mut self, - name: &str, - cursor: bool, - stop: &AtomicBool, - mut pixels: Vec, - ) -> Result { - self.state.result = None; - self.state.flipped = false; - self.state.logical_size = None; - self.state.busy = false; - let (frame, transform) = if let Some(id) = name - .strip_prefix("window:") - .and_then(|id| id.parse::().ok()) - { - let manager = self - .state - .window_manager - .as_ref() - .ok_or("Window capture is unavailable")?; - ( - manager.capture_window( - (id >> 32) as u32, - id as u32, - cursor as u32, - &self.queue.handle(), - (), - ), - wl_output::Transform::Normal, - ) - } else { - let output = self - .state - .outputs - .iter() - .find(|o| o.source.name == name) - .ok_or("Shared monitor disconnected")?; - if self - .selected_output - .is_some_and(|selected| selected != output.global) - { - return Err("Shared monitor was replaced".into()); - } - self.selected_output = Some(output.global); - let manager = self - .state - .manager - .as_ref() - .ok_or("Monitor capture is unavailable")?; - ( - manager.capture_output(cursor as i32, &output.proxy, &self.queue.handle(), ()), - output.transform, - ) - }; - let deadline = Instant::now() + Duration::from_secs(5); - - while self.state.result.is_none() { - if let Err(error) = self.dispatch(stop, deadline) { - frame.destroy(); - return Err(error); - } - } - - frame.destroy(); - self.state - .result - .take() - .expect("the capture dispatch loop exits after storing its result")?; - - let buffer = self - .state - .buffer - .as_ref() - .ok_or("Capture supplied no buffer")?; - let (width, height) = copy_oriented_pixels( - &buffer.map, - buffer.width, - buffer.height, - buffer.stride, - transform, - self.state.flipped, - &mut pixels, - ); - - Ok(Frame { - logical_size: self.state.logical_size, - width, - height, - stride: width * 4, - pixels, - }) - } - fn dispatch(&mut self, stop: &AtomicBool, deadline: Instant) -> Result<(), String> { if stop.load(Ordering::Relaxed) { return Err("Capture cancelled".into()); @@ -321,7 +183,7 @@ impl Dispatch for State { "wl_output" => state.outputs.push(Output { global: name, transform: wl_output::Transform::Normal, - proxy: registry.bind(name, version.min(4), qh, name), + _proxy: registry.bind(name, version.min(4), qh, name), source: Source { name: format!("output-{name}"), label: "Display".into(), @@ -387,100 +249,6 @@ impl Dispatch for State { } } -impl Dispatch for State { - fn event( - state: &mut Self, - frame: &frame::ZwlrScreencopyFrameV1, - event: frame::Event, - _: &(), - _: &Connection, - qh: &QueueHandle, - ) { - match event { - frame::Event::Buffer { - format: WEnum::Value(format), - width, - height, - stride, - } => { - let result = (|| { - if format != wl_shm::Format::Argb8888 && format != wl_shm::Format::Xrgb8888 { - return Err("Unsupported capture pixel format".into()); - } - - let bytes = buffer_size(width, height, stride)?; - if let Some(buffer) = &state.buffer - && (buffer.width, buffer.height, buffer.stride, buffer.format) - == (width, height, stride, format) - { - frame.copy(&buffer.proxy); - return Ok(()); - } - // SAFETY: fixed NUL-terminated name and a newly owned memfd. - let fd = unsafe { - libc::memfd_create(c"ferese-screencast".as_ptr(), libc::MFD_CLOEXEC) - }; - - if fd < 0 { - return Err(std::io::Error::last_os_error().to_string()); - } - - // SAFETY: memfd_create succeeded above and returned a fresh descriptor whose - // ownership transfers to File. - let file = unsafe { File::from_raw_fd(fd) }; - file.set_len(bytes as u64).map_err(|e| e.to_string())?; - // SAFETY: the file has the checked mapping length and lives with the mapping. - let map = unsafe { MmapMut::map_mut(&file) }.map_err(|e| e.to_string())?; - let pool = state - .shm - .as_ref() - .ok_or("Missing shared memory global")? - .create_pool(file.as_fd(), bytes as i32, qh, ()); - let proxy = pool.create_buffer( - 0, - width as i32, - height as i32, - stride as i32, - format, - qh, - (), - ); - - pool.destroy(); - frame.copy(&proxy); - state.buffer = Some(Buffer { - _file: file, - map, - proxy, - width, - height, - stride, - format, - }); - - Ok::<(), String>(()) - })(); - - if let Err(e) = result { - state.result = Some(Err(e)); - } - } - frame::Event::Ready { .. } => state.result = Some(Ok(())), - frame::Event::Failed => { - state.result = Some(Err(if state.busy { - BUSY.into() - } else { - "Capture ended: session locked, source closed, or compositor unavailable".into() - })) - } - frame::Event::Flags { - flags: WEnum::Value(flags), - } => state.flipped = flags.contains(frame::Flags::YInvert), - _ => (), - } - } -} - fn swaps_axes(transform: wl_output::Transform) -> bool { matches!( transform, @@ -494,6 +262,7 @@ fn swaps_axes(transform: wl_output::Transform) -> bool { // Screencopy supplies output-buffer coordinates. Normalize both the output's // presentation transform and the independent YInvert storage flag before // publishing upright, tightly packed BGRx frames to PipeWire. +#[cfg(test)] fn copy_oriented_pixels( source: &[u8], width: u32, @@ -545,6 +314,7 @@ fn copy_oriented_pixels( (out_width, out_height) } +#[cfg(test)] fn buffer_size(width: u32, height: u32, stride: u32) -> Result { let row = width.checked_mul(4).ok_or("Capture dimensions overflow")?; let bytes = stride @@ -572,27 +342,8 @@ impl Dispatch } wayland_client::delegate_noop!(State: ignore wl_shm::WlShm); -wayland_client::delegate_noop!(State: ignore wl_shm_pool::WlShmPool); -wayland_client::delegate_noop!(State: ignore wl_buffer::WlBuffer); wayland_client::delegate_noop!(State: ignore manager::ZwlrScreencopyManagerV1); -impl Dispatch for State { - fn event( - state: &mut Self, - _: &window_manager::FereseWindowCaptureManagerV1, - event: window_manager::Event, - _: &(), - _: &Connection, - _: &QueueHandle, - ) { - match event { - window_manager::Event::Geometry { width, height, .. } => { - state.logical_size = Some((width, height)) - } - window_manager::Event::Busy { .. } => state.busy = true, - _ => (), - } - } -} +wayland_client::delegate_noop!(State: ignore window_manager::FereseWindowCaptureManagerV1); #[cfg(test)] mod tests { @@ -622,7 +373,6 @@ mod tests { connection, queue, state: State::default(), - selected_output: None, }; capture.validate_sources(&[], &flag).unwrap_err() } diff --git a/portal/src/permissions.rs b/portal/src/permissions.rs index f848af8..a09ccc6 100644 --- a/portal/src/permissions.rs +++ b/portal/src/permissions.rs @@ -3,6 +3,7 @@ use std::path::Path; use std::sync::Arc; use std::time::Duration; +use ferese_ipc::events::Window; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::UnixStream; use tokio::sync::RwLock; @@ -197,7 +198,7 @@ impl Background { // Tokio owns the socket so cancelling the watch task also closes an idle stream. struct BackgroundEvents { stream: UnixStream, - windows: serde_json::Value, + windows: Vec, changed: bool, } @@ -238,7 +239,7 @@ impl BackgroundEvents { .map_err(|_| "Event subscription timed out".to_owned())??; Ok(Self { stream, - windows: serde_json::json!([]), + windows: Vec::new(), changed: false, }) } @@ -257,10 +258,9 @@ impl BackgroundEvents { self.changed = true; } Change::FocusChanged { focus } => { - for window in self.windows.as_array_mut().into_iter().flatten() { - window["focused"] = serde_json::json!( - focus["window"].is_number() && window["id"] == focus["window"] - ); + let focused = focus["window"].as_u64(); + for window in &mut self.windows { + window.focused = focused == Some(window.id); } self.changed = true; } @@ -364,14 +364,15 @@ async fn background_consent(app: &str, name: &str) -> Result, St )]))) } -fn app_states(windows: &serde_json::Value) -> HashMap { +fn app_states(windows: &[Window]) -> HashMap { let mut apps: HashMap = HashMap::new(); - for window in windows.as_array().into_iter().flatten() { - let Some(app) = window["app_id"].as_str().filter(|app| !app.is_empty()) else { + for window in windows { + let app = window.app_id.as_str(); + if app.is_empty() { continue; - }; + } let app = app.strip_suffix(".desktop").unwrap_or(app).to_owned(); - let state = if window["focused"] == true { 2 } else { 1 }; + let state = if window.focused { 2 } else { 1 }; apps.entry(app) .and_modify(|old| *old = (*old).max(state)) .or_insert(state); @@ -480,6 +481,18 @@ fn desktop_argument(argument: &str) -> Result { mod tests { use super::*; + fn window(id: u64, app: &str, focused: bool, workspace: u64) -> Window { + Window { + id, + app_id: app.into(), + title: format!("Window {id}"), + focused, + workspace, + floating: false, + fullscreen: false, + } + } + async fn send_frame(peer: &mut UnixStream, value: &impl serde::Serialize) { let mut bytes = Vec::new(); ferese_ipc::write_frame(&mut bytes, value).unwrap(); @@ -516,10 +529,10 @@ mod tests { desktop: Box::new(ferese_ipc::events::Snapshot { outputs: serde_json::json!([]), workspaces: serde_json::json!([]), - windows: serde_json::json!([{"id":1,"app_id":"one.desktop","focused":true}]), + windows: vec![window(1, "one.desktop", true, 1)], focus: serde_json::json!({"window":1}), config: Default::default(), - theme: serde_json::Value::Null, + theme: serde_json::Value::Null.into(), lock: ferese_ipc::events::LockState { phase: "unlocked".into(), sleeping: false, @@ -533,10 +546,10 @@ mod tests { events.next().await.unwrap().unwrap(), HashMap::from([("one".into(), 2)]) ); - let windows = serde_json::json!([ - {"id":1,"app_id":"one.desktop","focused":true}, - {"id":2,"app_id":"two","focused":false} - ]); + let windows = vec![ + window(1, "one.desktop", true, 1), + window(2, "two", false, 2), + ]; send_frame( &mut peer, &Event { @@ -577,7 +590,7 @@ mod tests { generation: 2, last: true, change: Change::WindowsChanged { - windows: serde_json::json!([]), + windows: Vec::new(), }, }, ) @@ -595,7 +608,7 @@ mod tests { let (stream, mut peer) = UnixStream::pair().unwrap(); let mut events = BackgroundEvents { stream, - windows: serde_json::json!([]), + windows: Vec::new(), changed: false, }; peer.write_u32((ferese_ipc::MAX_PAYLOAD_SIZE + 1) as u32) @@ -608,17 +621,17 @@ mod tests { #[test] fn application_state_groups_windows_and_keeps_inactive_workspaces_running() { - let windows = serde_json::json!([ - {"app_id":"org.test.App.desktop","focused":false,"mapped":false}, - {"app_id":"org.test.App","focused":true,"mapped":true}, - {"app_id":"org.test.Other","focused":false,"mapped":true}, - {"app_id":"","focused":false} - ]); + let windows = vec![ + window(1, "org.test.App.desktop", false, 2), + window(2, "org.test.App", true, 1), + window(3, "org.test.Other", false, 3), + window(4, "", false, 1), + ]; assert_eq!( app_states(&windows), HashMap::from([("org.test.App".into(), 2), ("org.test.Other".into(), 1)]) ); - assert!(app_states(&serde_json::json!([])).is_empty()); + assert!(app_states(&[]).is_empty()); } #[test] diff --git a/portal/src/recorder.rs b/portal/src/recorder.rs index 6f3b049..c535ab9 100644 --- a/portal/src/recorder.rs +++ b/portal/src/recorder.rs @@ -4,8 +4,8 @@ use std::io::{Read, Write}; use std::os::fd::AsRawFd; use std::os::unix::fs::OpenOptionsExt; use std::path::{Path, PathBuf}; -use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use crate::recording::{Audio, Recording}; @@ -515,20 +515,29 @@ fn encode( .is_some_and(|event| matches!(event.view(), gst::EventView::Eos(_))) { ended.store(true, Ordering::Relaxed); + // Finish through the encoder queue after appending the last + // image's timestamp, even when the capture source vanished. + return gst::PadProbeReturn::Drop; } gst::PadProbeReturn::Ok }); let announced = Arc::new(AtomicBool::new(false)); let notice = announced.clone(); + let last_frame = Arc::new(Mutex::new(None)); + let observed_frame = last_frame.clone(); source .static_pad("src") .ok_or("Capture has no video output")? - .add_probe(gst::PadProbeType::BUFFER, move |_, info| { - if info.buffer().is_some_and(|b| b.size() > 0) - && !notice.swap(true, Ordering::Relaxed) - { - event("recording", None, None); + .add_probe(gst::PadProbeType::BUFFER, move |pad, info| { + if let Some(buffer) = info.buffer().filter(|buffer| buffer.size() > 0) { + *observed_frame + .lock() + .expect("last recording frame mutex poisoned") = + Some((buffer.to_owned(), pad.current_caps())); + if !notice.swap(true, Ordering::Relaxed) { + event("recording", None, None); + } } gst::PadProbeReturn::Ok }); @@ -548,10 +557,35 @@ fn encode( finishing = Some(Instant::now()); // Inject after the capture source: it may already have disappeared // when the portal closes on lock or monitor removal. - let _ = queue + let input = queue .static_pad("sink") - .ok_or("Encoder queue has no input")? - .send_event(gst::event::Eos::new()); + .ok_or("Encoder queue has no input")?; + { + // Serialize at the queue input, where an idle source holds + // no stream lock. The source pad itself can stay locked while + // waiting for its next frame, so locking it would stall stop. + let _input_lock = input.stream_lock(); + let last = last_frame + .lock() + .expect("last recording frame mutex poisoned") + .take(); + if let (Some((mut buffer, caps)), Some(now)) = + (last, pipeline.current_running_time()) + && caps == input.current_caps() + && buffer.pts().is_some_and(|pts| { + now > pts + buffer.duration().unwrap_or(gst::ClockTime::ZERO) + }) + { + // make_mut copies the buffer header while sharing its + // owned pixel memory; it does not copy a full video frame. + let tail = buffer.make_mut(); + tail.set_pts(now); + tail.set_dts(None); + tail.set_duration(gst::ClockTime::from_nseconds(1)); + let _ = input.chain(buffer); + } + let _ = input.send_event(gst::event::Eos::new()); + } for (source, queue) in &audio_inputs { // Audio devices are still alive when video is revoked. // Let each source order EOS after its own startup events and @@ -656,6 +690,131 @@ mod tests { assert_video_decodes(&result.unwrap()); } + #[test] + #[ignore = "requires GStreamer video plugins"] + fn encoder_preserves_the_duration_of_a_static_capture() { + gst::init().unwrap(); + for (source_eos, with_audio) in [(false, false), (true, false), (false, true)] { + let directory = tempfile::tempdir().unwrap(); + let source = gst::ElementFactory::make("appsrc") + .property("is-live", true) + .property("do-timestamp", true) + .property("format", gst::Format::Time) + .property( + "caps", + gst::Caps::builder("video/x-raw") + .field("format", "BGRx") + .field("width", 64i32) + .field("height", 48i32) + .field("framerate", gst::Fraction::new(30, 1)) + .build(), + ) + .build() + .unwrap(); + let stop = Arc::new(AtomicBool::new(false)); + let producer = source.clone(); + let request_stop = stop.clone(); + let worker = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(150)); + let mut image = gst::Buffer::with_size(64 * 48 * 4).unwrap(); + image + .get_mut() + .unwrap() + .map_writable() + .unwrap() + .as_mut_slice() + .fill(0x80); + assert_eq!( + producer.emit_by_name::("push-buffer", &[&image]), + gst::FlowReturn::Ok + ); + // The source stays alive without producing a duplicate image. + std::thread::sleep(Duration::from_millis(if with_audio { 4000 } else { 1200 })); + if source_eos { + assert_eq!( + producer.emit_by_name::("end-of-stream", &[]), + gst::FlowReturn::Ok + ); + } else { + request_stop.store(true, Ordering::Relaxed); + } + }); + let audio = if with_audio { + vec![ + gst::ElementFactory::make("audiotestsrc") + .property("is-live", true) + .build() + .unwrap(), + ] + } else { + Vec::new() + }; + let path = encode(source, audio, None, directory.path(), stop).unwrap(); + worker.join().unwrap(); + let audio_branch = if with_audio { + "demux.audio_0 ! queue ! opusdec ! audioconvert ! audioresample ! audio/x-raw,format=F32LE,rate=48000,channels=1 ! fakesink name=audio sync=false" + } else { + "" + }; + let decoder = gst::parse::launch(&format!("filesrc location={} ! matroskademux name=demux demux.video_0 ! queue ! vp8dec ! fakesink name=video sync=false {audio_branch}", path.display())).unwrap().downcast::().unwrap(); + let audio_bytes = Arc::new(std::sync::atomic::AtomicU64::new(0)); + if with_audio { + let bytes = audio_bytes.clone(); + decoder + .by_name("audio") + .unwrap() + .static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if let Some(buffer) = info.buffer() { + bytes.fetch_add(buffer.size() as u64, Ordering::Relaxed); + } + gst::PadProbeReturn::Ok + }); + } + let end = Arc::new(std::sync::atomic::AtomicU64::new(0)); + let observed = end.clone(); + decoder + .by_name("video") + .unwrap() + .static_pad("sink") + .unwrap() + .add_probe(gst::PadProbeType::BUFFER, move |_, info| { + if let Some(buffer) = info.buffer() { + let time = buffer.pts().unwrap_or(gst::ClockTime::ZERO) + + buffer.duration().unwrap_or(gst::ClockTime::ZERO); + observed.fetch_max(time.nseconds(), Ordering::Relaxed); + } + gst::PadProbeReturn::Ok + }); + decoder.set_state(gst::State::Playing).unwrap(); + let result = decoder.bus().unwrap().timed_pop_filtered( + gst::ClockTime::from_seconds(5), + &[gst::MessageType::Eos, gst::MessageType::Error], + ); + decoder.set_state(gst::State::Null).unwrap(); + assert!( + matches!( + result.as_ref().map(|message| message.view()), + Some(gst::MessageView::Eos(_)) + ), + "static recording did not decode: {result:?}" + ); + if with_audio { + assert!( + audio_bytes.load(Ordering::Relaxed) >= 3 * 48_000 * 4, + "static video interrupted audio: {} decoded bytes", + audio_bytes.load(Ordering::Relaxed) + ); + } + assert!( + end.load(Ordering::Relaxed) >= 1_000_000_000, + "recording lost the idle interval: last decoded frame ends at {} ns", + end.load(Ordering::Relaxed) + ); + } + } + #[test] #[ignore = "requires GStreamer video plugins"] fn encoder_finishes_a_real_webm() { diff --git a/portal/src/stream.rs b/portal/src/stream.rs index a9c80c9..2d44f79 100644 --- a/portal/src/stream.rs +++ b/portal/src/stream.rs @@ -1,7 +1,12 @@ +use std::cell::RefCell; +use std::collections::HashMap; use std::io::{Read, Write}; +use std::os::fd::AsRawFd; +use std::ptr::NonNull; +use std::rc::Rc; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; +use std::time::Instant; use pipewire as pw; use pipewire::properties::properties; @@ -9,7 +14,10 @@ use pipewire::spa; use serde::{Deserialize, Serialize}; use spa::pod::Pod; -use crate::capture::{Capture, Frame}; +mod capture; +mod rate; +use capture::{Capture, Completion}; +use rate::FrameRate; #[derive(Debug, Serialize, Deserialize)] pub struct Ready { @@ -20,80 +28,94 @@ pub struct Ready { pub height: u32, } +struct PoolBuffer { + generation: u64, + buffer: capture::Buffer, +} + +#[derive(Clone, Copy)] +struct HeldBuffer { + raw: NonNull, + generation: u64, +} + struct Data { - latest: Arc>, + capture: Rc>, stop: Arc, - streaming: Arc, - fresh: Arc, - announced: bool, + rate: Arc, + buffers: HashMap, + held: Option, + generation: u64, sequence: u64, + active: bool, + announced: bool, negotiated: (u32, u32), + dma: bool, + force_shm: bool, + renegotiate: bool, + negotiating: bool, + next_capture: Instant, } -// pipewire-rs exposes read-only metadata through Buffer. Own a dequeued raw -// buffer here so header writes retain exclusive access and every exit requeues it. -struct OutputBuffer<'a> { - stream: &'a pw::stream::Stream, - raw: std::ptr::NonNull, -} -impl<'a> OutputBuffer<'a> { - fn dequeue(stream: &'a pw::stream::Stream) -> Option { - // SAFETY: called on the owning PipeWire loop; the guard queues exactly once. - let raw = std::ptr::NonNull::new(unsafe { stream.dequeue_raw_buffer() })?; - Some(Self { stream, raw }) - } - fn data(&mut self) -> Option<&mut spa::buffer::Data> { - // SAFETY: the dequeued buffer is exclusively ours until Drop. SPA Data - // is repr(transparent), and PipeWire owns the mapped allocation. - unsafe { - let buffer = self.raw.as_ref().buffer.as_mut()?; - if buffer.n_datas == 0 || buffer.datas.is_null() { - return None; +impl Data { + fn complete(&mut self, stream: &pw::stream::Stream, completion: Completion) { + let Some(held) = self.held.take() else { + return; + }; + let Some(pool) = self + .buffers + .get(&(held.raw.as_ptr() as usize)) + .filter(|pool| pool.generation == held.generation) + else { + // The completed image belongs to a retired pool. Refresh once more + // so the replacement pool receives its own initial image, then + // resume damage waiting. Never queue the retired raw pointer. + if let Err(error) = self.capture.borrow_mut().refresh() { + eprintln!("Could not refresh replacement capture buffers: {error}"); + self.stop.store(true, Ordering::Relaxed); } - buffer.datas.cast::().as_mut() - } - } - fn timestamp(&mut self, sequence: u64) { - // SAFETY: mutable access is exclusive, the metadata lookup checks its size, - // and clock_gettime writes only to our initialized stack timespec. - unsafe { - let buffer = self.raw.as_ref().buffer; - if buffer.is_null() { - return; + return; + }; + let mut deliver = matches!(completion, Completion::Ready) + && self.active + && !self.capture.borrow().stopped(); + match completion { + Completion::Retry => { + self.renegotiate = self.negotiated != self.capture.borrow().constraints().size } - let header = spa::sys::spa_buffer_find_meta_data( - buffer, - spa::sys::SPA_META_Header, - std::mem::size_of::(), - ) - .cast::(); - if let Some(header) = header.as_mut() { - let mut now = libc::timespec { - tv_sec: 0, - tv_nsec: 0, - }; - header.pts = if libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut now) == 0 { - now.tv_sec * 1_000_000_000 + now.tv_nsec + Completion::Failed(error) => { + if self.dma && !self.capture.borrow().stopped() { + eprintln!("DMA-BUF capture unavailable ({error}); using shared memory"); + self.force_shm = true; + self.renegotiate = true; } else { - -1 - }; - header.flags = 0; - header.offset = 0; - header.dts_offset = 0; - header.seq = sequence; + eprintln!("{error}"); + self.stop.store(true, Ordering::Relaxed); + } + deliver = false; } + Completion::Ready => (), } - } -} -impl Drop for OutputBuffer<'_> { - fn drop(&mut self) { - // SAFETY: raw came from this stream and has not yet been queued. + let raw = held.raw; + // SAFETY: this pointer is still registered with the same allocation + // generation and exclusively dequeued by us. All callbacks run on the + // owning PipeWire loop; remove_buffer invalidates the registry entry. unsafe { - self.stream.queue_raw_buffer(self.raw.as_ptr()); + let buffer = raw.as_ref().buffer.as_mut().expect("registered SPA buffer"); + let data = &mut *buffer.datas; + if let Some(chunk) = data.chunk.as_mut() { + chunk.offset = 0; + chunk.size = if deliver { pool.buffer.bytes } else { 0 }; + chunk.stride = pool.buffer.stride as i32; + } + if deliver { + timestamp(raw, self.sequence); + self.sequence = self.sequence.wrapping_add(1); + } + stream.queue_raw_buffer(raw.as_ptr()); } } } - fn int_property(key: u32, value: i32) -> spa::pod::Property { spa::pod::Property { key, @@ -111,7 +133,7 @@ fn pod(object: spa::pod::Object) -> Vec { .into_inner() } -fn video_format(width: u32, height: u32) -> Vec { +fn video_format(width: u32, height: u32, rate: &FrameRate) -> Vec { pod(spa::pod::object!( spa::utils::SpaTypes::ObjectParamFormat, spa::param::ParamType::EnumFormat, @@ -137,46 +159,136 @@ fn video_format(width: u32, height: u32) -> Vec { ), spa::pod::property!( spa::param::format::FormatProperties::VideoFramerate, + Choice, + Range, Fraction, - spa::utils::Fraction { num: 30, denom: 1 } + rate.fraction(), + spa::utils::Fraction { num: 1, denom: 1 }, + spa::utils::Fraction { + num: rate.maximum, + denom: 1 + } ) )) } +fn timestamp(raw: std::ptr::NonNull, sequence: u64) { + // SAFETY: mutable access is exclusive, the metadata lookup checks its size, + // and clock_gettime writes only to our initialized stack timespec. + unsafe { + let buffer = raw.as_ref().buffer; + if buffer.is_null() { + return; + } + let header = spa::sys::spa_buffer_find_meta_data( + buffer, + spa::sys::SPA_META_Header, + std::mem::size_of::(), + ) + .cast::(); + if let Some(header) = header.as_mut() { + let mut now = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + header.pts = if libc::clock_gettime(libc::CLOCK_MONOTONIC, &mut now) == 0 { + now.tv_sec * 1_000_000_000 + now.tv_nsec + } else { + -1 + }; + header.flags = 0; + header.offset = 0; + header.dts_offset = 0; + header.seq = sequence; + } + } +} + +fn buffer_params((width, height): (u32, u32), dma: bool) -> Vec { + pod(spa::pod::object!( + spa::utils::SpaTypes::ObjectParamBuffers, + spa::param::ParamType::Buffers, + int_property(spa::sys::SPA_PARAM_BUFFERS_buffers, 4), + int_property(spa::sys::SPA_PARAM_BUFFERS_blocks, 1), + int_property( + spa::sys::SPA_PARAM_BUFFERS_size, + (width * height * 4) as i32 + ), + int_property(spa::sys::SPA_PARAM_BUFFERS_stride, (width * 4) as i32), + int_property(spa::sys::SPA_PARAM_BUFFERS_align, 16), + // This is a set of supported memory types, not an exact integer. + // Consumers such as OBS offer a scalar mask with several bits; + // a flags choice lets SPA intersect it with our one supported type. + spa::pod::Property { + key: spa::sys::SPA_PARAM_BUFFERS_dataType, + flags: spa::pod::PropertyFlags::empty(), + value: spa::pod::Value::Choice(spa::pod::ChoiceValue::Int(spa::utils::Choice( + spa::utils::ChoiceFlags::empty(), + spa::utils::ChoiceEnum::Flags { + default: 1 + << if dma { + spa::sys::SPA_DATA_DmaBuf + } else { + spa::sys::SPA_DATA_MemFd + }, + flags: Vec::new(), + }, + ),)), + } + )) +} + +fn formats(size: (u32, u32), rate: &FrameRate, dma: bool) -> Vec> { + let shm = video_format(size.0, size.1, rate); + if !dma { + return vec![shm]; + } + let (_, mut format) = spa::pod::deserialize::PodDeserializer::deserialize_any_from(&shm) + .expect("serialized video format"); + if let spa::pod::Value::Object(ref mut object) = format { + object.properties.push(spa::pod::Property { + key: spa::sys::SPA_FORMAT_VIDEO_modifier, + flags: spa::pod::PropertyFlags::MANDATORY, + value: spa::pod::Value::Long(0), + }); + } + let dma = + spa::pod::serialize::PodSerializer::serialize(std::io::Cursor::new(Vec::new()), &format) + .expect("serializable video format") + .0 + .into_inner(); + vec![dma, shm] +} + pub fn run( name: String, cursor: bool, generation: Option, ) -> Result<(), Box> { + let configured_rate = std::env::var("FERESE_CAPTURE_MAX_FPS").ok(); + let rate = Arc::new(FrameRate::new(configured_rate.as_deref())?); let stop = Arc::new(AtomicBool::new(false)); + let (cancel_read, mut cancel_write) = std::os::unix::net::UnixStream::pair()?; + cancel_read.set_nonblocking(true)?; let pipe_stop = stop.clone(); std::thread::spawn(move || { let mut byte = [0u8]; let _ = std::io::stdin().read(&mut byte); pipe_stop.store(true, Ordering::Relaxed); + let _ = cancel_write.write_all(&[1]); }); - let mut capture = Capture::connect(&stop)?; - if let Some(generation) = generation { - capture.pin_output(&name, generation)?; - } - let deadline = Instant::now() + Duration::from_secs(5); - let first = loop { - match capture.frame(&name, cursor, &stop, Vec::new()) { - Err(error) if error == crate::capture::BUSY && Instant::now() < deadline => { - std::thread::sleep(Duration::from_millis(10)); - } - result => break result?, - } - }; - let (width, height) = (first.width, first.height); - let logical_size = first.logical_size; - let resizable = name.starts_with("window:"); - let latest = Arc::new(Mutex::new(first)); - let streaming = Arc::new(AtomicBool::new(false)); - - let fresh = Arc::new(AtomicBool::new(false)); + let capture = Rc::new(RefCell::new(Capture::connect( + &name, cursor, generation, &stop, + )?)); + let size = capture.borrow().constraints().size; + let logical_size = capture.borrow().logical_size(); + let force_shm = std::env::var_os("FERESE_CAPTURE_FORCE_SHM").is_some_and(|value| value == "1"); + let attempted_dma = !force_shm && capture.borrow().dma_supported(); + // Verify GBM allocation before advertising DMA-BUF to the consumer. Failure + // here leaves the same capture session usable through its SHM constraints. + let dma = attempted_dma && capture.borrow_mut().allocate(true).is_ok(); + let force_shm = force_shm || (attempted_dma && !dma); pw::init(); - let mainloop = pw::main_loop::MainLoopRc::new(None)?; let context = pw::context::ContextRc::new(&mainloop, None)?; let core = context.connect_rc(None)?; @@ -184,37 +296,46 @@ pub fn run( &core, "Ferese ScreenCast", properties! { - "media.class" => "Video/Source", - "media.type" => "Video", - "media.category" => "Capture", - "media.role" => "Screen", - "node.name" => format!("ferese.screencast.{}", std::process::id()), - "node.description" => format!("Ferese — {name}"), - "node.virtual" => "true", - "node.want-driver" => "true", - "node.always-process" => "false", - "node.pause-on-idle" => "true", - "node.latency" => "1/30", + "media.class" => "Video/Source", "media.type" => "Video", "media.category" => "Capture", + "media.role" => "Screen", "node.name" => format!("ferese.screencast.{}", std::process::id()), + "node.description" => format!("Ferese — {name}"), "node.virtual" => "true", + "node.want-driver" => "true", "node.always-process" => "false", "node.pause-on-idle" => "true", + "node.latency" => format!("{}/{}", rate.fraction().denom, rate.fraction().num), }, )?; - let format_frames = latest.clone(); - let data = Data { - latest: latest.clone(), + let data = Rc::new(RefCell::new(Data { + capture: capture.clone(), stop: stop.clone(), - streaming: streaming.clone(), - fresh: fresh.clone(), - announced: false, + rate: rate.clone(), + buffers: HashMap::new(), + held: None, + generation: 0, sequence: 0, - negotiated: (width, height), - }; + active: false, + announced: false, + negotiated: size, + dma, + force_shm, + renegotiate: false, + negotiating: false, + next_capture: Instant::now(), + })); let _listener = stream - .add_local_listener_with_user_data(data) + .add_local_listener_with_user_data(data.clone()) .state_changed(move |stream, data, _, state| { - data.fresh.store(false, Ordering::SeqCst); - data.streaming.store( - matches!(state, pw::stream::StreamState::Streaming), - Ordering::Relaxed, - ); + let mut data = data.borrow_mut(); + let active = matches!(state, pw::stream::StreamState::Streaming); + if active + && !data.active + && let Err(error) = data.capture.borrow_mut().refresh() + { + eprintln!("Could not resume capture: {error}"); + data.stop.store(true, Ordering::Relaxed); + } + if active && !data.active { + data.next_capture = Instant::now(); + } + data.active = active; match state { pw::stream::StreamState::Paused | pw::stream::StreamState::Streaming if !data.announced => @@ -227,16 +348,16 @@ pub fn run( serde_json::to_string(&Ready { logical_size, node, - width, - height + width: size.0, + height: size.1 }) - .expect("typed configuration and IPC values must serialize to JSON") + .expect("serializable capture readiness") ); let _ = std::io::stdout().flush(); } } pw::stream::StreamState::Unconnected if data.announced => { - data.stop.store(true, Ordering::Relaxed); + data.stop.store(true, Ordering::Relaxed) } pw::stream::StreamState::Error(error) => { eprintln!("PipeWire stream: {error}"); @@ -245,44 +366,51 @@ pub fn run( _ => (), } }) - .param_changed(move |stream, data, id, param| { + .param_changed(|stream, shared, id, param| { if id != spa::param::ParamType::Format.as_raw() { return; } - let Some(param) = param else { return }; + let Some(param) = param else { + return; + }; let mut format = spa::param::video::VideoInfoRaw::default(); + let mut data = shared.borrow_mut(); if format.parse(param).is_err() - || format.size().width == 0 - || format.size().height == 0 - || (!resizable && (format.size().width, format.size().height) != (width, height)) - || u64::from(format.size().width) * u64::from(format.size().height) * 4 - > 128 * 1024 * 1024 || format.format() != spa::param::video::VideoFormat::BGRx + || (format.size().width, format.size().height) + != data.capture.borrow().constraints().size + || data.rate.negotiate(format.framerate()).is_err() { data.stop.store(true, Ordering::Relaxed); return; } - let (width, height) = (format.size().width, format.size().height); - data.negotiated = (width, height); - let buffers = pod(spa::pod::object!( - spa::utils::SpaTypes::ObjectParamBuffers, - spa::param::ParamType::Buffers, - int_property(spa::sys::SPA_PARAM_BUFFERS_buffers, 4), - int_property(spa::sys::SPA_PARAM_BUFFERS_blocks, 1), - int_property( - spa::sys::SPA_PARAM_BUFFERS_size, - (width * height * 4) as i32 - ), - int_property(spa::sys::SPA_PARAM_BUFFERS_stride, (width * 4) as i32), - int_property(spa::sys::SPA_PARAM_BUFFERS_align, 16) - )); + data.negotiated = (format.size().width, format.size().height); + data.dma = format + .flags() + .contains(spa::param::video::VideoFlags::MODIFIER); + if data.dma && (data.force_shm || format.modifier() != 0) { + data.stop.store(true, Ordering::Relaxed); + return; + } + data.negotiating = false; + let selected = data.rate.fraction(); + let properties = + properties! { "node.latency" => format!("{}/{}", selected.denom, selected.num) }; + // SAFETY: this loop owns the stream; PipeWire copies the live dictionary. + unsafe { + pw::sys::pw_stream_update_properties( + stream.as_raw_ptr(), + properties.dict().as_raw_ptr(), + ); + } + let buffers = buffer_params(data.negotiated, data.dma); let header = pod(spa::pod::object!( spa::utils::SpaTypes::ObjectParamMeta, spa::param::ParamType::Meta, spa::pod::Property { key: spa::sys::SPA_PARAM_META_type, flags: spa::pod::PropertyFlags::empty(), - value: spa::pod::Value::Id(spa::utils::Id(spa::sys::SPA_META_Header)), + value: spa::pod::Value::Id(spa::utils::Id(spa::sys::SPA_META_Header)) }, int_property( spa::sys::SPA_PARAM_META_size, @@ -291,145 +419,298 @@ pub fn run( )); if stream .update_params(&mut [ - Pod::from_bytes(&buffers) - .expect("SPA pod bytes were produced by the matching pod serializer"), - Pod::from_bytes(&header) - .expect("SPA pod bytes were produced by the matching pod serializer"), + Pod::from_bytes(&buffers).expect("serialized buffer constraints"), + Pod::from_bytes(&header).expect("serialized metadata constraints"), ]) .is_err() { data.stop.store(true, Ordering::Relaxed); } }) - .process(|stream, data| { - let Some(mut buffer) = OutputBuffer::dequeue(stream) else { - return; - }; - let Some(target) = buffer.data() else { - return; - }; - if data.stop.load(Ordering::Relaxed) || !data.fresh.load(Ordering::SeqCst) { - *target.chunk_mut().size_mut() = 0; - return; - } - let frame = data.latest.lock().expect( - "capture frame state must remain consistent across producer and PipeWire callbacks", - ); - if (frame.width, frame.height) != data.negotiated { - *target.chunk_mut().size_mut() = 0; - return; - } - let row = frame.width as usize * 4; - let bytes = row * frame.height as usize; - let Some(destination) = target.data() else { - return; - }; - if destination.len() < bytes { + .add_buffer(|_, shared, raw| { + let mut data = shared.borrow_mut(); + if raw.is_null() || data.buffers.len() >= 8 { data.stop.store(true, Ordering::Relaxed); return; } - for (dst, src) in destination[..bytes] - .chunks_exact_mut(row) - .zip(frame.pixels.chunks_exact(frame.stride as usize)) + let result = data.capture.borrow_mut().allocate(data.dma); + match result { + Ok(buffer) => { + // SAFETY: PipeWire grants mutable access during add_buffer. + // ALLOC_BUFFERS delegates fd ownership to us until remove_buffer. + unsafe { + let Some(spa) = (*raw).buffer.as_mut() else { + data.stop.store(true, Ordering::Relaxed); + return; + }; + if spa.n_datas != 1 || spa.datas.is_null() { + data.stop.store(true, Ordering::Relaxed); + return; + } + let target = &mut *spa.datas; + target.type_ = if data.dma { + spa::sys::SPA_DATA_DmaBuf + } else { + spa::sys::SPA_DATA_MemFd + }; + target.flags = spa::sys::SPA_DATA_FLAG_READWRITE; + target.fd = i64::from(buffer.fd.as_raw_fd()); + target.mapoffset = 0; + target.maxsize = buffer.bytes; + target.data = std::ptr::null_mut(); + if let Some(chunk) = target.chunk.as_mut() { + chunk.offset = 0; + chunk.size = 0; + chunk.stride = buffer.stride as i32; + } + } + data.generation = data.generation.wrapping_add(1); + let generation = data.generation; + data.buffers + .insert(raw as usize, PoolBuffer { generation, buffer }); + } + Err(error) if data.dma => { + eprintln!("DMA-BUF allocation failed ({error}); using shared memory"); + data.force_shm = true; + data.renegotiate = true; + } + Err(error) => { + eprintln!("Capture allocation failed: {error}"); + data.stop.store(true, Ordering::Relaxed); + } + } + }) + .remove_buffer(|_, shared, raw| { + shared.borrow_mut().buffers.remove(&(raw as usize)); + }) + .process(|stream, shared| { + let mut data = shared.borrow_mut(); + if !data.active + || data.held.is_some() + || data.negotiating + || Instant::now() < data.next_capture + || data.stop.load(Ordering::Relaxed) { - dst.copy_from_slice(&src[..row]); + return; + } + // SAFETY: called only on the owning loop. The registry and held token + // retain exclusive ownership until completion, removal or disconnect. + let Some(raw) = NonNull::new(unsafe { stream.dequeue_raw_buffer() }) else { + return; + }; + let key = raw.as_ptr() as usize; + let result = data + .buffers + .get(&key) + .filter(|pool| pool.buffer.size == data.capture.borrow().constraints().size) + .map(|pool| { + ( + pool.generation, + data.capture.borrow_mut().submit(&pool.buffer), + ) + }); + if let Some((generation, Ok(()))) = result { + data.held = Some(HeldBuffer { raw, generation }); + data.next_capture = Instant::now() + data.rate.interval(); + } else { + // SAFETY: this buffer was just dequeued and hasn't been handed to + // Wayland. Return it empty; old dimensions cannot reach consumers. + unsafe { + if let Some(chunk) = (*(*raw.as_ptr()).buffer) + .datas + .as_mut() + .and_then(|data| data.chunk.as_mut()) + { + chunk.size = 0; + } + stream.queue_raw_buffer(raw.as_ptr()); + } } - *target.chunk_mut().offset_mut() = 0; - *target.chunk_mut().size_mut() = bytes as u32; - *target.chunk_mut().stride_mut() = row as i32; - buffer.timestamp(data.sequence); - data.sequence = data.sequence.wrapping_add(1); }) .register()?; - - let format = video_format(width, height); - + let capture_io = capture.clone(); + let _capture_io = mainloop.loop_().add_io( + capture.borrow().poll_fd()?, + spa::support::system::IoFlags::IN, + move |_| capture_io.borrow_mut().dispatch(), + ); + let _cancel_io = + mainloop + .loop_() + .add_io(cancel_read, spa::support::system::IoFlags::IN, |_| {}); + let advertised = formats(size, &rate, dma); + let mut parameters: Vec<_> = advertised + .iter() + .map(|format| Pod::from_bytes(format).expect("serialized video format")) + .collect(); + // RT_PROCESS stays disabled: PipeWire callbacks, Wayland dispatch and the + // buffer registry all belong to this single main loop. stream.connect( spa::utils::Direction::Output, None, - pw::stream::StreamFlags::MAP_BUFFERS | pw::stream::StreamFlags::DRIVER, - &mut [Pod::from_bytes(&format) - .expect("SPA pod bytes were produced by the matching pod serializer")], + pw::stream::StreamFlags::ALLOC_BUFFERS | pw::stream::StreamFlags::DRIVER, + &mut parameters, )?; - - let interval = Duration::from_nanos(1_000_000_000 / 30); - let capture_stop = stop.clone(); - let capture_streaming = streaming.clone(); - let capture_thread = std::thread::spawn(move || { - let mut spare = Vec::new(); - let mut checked = Instant::now() - Duration::from_secs(1); - while !capture_stop.load(Ordering::Relaxed) { - let start = Instant::now(); - - if capture_streaming.load(Ordering::Relaxed) - || checked.elapsed() >= Duration::from_secs(1) - { - checked = Instant::now(); - match capture.frame(&name, cursor, &capture_stop, std::mem::take(&mut spare)) { - Ok(frame) if resizable || (frame.width == width && frame.height == height) => { - spare = std::mem::replace(&mut *latest.lock().expect("capture frame state must remain consistent across producer and PipeWire callbacks"), frame).pixels; - fresh.store(capture_streaming.load(Ordering::Relaxed), Ordering::SeqCst); - } - Ok(_) => { - eprintln!("Shared monitor changed size; start sharing again"); - capture_stop.store(true, Ordering::Relaxed); - } - Err(error) if error == crate::capture::BUSY => (), - Err(error) => { - eprintln!("{error}"); - capture_stop.store(true, Ordering::Relaxed); - } - } - } - std::thread::sleep(interval.saturating_sub(start.elapsed())); - } - }); let mut next_frame = Instant::now(); - let mut requested_size = (width, height); - let mut renegotiation_error = None; - - while !stop.load(Ordering::Relaxed) { - let size = { - let frame = format_frames.lock().expect( - "capture frame state must remain consistent across producer and PipeWire callbacks", + while !stop.load(Ordering::Relaxed) && !capture.borrow().stopped() { + capture.borrow_mut().dispatch(); + let completion = capture.borrow_mut().completion(); + if let Some(completion) = completion { + data.borrow_mut().complete(&stream, completion); + } + let changed = capture.borrow_mut().take_changed(); + if changed && !name.starts_with("window:") && capture.borrow().constraints().size != size { + eprintln!("Shared monitor changed size; start sharing again"); + break; + } + let renegotiate = { + let mut data = data.borrow_mut(); + std::mem::take(&mut data.renegotiate) || changed + }; + if renegotiate { + data.borrow_mut().negotiating = true; + let advertised = formats( + capture.borrow().constraints().size, + &rate, + !data.borrow().force_shm && capture.borrow().dma_supported(), ); - (frame.width, frame.height) - }; - if resizable && size != requested_size { - let format = video_format(size.0, size.1); - if let Err(error) = stream.update_params(&mut [Pod::from_bytes(&format) - .expect("SPA pod bytes were produced by the matching pod serializer")]) - { - renegotiation_error = Some(error); - stop.store(true, Ordering::Relaxed); - break; - } - requested_size = size; + let mut params: Vec<_> = advertised + .iter() + .map(|format| Pod::from_bytes(format).expect("serialized video format")) + .collect(); + stream.update_params(&mut params)?; } - let active = streaming.load(Ordering::Relaxed); - let timeout = if active { - next_frame - .saturating_duration_since(Instant::now()) - .min(Duration::from_millis(10)) + let can_capture = { + let data = data.borrow(); + data.active && data.held.is_none() && !data.renegotiate && !data.negotiating + }; + let deadline = next_frame.max(data.borrow().next_capture); + if can_capture && Instant::now() >= deadline { + stream.trigger_process()?; + next_frame = Instant::now() + rate.interval(); + } + let timeout = if can_capture { + pw::loop_::Timeout::Finite( + next_frame + .max(data.borrow().next_capture) + .saturating_duration_since(Instant::now()), + ) } else { - Duration::from_millis(10) + pw::loop_::Timeout::Infinite }; - mainloop - .loop_() - .iterate(pw::loop_::Timeout::Finite(timeout)); - if streaming.load(Ordering::Relaxed) && Instant::now() >= next_frame { - let _ = stream.trigger_process(); - next_frame = Instant::now() + interval; - } else if !streaming.load(Ordering::Relaxed) { - next_frame = Instant::now(); - } + mainloop.loop_().iterate(timeout); } - let _ = stream.disconnect(); - let _ = capture_thread.join(); - if let Some(error) = renegotiation_error { - return Err(error.into()); - } - Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn buffer_types_intersect_obs_masks_in_either_negotiation_order() { + // PipeWire expands OBS's MemPtr bit to also permit MemFd transport. + let obs_mask = (1 << spa::sys::SPA_DATA_MemPtr) + | (1 << spa::sys::SPA_DATA_MemFd) + | (1 << spa::sys::SPA_DATA_DmaBuf); + let consumer = pod(spa::pod::object!( + spa::utils::SpaTypes::ObjectParamBuffers, + spa::param::ParamType::Buffers, + int_property(spa::sys::SPA_PARAM_BUFFERS_dataType, obs_mask) + )); + for dma in [false, true] { + let producer = buffer_params((735, 840), dma); + for reverse in [false, true] { + let (offer, filter) = if reverse { + (&consumer, &producer) + } else { + (&producer, &consumer) + }; + let mut bytes = vec![0; 1024]; + let builder = spa::pod::builder::Builder::new(&mut bytes); + let mut result = std::ptr::null_mut(); + // SAFETY: both input pods and the builder's writable storage + // remain alive throughout filtering and reading the result. + let status = unsafe { + spa::sys::spa_pod_filter( + builder.as_raw_ptr(), + &mut result, + Pod::from_bytes(offer).unwrap().as_raw_ptr(), + Pod::from_bytes(filter).unwrap().as_raw_ptr(), + ) + }; + assert!( + status >= 0, + "SPA rejected dma={dma} reverse={reverse}: {status}" + ); + assert!(!result.is_null()); + // SAFETY: successful filtering initialized this pod in the + // still-live builder storage; no subsequent writes occur. + let filtered = unsafe { Pod::from_raw(result) }; + let (_, value) = spa::pod::deserialize::PodDeserializer::deserialize_any_from( + filtered.as_bytes(), + ) + .unwrap(); + let spa::pod::Value::Object(object) = value else { + panic!("buffer parameters must remain an object") + }; + let types = object + .properties + .iter() + .find(|property| property.key == spa::sys::SPA_PARAM_BUFFERS_dataType) + .unwrap(); + let expected = 1 + << if dma { + spa::sys::SPA_DATA_DmaBuf + } else { + spa::sys::SPA_DATA_MemFd + }; + match &types.value { + spa::pod::Value::Int(mask) => assert_eq!(*mask, expected), + spa::pod::Value::Choice(spa::pod::ChoiceValue::Int(spa::utils::Choice( + _, + choice, + ))) => match choice { + spa::utils::ChoiceEnum::Flags { default, .. } + | spa::utils::ChoiceEnum::None(default) => assert_eq!(*default, expected), + other => panic!("unexpected filtered choice: {other:?}"), + }, + other => panic!("unexpected filtered mask: {other:?}"), + } + } + } + } + + #[test] + fn dma_is_an_explicit_format_with_a_shared_memory_alternative() { + let rate = FrameRate::new(None).unwrap(); + let offers = formats((640, 480), &rate, true); + assert_eq!(offers.len(), 2); + for (index, offer) in offers.iter().enumerate() { + let (_, value) = + spa::pod::deserialize::PodDeserializer::deserialize_any_from(offer).unwrap(); + let spa::pod::Value::Object(object) = value else { + panic!("video format must be an object"); + }; + let modifier = object + .properties + .iter() + .find(|property| property.key == spa::sys::SPA_FORMAT_VIDEO_modifier); + if index == 0 { + let modifier = modifier.expect("DMA-BUF requires an explicit modifier"); + assert!(modifier.flags.contains(spa::pod::PropertyFlags::MANDATORY)); + assert_eq!(modifier.value, spa::pod::Value::Long(0)); + } else { + assert!( + modifier.is_none(), + "SHM must not inherit the DMA-BUF modifier" + ); + } + assert!(object.properties.iter().any(|property| property.key + == spa::sys::SPA_FORMAT_VIDEO_framerate + && matches!(property.value, spa::pod::Value::Choice(_)))); + } + assert_eq!(formats((640, 480), &rate, false), vec![offers[1].clone()]); + } +} diff --git a/portal/src/stream/capture.rs b/portal/src/stream/capture.rs new file mode 100644 index 0000000..c043e78 --- /dev/null +++ b/portal/src/stream/capture.rs @@ -0,0 +1,627 @@ +use std::collections::HashMap; +use std::fs::File; +use std::os::fd::{AsFd, AsRawFd, FromRawFd, OwnedFd}; +use std::os::unix::fs::MetadataExt; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use ferese_protocols::window_capture::v1::client::ferese_window_capture_manager_v1 as window_manager; +use wayland_client::protocol::{wl_buffer, wl_output, wl_registry, wl_shm, wl_shm_pool}; +use wayland_client::{Connection, Dispatch, EventQueue, QueueHandle, WEnum}; +use wayland_protocols::ext::image_capture_source::v1::client::{ + ext_image_capture_source_v1 as source, + ext_output_image_capture_source_manager_v1 as output_manager, +}; +use wayland_protocols::ext::image_copy_capture::v1::client::{ + ext_image_copy_capture_frame_v1 as frame, ext_image_copy_capture_manager_v1 as manager, + ext_image_copy_capture_session_v1 as session, +}; +use wayland_protocols::wp::linux_dmabuf::zv1::client::{ + zwp_linux_buffer_params_v1 as params, zwp_linux_dmabuf_v1 as dmabuf, +}; + +#[derive(Clone, Debug, Default, PartialEq)] +pub(super) struct Constraints { + pub size: (u32, u32), + pub shm: bool, + pub device: Option, + pub linear: bool, +} + +impl Constraints { + pub(super) fn validate(&self) -> Result<(), String> { + let (width, height) = self.size; + if width == 0 + || height == 0 + || u64::from(width) + .checked_mul(u64::from(height)) + .and_then(|pixels| pixels.checked_mul(4)) + .is_none_or(|bytes| bytes > 128 * 1024 * 1024) + || !self.shm + { + return Err("Invalid capture buffer constraints".into()); + } + Ok(()) + } +} + +#[derive(Debug)] +pub(super) enum Completion { + Ready, + Retry, + Failed(String), +} + +pub(super) struct Buffer { + pub proxy: wl_buffer::WlBuffer, + pub fd: OwnedFd, + pub stride: u32, + pub bytes: u32, + pub size: (u32, u32), + _bo: Option>, +} + +impl Drop for Buffer { + fn drop(&mut self) { + self.proxy.destroy(); + } +} + +#[derive(Default)] +struct State { + outputs: HashMap, + manager: Option, + windows: Option, + output_manager: Option, + shm: Option, + dma: Option, + pending_constraints: Constraints, + constraints: Option, + logical_size: Option<(u32, u32)>, + changed: bool, + stopped: bool, + completion: Option, + error: Option, + selected: Option, +} + +pub(super) struct Capture { + connection: Connection, + queue: EventQueue, + state: State, + session: session::ExtImageCopyCaptureSessionV1, + source: source::ExtImageCaptureSourceV1, + frame: Option, + pub device: Option>, + device_id: Option, +} + +impl Capture { + pub(super) fn connect( + name: &str, + cursor: bool, + generation: Option, + stop: &AtomicBool, + ) -> Result { + let connection = Connection::connect_to_env().map_err(|error| error.to_string())?; + let mut queue = connection.new_event_queue(); + let qh = queue.handle(); + let mut state = State::default(); + connection.display().get_registry(&qh, ()); + // Startup alone uses bounded roundtrips. Streaming dispatch is integrated + // with PipeWire's fd loop and never waits for an unchanged source. + let deadline = Instant::now() + Duration::from_secs(5); + for _ in 0..2 { + let done = std::sync::Arc::new(AtomicBool::new(false)); + connection.display().sync(&qh, done.clone()); + while !done.load(Ordering::Relaxed) { + if stop.load(Ordering::Relaxed) || Instant::now() >= deadline { + return Err("Capture startup cancelled or timed out".into()); + } + dispatch(&connection, &mut queue, &mut state, 20)?; + } + } + let source = if let Some(id) = name + .strip_prefix("window:") + .and_then(|value| value.parse::().ok()) + { + state + .windows + .as_ref() + .ok_or("Persistent window capture is unavailable")? + .create_source((id >> 32) as u32, id as u32, &qh, ()) + } else { + let (&id, (output, _)) = state + .outputs + .iter() + .find(|(_, (_, output_name))| output_name == name) + .ok_or("Shared display disconnected")?; + if generation.is_some_and(|generation| generation != id) { + return Err("Shared display was replaced".into()); + } + state.selected = Some(id); + state + .output_manager + .as_ref() + .ok_or("Persistent display capture is unavailable")? + .create_source(output, &qh, ()) + }; + let options = if cursor { + manager::Options::PaintCursors + } else { + manager::Options::empty() + }; + let session = state + .manager + .as_ref() + .ok_or("Persistent screen capture is unavailable")? + .create_session(&source, options, &qh, ()); + while state.constraints.is_none() && !state.stopped { + if stop.load(Ordering::Relaxed) || Instant::now() >= deadline { + return Err("Capture startup cancelled or timed out".into()); + } + dispatch(&connection, &mut queue, &mut state, 20)?; + } + if state.stopped { + return Err("Capture source is unavailable".into()); + } + let constraints = state + .constraints + .as_ref() + .expect("capture constraints received"); + constraints.validate()?; + let device_id = constraints.device; + let device = constraints + .device + .and_then(open_device) + .and_then(|file| gbm::Device::new(file).ok()); + Ok(Self { + connection, + queue, + state, + session, + source, + frame: None, + device, + device_id, + }) + } + + pub(super) fn poll_fd(&self) -> Result { + self.connection + .backend() + .poll_fd() + .try_clone_to_owned() + .map_err(|error| error.to_string()) + } + pub(super) fn constraints(&self) -> &Constraints { + self.state + .constraints + .as_ref() + .expect("capture constraints received") + } + pub(super) fn logical_size(&self) -> Option<(u32, u32)> { + self.state.logical_size + } + pub(super) fn stopped(&self) -> bool { + self.state.stopped || self.state.error.is_some() + } + pub(super) fn take_changed(&mut self) -> bool { + std::mem::take(&mut self.state.changed) + } + pub(super) fn dma_supported(&self) -> bool { + self.constraints().linear && self.device.is_some() && self.state.dma.is_some() + } + pub(super) fn dispatch(&mut self) { + if let Err(error) = dispatch(&self.connection, &mut self.queue, &mut self.state, 0) { + self.state.error = Some(error); + } + let device = self.constraints().device; + if device != self.device_id { + self.device = device + .and_then(open_device) + .and_then(|file| gbm::Device::new(file).ok()); + self.device_id = device; + } + } + pub(super) fn completion(&mut self) -> Option { + let completion = self.state.completion.take()?; + if let Some(frame) = self.frame.take() { + frame.destroy(); + } + Some(completion) + } + pub(super) fn refresh(&mut self) -> Result<(), String> { + self.state + .windows + .as_ref() + .ok_or("Capture refresh is unavailable")? + .refresh_source(&self.source); + self.connection.flush().map_err(|error| error.to_string()) + } + + pub(super) fn submit(&mut self, buffer: &Buffer) -> Result<(), String> { + if self.frame.is_some() || self.stopped() { + return Err("Capture is already pending or stopped".into()); + } + let frame = self.session.create_frame(&self.queue.handle(), ()); + frame.attach_buffer(&buffer.proxy); + // PipeWire may return any pool buffer. Mark it fully stale; only source + // damage wakes the session, then the compositor fills this whole buffer. + frame.damage_buffer(0, 0, buffer.size.0 as i32, buffer.size.1 as i32); + frame.capture(); + self.frame = Some(frame); + self.connection.flush().map_err(|error| error.to_string()) + } + pub(super) fn allocate(&mut self, dma: bool) -> Result { + let (width, height) = self.constraints().size; + let qh = self.queue.handle(); + if dma { + let device = self.device.as_ref().ok_or("No DMA-BUF allocation device")?; + let bo = device + .create_buffer_object::<()>( + width, + height, + gbm::Format::Xrgb8888, + gbm::BufferObjectFlags::RENDERING | gbm::BufferObjectFlags::LINEAR, + ) + .map_err(|error| error.to_string())?; + if bo.plane_count() != 1 || bo.modifier() != gbm::Modifier::Linear { + return Err("Allocator did not supply a linear single-plane buffer".into()); + } + let fd = bo.fd().map_err(|error| error.to_string())?; + let stride = bo.stride_for_plane(0); + let offset = bo.offset(0); + if offset != 0 { + return Err("Unsupported DMA-BUF plane offset".into()); + } + let bytes = stride + .checked_mul(height) + .ok_or("DMA-BUF allocation is too large")?; + let params = self + .state + .dma + .as_ref() + .ok_or("Missing DMA-BUF global")? + .create_params(&qh, ()); + params.add(fd.as_fd(), 0, offset, stride, 0, 0); + let proxy = params.create_immed( + width as i32, + height as i32, + gbm::Format::Xrgb8888 as u32, + params::Flags::empty(), + &qh, + (), + ); + params.destroy(); + return Ok(Buffer { + proxy, + fd, + stride, + bytes, + size: (width, height), + _bo: Some(bo), + }); + } + let stride = width.checked_mul(4).ok_or("SHM row is too large")?; + let bytes = stride + .checked_mul(height) + .filter(|bytes| *bytes <= 128 * 1024 * 1024) + .ok_or("SHM buffer is too large")?; + // SAFETY: this returns a new owned anonymous file with a fixed C name. + let fd = unsafe { + libc::memfd_create( + c"ferese-stream".as_ptr(), + libc::MFD_CLOEXEC | libc::MFD_ALLOW_SEALING, + ) + }; + if fd < 0 { + return Err(std::io::Error::last_os_error().to_string()); + } + // SAFETY: memfd_create succeeded and ownership transfers to File. + let file = unsafe { File::from_raw_fd(fd) }; + file.set_len(u64::from(bytes)) + .map_err(|error| error.to_string())?; + // SAFETY: this file is owned here and has its final size. The compositor + // can map it but neither process needs to resize a pool buffer. + if unsafe { + libc::fcntl( + file.as_raw_fd(), + libc::F_ADD_SEALS, + libc::F_SEAL_SHRINK | libc::F_SEAL_GROW | libc::F_SEAL_SEAL, + ) + } < 0 + { + return Err(std::io::Error::last_os_error().to_string()); + } + let pool = self + .state + .shm + .as_ref() + .ok_or("Missing SHM global")? + .create_pool(file.as_fd(), bytes as i32, &qh, ()); + let proxy = pool.create_buffer( + 0, + width as i32, + height as i32, + stride as i32, + wl_shm::Format::Xrgb8888, + &qh, + (), + ); + pool.destroy(); + Ok(Buffer { + proxy, + fd: file.into(), + stride, + bytes, + size: (width, height), + _bo: None, + }) + } +} + +impl Drop for Capture { + fn drop(&mut self) { + if let Some(frame) = self.frame.take() { + frame.destroy(); + } + self.session.destroy(); + self.source.destroy(); + let _ = self.connection.flush(); + } +} + +fn open_device(device: u64) -> Option { + std::fs::read_dir("/dev/dri") + .ok()? + .filter_map(Result::ok) + .find_map(|entry| { + if !entry.file_name().as_encoded_bytes().starts_with(b"renderD") + || entry.metadata().ok()?.rdev() != device + { + return None; + } + File::options() + .read(true) + .write(true) + .open(entry.path()) + .ok() + }) +} + +fn dispatch( + connection: &Connection, + queue: &mut EventQueue, + state: &mut State, + timeout: i32, +) -> Result<(), String> { + queue + .dispatch_pending(state) + .map_err(|error| error.to_string())?; + connection.flush().map_err(|error| error.to_string())?; + if let Some(read) = queue.prepare_read() { + let mut fd = libc::pollfd { + fd: read.connection_fd().as_raw_fd(), + events: libc::POLLIN, + revents: 0, + }; + // SAFETY: poll receives one live descriptor owned by the read guard. + let ready = unsafe { libc::poll(&mut fd, 1, timeout) }; + if ready > 0 { + read.read().map_err(|error| error.to_string())?; + } else if ready < 0 + && std::io::Error::last_os_error().kind() != std::io::ErrorKind::Interrupted + { + return Err(std::io::Error::last_os_error().to_string()); + } + } + queue + .dispatch_pending(state) + .map_err(|error| error.to_string())?; + Ok(()) +} + +impl Dispatch for State { + fn event( + state: &mut Self, + registry: &wl_registry::WlRegistry, + event: wl_registry::Event, + _: &(), + _: &Connection, + qh: &QueueHandle, + ) { + match event { + wl_registry::Event::Global { + name, + interface, + version, + } => match interface.as_str() { + "wl_output" => { + state.outputs.insert( + name, + ( + registry.bind(name, version.min(4), qh, name), + format!("output-{name}"), + ), + ); + } + "ext_image_copy_capture_manager_v1" => { + state.manager = Some(registry.bind(name, 1, qh, ())) + } + "ext_output_image_capture_source_manager_v1" => { + state.output_manager = Some(registry.bind(name, 1, qh, ())) + } + "ferese_window_capture_manager_v1" if version >= 2 => { + state.windows = Some(registry.bind(name, 2, qh, ())) + } + "wl_shm" => state.shm = Some(registry.bind(name, 1, qh, ())), + "zwp_linux_dmabuf_v1" if version >= 3 => { + state.dma = Some(registry.bind(name, version.min(4), qh, ())) + } + _ => (), + }, + wl_registry::Event::GlobalRemove { name } => { + state.outputs.remove(&name); + if state.selected == Some(name) { + state.stopped = true; + } + } + _ => (), + } + } +} + +impl Dispatch for State { + fn event( + state: &mut Self, + _: &session::ExtImageCopyCaptureSessionV1, + event: session::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + session::Event::BufferSize { width, height } => { + state.pending_constraints.size = (width, height) + } + session::Event::ShmFormat { + format: WEnum::Value(wl_shm::Format::Xrgb8888), + } => state.pending_constraints.shm = true, + session::Event::DmabufDevice { device } + if device.len() == std::mem::size_of::() => + { + let mut bytes = [0u8; 8]; + bytes[..device.len()].copy_from_slice(&device); + state.pending_constraints.device = Some(u64::from_ne_bytes(bytes)); + } + session::Event::DmabufFormat { format, modifiers } + if format == gbm::Format::Xrgb8888 as u32 => + { + state.pending_constraints.linear |= + modifiers.chunks_exact(8).any(|bytes| bytes == [0; 8]); + } + session::Event::Done => { + let constraints = std::mem::take(&mut state.pending_constraints); + match constraints.validate() { + Ok(()) => { + state.changed |= state + .constraints + .as_ref() + .is_some_and(|old| old != &constraints); + state.constraints = Some(constraints); + } + Err(error) => state.error = Some(error), + } + } + session::Event::Stopped => state.stopped = true, + _ => (), + } + } +} + +impl Dispatch for State { + fn event( + state: &mut Self, + _: &frame::ExtImageCopyCaptureFrameV1, + event: frame::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + match event { + frame::Event::Ready => state.completion = Some(Completion::Ready), + frame::Event::Failed { + reason: WEnum::Value(frame::FailureReason::BufferConstraints), + } => state.completion = Some(Completion::Retry), + frame::Event::Failed { reason } => { + state.completion = Some(Completion::Failed(format!("Capture failed: {reason:?}"))) + } + frame::Event::Transform { transform } + if transform != WEnum::Value(wl_output::Transform::Normal) => + { + state.error = Some("Capture returned a non-upright buffer".into()) + } + _ => (), + } + } +} + +impl Dispatch for State { + fn event( + state: &mut Self, + _: &window_manager::FereseWindowCaptureManagerV1, + event: window_manager::Event, + _: &(), + _: &Connection, + _: &QueueHandle, + ) { + if let window_manager::Event::SourceGeometry { width, height, .. } = event { + state.logical_size = Some((width, height)); + } + } +} + +impl Dispatch> + for State +{ + fn event( + _: &mut Self, + _: &wayland_client::protocol::wl_callback::WlCallback, + _: wayland_client::protocol::wl_callback::Event, + done: &std::sync::Arc, + _: &Connection, + _: &QueueHandle, + ) { + done.store(true, Ordering::Relaxed); + } +} + +impl Dispatch for State { + fn event( + state: &mut Self, + _: &wl_output::WlOutput, + event: wl_output::Event, + id: &u32, + _: &Connection, + _: &QueueHandle, + ) { + if let wl_output::Event::Name { name } = event + && let Some((_, output_name)) = state.outputs.get_mut(id) + { + *output_name = name; + } + } +} +wayland_client::delegate_noop!(State: ignore wl_shm::WlShm); +wayland_client::delegate_noop!(State: ignore wl_shm_pool::WlShmPool); +wayland_client::delegate_noop!(State: ignore wl_buffer::WlBuffer); +wayland_client::delegate_noop!(State: ignore manager::ExtImageCopyCaptureManagerV1); +wayland_client::delegate_noop!(State: ignore output_manager::ExtOutputImageCaptureSourceManagerV1); +wayland_client::delegate_noop!(State: ignore source::ExtImageCaptureSourceV1); +wayland_client::delegate_noop!(State: ignore dmabuf::ZwpLinuxDmabufV1); +wayland_client::delegate_noop!(State: ignore params::ZwpLinuxBufferParamsV1); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn buffer_constraints_are_bounded_without_integer_overflow() { + let mut constraints = Constraints { + size: (640, 480), + shm: true, + ..Constraints::default() + }; + assert!(constraints.validate().is_ok()); + constraints.size = (16384, 2048); + assert!(constraints.validate().is_ok()); + for size in [(0, 1), (1, 0), (16384, 2049), (u32::MAX, u32::MAX)] { + constraints.size = size; + assert!(constraints.validate().is_err(), "{size:?}"); + } + constraints.size = (640, 480); + constraints.shm = false; + assert!(constraints.validate().is_err()); + } +} diff --git a/portal/src/stream/rate.rs b/portal/src/stream/rate.rs new file mode 100644 index 0000000..000da16 --- /dev/null +++ b/portal/src/stream/rate.rs @@ -0,0 +1,85 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; + +use pipewire::spa::utils::Fraction; + +/// The negotiated capture clock, independent of the output's refresh rate. +pub(super) struct FrameRate { + current: AtomicU64, + pub(super) maximum: u32, +} + +impl FrameRate { + pub(super) fn new(value: Option<&str>) -> Result { + let maximum = match value { + None => 60, + Some(value) => value + .parse::() + .ok() + .filter(|value| (1..=240).contains(value)) + .ok_or("FERESE_CAPTURE_MAX_FPS must be an integer from 1 to 240")?, + }; + Ok(Self { + current: AtomicU64::new((u64::from(maximum.min(30)) << 32) | 1), + maximum, + }) + } + + pub(super) fn negotiate(&self, rate: Fraction) -> Result<(), &'static str> { + if rate.denom == 0 + || rate.num < rate.denom + || u64::from(rate.num) > u64::from(self.maximum) * u64::from(rate.denom) + { + return Err("Negotiated capture rate is outside the advertised range"); + } + self.current.store( + (u64::from(rate.num) << 32) | u64::from(rate.denom), + Ordering::Relaxed, + ); + Ok(()) + } + + pub(super) fn fraction(&self) -> Fraction { + let packed = self.current.load(Ordering::Relaxed); + Fraction { + num: (packed >> 32) as u32, + denom: packed as u32, + } + } + + pub(super) fn interval(&self) -> Duration { + let rate = self.fraction(); + Duration::from_nanos(1_000_000_000 * u64::from(rate.denom) / u64::from(rate.num)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configuration_bounds_and_default_clock() { + let rate = FrameRate::new(None).unwrap(); + assert_eq!(rate.maximum, 60); + assert_eq!(rate.interval(), Duration::from_nanos(33_333_333)); + assert_eq!(FrameRate::new(Some("15")).unwrap().fraction().num, 15); + for invalid in ["", "0", "241", "-1", "60.0", " 60"] { + assert!(FrameRate::new(Some(invalid)).is_err(), "{invalid}"); + } + } + + #[test] + fn consumer_negotiation_preserves_fractional_rates() { + let rate = FrameRate::new(None).unwrap(); + rate.negotiate(Fraction { + num: 60_000, + denom: 1001, + }) + .unwrap(); + assert_eq!(rate.interval(), Duration::from_nanos(16_683_333)); + for (num, denom) in [(0, 1), (1, 0), (1, 2), (61, 1), (u32::MAX, 1)] { + assert!(rate.negotiate(Fraction { num, denom }).is_err()); + assert_eq!(rate.fraction().num, 60_000); + } + } +} diff --git a/protocols/ferese-window-capture-v1.xml b/protocols/ferese-window-capture-v1.xml index 13e3ef5..c52245b 100644 --- a/protocols/ferese-window-capture-v1.xml +++ b/protocols/ferese-window-capture-v1.xml @@ -7,7 +7,7 @@ requires session opt-in and fails while locked. Window IDs are compositor lifetime IDs and never resolve to replacement windows. - + @@ -39,5 +39,34 @@ + + + Creates a source for ext-image-copy-capture-v1. Unlike a snapshot, + a session waits for content damage and negotiates reusable buffers. + Invalid, closed or protected windows produce a stopped session. + + + + + + + + Invalidates the last delivered image for this source's sessions, so + a resumed PipeWire consumer receives an initial image even when the + source is unchanged. Outstanding buffers keep their ownership and GPU + synchronization; this request never cancels a capture in flight. + Subsequent frames continue to wait for source damage. + + + + + + The current logical window size, sent when the source is created. + Subsequent pixel-size changes are announced by the capture session. + + + + + diff --git a/scripts/tests/fixtures/capture-consumer.c b/scripts/tests/fixtures/capture-consumer.c new file mode 100644 index 0000000..57d3eb3 --- /dev/null +++ b/scripts/tests/fixtures/capture-consumer.c @@ -0,0 +1,179 @@ +/* Native PipeWire consumer: require the selected memory type and write packed + * BGRx frames only after the producer has completed GPU work. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct capture { + struct pw_main_loop *loop; + struct pw_stream *stream; + struct spa_hook listener; + struct spa_video_info_raw format; + FILE *file; + unsigned expected_type, frames, wanted; + uint64_t last_sequence; + int64_t last_pts; + int failed; + int obs_buffers; +}; + +static void fail(struct capture *capture, const char *error) { + fprintf(stderr, "capture-consumer: %s\n", error); + capture->failed = 1; + pw_main_loop_quit(capture->loop); +} + +static void state_changed(void *data, enum pw_stream_state old, + enum pw_stream_state state, const char *error) { + (void)old; + struct capture *capture = data; + if (state == PW_STREAM_STATE_ERROR) fail(capture, error ? error : "stream error"); +} + +static void param_changed(void *data, uint32_t id, const struct spa_pod *param) { + struct capture *capture = data; + if (id != SPA_PARAM_Format || !param) return; + if (spa_format_video_raw_parse(param, &capture->format) < 0 || + capture->format.format != SPA_VIDEO_FORMAT_BGRx || + !capture->format.size.width || !capture->format.size.height || + !!(capture->format.flags & SPA_VIDEO_FLAG_MODIFIER) != (capture->expected_type == SPA_DATA_DmaBuf)) { + fail(capture, "incorrect negotiated video format or memory contract"); + return; + } + uint8_t bytes[1024]; + struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(bytes, sizeof bytes); + const struct spa_pod *buffers; + if (capture->obs_buffers) { + /* OBS supplies a scalar bitmask containing multiple accepted types. + * Keep this separate from the strict transport test's flags choice. */ + buffers = spa_pod_builder_add_object(&builder, + SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, + SPA_PARAM_BUFFERS_dataType, + SPA_POD_Int((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_DmaBuf))); + } else { + buffers = spa_pod_builder_add_object(&builder, + SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, + SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(4, 2, 8), + SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(1 << capture->expected_type)); + } + const struct spa_pod *header = spa_pod_builder_add_object(&builder, + SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta, + SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header), + SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_header))); + const struct spa_pod *parameters[] = {buffers, header}; + pw_stream_update_params(capture->stream, parameters, 2); + fprintf(stderr, "capture-consumer: %ux%u %u/%u fps %s\n", capture->format.size.width, + capture->format.size.height, capture->format.framerate.num, + capture->format.framerate.denom, capture->expected_type == SPA_DATA_DmaBuf ? "DMA-BUF" : "SHM"); +} + +static void process(void *data) { + struct capture *capture = data; + struct pw_buffer *buffer = pw_stream_dequeue_buffer(capture->stream); + if (!buffer) return; + struct spa_buffer *spa = buffer->buffer; + if (spa->n_datas != 1 || !spa->datas[0].chunk) { + fail(capture, "invalid plane count or missing chunk"); + goto done; + } + struct spa_data *plane = &spa->datas[0]; + if (!plane->chunk->size) goto done; + if (capture->obs_buffers && capture->expected_type == SPA_DATA_MemFd && !plane->data) { + fail(capture, "OBS shared-memory input was not mapped by PipeWire"); + goto done; + } + const struct spa_meta_header *header = spa_buffer_find_meta_data(spa, SPA_META_Header, sizeof *header); + if (!header || header->pts < 0 || (capture->frames && (header->seq <= capture->last_sequence || header->pts <= capture->last_pts))) { + fprintf(stderr, "capture-consumer: header=%p frames=%u pts=%lld last_pts=%lld sequence=%llu last_sequence=%llu\n", (void *)header, capture->frames, + (long long)(header ? header->pts : -1), (long long)capture->last_pts, + (unsigned long long)(header ? header->seq : 0), (unsigned long long)capture->last_sequence); + fail(capture, "missing or stale frame sequence/timestamp"); + goto done; + } + capture->last_sequence = header->seq; + capture->last_pts = header->pts; + uint32_t row = capture->format.size.width * 4; + uint32_t height = capture->format.size.height; + if (plane->type != capture->expected_type || plane->fd < 0 || plane->chunk->stride < (int)row || + (uint64_t)plane->chunk->offset + (uint64_t)(height - 1) * plane->chunk->stride + row > plane->maxsize) { + fail(capture, "producer supplied the wrong memory type or invalid plane bounds"); + goto done; + } + size_t length = (size_t)plane->mapoffset + plane->maxsize; + void *mapping = mmap(NULL, length, PROT_READ, MAP_SHARED, plane->fd, 0); + if (mapping == MAP_FAILED) { fail(capture, "buffer mapping failed"); goto done; } + struct dma_buf_sync sync = {.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ}; + if (plane->type == SPA_DATA_DmaBuf && ioctl(plane->fd, DMA_BUF_IOCTL_SYNC, &sync)) { + munmap(mapping, length); fail(capture, "DMA-BUF read synchronization failed"); goto done; + } + const uint8_t *pixels = (const uint8_t *)mapping + plane->mapoffset + plane->chunk->offset; + for (uint32_t y = 0; y < height; y++) { + if (fwrite(pixels + (size_t)y * plane->chunk->stride, 1, row, capture->file) != row) { + capture->failed = 1; + break; + } + } + if (plane->type == SPA_DATA_DmaBuf) { + sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ; + if (ioctl(plane->fd, DMA_BUF_IOCTL_SYNC, &sync)) capture->failed = 1; + } + munmap(mapping, length); + fflush(capture->file); + capture->frames++; + if (capture->failed || capture->frames >= capture->wanted) pw_main_loop_quit(capture->loop); + done: + pw_stream_queue_buffer(capture->stream, buffer); +} + +int main(int argc, char **argv) { + if (argc != 5) return 2; + pw_init(&argc, &argv); + struct capture capture = {.expected_type = !strcmp(argv[2], "dma") ? SPA_DATA_DmaBuf : SPA_DATA_MemFd, + .wanted = strtoul(argv[3], NULL, 10), + .obs_buffers = getenv("FERESE_TEST_CAPTURE_OBS_BUFFERS") != NULL}; + capture.file = fopen(argv[4], "wb"); + capture.loop = pw_main_loop_new(NULL); + if (!capture.file || !capture.loop || !capture.wanted) return 3; + struct pw_context *context = pw_context_new(pw_main_loop_get_loop(capture.loop), NULL, 0); + struct pw_core *core = pw_context_connect(context, NULL, 0); + if (!core) return 4; + capture.stream = pw_stream_new(core, "Ferese capture regression", pw_properties_new(PW_KEY_MEDIA_TYPE, "Video", PW_KEY_MEDIA_CATEGORY, "Capture", NULL)); + const struct pw_stream_events events = {.version = PW_VERSION_STREAM_EVENTS, + .state_changed = state_changed, .param_changed = param_changed, .process = process}; + pw_stream_add_listener(capture.stream, &capture.listener, &events, &capture); + uint8_t bytes[1024]; + struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(bytes, sizeof bytes); + const struct spa_pod *format; + if (capture.expected_type == SPA_DATA_DmaBuf) { + format = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat, + SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), + SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), + SPA_FORMAT_VIDEO_format, SPA_POD_Id(SPA_VIDEO_FORMAT_BGRx), + SPA_FORMAT_VIDEO_modifier, SPA_POD_Long(0), + SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&SPA_FRACTION(60, 1))); + } else { + format = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat, + SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), + SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), + SPA_FORMAT_VIDEO_format, SPA_POD_Id(SPA_VIDEO_FORMAT_BGRx), + SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&SPA_FRACTION(60, 1))); + } + enum pw_stream_flags stream_flags = PW_STREAM_FLAG_AUTOCONNECT; + if (capture.obs_buffers) stream_flags |= PW_STREAM_FLAG_MAP_BUFFERS; + int result = pw_stream_connect(capture.stream, PW_DIRECTION_INPUT, strtoul(argv[1], NULL, 10), stream_flags, &format, 1); + if (result >= 0) pw_main_loop_run(capture.loop); + else capture.failed = 1; + pw_stream_destroy(capture.stream); + pw_core_disconnect(core); + pw_context_destroy(context); + pw_main_loop_destroy(capture.loop); + fclose(capture.file); + return capture.failed || capture.frames != capture.wanted; +} diff --git a/scripts/tests/fixtures/window-capture-scale.c b/scripts/tests/fixtures/window-capture-scale.c new file mode 100644 index 0000000..0e3df0f --- /dev/null +++ b/scripts/tests/fixtures/window-capture-scale.c @@ -0,0 +1,148 @@ +/* Native-pixel stripes distinguish a preserved backing buffer from a resample. */ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include "xdg-shell-client-protocol.h" +#include "viewporter-client-protocol.h" + +static struct wl_compositor *compositor; +static struct wl_subcompositor *subcompositor; +static struct wl_shm *shm; +static struct xdg_wm_base *wm; +static struct wp_viewporter *viewporter; +static struct wl_surface *surface; +static struct wl_buffer *buffer; + +static void ping(void *data, struct xdg_wm_base *base, uint32_t serial) { + (void)data; + xdg_wm_base_pong(base, serial); +} +static const struct xdg_wm_base_listener wm_listener = {.ping = ping}; + +static void global(void *data, struct wl_registry *registry, uint32_t name, + const char *interface, uint32_t version) { + (void)data; (void)version; + if (!strcmp(interface, "wl_compositor")) + compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 4); + else if (!strcmp(interface, "wl_subcompositor")) + subcompositor = wl_registry_bind(registry, name, &wl_subcompositor_interface, 1); + else if (!strcmp(interface, "wl_shm")) + shm = wl_registry_bind(registry, name, &wl_shm_interface, 1); + else if (!strcmp(interface, "wp_viewporter")) + viewporter = wl_registry_bind(registry, name, &wp_viewporter_interface, 1); + else if (!strcmp(interface, "xdg_wm_base")) { + wm = wl_registry_bind(registry, name, &xdg_wm_base_interface, 1); + xdg_wm_base_add_listener(wm, &wm_listener, NULL); + } +} +static void removed(void *data, struct wl_registry *registry, uint32_t name) { + (void)data; (void)registry; (void)name; +} +static const struct wl_registry_listener registry_listener = {global, removed}; + +static void configure(void *data, struct xdg_surface *xdg, uint32_t serial) { + (void)data; + xdg_surface_ack_configure(xdg, serial); + xdg_surface_set_window_geometry(xdg, 0, 0, 640, 480); + wl_surface_attach(surface, buffer, 0, 0); + wl_surface_damage_buffer(surface, 0, 0, INT32_MAX, INT32_MAX); + wl_surface_commit(surface); +} +static const struct xdg_surface_listener surface_listener = {.configure = configure}; +static void size(void *data, struct xdg_toplevel *top, int32_t w, int32_t h, + struct wl_array *states) { + (void)data; (void)top; (void)w; (void)h; (void)states; +} +static void close_window(void *data, struct xdg_toplevel *top) { + (void)data; (void)top; exit(0); +} +static const struct xdg_toplevel_listener top_listener = {.configure = size, .close = close_window}; + +int main(int argc, char **argv) { + if (argc != 2) return 1; + struct wl_display *display = wl_display_connect(NULL); + if (!display) return 2; + struct wl_registry *registry = wl_display_get_registry(display); + wl_registry_add_listener(registry, ®istry_listener, NULL); + if (wl_display_roundtrip(display) < 0 || !compositor || !shm || !wm || !viewporter) return 3; + int integer = !strcmp(argv[1], "integer"); + int fractional = !strcmp(argv[1], "fractional"); + int cropped = !strcmp(argv[1], "cropped"); + int width = fractional ? 1120 : 1280; + int height = fractional ? 840 : 960; + int fd = memfd_create("window-capture-scale", MFD_CLOEXEC); + size_t bytes = (size_t)width * height * 4; + if (fd < 0 || ftruncate(fd, bytes)) return 4; + uint32_t *pixels = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + if (pixels == MAP_FAILED) return 5; + for (int y = 0; y < height; y++) + for (int x = 0; x < width; x++) + pixels[y * width + x] = x % 2 ? 0xffffffff : 0xff000000; + struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, bytes); + buffer = wl_shm_pool_create_buffer(pool, 0, width, height, width * 4, WL_SHM_FORMAT_ARGB8888); + wl_shm_pool_destroy(pool); + munmap(pixels, bytes); + close(fd); + surface = wl_compositor_create_surface(compositor); + if (!strcmp(argv[1], "subsurface") || !strcmp(argv[1], "subsurface-outside")) { + if (!subcompositor) return 6; + struct wl_surface *child = wl_compositor_create_surface(compositor); + struct wl_subsurface *sub = wl_subcompositor_get_subsurface(subcompositor, child, surface); + if (!strcmp(argv[1], "subsurface-outside")) + wl_subsurface_set_position(sub, 1280, 0); + wl_surface_set_buffer_scale(child, 2); + wl_surface_attach(child, buffer, 0, 0); + wl_surface_damage_buffer(child, 0, 0, INT32_MAX, INT32_MAX); + wl_surface_commit(child); + // A transparent placeholder makes the child's density authoritative. + int root_fd = memfd_create("capture-scale-root", MFD_CLOEXEC); + if (root_fd < 0 || ftruncate(root_fd, 4)) return 7; + uint32_t *root_pixels = mmap(NULL, 4, PROT_READ | PROT_WRITE, MAP_SHARED, root_fd, 0); + if (root_pixels == MAP_FAILED) return 8; + *root_pixels = 0; + struct wl_shm_pool *root_pool = wl_shm_create_pool(shm, root_fd, 4); + buffer = wl_shm_pool_create_buffer(root_pool, 0, 1, 1, 4, WL_SHM_FORMAT_ARGB8888); + wl_shm_pool_destroy(root_pool); + munmap(root_pixels, 4); + close(root_fd); + } + if (integer) { + wl_surface_set_buffer_scale(surface, 2); + } else { + struct wp_viewport *viewport = wp_viewporter_get_viewport(viewporter, surface); + wp_viewport_set_destination(viewport, 640, 480); + if (cropped) + wp_viewport_set_source(viewport, wl_fixed_from_int(320), wl_fixed_from_int(240), + wl_fixed_from_int(640), wl_fixed_from_int(480)); + } + struct xdg_surface *xdg = xdg_wm_base_get_xdg_surface(wm, surface); + xdg_surface_add_listener(xdg, &surface_listener, NULL); + struct xdg_toplevel *top = xdg_surface_get_toplevel(xdg); + xdg_toplevel_add_listener(top, &top_listener, NULL); + xdg_toplevel_set_app_id(top, "ferese.test.capture-scale"); + xdg_toplevel_set_title(top, argv[1]); + wl_surface_commit(surface); + while (1) { + wl_display_flush(display); + struct pollfd fds[2] = {{wl_display_get_fd(display), POLLIN, 0}, {STDIN_FILENO, POLLIN, 0}}; + if (poll(fds, 2, -1) < 0) break; + if ((fds[0].revents | fds[1].revents) & (POLLERR | POLLHUP | POLLNVAL)) break; + if (fds[0].revents & POLLIN && wl_display_dispatch(display) < 0) break; + if (fds[1].revents & POLLIN) { + char command; + if (read(STDIN_FILENO, &command, 1) != 1) break; + if (command == 'd') { + wl_surface_attach(surface, buffer, 0, 0); + wl_surface_damage(surface, 0, 0, 640, 480); + wl_surface_commit(surface); + } + } + } + return 0; +} diff --git a/scripts/tests/fixtures/window-capture.c b/scripts/tests/fixtures/window-capture.c index cacd34f..fbaa102 100644 --- a/scripts/tests/fixtures/window-capture.c +++ b/scripts/tests/fixtures/window-capture.c @@ -18,6 +18,9 @@ static struct wl_buffer *buffer; static int width = 640, height = 480; static int private_capture_seen; static int covered; +static unsigned updates; +static unsigned animation_frames; +static void make_buffer(void); static void ping(void *data, struct xdg_wm_base *base, uint32_t serial) { (void)data; @@ -28,7 +31,9 @@ static const struct xdg_wm_base_listener wm_listener = {.ping = ping}; static void global(void *data, struct wl_registry *registry, uint32_t name, const char *interface, uint32_t version) { (void)data; (void)version; - if (!strcmp(interface, "ferese_window_capture_manager_v1")) private_capture_seen = 1; + if (!strcmp(interface, "ferese_window_capture_manager_v1") || + !strcmp(interface, "ext_image_copy_capture_manager_v1") || + !strcmp(interface, "ext_output_image_capture_source_manager_v1")) private_capture_seen = 1; if (!strcmp(interface, "wl_compositor")) compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 4); else if (!strcmp(interface, "wl_shm")) @@ -73,6 +78,7 @@ static void make_buffer(void) { for (size_t i = 0; i < (size_t)width * height; i++) { pixels[i] = covered ? 0xff00ff00 : (i % width < 80 ? 0x80800000 : (i / width < (size_t)height / 2 ? 0xffff0000 : 0xff0000ff)); } + if (updates) pixels[(size_t)(height / 2) * width + width / 2] = 0xff000000 | updates; struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, bytes); buffer = wl_shm_pool_create_buffer(pool, 0, width, height, width * 4, WL_SHM_FORMAT_ARGB8888); wl_shm_pool_destroy(pool); @@ -80,6 +86,23 @@ static void make_buffer(void) { munmap(pixels, bytes); } +static void animate(void *data, struct wl_callback *callback, uint32_t time) { + (void)data; (void)time; + wl_callback_destroy(callback); + updates++; + wl_buffer_destroy(buffer); + make_buffer(); + wl_surface_attach(surface, buffer, 0, 0); + wl_surface_damage(surface, width / 2, height / 2, 1, 1); + if (--animation_frames) { + struct wl_callback *next = wl_surface_frame(surface); + static const struct wl_callback_listener listener = {.done = animate}; + wl_callback_add_listener(next, &listener, NULL); + } + wl_surface_commit(surface); +} +static const struct wl_callback_listener animation_listener = {.done = animate}; + int main(int argc, char **argv) { struct wl_display *display = wl_display_connect(NULL); if (!display) return 1; @@ -97,17 +120,34 @@ int main(int argc, char **argv) { xdg_toplevel_set_app_id(top, "ferese.test.window-capture"); xdg_toplevel_set_title(top, covered ? "Occluding window" : "Capture target"); wl_surface_commit(surface); - if (argc > 1 && !strcmp(argv[1], "resize")) { + if ((argc > 1 && (!strcmp(argv[1], "resize") || !strcmp(argv[1], "interactive"))) || (argc > 2 && !strcmp(argv[2], "interactive"))) { while (1) { wl_display_flush(display); struct pollfd fds[2] = {{wl_display_get_fd(display), POLLIN, 0}, {STDIN_FILENO, POLLIN, 0}}; if (poll(fds, 2, -1) < 0) break; + if ((fds[0].revents | fds[1].revents) & (POLLERR | POLLHUP | POLLNVAL)) break; if (fds[0].revents & POLLIN && wl_display_dispatch(display) < 0) break; if (fds[1].revents & POLLIN) { char command; if (read(STDIN_FILENO, &command, 1) != 1) break; - if (command == 'r') { - width = 480; height = 320; + if (command == 'a') { + animation_frames = 3; + struct wl_callback *next = wl_surface_frame(surface); + wl_callback_add_listener(next, &animation_listener, NULL); + // Start an animation by requesting a callback; pixels have + // not changed yet. A capture must not leave this waiting. + wl_surface_commit(surface); + } + if (command == 'd') { + updates++; + wl_buffer_destroy(buffer); + make_buffer(); + wl_surface_attach(surface, buffer, 0, 0); + wl_surface_damage(surface, width / 2, height / 2, 1, 1); + wl_surface_commit(surface); + } + if (command == 'r' || command == 'o') { + width = 480 + (command == 'o'); height = 320 + (command == 'o'); wl_buffer_destroy(buffer); make_buffer(); xdg_surface_set_window_geometry(xdg, 0, 0, width, height); diff --git a/scripts/tests/test_portal_isolated.py b/scripts/tests/test_portal_isolated.py index 548997a..bf420ce 100644 --- a/scripts/tests/test_portal_isolated.py +++ b/scripts/tests/test_portal_isolated.py @@ -34,11 +34,12 @@ class PortalTest(unittest.TestCase): runtime.mkdir(mode=0o700) config = root / "config/ferese" config.mkdir(parents=True) - (config / "config.kdl").write_text('window-rule app-id="dev.ferese.ScreenShare" floating=#true\n') + (config / "config.kdl").write_text('window-rule app-id="dev.ferese.ScreenShare" floating=#true\nwindow-rule app-id="ferese.test.window-capture" floating=#true\n') display = Path(os.environ["WAYLAND_DISPLAY"]) if not display.is_absolute(): display = Path(os.environ["XDG_RUNTIME_DIR"]) / display env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"), + XDG_STATE_HOME=str(root / "state"), XDG_CACHE_HOME=str(root / "cache"), XDG_DATA_HOME=str(root / "data"), WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1", PIPEWIRE_RUNTIME_DIR=os.environ["XDG_RUNTIME_DIR"], XDG_CURRENT_DESKTOP="Ferese", XDG_DESKTOP_PORTAL_DIR=str(repo / "packaging/portal")) @@ -68,6 +69,14 @@ class PortalTest(unittest.TestCase): return json.loads(worker.stdout.readline())["node"] def consume(node, fd=None): + import threading + producing = threading.Event() + def update_source(): + while not producing.wait(.05): + fixture.stdin.write(b"d") + fixture.stdin.flush() + updates = threading.Thread(target=update_source) + updates.start() source = f'pipewiresrc path={node} num-buffers=15' if fd is not None: source += f' fd={fd}' @@ -90,6 +99,8 @@ class PortalTest(unittest.TestCase): self.assertGreater(timestamps[-1] - timestamps[0], Gst.SECOND // 5) finally: pipeline.set_state(Gst.State.NULL) + producing.set() + updates.join(timeout=2) try: launch("compositor", [str(compositor_binary), "--backend", "nested"]) @@ -98,6 +109,14 @@ class PortalTest(unittest.TestCase): until(lambda: ipc_socket(runtime).is_socket()) env["WAYLAND_DISPLAY"] = str(sockets()[0]) env = ipc_environment(env) + protocols = subprocess.check_output(["pkg-config", "--variable=pkgdatadir", "wayland-protocols"], text=True).strip() + xml = str(Path(protocols) / "stable/xdg-shell/xdg-shell.xml") + for kind, output in [("client-header", "xdg-shell-client-protocol.h"), ("private-code", "xdg-shell-protocol.c")]: + subprocess.run(["wayland-scanner", kind, xml, str(root / output)], check=True) + flags = subprocess.check_output(["pkg-config", "--cflags", "--libs", "wayland-client"], text=True).split() + subprocess.run(["cc", "-Wall", "-Wextra", "-I", str(root), str(repo / "scripts/tests/fixtures/window-capture.c"), + str(root / "xdg-shell-protocol.c"), "-o", str(root / "source"), *flags], check=True) + fixture = launch("source", [str(root / "source"), "interactive"], stdin=subprocess.PIPE) # Private bus, including the real portal frontend and permission store. bus = launch("bus", ["dbus-daemon", "--session", "--nofork", "--print-address=1"], stdout=subprocess.PIPE) env["DBUS_SESSION_BUS_ADDRESS"] = bus.stdout.readline().decode().strip() diff --git a/scripts/tests/test_window_capture_isolated.py b/scripts/tests/test_window_capture_isolated.py index 568a3a8..977e3e8 100644 --- a/scripts/tests/test_window_capture_isolated.py +++ b/scripts/tests/test_window_capture_isolated.py @@ -9,6 +9,7 @@ import io import json import os from pathlib import Path +import re from session_socket import ipc_environment, ipc_socket import signal import selectors @@ -54,7 +55,7 @@ class WindowCapture(unittest.TestCase): display = Path(os.environ["WAYLAND_DISPLAY"]) if not display.is_absolute(): display = Path(os.environ["XDG_RUNTIME_DIR"]) / display - env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"), WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1") + env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"), WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1", FERESE_TRACE_PERFORMANCE="1") env.pop("WAYLAND_SOCKET", None) env.pop("FERESE_SHELL_CONTROL_SOCKET", None) log = (root / "compositor.log").open("w") @@ -72,18 +73,20 @@ class WindowCapture(unittest.TestCase): def call(*args): return subprocess.check_output([str(CTL), "-j", *args], env=ipc_environment(env), timeout=10) - def windows(count): + def windows(count, width=640): deadline = time.monotonic() + 10 while time.monotonic() < deadline: result = json.loads(call("get-windows")) - if len(result) == count and all(window["mapped"] and window["width"] == 640 for window in result): + if len(result) == count and all(window["mapped"] and (width is None or window["width"] == width) for window in result): return result time.sleep(0.02) self.fail((root / "compositor.log").read_text()) - processes.append(subprocess.Popen([str(root / "client")], env=ipc_environment(env), stdout=log, stderr=log, start_new_session=True)) + target_client = subprocess.Popen([str(root / "client"), "interactive"], env=ipc_environment(env), stdin=subprocess.PIPE, stdout=log, stderr=log, start_new_session=True) + processes.append(target_client) target = windows(1)[0] - processes.append(subprocess.Popen([str(root / "client"), "cover"], env=ipc_environment(env), stdout=log, stderr=log, start_new_session=True)) + cover_client = subprocess.Popen([str(root / "client"), "cover", "interactive"], env=ipc_environment(env), stdin=subprocess.PIPE, stdout=log, stderr=log, start_new_session=True) + processes.append(cover_client) current = windows(2) cover = next(window for window in current if window["title"] == "Occluding window") self.assertEqual((target["x"], target["y"], target["width"], target["height"]), @@ -96,8 +99,12 @@ class WindowCapture(unittest.TestCase): self.assertEqual(image.convert("RGBA").getpixel((round(40 * scale), round(80 * scale))), (255, 0, 0, 128)) with Image.open(io.BytesIO(call("screenshot-window", str(cover["id"])))) as image: self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 2)), (0, 255, 0)) - # Exercise the asynchronous conversion worker and reuse its render - # target across a stream, not just the synchronous screenshot path. + # Capture an inactive workspace, where display repaint callbacks + # cannot conceal a stalled capture-driven animation. + call("workspace", "2") + # Verify real streaming updates and silence between commits. Each + # requested frame changes a pixel; static content must not be copied + # repeatedly just to satisfy a consumer's process callbacks. stream_env = dict(env, PIPEWIRE_REMOTE=str(Path(os.environ["XDG_RUNTIME_DIR"]) / "pipewire-0")) worker = subprocess.Popen([str(PORTAL), "--stream-window", str(target["id"]), "hidden"], @@ -110,9 +117,37 @@ class WindowCapture(unittest.TestCase): ready = json.loads(worker.stdout.readline()) self.assertEqual((ready["width"], ready["height"]), (round(640 * scale), round(480 * scale))) raw = root / "frames.bgrx" - subprocess.run(["gst-launch-1.0", "-q", "pipewiresrc", "min-buffers=4", f'path={ready["node"]}', - "num-buffers=15", "!", "video/x-raw,format=BGRx", "!", "filesink", - f"location={raw}"], env=stream_env, check=True, timeout=20) + transport = os.environ.get("FERESE_TEST_CAPTURE_TRANSPORT") + if transport: + flags = subprocess.check_output(["pkg-config", "--cflags", "--libs", "libpipewire-0.3"], text=True).split() + subprocess.run(["cc", "-Wall", "-Wextra", str(REPO / "scripts/tests/fixtures/capture-consumer.c"), "-o", str(root / "consumer"), *flags], check=True) + command = [str(root / "consumer"), str(ready["node"]), transport, "15", str(raw)] + else: + command = ["gst-launch-1.0", "-q", "pipewiresrc", "min-buffers=4", f'path={ready["node"]}', + "num-buffers=15", "!", "video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"] + consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True) + processes.append(consumer) + frame_bytes = ready["width"] * ready["height"] * 4 + + def wait_frames(count): + deadline = time.monotonic() + 7 + while not raw.exists() or raw.stat().st_size < count * frame_bytes: + if consumer.poll() is not None or worker.poll() is not None or time.monotonic() >= deadline: + self.fail((root / "compositor.log").read_text()) + time.sleep(0.01) + + wait_frames(1) + time.sleep(0.35) + self.assertEqual(raw.stat().st_size, frame_bytes, "unchanged window produced duplicate frames") + target_client.stdin.write(b"a") + target_client.stdin.flush() + wait_frames(4) + for count in range(5, 16): + target_client.stdin.write(b"d") + target_client.stdin.flush() + wait_frames(count) + consumer.wait(timeout=7) + self.assertEqual(consumer.returncode, 0) pixels = raw.read_bytes() width, height = ready["width"], ready["height"] frame_bytes = width * height * 4 @@ -123,6 +158,59 @@ class WindowCapture(unittest.TestCase): blue = start + ((height * 3 // 4) * width + width // 2) * 4 self.assertEqual(pixels[red:red + 3], bytes([0, 0, 255])) self.assertEqual(pixels[blue:blue + 3], bytes([255, 0, 0])) + call("workspace", "1") + # A new consumer must receive an initial image even when the + # source has stayed unchanged since the previous consumer left. + original_command = command.copy() + raw = root / "reconnected.bgrx" + if transport: + command[3] = "1" + command[-1] = str(raw) + else: + command[command.index("num-buffers=15")] = "num-buffers=1" + command[-1] = f"location={raw}" + consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True) + processes.append(consumer) + wait_frames(1) + consumer.wait(timeout=7) + self.assertEqual(consumer.returncode, 0) + self.assertEqual(raw.stat().st_size, frame_bytes) + command = original_command + # Resize while the consumer is disconnected, then reconnect to + # the same node. Retired pool buffers must never be queued again. + resize_width, resize_height = ((481, 321) if transport else (480, 320)) + target_client.stdin.write(b"o" if transport else b"r") + target_client.stdin.flush() + deadline = time.monotonic() + 7 + while not any(w["id"] == target["id"] and w["width"] == resize_width for w in json.loads(call("windows"))): + if time.monotonic() >= deadline: + self.fail("Window did not resize") + time.sleep(.02) + raw = root / "resized.bgrx" + if transport: + command[-1] = str(raw) + else: + command[-1] = f"location={raw}" + consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True) + processes.append(consumer) + width, height = round(resize_width * scale), round(resize_height * scale) + frame_bytes = width * height * 4 + wait_frames(1) + time.sleep(.35) + self.assertEqual(raw.stat().st_size, frame_bytes, "paused resize left a stale frame in the stream") + for count in range(2, 16): + target_client.stdin.write(b"d") + target_client.stdin.flush() + wait_frames(count) + consumer.wait(timeout=7) + self.assertEqual(consumer.returncode, 0) + resized = raw.read_bytes() + self.assertEqual(len(resized), frame_bytes * 15) + for index in (0, 14): + red = index * frame_bytes + ((height // 4) * width + width // 2) * 4 + blue = index * frame_bytes + ((height * 3 // 4) * width + width // 2) * 4 + self.assertEqual(resized[red:red + 3], bytes([0, 0, 255])) + self.assertEqual(resized[blue:blue + 3], bytes([255, 0, 0])) # A live rule change revokes the window stream and synchronous # window screenshot route, while normal display mapping remains. config.write_text('window-rule app-id="ferese.test.window-capture" floating=#true\n' @@ -135,12 +223,12 @@ class WindowCapture(unittest.TestCase): env=ipc_environment(env), capture_output=True, timeout=10) self.assertNotEqual(rejected.returncode, 0) self.assertIn(b"protected", rejected.stderr) - self.assertTrue(next(window for window in windows(2) if window["id"] == target["id"])["mapped"]) + self.assertTrue(next(window for window in windows(2, width=None) if window["id"] == target["id"])["mapped"]) with Image.open(io.BytesIO(call("screenshot-window", str(cover["id"])))) as image: self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 2)), (0, 255, 0)) os.killpg(processes[2].pid, signal.SIGTERM) processes[2].wait(timeout=5) - protected = windows(1)[0] + protected = windows(1, width=resize_width)[0] self.assertTrue(protected["mapped"]) for args in [("screenshot",), ("screenshot", "--geometry", f'{protected["x"]},{protected["y"]} {protected["width"]}x{protected["height"]}')]: @@ -152,11 +240,103 @@ class WindowCapture(unittest.TestCase): call("reload-config") with Image.open(io.BytesIO(call("screenshot-window", str(target["id"])))) as image: self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 4)), (255, 0, 0)) + # Updates behind an opaque private window still belong in the + # filtered capture, even when display occlusion hides them. + config.write_text('window-rule app-id="ferese.test.window-capture" floating=#true\n' + 'window-rule title="Occluding window" block-out-from-screencasts=#true\n') + call("reload-config") + private_cover = subprocess.Popen([str(root / "client"), "cover", "interactive"], + env=ipc_environment(env), stdin=subprocess.PIPE, + stdout=log, stderr=log, start_new_session=True) + processes.append(private_cover) + pair = windows(2, width=None) + cover = next(window for window in pair if window["title"] == "Occluding window") + target = next(window for window in pair if window["id"] == target["id"]) + marker_x = target["x"] + target["width"] / 2 + marker_y = target["y"] + target["height"] / 2 + self.assertLess(cover["x"], marker_x) + self.assertGreater(cover["x"] + cover["width"], marker_x) + self.assertLess(cover["y"], marker_y) + self.assertGreater(cover["y"] + cover["height"], marker_y) + # Monitor capture uses an independent filtered scene and the same + # buffer transport. It must also wait quietly between real updates. + # Adding the occluder and reloading rules can animate focus + # borders. Those are real damage, so establish a settled scene + # before asserting that an unchanged monitor produces no frames. + previous, stable_since = None, time.monotonic() + deadline = time.monotonic() + 7 + while time.monotonic() < deadline: + image = call("screenshot") + if image != previous: + previous, stable_since = image, time.monotonic() + elif time.monotonic() - stable_since >= .35: + break + time.sleep(.04) + else: + self.fail("Monitor did not settle before the damage-waiting check") + sources = json.loads(subprocess.check_output([str(PORTAL), "--sources"], env=stream_env, timeout=7)) + worker = subprocess.Popen([str(PORTAL), "--stream", sources[0]["name"], "hidden"], env=stream_env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log, start_new_session=True) + processes.append(worker) + with selectors.DefaultSelector() as selector: + selector.register(worker.stdout, selectors.EVENT_READ) + self.assertTrue(selector.select(10), log.name) + ready = json.loads(worker.stdout.readline()) + raw = root / "monitor.bgrx" + frame_bytes = ready["width"] * ready["height"] * 4 + if transport: + command = [str(root / "consumer"), str(ready["node"]), transport, "15", str(raw)] + else: + command = ["gst-launch-1.0", "-q", "pipewiresrc", f'path={ready["node"]}', "num-buffers=15", "!", + "video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"] + consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True) + processes.append(consumer) + wait_frames(1) + # Wait for a real compositor report, rather than inferring GPU + # reuse merely from the absence of delivered duplicate frames. + deadline = time.monotonic() + 7 + while time.monotonic() < deadline: + reports = [line for line in (root / "compositor.log").read_text().splitlines() + if "capture scene outcomes" in line and "include_cursor=false" in line] + if any(int(match.group(1)) > 0 for line in reports + if (match := re.search(r"scene_reuses=(\d+)", line))): + break + time.sleep(.05) + else: + self.fail("Unchanged capture never reported reusing its filtered scene:\n" + + (root / "compositor.log").read_text()) + monitor_idle = raw.read_bytes() + self.assertEqual(len(monitor_idle), frame_bytes, + "unchanged monitor produced extra frames; first two images equal=" + + str(monitor_idle[:frame_bytes] == monitor_idle[frame_bytes:2 * frame_bytes])) + for count in range(2, 16): + target_client.stdin.write(b"d") + target_client.stdin.flush() + wait_frames(count) + consumer.wait(timeout=7) + self.assertEqual(consumer.returncode, 0) + monitor = raw.read_bytes() + self.assertEqual(len(monitor), frame_bytes * 15) + probe_x = round((cover["x"] + cover["width"] / 2) * scale) + probe_y = round((cover["y"] + cover["height"] / 4) * scale) + probe = (probe_y * ready["width"] + probe_x) * 4 + for index in (0, 14): + start = index * frame_bytes + probe + self.assertNotEqual(monitor[start:start + 3], bytes([0, 255, 0]), + "private occluder leaked into the filtered capture") + for index in range(1, 15): + self.assertNotEqual(monitor[(index - 1) * frame_bytes:index * frame_bytes], monitor[index * frame_bytes:(index + 1) * frame_bytes], + "monitor stream reused stale pixels after source damage") + worker.stdin.close() + self.assertEqual(worker.wait(timeout=7), 0) finally: for process in reversed(processes): if process.poll() is None: os.killpg(process.pid, signal.SIGTERM) process.wait(timeout=5) + if process.stdin: + process.stdin.close() + if process.stdout and process.stdout is not log: + process.stdout.close() log.close() diff --git a/scripts/tests/test_window_capture_scale_isolated.py b/scripts/tests/test_window_capture_scale_isolated.py new file mode 100644 index 0000000..1ff9808 --- /dev/null +++ b/scripts/tests/test_window_capture_scale_isolated.py @@ -0,0 +1,159 @@ +"""Keep native backing pixels in window streams on a fractionally scaled host. + +FERESE_TEST_WINDOW_CAPTURE_SCALE=1 python3 scripts/tests/test_window_capture_scale_isolated.py +Requires a Wayland host, PipeWire, GStreamer, wayland-scanner, cc, and Pillow. +""" +import json +import os +from pathlib import Path +import selectors +import signal +import subprocess +import tempfile +import time +import unittest + +from session_socket import ipc_environment, ipc_socket + +REPO = Path(__file__).resolve().parents[2] +COMPOSITOR = Path(os.environ.get("FERESE_TEST_BINARY", REPO / "target/release/ferese")) +CTL = Path(os.environ.get("FERESE_TEST_CTL", REPO / "target/release/feresectl")) +PORTAL = Path(os.environ.get("FERESE_TEST_PORTAL", REPO / "target/release/xdg-desktop-portal-ferese")) + + +@unittest.skipUnless(os.environ.get("FERESE_TEST_WINDOW_CAPTURE_SCALE") == "1", "opt-in capture test") +class WindowCaptureScale(unittest.TestCase): + def test_backing_pixels_survive_fractional_output_scale(self): + children = [] + with tempfile.TemporaryDirectory(prefix="ferese-capture-scale-") as temporary: + root = Path(temporary) + protocols = Path(subprocess.check_output( + ["pkg-config", "--variable=pkgdatadir", "wayland-protocols"], text=True).strip()) + sources = [] + for protocol, relative in [("xdg-shell", "stable/xdg-shell/xdg-shell.xml"), + ("viewporter", "stable/viewporter/viewporter.xml")]: + source = root / f"{protocol}-protocol.c" + subprocess.run(["wayland-scanner", "client-header", str(protocols / relative), + str(root / f"{protocol}-client-protocol.h")], check=True) + subprocess.run(["wayland-scanner", "private-code", str(protocols / relative), + str(source)], check=True) + sources.append(str(source)) + flags = subprocess.check_output( + ["pkg-config", "--cflags", "--libs", "wayland-client"], text=True).split() + subprocess.run(["cc", "-Wall", "-Wextra", "-Werror", "-I", str(root), + str(REPO / "scripts/tests/fixtures/window-capture-scale.c"), + *sources, "-o", str(root / "client"), *flags], check=True) + runtime = root / "runtime" + runtime.mkdir(mode=0o700) + config = root / "config/ferese" + config.mkdir(parents=True) + (config / "config.kdl").write_text( + 'animations { reduced-motion #true; }\n' + 'window-rule app-id="ferese.test.capture-scale" floating=#true\n') + host_runtime = Path(os.environ["XDG_RUNTIME_DIR"]) + display = Path(os.environ["WAYLAND_DISPLAY"]) + if not display.is_absolute(): + display = host_runtime / display + env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), WAYLAND_DISPLAY=str(display), + XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"), + XDG_DATA_HOME=str(root / "data"), FERESE_ENABLE_SCREENCOPY="1", + PIPEWIRE_REMOTE=str(host_runtime / "pipewire-0")) + for key in ("WAYLAND_SOCKET", "FERESE_SHELL_CONTROL_SOCKET", "DISPLAY", "XAUTHORITY"): + env.pop(key, None) + log_path = root / "session.log" + with log_path.open("w") as log: + def launch(command, **kwargs): + child = subprocess.Popen(command, env=ipc_environment(env), stderr=log, + stdout=kwargs.pop("stdout", log), + start_new_session=True, **kwargs) + children.append(child) + return child + + def wait_for(check): + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + result = check() + if result: + return result + if compositor.poll() is not None: + self.fail(log_path.read_text()) + time.sleep(.03) + self.fail("Timed out:\n" + log_path.read_text()) + + def call(*args): + return json.loads(subprocess.check_output( + [str(CTL), "-j", *args], env=ipc_environment(env), timeout=5)) + + try: + compositor = launch([str(COMPOSITOR), "--backend=nested"]) + wait_for(lambda: ipc_socket(runtime).is_socket()) + env["WAYLAND_DISPLAY"] = str(wait_for( + lambda: next((p for p in runtime.glob("wayland-*") if p.is_socket()), None))) + scale = call("outputs")[0]["scale"] + print(f"Isolated capture output scale: {scale}", flush=True) + self.assertTrue(1 < scale < 2, + "This regression needs a fractional host scale between 1 and 2; " + "an integer 2x host masks the original downsampling bug") + for mode, density in [("integer", 2), ("viewport", 2), + ("fractional", 1.75), ("cropped", 1), + ("subsurface", 2), ("subsurface-outside", 1)]: + with self.subTest(mode=mode, output_scale=scale): + client = launch([str(root / "client"), mode], stdin=subprocess.PIPE) + window = wait_for(lambda: next((w for w in call("windows") + if w["title"] == mode and w["mapped"]), None)) + worker = launch([str(PORTAL), "--stream-window", str(window["id"]), "hidden"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + with selectors.DefaultSelector() as selector: + selector.register(worker.stdout, selectors.EVENT_READ) + self.assertTrue(selector.select(10), log_path.read_text()) + ready = json.loads(worker.stdout.readline()) + expected = (round(640 * max(scale, density)), + round(480 * max(scale, density))) + self.assertEqual((ready["width"], ready["height"]), expected, + "window sharing discarded native backing pixels") + raw = root / f"{mode}.bgrx" + consumer = launch(["gst-launch-1.0", "-q", "pipewiresrc", + f'path={ready["node"]}', "num-buffers=3", "!", + "video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"]) + frame_bytes = expected[0] * expected[1] * 4 + wait_for(lambda: raw.exists() and raw.stat().st_size >= frame_bytes) + # A static source waits for damage. Generate each next + # commit explicitly instead of relying on duplicate frames. + for count in (2, 3): + client.stdin.write(b"d") + client.stdin.flush() + wait_for(lambda: raw.stat().st_size >= count * frame_bytes) + consumer.wait(timeout=5) + self.assertEqual(consumer.returncode, 0) + width, height = expected + pixels = raw.read_bytes() + self.assertEqual(len(pixels), width * height * 4 * 3) + if mode == "subsurface-outside": + self.assertFalse(any(pixels[0::4]) or any(pixels[1::4]) + or any(pixels[2::4]), + "content outside the window was captured") + elif density >= scale: + for index in (0, 2): + start = (index * width * height + height // 2 * width) * 4 + row = pixels[start:start + width * 4] + self.assertEqual(row, bytes((0, 0, 0, 255, 255, 255, 255, 255)) * (width // 2), + "native one-pixel stripes were filtered") + worker.stdin.close() + worker.wait(timeout=5) + worker.stdout.close() + client.terminate() + client.wait(timeout=5) + client.stdin.close() + wait_for(lambda: not any(w["id"] == window["id"] for w in call("windows"))) + finally: + for child in reversed(children): + if child.poll() is None: + os.killpg(child.pid, signal.SIGTERM) + child.wait(timeout=5) + for pipe in (child.stdin, child.stdout, child.stderr): + if pipe: + pipe.close() + + +if __name__ == "__main__": + unittest.main()