perf(capture): add SceneLens GPU streams and retained scene reuse

Use persistent privacy-filtered capture targets and client-owned GPU buffers with fence completion, backpressure, negotiated frame rates and shared-memory fallback.

Invalidate output views on scene changes while preserving inactive-window callbacks and capture revocation. Include OBS memory-mask negotiation and native pixel, transport and recording regressions.
This commit is contained in:
Samuel Onoja committed 2026-10-24 19:58:54 +01:00
1 parent f49f62583d
commit 533cd74c94
40 files changed
+4704 -704

No files matched your search

+179
View File
@@ -0,0 +1,179 @@
/* Native PipeWire consumer: require the selected memory type and write packed
* BGRx frames only after the producer has completed GPU work. */
#define _GNU_SOURCE
#include <errno.h>
#include <linux/dma-buf.h>
#include <pipewire/pipewire.h>
#include <spa/param/video/format-utils.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
struct capture {
struct pw_main_loop *loop;
struct pw_stream *stream;
struct spa_hook listener;
struct spa_video_info_raw format;
FILE *file;
unsigned expected_type, frames, wanted;
uint64_t last_sequence;
int64_t last_pts;
int failed;
int obs_buffers;
};
static void fail(struct capture *capture, const char *error) {
fprintf(stderr, "capture-consumer: %s\n", error);
capture->failed = 1;
pw_main_loop_quit(capture->loop);
}
static void state_changed(void *data, enum pw_stream_state old,
enum pw_stream_state state, const char *error) {
(void)old;
struct capture *capture = data;
if (state == PW_STREAM_STATE_ERROR) fail(capture, error ? error : "stream error");
}
static void param_changed(void *data, uint32_t id, const struct spa_pod *param) {
struct capture *capture = data;
if (id != SPA_PARAM_Format || !param) return;
if (spa_format_video_raw_parse(param, &capture->format) < 0 ||
capture->format.format != SPA_VIDEO_FORMAT_BGRx ||
!capture->format.size.width || !capture->format.size.height ||
!!(capture->format.flags & SPA_VIDEO_FLAG_MODIFIER) != (capture->expected_type == SPA_DATA_DmaBuf)) {
fail(capture, "incorrect negotiated video format or memory contract");
return;
}
uint8_t bytes[1024];
struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(bytes, sizeof bytes);
const struct spa_pod *buffers;
if (capture->obs_buffers) {
/* OBS supplies a scalar bitmask containing multiple accepted types.
* Keep this separate from the strict transport test's flags choice. */
buffers = spa_pod_builder_add_object(&builder,
SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers,
SPA_PARAM_BUFFERS_dataType,
SPA_POD_Int((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_DmaBuf)));
} else {
buffers = spa_pod_builder_add_object(&builder,
SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers,
SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(4, 2, 8),
SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(1 << capture->expected_type));
}
const struct spa_pod *header = spa_pod_builder_add_object(&builder,
SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta,
SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header),
SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_header)));
const struct spa_pod *parameters[] = {buffers, header};
pw_stream_update_params(capture->stream, parameters, 2);
fprintf(stderr, "capture-consumer: %ux%u %u/%u fps %s\n", capture->format.size.width,
capture->format.size.height, capture->format.framerate.num,
capture->format.framerate.denom, capture->expected_type == SPA_DATA_DmaBuf ? "DMA-BUF" : "SHM");
}
static void process(void *data) {
struct capture *capture = data;
struct pw_buffer *buffer = pw_stream_dequeue_buffer(capture->stream);
if (!buffer) return;
struct spa_buffer *spa = buffer->buffer;
if (spa->n_datas != 1 || !spa->datas[0].chunk) {
fail(capture, "invalid plane count or missing chunk");
goto done;
}
struct spa_data *plane = &spa->datas[0];
if (!plane->chunk->size) goto done;
if (capture->obs_buffers && capture->expected_type == SPA_DATA_MemFd && !plane->data) {
fail(capture, "OBS shared-memory input was not mapped by PipeWire");
goto done;
}
const struct spa_meta_header *header = spa_buffer_find_meta_data(spa, SPA_META_Header, sizeof *header);
if (!header || header->pts < 0 || (capture->frames && (header->seq <= capture->last_sequence || header->pts <= capture->last_pts))) {
fprintf(stderr, "capture-consumer: header=%p frames=%u pts=%lld last_pts=%lld sequence=%llu last_sequence=%llu\n", (void *)header, capture->frames,
(long long)(header ? header->pts : -1), (long long)capture->last_pts,
(unsigned long long)(header ? header->seq : 0), (unsigned long long)capture->last_sequence);
fail(capture, "missing or stale frame sequence/timestamp");
goto done;
}
capture->last_sequence = header->seq;
capture->last_pts = header->pts;
uint32_t row = capture->format.size.width * 4;
uint32_t height = capture->format.size.height;
if (plane->type != capture->expected_type || plane->fd < 0 || plane->chunk->stride < (int)row ||
(uint64_t)plane->chunk->offset + (uint64_t)(height - 1) * plane->chunk->stride + row > plane->maxsize) {
fail(capture, "producer supplied the wrong memory type or invalid plane bounds");
goto done;
}
size_t length = (size_t)plane->mapoffset + plane->maxsize;
void *mapping = mmap(NULL, length, PROT_READ, MAP_SHARED, plane->fd, 0);
if (mapping == MAP_FAILED) { fail(capture, "buffer mapping failed"); goto done; }
struct dma_buf_sync sync = {.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ};
if (plane->type == SPA_DATA_DmaBuf && ioctl(plane->fd, DMA_BUF_IOCTL_SYNC, &sync)) {
munmap(mapping, length); fail(capture, "DMA-BUF read synchronization failed"); goto done;
}
const uint8_t *pixels = (const uint8_t *)mapping + plane->mapoffset + plane->chunk->offset;
for (uint32_t y = 0; y < height; y++) {
if (fwrite(pixels + (size_t)y * plane->chunk->stride, 1, row, capture->file) != row) {
capture->failed = 1;
break;
}
}
if (plane->type == SPA_DATA_DmaBuf) {
sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ;
if (ioctl(plane->fd, DMA_BUF_IOCTL_SYNC, &sync)) capture->failed = 1;
}
munmap(mapping, length);
fflush(capture->file);
capture->frames++;
if (capture->failed || capture->frames >= capture->wanted) pw_main_loop_quit(capture->loop);
done:
pw_stream_queue_buffer(capture->stream, buffer);
}
int main(int argc, char **argv) {
if (argc != 5) return 2;
pw_init(&argc, &argv);
struct capture capture = {.expected_type = !strcmp(argv[2], "dma") ? SPA_DATA_DmaBuf : SPA_DATA_MemFd,
.wanted = strtoul(argv[3], NULL, 10),
.obs_buffers = getenv("FERESE_TEST_CAPTURE_OBS_BUFFERS") != NULL};
capture.file = fopen(argv[4], "wb");
capture.loop = pw_main_loop_new(NULL);
if (!capture.file || !capture.loop || !capture.wanted) return 3;
struct pw_context *context = pw_context_new(pw_main_loop_get_loop(capture.loop), NULL, 0);
struct pw_core *core = pw_context_connect(context, NULL, 0);
if (!core) return 4;
capture.stream = pw_stream_new(core, "Ferese capture regression", pw_properties_new(PW_KEY_MEDIA_TYPE, "Video", PW_KEY_MEDIA_CATEGORY, "Capture", NULL));
const struct pw_stream_events events = {.version = PW_VERSION_STREAM_EVENTS,
.state_changed = state_changed, .param_changed = param_changed, .process = process};
pw_stream_add_listener(capture.stream, &capture.listener, &events, &capture);
uint8_t bytes[1024];
struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(bytes, sizeof bytes);
const struct spa_pod *format;
if (capture.expected_type == SPA_DATA_DmaBuf) {
format = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
SPA_FORMAT_VIDEO_format, SPA_POD_Id(SPA_VIDEO_FORMAT_BGRx),
SPA_FORMAT_VIDEO_modifier, SPA_POD_Long(0),
SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&SPA_FRACTION(60, 1)));
} else {
format = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
SPA_FORMAT_VIDEO_format, SPA_POD_Id(SPA_VIDEO_FORMAT_BGRx),
SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&SPA_FRACTION(60, 1)));
}
enum pw_stream_flags stream_flags = PW_STREAM_FLAG_AUTOCONNECT;
if (capture.obs_buffers) stream_flags |= PW_STREAM_FLAG_MAP_BUFFERS;
int result = pw_stream_connect(capture.stream, PW_DIRECTION_INPUT, strtoul(argv[1], NULL, 10), stream_flags, &format, 1);
if (result >= 0) pw_main_loop_run(capture.loop);
else capture.failed = 1;
pw_stream_destroy(capture.stream);
pw_core_disconnect(core);
pw_context_destroy(context);
pw_main_loop_destroy(capture.loop);
fclose(capture.file);
return capture.failed || capture.frames != capture.wanted;
}
+148
View File
@@ -0,0 +1,148 @@
/* Native-pixel stripes distinguish a preserved backing buffer from a resample. */
#define _GNU_SOURCE
#include <stdint.h>
#include <poll.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <unistd.h>
#include <wayland-client.h>
#include "xdg-shell-client-protocol.h"
#include "viewporter-client-protocol.h"
static struct wl_compositor *compositor;
static struct wl_subcompositor *subcompositor;
static struct wl_shm *shm;
static struct xdg_wm_base *wm;
static struct wp_viewporter *viewporter;
static struct wl_surface *surface;
static struct wl_buffer *buffer;
static void ping(void *data, struct xdg_wm_base *base, uint32_t serial) {
(void)data;
xdg_wm_base_pong(base, serial);
}
static const struct xdg_wm_base_listener wm_listener = {.ping = ping};
static void global(void *data, struct wl_registry *registry, uint32_t name,
const char *interface, uint32_t version) {
(void)data; (void)version;
if (!strcmp(interface, "wl_compositor"))
compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 4);
else if (!strcmp(interface, "wl_subcompositor"))
subcompositor = wl_registry_bind(registry, name, &wl_subcompositor_interface, 1);
else if (!strcmp(interface, "wl_shm"))
shm = wl_registry_bind(registry, name, &wl_shm_interface, 1);
else if (!strcmp(interface, "wp_viewporter"))
viewporter = wl_registry_bind(registry, name, &wp_viewporter_interface, 1);
else if (!strcmp(interface, "xdg_wm_base")) {
wm = wl_registry_bind(registry, name, &xdg_wm_base_interface, 1);
xdg_wm_base_add_listener(wm, &wm_listener, NULL);
}
}
static void removed(void *data, struct wl_registry *registry, uint32_t name) {
(void)data; (void)registry; (void)name;
}
static const struct wl_registry_listener registry_listener = {global, removed};
static void configure(void *data, struct xdg_surface *xdg, uint32_t serial) {
(void)data;
xdg_surface_ack_configure(xdg, serial);
xdg_surface_set_window_geometry(xdg, 0, 0, 640, 480);
wl_surface_attach(surface, buffer, 0, 0);
wl_surface_damage_buffer(surface, 0, 0, INT32_MAX, INT32_MAX);
wl_surface_commit(surface);
}
static const struct xdg_surface_listener surface_listener = {.configure = configure};
static void size(void *data, struct xdg_toplevel *top, int32_t w, int32_t h,
struct wl_array *states) {
(void)data; (void)top; (void)w; (void)h; (void)states;
}
static void close_window(void *data, struct xdg_toplevel *top) {
(void)data; (void)top; exit(0);
}
static const struct xdg_toplevel_listener top_listener = {.configure = size, .close = close_window};
int main(int argc, char **argv) {
if (argc != 2) return 1;
struct wl_display *display = wl_display_connect(NULL);
if (!display) return 2;
struct wl_registry *registry = wl_display_get_registry(display);
wl_registry_add_listener(registry, &registry_listener, NULL);
if (wl_display_roundtrip(display) < 0 || !compositor || !shm || !wm || !viewporter) return 3;
int integer = !strcmp(argv[1], "integer");
int fractional = !strcmp(argv[1], "fractional");
int cropped = !strcmp(argv[1], "cropped");
int width = fractional ? 1120 : 1280;
int height = fractional ? 840 : 960;
int fd = memfd_create("window-capture-scale", MFD_CLOEXEC);
size_t bytes = (size_t)width * height * 4;
if (fd < 0 || ftruncate(fd, bytes)) return 4;
uint32_t *pixels = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
if (pixels == MAP_FAILED) return 5;
for (int y = 0; y < height; y++)
for (int x = 0; x < width; x++)
pixels[y * width + x] = x % 2 ? 0xffffffff : 0xff000000;
struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, bytes);
buffer = wl_shm_pool_create_buffer(pool, 0, width, height, width * 4, WL_SHM_FORMAT_ARGB8888);
wl_shm_pool_destroy(pool);
munmap(pixels, bytes);
close(fd);
surface = wl_compositor_create_surface(compositor);
if (!strcmp(argv[1], "subsurface") || !strcmp(argv[1], "subsurface-outside")) {
if (!subcompositor) return 6;
struct wl_surface *child = wl_compositor_create_surface(compositor);
struct wl_subsurface *sub = wl_subcompositor_get_subsurface(subcompositor, child, surface);
if (!strcmp(argv[1], "subsurface-outside"))
wl_subsurface_set_position(sub, 1280, 0);
wl_surface_set_buffer_scale(child, 2);
wl_surface_attach(child, buffer, 0, 0);
wl_surface_damage_buffer(child, 0, 0, INT32_MAX, INT32_MAX);
wl_surface_commit(child);
// A transparent placeholder makes the child's density authoritative.
int root_fd = memfd_create("capture-scale-root", MFD_CLOEXEC);
if (root_fd < 0 || ftruncate(root_fd, 4)) return 7;
uint32_t *root_pixels = mmap(NULL, 4, PROT_READ | PROT_WRITE, MAP_SHARED, root_fd, 0);
if (root_pixels == MAP_FAILED) return 8;
*root_pixels = 0;
struct wl_shm_pool *root_pool = wl_shm_create_pool(shm, root_fd, 4);
buffer = wl_shm_pool_create_buffer(root_pool, 0, 1, 1, 4, WL_SHM_FORMAT_ARGB8888);
wl_shm_pool_destroy(root_pool);
munmap(root_pixels, 4);
close(root_fd);
}
if (integer) {
wl_surface_set_buffer_scale(surface, 2);
} else {
struct wp_viewport *viewport = wp_viewporter_get_viewport(viewporter, surface);
wp_viewport_set_destination(viewport, 640, 480);
if (cropped)
wp_viewport_set_source(viewport, wl_fixed_from_int(320), wl_fixed_from_int(240),
wl_fixed_from_int(640), wl_fixed_from_int(480));
}
struct xdg_surface *xdg = xdg_wm_base_get_xdg_surface(wm, surface);
xdg_surface_add_listener(xdg, &surface_listener, NULL);
struct xdg_toplevel *top = xdg_surface_get_toplevel(xdg);
xdg_toplevel_add_listener(top, &top_listener, NULL);
xdg_toplevel_set_app_id(top, "ferese.test.capture-scale");
xdg_toplevel_set_title(top, argv[1]);
wl_surface_commit(surface);
while (1) {
wl_display_flush(display);
struct pollfd fds[2] = {{wl_display_get_fd(display), POLLIN, 0}, {STDIN_FILENO, POLLIN, 0}};
if (poll(fds, 2, -1) < 0) break;
if ((fds[0].revents | fds[1].revents) & (POLLERR | POLLHUP | POLLNVAL)) break;
if (fds[0].revents & POLLIN && wl_display_dispatch(display) < 0) break;
if (fds[1].revents & POLLIN) {
char command;
if (read(STDIN_FILENO, &command, 1) != 1) break;
if (command == 'd') {
wl_surface_attach(surface, buffer, 0, 0);
wl_surface_damage(surface, 0, 0, 640, 480);
wl_surface_commit(surface);
}
}
}
return 0;
}
+44 -4
View File
@@ -18,6 +18,9 @@ static struct wl_buffer *buffer;
static int width = 640, height = 480;
static int private_capture_seen;
static int covered;
static unsigned updates;
static unsigned animation_frames;
static void make_buffer(void);
static void ping(void *data, struct xdg_wm_base *base, uint32_t serial) {
(void)data;
@@ -28,7 +31,9 @@ static const struct xdg_wm_base_listener wm_listener = {.ping = ping};
static void global(void *data, struct wl_registry *registry, uint32_t name,
const char *interface, uint32_t version) {
(void)data; (void)version;
if (!strcmp(interface, "ferese_window_capture_manager_v1")) private_capture_seen = 1;
if (!strcmp(interface, "ferese_window_capture_manager_v1") ||
!strcmp(interface, "ext_image_copy_capture_manager_v1") ||
!strcmp(interface, "ext_output_image_capture_source_manager_v1")) private_capture_seen = 1;
if (!strcmp(interface, "wl_compositor"))
compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 4);
else if (!strcmp(interface, "wl_shm"))
@@ -73,6 +78,7 @@ static void make_buffer(void) {
for (size_t i = 0; i < (size_t)width * height; i++) {
pixels[i] = covered ? 0xff00ff00 : (i % width < 80 ? 0x80800000 : (i / width < (size_t)height / 2 ? 0xffff0000 : 0xff0000ff));
}
if (updates) pixels[(size_t)(height / 2) * width + width / 2] = 0xff000000 | updates;
struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, bytes);
buffer = wl_shm_pool_create_buffer(pool, 0, width, height, width * 4, WL_SHM_FORMAT_ARGB8888);
wl_shm_pool_destroy(pool);
@@ -80,6 +86,23 @@ static void make_buffer(void) {
munmap(pixels, bytes);
}
static void animate(void *data, struct wl_callback *callback, uint32_t time) {
(void)data; (void)time;
wl_callback_destroy(callback);
updates++;
wl_buffer_destroy(buffer);
make_buffer();
wl_surface_attach(surface, buffer, 0, 0);
wl_surface_damage(surface, width / 2, height / 2, 1, 1);
if (--animation_frames) {
struct wl_callback *next = wl_surface_frame(surface);
static const struct wl_callback_listener listener = {.done = animate};
wl_callback_add_listener(next, &listener, NULL);
}
wl_surface_commit(surface);
}
static const struct wl_callback_listener animation_listener = {.done = animate};
int main(int argc, char **argv) {
struct wl_display *display = wl_display_connect(NULL);
if (!display) return 1;
@@ -97,17 +120,34 @@ int main(int argc, char **argv) {
xdg_toplevel_set_app_id(top, "ferese.test.window-capture");
xdg_toplevel_set_title(top, covered ? "Occluding window" : "Capture target");
wl_surface_commit(surface);
if (argc > 1 && !strcmp(argv[1], "resize")) {
if ((argc > 1 && (!strcmp(argv[1], "resize") || !strcmp(argv[1], "interactive"))) || (argc > 2 && !strcmp(argv[2], "interactive"))) {
while (1) {
wl_display_flush(display);
struct pollfd fds[2] = {{wl_display_get_fd(display), POLLIN, 0}, {STDIN_FILENO, POLLIN, 0}};
if (poll(fds, 2, -1) < 0) break;
if ((fds[0].revents | fds[1].revents) & (POLLERR | POLLHUP | POLLNVAL)) break;
if (fds[0].revents & POLLIN && wl_display_dispatch(display) < 0) break;
if (fds[1].revents & POLLIN) {
char command;
if (read(STDIN_FILENO, &command, 1) != 1) break;
if (command == 'r') {
width = 480; height = 320;
if (command == 'a') {
animation_frames = 3;
struct wl_callback *next = wl_surface_frame(surface);
wl_callback_add_listener(next, &animation_listener, NULL);
// Start an animation by requesting a callback; pixels have
// not changed yet. A capture must not leave this waiting.
wl_surface_commit(surface);
}
if (command == 'd') {
updates++;
wl_buffer_destroy(buffer);
make_buffer();
wl_surface_attach(surface, buffer, 0, 0);
wl_surface_damage(surface, width / 2, height / 2, 1, 1);
wl_surface_commit(surface);
}
if (command == 'r' || command == 'o') {
width = 480 + (command == 'o'); height = 320 + (command == 'o');
wl_buffer_destroy(buffer);
make_buffer();
xdg_surface_set_window_geometry(xdg, 0, 0, width, height);
+20 -1
View File
@@ -34,11 +34,12 @@ class PortalTest(unittest.TestCase):
runtime.mkdir(mode=0o700)
config = root / "config/ferese"
config.mkdir(parents=True)
(config / "config.kdl").write_text('window-rule app-id="dev.ferese.ScreenShare" floating=#true\n')
(config / "config.kdl").write_text('window-rule app-id="dev.ferese.ScreenShare" floating=#true\nwindow-rule app-id="ferese.test.window-capture" floating=#true\n')
display = Path(os.environ["WAYLAND_DISPLAY"])
if not display.is_absolute():
display = Path(os.environ["XDG_RUNTIME_DIR"]) / display
env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"),
XDG_STATE_HOME=str(root / "state"), XDG_CACHE_HOME=str(root / "cache"), XDG_DATA_HOME=str(root / "data"),
WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1",
PIPEWIRE_RUNTIME_DIR=os.environ["XDG_RUNTIME_DIR"], XDG_CURRENT_DESKTOP="Ferese",
XDG_DESKTOP_PORTAL_DIR=str(repo / "packaging/portal"))
@@ -68,6 +69,14 @@ class PortalTest(unittest.TestCase):
return json.loads(worker.stdout.readline())["node"]
def consume(node, fd=None):
import threading
producing = threading.Event()
def update_source():
while not producing.wait(.05):
fixture.stdin.write(b"d")
fixture.stdin.flush()
updates = threading.Thread(target=update_source)
updates.start()
source = f'pipewiresrc path={node} num-buffers=15'
if fd is not None:
source += f' fd={fd}'
@@ -90,6 +99,8 @@ class PortalTest(unittest.TestCase):
self.assertGreater(timestamps[-1] - timestamps[0], Gst.SECOND // 5)
finally:
pipeline.set_state(Gst.State.NULL)
producing.set()
updates.join(timeout=2)
try:
launch("compositor", [str(compositor_binary), "--backend", "nested"])
@@ -98,6 +109,14 @@ class PortalTest(unittest.TestCase):
until(lambda: ipc_socket(runtime).is_socket())
env["WAYLAND_DISPLAY"] = str(sockets()[0])
env = ipc_environment(env)
protocols = subprocess.check_output(["pkg-config", "--variable=pkgdatadir", "wayland-protocols"], text=True).strip()
xml = str(Path(protocols) / "stable/xdg-shell/xdg-shell.xml")
for kind, output in [("client-header", "xdg-shell-client-protocol.h"), ("private-code", "xdg-shell-protocol.c")]:
subprocess.run(["wayland-scanner", kind, xml, str(root / output)], check=True)
flags = subprocess.check_output(["pkg-config", "--cflags", "--libs", "wayland-client"], text=True).split()
subprocess.run(["cc", "-Wall", "-Wextra", "-I", str(root), str(repo / "scripts/tests/fixtures/window-capture.c"),
str(root / "xdg-shell-protocol.c"), "-o", str(root / "source"), *flags], check=True)
fixture = launch("source", [str(root / "source"), "interactive"], stdin=subprocess.PIPE)
# Private bus, including the real portal frontend and permission store.
bus = launch("bus", ["dbus-daemon", "--session", "--nofork", "--print-address=1"], stdout=subprocess.PIPE)
env["DBUS_SESSION_BUS_ADDRESS"] = bus.stdout.readline().decode().strip()
+192 -12
View File
@@ -9,6 +9,7 @@ import io
import json
import os
from pathlib import Path
import re
from session_socket import ipc_environment, ipc_socket
import signal
import selectors
@@ -54,7 +55,7 @@ class WindowCapture(unittest.TestCase):
display = Path(os.environ["WAYLAND_DISPLAY"])
if not display.is_absolute():
display = Path(os.environ["XDG_RUNTIME_DIR"]) / display
env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"), WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1")
env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"), WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1", FERESE_TRACE_PERFORMANCE="1")
env.pop("WAYLAND_SOCKET", None)
env.pop("FERESE_SHELL_CONTROL_SOCKET", None)
log = (root / "compositor.log").open("w")
@@ -72,18 +73,20 @@ class WindowCapture(unittest.TestCase):
def call(*args):
return subprocess.check_output([str(CTL), "-j", *args], env=ipc_environment(env), timeout=10)
def windows(count):
def windows(count, width=640):
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
result = json.loads(call("get-windows"))
if len(result) == count and all(window["mapped"] and window["width"] == 640 for window in result):
if len(result) == count and all(window["mapped"] and (width is None or window["width"] == width) for window in result):
return result
time.sleep(0.02)
self.fail((root / "compositor.log").read_text())
processes.append(subprocess.Popen([str(root / "client")], env=ipc_environment(env), stdout=log, stderr=log, start_new_session=True))
target_client = subprocess.Popen([str(root / "client"), "interactive"], env=ipc_environment(env), stdin=subprocess.PIPE, stdout=log, stderr=log, start_new_session=True)
processes.append(target_client)
target = windows(1)[0]
processes.append(subprocess.Popen([str(root / "client"), "cover"], env=ipc_environment(env), stdout=log, stderr=log, start_new_session=True))
cover_client = subprocess.Popen([str(root / "client"), "cover", "interactive"], env=ipc_environment(env), stdin=subprocess.PIPE, stdout=log, stderr=log, start_new_session=True)
processes.append(cover_client)
current = windows(2)
cover = next(window for window in current if window["title"] == "Occluding window")
self.assertEqual((target["x"], target["y"], target["width"], target["height"]),
@@ -96,8 +99,12 @@ class WindowCapture(unittest.TestCase):
self.assertEqual(image.convert("RGBA").getpixel((round(40 * scale), round(80 * scale))), (255, 0, 0, 128))
with Image.open(io.BytesIO(call("screenshot-window", str(cover["id"])))) as image:
self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 2)), (0, 255, 0))
# Exercise the asynchronous conversion worker and reuse its render
# target across a stream, not just the synchronous screenshot path.
# Capture an inactive workspace, where display repaint callbacks
# cannot conceal a stalled capture-driven animation.
call("workspace", "2")
# Verify real streaming updates and silence between commits. Each
# requested frame changes a pixel; static content must not be copied
# repeatedly just to satisfy a consumer's process callbacks.
stream_env = dict(env, PIPEWIRE_REMOTE=str(Path(os.environ["XDG_RUNTIME_DIR"]) / "pipewire-0"))
worker = subprocess.Popen([str(PORTAL),
"--stream-window", str(target["id"]), "hidden"],
@@ -110,9 +117,37 @@ class WindowCapture(unittest.TestCase):
ready = json.loads(worker.stdout.readline())
self.assertEqual((ready["width"], ready["height"]), (round(640 * scale), round(480 * scale)))
raw = root / "frames.bgrx"
subprocess.run(["gst-launch-1.0", "-q", "pipewiresrc", "min-buffers=4", f'path={ready["node"]}',
"num-buffers=15", "!", "video/x-raw,format=BGRx", "!", "filesink",
f"location={raw}"], env=stream_env, check=True, timeout=20)
transport = os.environ.get("FERESE_TEST_CAPTURE_TRANSPORT")
if transport:
flags = subprocess.check_output(["pkg-config", "--cflags", "--libs", "libpipewire-0.3"], text=True).split()
subprocess.run(["cc", "-Wall", "-Wextra", str(REPO / "scripts/tests/fixtures/capture-consumer.c"), "-o", str(root / "consumer"), *flags], check=True)
command = [str(root / "consumer"), str(ready["node"]), transport, "15", str(raw)]
else:
command = ["gst-launch-1.0", "-q", "pipewiresrc", "min-buffers=4", f'path={ready["node"]}',
"num-buffers=15", "!", "video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"]
consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
processes.append(consumer)
frame_bytes = ready["width"] * ready["height"] * 4
def wait_frames(count):
deadline = time.monotonic() + 7
while not raw.exists() or raw.stat().st_size < count * frame_bytes:
if consumer.poll() is not None or worker.poll() is not None or time.monotonic() >= deadline:
self.fail((root / "compositor.log").read_text())
time.sleep(0.01)
wait_frames(1)
time.sleep(0.35)
self.assertEqual(raw.stat().st_size, frame_bytes, "unchanged window produced duplicate frames")
target_client.stdin.write(b"a")
target_client.stdin.flush()
wait_frames(4)
for count in range(5, 16):
target_client.stdin.write(b"d")
target_client.stdin.flush()
wait_frames(count)
consumer.wait(timeout=7)
self.assertEqual(consumer.returncode, 0)
pixels = raw.read_bytes()
width, height = ready["width"], ready["height"]
frame_bytes = width * height * 4
@@ -123,6 +158,59 @@ class WindowCapture(unittest.TestCase):
blue = start + ((height * 3 // 4) * width + width // 2) * 4
self.assertEqual(pixels[red:red + 3], bytes([0, 0, 255]))
self.assertEqual(pixels[blue:blue + 3], bytes([255, 0, 0]))
call("workspace", "1")
# A new consumer must receive an initial image even when the
# source has stayed unchanged since the previous consumer left.
original_command = command.copy()
raw = root / "reconnected.bgrx"
if transport:
command[3] = "1"
command[-1] = str(raw)
else:
command[command.index("num-buffers=15")] = "num-buffers=1"
command[-1] = f"location={raw}"
consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
processes.append(consumer)
wait_frames(1)
consumer.wait(timeout=7)
self.assertEqual(consumer.returncode, 0)
self.assertEqual(raw.stat().st_size, frame_bytes)
command = original_command
# Resize while the consumer is disconnected, then reconnect to
# the same node. Retired pool buffers must never be queued again.
resize_width, resize_height = ((481, 321) if transport else (480, 320))
target_client.stdin.write(b"o" if transport else b"r")
target_client.stdin.flush()
deadline = time.monotonic() + 7
while not any(w["id"] == target["id"] and w["width"] == resize_width for w in json.loads(call("windows"))):
if time.monotonic() >= deadline:
self.fail("Window did not resize")
time.sleep(.02)
raw = root / "resized.bgrx"
if transport:
command[-1] = str(raw)
else:
command[-1] = f"location={raw}"
consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
processes.append(consumer)
width, height = round(resize_width * scale), round(resize_height * scale)
frame_bytes = width * height * 4
wait_frames(1)
time.sleep(.35)
self.assertEqual(raw.stat().st_size, frame_bytes, "paused resize left a stale frame in the stream")
for count in range(2, 16):
target_client.stdin.write(b"d")
target_client.stdin.flush()
wait_frames(count)
consumer.wait(timeout=7)
self.assertEqual(consumer.returncode, 0)
resized = raw.read_bytes()
self.assertEqual(len(resized), frame_bytes * 15)
for index in (0, 14):
red = index * frame_bytes + ((height // 4) * width + width // 2) * 4
blue = index * frame_bytes + ((height * 3 // 4) * width + width // 2) * 4
self.assertEqual(resized[red:red + 3], bytes([0, 0, 255]))
self.assertEqual(resized[blue:blue + 3], bytes([255, 0, 0]))
# A live rule change revokes the window stream and synchronous
# window screenshot route, while normal display mapping remains.
config.write_text('window-rule app-id="ferese.test.window-capture" floating=#true\n'
@@ -135,12 +223,12 @@ class WindowCapture(unittest.TestCase):
env=ipc_environment(env), capture_output=True, timeout=10)
self.assertNotEqual(rejected.returncode, 0)
self.assertIn(b"protected", rejected.stderr)
self.assertTrue(next(window for window in windows(2) if window["id"] == target["id"])["mapped"])
self.assertTrue(next(window for window in windows(2, width=None) if window["id"] == target["id"])["mapped"])
with Image.open(io.BytesIO(call("screenshot-window", str(cover["id"])))) as image:
self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 2)), (0, 255, 0))
os.killpg(processes[2].pid, signal.SIGTERM)
processes[2].wait(timeout=5)
protected = windows(1)[0]
protected = windows(1, width=resize_width)[0]
self.assertTrue(protected["mapped"])
for args in [("screenshot",), ("screenshot", "--geometry",
f'{protected["x"]},{protected["y"]} {protected["width"]}x{protected["height"]}')]:
@@ -152,11 +240,103 @@ class WindowCapture(unittest.TestCase):
call("reload-config")
with Image.open(io.BytesIO(call("screenshot-window", str(target["id"])))) as image:
self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 4)), (255, 0, 0))
# Updates behind an opaque private window still belong in the
# filtered capture, even when display occlusion hides them.
config.write_text('window-rule app-id="ferese.test.window-capture" floating=#true\n'
'window-rule title="Occluding window" block-out-from-screencasts=#true\n')
call("reload-config")
private_cover = subprocess.Popen([str(root / "client"), "cover", "interactive"],
env=ipc_environment(env), stdin=subprocess.PIPE,
stdout=log, stderr=log, start_new_session=True)
processes.append(private_cover)
pair = windows(2, width=None)
cover = next(window for window in pair if window["title"] == "Occluding window")
target = next(window for window in pair if window["id"] == target["id"])
marker_x = target["x"] + target["width"] / 2
marker_y = target["y"] + target["height"] / 2
self.assertLess(cover["x"], marker_x)
self.assertGreater(cover["x"] + cover["width"], marker_x)
self.assertLess(cover["y"], marker_y)
self.assertGreater(cover["y"] + cover["height"], marker_y)
# Monitor capture uses an independent filtered scene and the same
# buffer transport. It must also wait quietly between real updates.
# Adding the occluder and reloading rules can animate focus
# borders. Those are real damage, so establish a settled scene
# before asserting that an unchanged monitor produces no frames.
previous, stable_since = None, time.monotonic()
deadline = time.monotonic() + 7
while time.monotonic() < deadline:
image = call("screenshot")
if image != previous:
previous, stable_since = image, time.monotonic()
elif time.monotonic() - stable_since >= .35:
break
time.sleep(.04)
else:
self.fail("Monitor did not settle before the damage-waiting check")
sources = json.loads(subprocess.check_output([str(PORTAL), "--sources"], env=stream_env, timeout=7))
worker = subprocess.Popen([str(PORTAL), "--stream", sources[0]["name"], "hidden"], env=stream_env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log, start_new_session=True)
processes.append(worker)
with selectors.DefaultSelector() as selector:
selector.register(worker.stdout, selectors.EVENT_READ)
self.assertTrue(selector.select(10), log.name)
ready = json.loads(worker.stdout.readline())
raw = root / "monitor.bgrx"
frame_bytes = ready["width"] * ready["height"] * 4
if transport:
command = [str(root / "consumer"), str(ready["node"]), transport, "15", str(raw)]
else:
command = ["gst-launch-1.0", "-q", "pipewiresrc", f'path={ready["node"]}', "num-buffers=15", "!",
"video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"]
consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
processes.append(consumer)
wait_frames(1)
# Wait for a real compositor report, rather than inferring GPU
# reuse merely from the absence of delivered duplicate frames.
deadline = time.monotonic() + 7
while time.monotonic() < deadline:
reports = [line for line in (root / "compositor.log").read_text().splitlines()
if "capture scene outcomes" in line and "include_cursor=false" in line]
if any(int(match.group(1)) > 0 for line in reports
if (match := re.search(r"scene_reuses=(\d+)", line))):
break
time.sleep(.05)
else:
self.fail("Unchanged capture never reported reusing its filtered scene:\n"
+ (root / "compositor.log").read_text())
monitor_idle = raw.read_bytes()
self.assertEqual(len(monitor_idle), frame_bytes,
"unchanged monitor produced extra frames; first two images equal="
+ str(monitor_idle[:frame_bytes] == monitor_idle[frame_bytes:2 * frame_bytes]))
for count in range(2, 16):
target_client.stdin.write(b"d")
target_client.stdin.flush()
wait_frames(count)
consumer.wait(timeout=7)
self.assertEqual(consumer.returncode, 0)
monitor = raw.read_bytes()
self.assertEqual(len(monitor), frame_bytes * 15)
probe_x = round((cover["x"] + cover["width"] / 2) * scale)
probe_y = round((cover["y"] + cover["height"] / 4) * scale)
probe = (probe_y * ready["width"] + probe_x) * 4
for index in (0, 14):
start = index * frame_bytes + probe
self.assertNotEqual(monitor[start:start + 3], bytes([0, 255, 0]),
"private occluder leaked into the filtered capture")
for index in range(1, 15):
self.assertNotEqual(monitor[(index - 1) * frame_bytes:index * frame_bytes], monitor[index * frame_bytes:(index + 1) * frame_bytes],
"monitor stream reused stale pixels after source damage")
worker.stdin.close()
self.assertEqual(worker.wait(timeout=7), 0)
finally:
for process in reversed(processes):
if process.poll() is None:
os.killpg(process.pid, signal.SIGTERM)
process.wait(timeout=5)
if process.stdin:
process.stdin.close()
if process.stdout and process.stdout is not log:
process.stdout.close()
log.close()
@@ -0,0 +1,159 @@
"""Keep native backing pixels in window streams on a fractionally scaled host.
FERESE_TEST_WINDOW_CAPTURE_SCALE=1 python3 scripts/tests/test_window_capture_scale_isolated.py
Requires a Wayland host, PipeWire, GStreamer, wayland-scanner, cc, and Pillow.
"""
import json
import os
from pathlib import Path
import selectors
import signal
import subprocess
import tempfile
import time
import unittest
from session_socket import ipc_environment, ipc_socket
REPO = Path(__file__).resolve().parents[2]
COMPOSITOR = Path(os.environ.get("FERESE_TEST_BINARY", REPO / "target/release/ferese"))
CTL = Path(os.environ.get("FERESE_TEST_CTL", REPO / "target/release/feresectl"))
PORTAL = Path(os.environ.get("FERESE_TEST_PORTAL", REPO / "target/release/xdg-desktop-portal-ferese"))
@unittest.skipUnless(os.environ.get("FERESE_TEST_WINDOW_CAPTURE_SCALE") == "1", "opt-in capture test")
class WindowCaptureScale(unittest.TestCase):
def test_backing_pixels_survive_fractional_output_scale(self):
children = []
with tempfile.TemporaryDirectory(prefix="ferese-capture-scale-") as temporary:
root = Path(temporary)
protocols = Path(subprocess.check_output(
["pkg-config", "--variable=pkgdatadir", "wayland-protocols"], text=True).strip())
sources = []
for protocol, relative in [("xdg-shell", "stable/xdg-shell/xdg-shell.xml"),
("viewporter", "stable/viewporter/viewporter.xml")]:
source = root / f"{protocol}-protocol.c"
subprocess.run(["wayland-scanner", "client-header", str(protocols / relative),
str(root / f"{protocol}-client-protocol.h")], check=True)
subprocess.run(["wayland-scanner", "private-code", str(protocols / relative),
str(source)], check=True)
sources.append(str(source))
flags = subprocess.check_output(
["pkg-config", "--cflags", "--libs", "wayland-client"], text=True).split()
subprocess.run(["cc", "-Wall", "-Wextra", "-Werror", "-I", str(root),
str(REPO / "scripts/tests/fixtures/window-capture-scale.c"),
*sources, "-o", str(root / "client"), *flags], check=True)
runtime = root / "runtime"
runtime.mkdir(mode=0o700)
config = root / "config/ferese"
config.mkdir(parents=True)
(config / "config.kdl").write_text(
'animations { reduced-motion #true; }\n'
'window-rule app-id="ferese.test.capture-scale" floating=#true\n')
host_runtime = Path(os.environ["XDG_RUNTIME_DIR"])
display = Path(os.environ["WAYLAND_DISPLAY"])
if not display.is_absolute():
display = host_runtime / display
env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), WAYLAND_DISPLAY=str(display),
XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"),
XDG_DATA_HOME=str(root / "data"), FERESE_ENABLE_SCREENCOPY="1",
PIPEWIRE_REMOTE=str(host_runtime / "pipewire-0"))
for key in ("WAYLAND_SOCKET", "FERESE_SHELL_CONTROL_SOCKET", "DISPLAY", "XAUTHORITY"):
env.pop(key, None)
log_path = root / "session.log"
with log_path.open("w") as log:
def launch(command, **kwargs):
child = subprocess.Popen(command, env=ipc_environment(env), stderr=log,
stdout=kwargs.pop("stdout", log),
start_new_session=True, **kwargs)
children.append(child)
return child
def wait_for(check):
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
result = check()
if result:
return result
if compositor.poll() is not None:
self.fail(log_path.read_text())
time.sleep(.03)
self.fail("Timed out:\n" + log_path.read_text())
def call(*args):
return json.loads(subprocess.check_output(
[str(CTL), "-j", *args], env=ipc_environment(env), timeout=5))
try:
compositor = launch([str(COMPOSITOR), "--backend=nested"])
wait_for(lambda: ipc_socket(runtime).is_socket())
env["WAYLAND_DISPLAY"] = str(wait_for(
lambda: next((p for p in runtime.glob("wayland-*") if p.is_socket()), None)))
scale = call("outputs")[0]["scale"]
print(f"Isolated capture output scale: {scale}", flush=True)
self.assertTrue(1 < scale < 2,
"This regression needs a fractional host scale between 1 and 2; "
"an integer 2x host masks the original downsampling bug")
for mode, density in [("integer", 2), ("viewport", 2),
("fractional", 1.75), ("cropped", 1),
("subsurface", 2), ("subsurface-outside", 1)]:
with self.subTest(mode=mode, output_scale=scale):
client = launch([str(root / "client"), mode], stdin=subprocess.PIPE)
window = wait_for(lambda: next((w for w in call("windows")
if w["title"] == mode and w["mapped"]), None))
worker = launch([str(PORTAL), "--stream-window", str(window["id"]), "hidden"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
with selectors.DefaultSelector() as selector:
selector.register(worker.stdout, selectors.EVENT_READ)
self.assertTrue(selector.select(10), log_path.read_text())
ready = json.loads(worker.stdout.readline())
expected = (round(640 * max(scale, density)),
round(480 * max(scale, density)))
self.assertEqual((ready["width"], ready["height"]), expected,
"window sharing discarded native backing pixels")
raw = root / f"{mode}.bgrx"
consumer = launch(["gst-launch-1.0", "-q", "pipewiresrc",
f'path={ready["node"]}', "num-buffers=3", "!",
"video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"])
frame_bytes = expected[0] * expected[1] * 4
wait_for(lambda: raw.exists() and raw.stat().st_size >= frame_bytes)
# A static source waits for damage. Generate each next
# commit explicitly instead of relying on duplicate frames.
for count in (2, 3):
client.stdin.write(b"d")
client.stdin.flush()
wait_for(lambda: raw.stat().st_size >= count * frame_bytes)
consumer.wait(timeout=5)
self.assertEqual(consumer.returncode, 0)
width, height = expected
pixels = raw.read_bytes()
self.assertEqual(len(pixels), width * height * 4 * 3)
if mode == "subsurface-outside":
self.assertFalse(any(pixels[0::4]) or any(pixels[1::4])
or any(pixels[2::4]),
"content outside the window was captured")
elif density >= scale:
for index in (0, 2):
start = (index * width * height + height // 2 * width) * 4
row = pixels[start:start + width * 4]
self.assertEqual(row, bytes((0, 0, 0, 255, 255, 255, 255, 255)) * (width // 2),
"native one-pixel stripes were filtered")
worker.stdin.close()
worker.wait(timeout=5)
worker.stdout.close()
client.terminate()
client.wait(timeout=5)
client.stdin.close()
wait_for(lambda: not any(w["id"] == window["id"] for w in call("windows")))
finally:
for child in reversed(children):
if child.poll() is None:
os.killpg(child.pid, signal.SIGTERM)
child.wait(timeout=5)
for pipe in (child.stdin, child.stdout, child.stderr):
if pipe:
pipe.close()
if __name__ == "__main__":
unittest.main()