perf(capture): add SceneLens GPU streams and retained scene reuse

Use persistent privacy-filtered capture targets and client-owned GPU buffers with fence completion, backpressure, negotiated frame rates and shared-memory fallback.

Invalidate output views on scene changes while preserving inactive-window callbacks and capture revocation. Include OBS memory-mask negotiation and native pixel, transport and recording regressions.

scripts/tests/fixtures/capture-consumer.c

@@ -0,0 +1,179 @@
+/* Native PipeWire consumer: require the selected memory type and write packed
+ * BGRx frames only after the producer has completed GPU work. */
+#define _GNU_SOURCE
+#include <errno.h>
+#include <linux/dma-buf.h>
+#include <pipewire/pipewire.h>
+#include <spa/param/video/format-utils.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+
+struct capture {
+    struct pw_main_loop *loop;
+    struct pw_stream *stream;
+    struct spa_hook listener;
+    struct spa_video_info_raw format;
+    FILE *file;
+    unsigned expected_type, frames, wanted;
+    uint64_t last_sequence;
+    int64_t last_pts;
+    int failed;
+    int obs_buffers;
+};
+
+static void fail(struct capture *capture, const char *error) {
+    fprintf(stderr, "capture-consumer: %s\n", error);
+    capture->failed = 1;
+    pw_main_loop_quit(capture->loop);
+}
+
+static void state_changed(void *data, enum pw_stream_state old,
+                          enum pw_stream_state state, const char *error) {
+    (void)old;
+    struct capture *capture = data;
+    if (state == PW_STREAM_STATE_ERROR) fail(capture, error ? error : "stream error");
+}
+
+static void param_changed(void *data, uint32_t id, const struct spa_pod *param) {
+    struct capture *capture = data;
+    if (id != SPA_PARAM_Format || !param) return;
+    if (spa_format_video_raw_parse(param, &capture->format) < 0 ||
+        capture->format.format != SPA_VIDEO_FORMAT_BGRx ||
+        !capture->format.size.width || !capture->format.size.height ||
+        !!(capture->format.flags & SPA_VIDEO_FLAG_MODIFIER) != (capture->expected_type == SPA_DATA_DmaBuf)) {
+        fail(capture, "incorrect negotiated video format or memory contract");
+        return;
+    }
+    uint8_t bytes[1024];
+    struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(bytes, sizeof bytes);
+    const struct spa_pod *buffers;
+    if (capture->obs_buffers) {
+        /* OBS supplies a scalar bitmask containing multiple accepted types.
+         * Keep this separate from the strict transport test's flags choice. */
+        buffers = spa_pod_builder_add_object(&builder,
+            SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers,
+            SPA_PARAM_BUFFERS_dataType,
+            SPA_POD_Int((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_DmaBuf)));
+    } else {
+        buffers = spa_pod_builder_add_object(&builder,
+            SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers,
+            SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(4, 2, 8),
+            SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(1 << capture->expected_type));
+    }
+    const struct spa_pod *header = spa_pod_builder_add_object(&builder,
+        SPA_TYPE_OBJECT_ParamMeta, SPA_PARAM_Meta,
+        SPA_PARAM_META_type, SPA_POD_Id(SPA_META_Header),
+        SPA_PARAM_META_size, SPA_POD_Int(sizeof(struct spa_meta_header)));
+    const struct spa_pod *parameters[] = {buffers, header};
+    pw_stream_update_params(capture->stream, parameters, 2);
+    fprintf(stderr, "capture-consumer: %ux%u %u/%u fps %s\n", capture->format.size.width,
+            capture->format.size.height, capture->format.framerate.num,
+            capture->format.framerate.denom, capture->expected_type == SPA_DATA_DmaBuf ? "DMA-BUF" : "SHM");
+}
+
+static void process(void *data) {
+    struct capture *capture = data;
+    struct pw_buffer *buffer = pw_stream_dequeue_buffer(capture->stream);
+    if (!buffer) return;
+    struct spa_buffer *spa = buffer->buffer;
+    if (spa->n_datas != 1 || !spa->datas[0].chunk) {
+        fail(capture, "invalid plane count or missing chunk");
+        goto done;
+    }
+    struct spa_data *plane = &spa->datas[0];
+    if (!plane->chunk->size) goto done;
+    if (capture->obs_buffers && capture->expected_type == SPA_DATA_MemFd && !plane->data) {
+        fail(capture, "OBS shared-memory input was not mapped by PipeWire");
+        goto done;
+    }
+    const struct spa_meta_header *header = spa_buffer_find_meta_data(spa, SPA_META_Header, sizeof *header);
+    if (!header || header->pts < 0 || (capture->frames && (header->seq <= capture->last_sequence || header->pts <= capture->last_pts))) {
+        fprintf(stderr, "capture-consumer: header=%p frames=%u pts=%lld last_pts=%lld sequence=%llu last_sequence=%llu\n", (void *)header, capture->frames,
+                (long long)(header ? header->pts : -1), (long long)capture->last_pts,
+                (unsigned long long)(header ? header->seq : 0), (unsigned long long)capture->last_sequence);
+        fail(capture, "missing or stale frame sequence/timestamp");
+        goto done;
+    }
+    capture->last_sequence = header->seq;
+    capture->last_pts = header->pts;
+    uint32_t row = capture->format.size.width * 4;
+    uint32_t height = capture->format.size.height;
+    if (plane->type != capture->expected_type || plane->fd < 0 || plane->chunk->stride < (int)row ||
+        (uint64_t)plane->chunk->offset + (uint64_t)(height - 1) * plane->chunk->stride + row > plane->maxsize) {
+        fail(capture, "producer supplied the wrong memory type or invalid plane bounds");
+        goto done;
+    }
+    size_t length = (size_t)plane->mapoffset + plane->maxsize;
+    void *mapping = mmap(NULL, length, PROT_READ, MAP_SHARED, plane->fd, 0);
+    if (mapping == MAP_FAILED) { fail(capture, "buffer mapping failed"); goto done; }
+    struct dma_buf_sync sync = {.flags = DMA_BUF_SYNC_START | DMA_BUF_SYNC_READ};
+    if (plane->type == SPA_DATA_DmaBuf && ioctl(plane->fd, DMA_BUF_IOCTL_SYNC, &sync)) {
+        munmap(mapping, length); fail(capture, "DMA-BUF read synchronization failed"); goto done;
+    }
+    const uint8_t *pixels = (const uint8_t *)mapping + plane->mapoffset + plane->chunk->offset;
+    for (uint32_t y = 0; y < height; y++) {
+        if (fwrite(pixels + (size_t)y * plane->chunk->stride, 1, row, capture->file) != row) {
+            capture->failed = 1;
+            break;
+        }
+    }
+    if (plane->type == SPA_DATA_DmaBuf) {
+        sync.flags = DMA_BUF_SYNC_END | DMA_BUF_SYNC_READ;
+        if (ioctl(plane->fd, DMA_BUF_IOCTL_SYNC, &sync)) capture->failed = 1;
+    }
+    munmap(mapping, length);
+    fflush(capture->file);
+    capture->frames++;
+    if (capture->failed || capture->frames >= capture->wanted) pw_main_loop_quit(capture->loop);
+ done:
+    pw_stream_queue_buffer(capture->stream, buffer);
+}
+
+int main(int argc, char **argv) {
+    if (argc != 5) return 2;
+    pw_init(&argc, &argv);
+    struct capture capture = {.expected_type = !strcmp(argv[2], "dma") ? SPA_DATA_DmaBuf : SPA_DATA_MemFd,
+                              .wanted = strtoul(argv[3], NULL, 10),
+                              .obs_buffers = getenv("FERESE_TEST_CAPTURE_OBS_BUFFERS") != NULL};
+    capture.file = fopen(argv[4], "wb");
+    capture.loop = pw_main_loop_new(NULL);
+    if (!capture.file || !capture.loop || !capture.wanted) return 3;
+    struct pw_context *context = pw_context_new(pw_main_loop_get_loop(capture.loop), NULL, 0);
+    struct pw_core *core = pw_context_connect(context, NULL, 0);
+    if (!core) return 4;
+    capture.stream = pw_stream_new(core, "Ferese capture regression", pw_properties_new(PW_KEY_MEDIA_TYPE, "Video", PW_KEY_MEDIA_CATEGORY, "Capture", NULL));
+    const struct pw_stream_events events = {.version = PW_VERSION_STREAM_EVENTS,
+        .state_changed = state_changed, .param_changed = param_changed, .process = process};
+    pw_stream_add_listener(capture.stream, &capture.listener, &events, &capture);
+    uint8_t bytes[1024];
+    struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(bytes, sizeof bytes);
+    const struct spa_pod *format;
+    if (capture.expected_type == SPA_DATA_DmaBuf) {
+        format = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
+            SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
+            SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
+            SPA_FORMAT_VIDEO_format, SPA_POD_Id(SPA_VIDEO_FORMAT_BGRx),
+            SPA_FORMAT_VIDEO_modifier, SPA_POD_Long(0),
+            SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&SPA_FRACTION(60, 1)));
+    } else {
+        format = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat,
+            SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video),
+            SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw),
+            SPA_FORMAT_VIDEO_format, SPA_POD_Id(SPA_VIDEO_FORMAT_BGRx),
+            SPA_FORMAT_VIDEO_framerate, SPA_POD_Fraction(&SPA_FRACTION(60, 1)));
+    }
+    enum pw_stream_flags stream_flags = PW_STREAM_FLAG_AUTOCONNECT;
+    if (capture.obs_buffers) stream_flags |= PW_STREAM_FLAG_MAP_BUFFERS;
+    int result = pw_stream_connect(capture.stream, PW_DIRECTION_INPUT, strtoul(argv[1], NULL, 10), stream_flags, &format, 1);
+    if (result >= 0) pw_main_loop_run(capture.loop);
+    else capture.failed = 1;
+    pw_stream_destroy(capture.stream);
+    pw_core_disconnect(core);
+    pw_context_destroy(context);
+    pw_main_loop_destroy(capture.loop);
+    fclose(capture.file);
+    return capture.failed || capture.frames != capture.wanted;
+}

scripts/tests/fixtures/window-capture-scale.c

@@ -0,0 +1,148 @@
+/* Native-pixel stripes distinguish a preserved backing buffer from a resample. */
+#define _GNU_SOURCE
+#include <stdint.h>
+#include <poll.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <wayland-client.h>
+#include "xdg-shell-client-protocol.h"
+#include "viewporter-client-protocol.h"
+
+static struct wl_compositor *compositor;
+static struct wl_subcompositor *subcompositor;
+static struct wl_shm *shm;
+static struct xdg_wm_base *wm;
+static struct wp_viewporter *viewporter;
+static struct wl_surface *surface;
+static struct wl_buffer *buffer;
+
+static void ping(void *data, struct xdg_wm_base *base, uint32_t serial) {
+    (void)data;
+    xdg_wm_base_pong(base, serial);
+}
+static const struct xdg_wm_base_listener wm_listener = {.ping = ping};
+
+static void global(void *data, struct wl_registry *registry, uint32_t name,
+                   const char *interface, uint32_t version) {
+    (void)data; (void)version;
+    if (!strcmp(interface, "wl_compositor"))
+        compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 4);
+    else if (!strcmp(interface, "wl_subcompositor"))
+        subcompositor = wl_registry_bind(registry, name, &wl_subcompositor_interface, 1);
+    else if (!strcmp(interface, "wl_shm"))
+        shm = wl_registry_bind(registry, name, &wl_shm_interface, 1);
+    else if (!strcmp(interface, "wp_viewporter"))
+        viewporter = wl_registry_bind(registry, name, &wp_viewporter_interface, 1);
+    else if (!strcmp(interface, "xdg_wm_base")) {
+        wm = wl_registry_bind(registry, name, &xdg_wm_base_interface, 1);
+        xdg_wm_base_add_listener(wm, &wm_listener, NULL);
+    }
+}
+static void removed(void *data, struct wl_registry *registry, uint32_t name) {
+    (void)data; (void)registry; (void)name;
+}
+static const struct wl_registry_listener registry_listener = {global, removed};
+
+static void configure(void *data, struct xdg_surface *xdg, uint32_t serial) {
+    (void)data;
+    xdg_surface_ack_configure(xdg, serial);
+    xdg_surface_set_window_geometry(xdg, 0, 0, 640, 480);
+    wl_surface_attach(surface, buffer, 0, 0);
+    wl_surface_damage_buffer(surface, 0, 0, INT32_MAX, INT32_MAX);
+    wl_surface_commit(surface);
+}
+static const struct xdg_surface_listener surface_listener = {.configure = configure};
+static void size(void *data, struct xdg_toplevel *top, int32_t w, int32_t h,
+                 struct wl_array *states) {
+    (void)data; (void)top; (void)w; (void)h; (void)states;
+}
+static void close_window(void *data, struct xdg_toplevel *top) {
+    (void)data; (void)top; exit(0);
+}
+static const struct xdg_toplevel_listener top_listener = {.configure = size, .close = close_window};
+
+int main(int argc, char **argv) {
+    if (argc != 2) return 1;
+    struct wl_display *display = wl_display_connect(NULL);
+    if (!display) return 2;
+    struct wl_registry *registry = wl_display_get_registry(display);
+    wl_registry_add_listener(registry, &registry_listener, NULL);
+    if (wl_display_roundtrip(display) < 0 || !compositor || !shm || !wm || !viewporter) return 3;
+    int integer = !strcmp(argv[1], "integer");
+    int fractional = !strcmp(argv[1], "fractional");
+    int cropped = !strcmp(argv[1], "cropped");
+    int width = fractional ? 1120 : 1280;
+    int height = fractional ? 840 : 960;
+    int fd = memfd_create("window-capture-scale", MFD_CLOEXEC);
+    size_t bytes = (size_t)width * height * 4;
+    if (fd < 0 || ftruncate(fd, bytes)) return 4;
+    uint32_t *pixels = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (pixels == MAP_FAILED) return 5;
+    for (int y = 0; y < height; y++)
+        for (int x = 0; x < width; x++)
+            pixels[y * width + x] = x % 2 ? 0xffffffff : 0xff000000;
+    struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, bytes);
+    buffer = wl_shm_pool_create_buffer(pool, 0, width, height, width * 4, WL_SHM_FORMAT_ARGB8888);
+    wl_shm_pool_destroy(pool);
+    munmap(pixels, bytes);
+    close(fd);
+    surface = wl_compositor_create_surface(compositor);
+    if (!strcmp(argv[1], "subsurface") || !strcmp(argv[1], "subsurface-outside")) {
+        if (!subcompositor) return 6;
+        struct wl_surface *child = wl_compositor_create_surface(compositor);
+        struct wl_subsurface *sub = wl_subcompositor_get_subsurface(subcompositor, child, surface);
+        if (!strcmp(argv[1], "subsurface-outside"))
+            wl_subsurface_set_position(sub, 1280, 0);
+        wl_surface_set_buffer_scale(child, 2);
+        wl_surface_attach(child, buffer, 0, 0);
+        wl_surface_damage_buffer(child, 0, 0, INT32_MAX, INT32_MAX);
+        wl_surface_commit(child);
+        // A transparent placeholder makes the child's density authoritative.
+        int root_fd = memfd_create("capture-scale-root", MFD_CLOEXEC);
+        if (root_fd < 0 || ftruncate(root_fd, 4)) return 7;
+        uint32_t *root_pixels = mmap(NULL, 4, PROT_READ | PROT_WRITE, MAP_SHARED, root_fd, 0);
+        if (root_pixels == MAP_FAILED) return 8;
+        *root_pixels = 0;
+        struct wl_shm_pool *root_pool = wl_shm_create_pool(shm, root_fd, 4);
+        buffer = wl_shm_pool_create_buffer(root_pool, 0, 1, 1, 4, WL_SHM_FORMAT_ARGB8888);
+        wl_shm_pool_destroy(root_pool);
+        munmap(root_pixels, 4);
+        close(root_fd);
+    }
+    if (integer) {
+        wl_surface_set_buffer_scale(surface, 2);
+    } else {
+        struct wp_viewport *viewport = wp_viewporter_get_viewport(viewporter, surface);
+        wp_viewport_set_destination(viewport, 640, 480);
+        if (cropped)
+            wp_viewport_set_source(viewport, wl_fixed_from_int(320), wl_fixed_from_int(240),
+                                   wl_fixed_from_int(640), wl_fixed_from_int(480));
+    }
+    struct xdg_surface *xdg = xdg_wm_base_get_xdg_surface(wm, surface);
+    xdg_surface_add_listener(xdg, &surface_listener, NULL);
+    struct xdg_toplevel *top = xdg_surface_get_toplevel(xdg);
+    xdg_toplevel_add_listener(top, &top_listener, NULL);
+    xdg_toplevel_set_app_id(top, "ferese.test.capture-scale");
+    xdg_toplevel_set_title(top, argv[1]);
+    wl_surface_commit(surface);
+    while (1) {
+        wl_display_flush(display);
+        struct pollfd fds[2] = {{wl_display_get_fd(display), POLLIN, 0}, {STDIN_FILENO, POLLIN, 0}};
+        if (poll(fds, 2, -1) < 0) break;
+        if ((fds[0].revents | fds[1].revents) & (POLLERR | POLLHUP | POLLNVAL)) break;
+        if (fds[0].revents & POLLIN && wl_display_dispatch(display) < 0) break;
+        if (fds[1].revents & POLLIN) {
+            char command;
+            if (read(STDIN_FILENO, &command, 1) != 1) break;
+            if (command == 'd') {
+                wl_surface_attach(surface, buffer, 0, 0);
+                wl_surface_damage(surface, 0, 0, 640, 480);
+                wl_surface_commit(surface);
+            }
+        }
+    }
+    return 0;
+}

scripts/tests/fixtures/window-capture.c

@@ -18,6 +18,9 @@ static struct wl_buffer *buffer;
 static int width = 640, height = 480;
 static int private_capture_seen;
 static int covered;
+static unsigned updates;
+static unsigned animation_frames;
+static void make_buffer(void);
 
 static void ping(void *data, struct xdg_wm_base *base, uint32_t serial) {
     (void)data;
@@ -28,7 +31,9 @@ static const struct xdg_wm_base_listener wm_listener = {.ping = ping};
 static void global(void *data, struct wl_registry *registry, uint32_t name,
                    const char *interface, uint32_t version) {
     (void)data; (void)version;
-    if (!strcmp(interface, "ferese_window_capture_manager_v1")) private_capture_seen = 1;
+    if (!strcmp(interface, "ferese_window_capture_manager_v1") ||
+        !strcmp(interface, "ext_image_copy_capture_manager_v1") ||
+        !strcmp(interface, "ext_output_image_capture_source_manager_v1")) private_capture_seen = 1;
     if (!strcmp(interface, "wl_compositor"))
         compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 4);
     else if (!strcmp(interface, "wl_shm"))
@@ -73,6 +78,7 @@ static void make_buffer(void) {
     for (size_t i = 0; i < (size_t)width * height; i++) {
         pixels[i] = covered ? 0xff00ff00 : (i % width < 80 ? 0x80800000 : (i / width < (size_t)height / 2 ? 0xffff0000 : 0xff0000ff));
     }
+    if (updates) pixels[(size_t)(height / 2) * width + width / 2] = 0xff000000 | updates;
     struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, bytes);
     buffer = wl_shm_pool_create_buffer(pool, 0, width, height, width * 4, WL_SHM_FORMAT_ARGB8888);
     wl_shm_pool_destroy(pool);
@@ -80,6 +86,23 @@ static void make_buffer(void) {
     munmap(pixels, bytes);
 }
 
+static void animate(void *data, struct wl_callback *callback, uint32_t time) {
+    (void)data; (void)time;
+    wl_callback_destroy(callback);
+    updates++;
+    wl_buffer_destroy(buffer);
+    make_buffer();
+    wl_surface_attach(surface, buffer, 0, 0);
+    wl_surface_damage(surface, width / 2, height / 2, 1, 1);
+    if (--animation_frames) {
+        struct wl_callback *next = wl_surface_frame(surface);
+        static const struct wl_callback_listener listener = {.done = animate};
+        wl_callback_add_listener(next, &listener, NULL);
+    }
+    wl_surface_commit(surface);
+}
+static const struct wl_callback_listener animation_listener = {.done = animate};
+
 int main(int argc, char **argv) {
     struct wl_display *display = wl_display_connect(NULL);
     if (!display) return 1;
@@ -97,17 +120,34 @@ int main(int argc, char **argv) {
     xdg_toplevel_set_app_id(top, "ferese.test.window-capture");
     xdg_toplevel_set_title(top, covered ? "Occluding window" : "Capture target");
     wl_surface_commit(surface);
-    if (argc > 1 && !strcmp(argv[1], "resize")) {
+    if ((argc > 1 && (!strcmp(argv[1], "resize") || !strcmp(argv[1], "interactive"))) || (argc > 2 && !strcmp(argv[2], "interactive"))) {
         while (1) {
             wl_display_flush(display);
             struct pollfd fds[2] = {{wl_display_get_fd(display), POLLIN, 0}, {STDIN_FILENO, POLLIN, 0}};
             if (poll(fds, 2, -1) < 0) break;
+            if ((fds[0].revents | fds[1].revents) & (POLLERR | POLLHUP | POLLNVAL)) break;
             if (fds[0].revents & POLLIN && wl_display_dispatch(display) < 0) break;
             if (fds[1].revents & POLLIN) {
                 char command;
                 if (read(STDIN_FILENO, &command, 1) != 1) break;
-                if (command == 'r') {
-                    width = 480; height = 320;
+                if (command == 'a') {
+                    animation_frames = 3;
+                    struct wl_callback *next = wl_surface_frame(surface);
+                    wl_callback_add_listener(next, &animation_listener, NULL);
+                    // Start an animation by requesting a callback; pixels have
+                    // not changed yet. A capture must not leave this waiting.
+                    wl_surface_commit(surface);
+                }
+                if (command == 'd') {
+                    updates++;
+                    wl_buffer_destroy(buffer);
+                    make_buffer();
+                    wl_surface_attach(surface, buffer, 0, 0);
+                    wl_surface_damage(surface, width / 2, height / 2, 1, 1);
+                    wl_surface_commit(surface);
+                }
+                if (command == 'r' || command == 'o') {
+                    width = 480 + (command == 'o'); height = 320 + (command == 'o');
                     wl_buffer_destroy(buffer);
                     make_buffer();
                     xdg_surface_set_window_geometry(xdg, 0, 0, width, height);