perf(capture): add SceneLens GPU streams and retained scene reuse

Use persistent privacy-filtered capture targets and client-owned GPU buffers with fence completion, backpressure, negotiated frame rates and shared-memory fallback.

Invalidate output views on scene changes while preserving inactive-window callbacks and capture revocation. Include OBS memory-mask negotiation and native pixel, transport and recording regressions.

scripts/tests/test_window_capture_isolated.py

@@ -9,6 +9,7 @@ import io
 import json
 import os
 from pathlib import Path
+import re
 from session_socket import ipc_environment, ipc_socket
 import signal
 import selectors
@@ -54,7 +55,7 @@ class WindowCapture(unittest.TestCase):
             display = Path(os.environ["WAYLAND_DISPLAY"])
             if not display.is_absolute():
                 display = Path(os.environ["XDG_RUNTIME_DIR"]) / display
-            env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"), WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1")
+            env = dict(os.environ, XDG_RUNTIME_DIR=str(runtime), XDG_CONFIG_HOME=str(root / "config"), XDG_STATE_HOME=str(root / "state"), WAYLAND_DISPLAY=str(display), FERESE_ENABLE_SCREENCOPY="1", FERESE_TRACE_PERFORMANCE="1")
             env.pop("WAYLAND_SOCKET", None)
             env.pop("FERESE_SHELL_CONTROL_SOCKET", None)
             log = (root / "compositor.log").open("w")
@@ -72,18 +73,20 @@ class WindowCapture(unittest.TestCase):
                 def call(*args):
                     return subprocess.check_output([str(CTL), "-j", *args], env=ipc_environment(env), timeout=10)
 
-                def windows(count):
+                def windows(count, width=640):
                     deadline = time.monotonic() + 10
                     while time.monotonic() < deadline:
                         result = json.loads(call("get-windows"))
-                        if len(result) == count and all(window["mapped"] and window["width"] == 640 for window in result):
+                        if len(result) == count and all(window["mapped"] and (width is None or window["width"] == width) for window in result):
                             return result
                         time.sleep(0.02)
                     self.fail((root / "compositor.log").read_text())
 
-                processes.append(subprocess.Popen([str(root / "client")], env=ipc_environment(env), stdout=log, stderr=log, start_new_session=True))
+                target_client = subprocess.Popen([str(root / "client"), "interactive"], env=ipc_environment(env), stdin=subprocess.PIPE, stdout=log, stderr=log, start_new_session=True)
+                processes.append(target_client)
                 target = windows(1)[0]
-                processes.append(subprocess.Popen([str(root / "client"), "cover"], env=ipc_environment(env), stdout=log, stderr=log, start_new_session=True))
+                cover_client = subprocess.Popen([str(root / "client"), "cover", "interactive"], env=ipc_environment(env), stdin=subprocess.PIPE, stdout=log, stderr=log, start_new_session=True)
+                processes.append(cover_client)
                 current = windows(2)
                 cover = next(window for window in current if window["title"] == "Occluding window")
                 self.assertEqual((target["x"], target["y"], target["width"], target["height"]),
@@ -96,8 +99,12 @@ class WindowCapture(unittest.TestCase):
                     self.assertEqual(image.convert("RGBA").getpixel((round(40 * scale), round(80 * scale))), (255, 0, 0, 128))
                 with Image.open(io.BytesIO(call("screenshot-window", str(cover["id"])))) as image:
                     self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 2)), (0, 255, 0))
-                # Exercise the asynchronous conversion worker and reuse its render
-                # target across a stream, not just the synchronous screenshot path.
+                # Capture an inactive workspace, where display repaint callbacks
+                # cannot conceal a stalled capture-driven animation.
+                call("workspace", "2")
+                # Verify real streaming updates and silence between commits. Each
+                # requested frame changes a pixel; static content must not be copied
+                # repeatedly just to satisfy a consumer's process callbacks.
                 stream_env = dict(env, PIPEWIRE_REMOTE=str(Path(os.environ["XDG_RUNTIME_DIR"]) / "pipewire-0"))
                 worker = subprocess.Popen([str(PORTAL),
                                            "--stream-window", str(target["id"]), "hidden"],
@@ -110,9 +117,37 @@ class WindowCapture(unittest.TestCase):
                 ready = json.loads(worker.stdout.readline())
                 self.assertEqual((ready["width"], ready["height"]), (round(640 * scale), round(480 * scale)))
                 raw = root / "frames.bgrx"
-                subprocess.run(["gst-launch-1.0", "-q", "pipewiresrc", "min-buffers=4", f'path={ready["node"]}',
-                                "num-buffers=15", "!", "video/x-raw,format=BGRx", "!", "filesink",
-                                f"location={raw}"], env=stream_env, check=True, timeout=20)
+                transport = os.environ.get("FERESE_TEST_CAPTURE_TRANSPORT")
+                if transport:
+                    flags = subprocess.check_output(["pkg-config", "--cflags", "--libs", "libpipewire-0.3"], text=True).split()
+                    subprocess.run(["cc", "-Wall", "-Wextra", str(REPO / "scripts/tests/fixtures/capture-consumer.c"), "-o", str(root / "consumer"), *flags], check=True)
+                    command = [str(root / "consumer"), str(ready["node"]), transport, "15", str(raw)]
+                else:
+                    command = ["gst-launch-1.0", "-q", "pipewiresrc", "min-buffers=4", f'path={ready["node"]}',
+                               "num-buffers=15", "!", "video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"]
+                consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
+                processes.append(consumer)
+                frame_bytes = ready["width"] * ready["height"] * 4
+
+                def wait_frames(count):
+                    deadline = time.monotonic() + 7
+                    while not raw.exists() or raw.stat().st_size < count * frame_bytes:
+                        if consumer.poll() is not None or worker.poll() is not None or time.monotonic() >= deadline:
+                            self.fail((root / "compositor.log").read_text())
+                        time.sleep(0.01)
+
+                wait_frames(1)
+                time.sleep(0.35)
+                self.assertEqual(raw.stat().st_size, frame_bytes, "unchanged window produced duplicate frames")
+                target_client.stdin.write(b"a")
+                target_client.stdin.flush()
+                wait_frames(4)
+                for count in range(5, 16):
+                    target_client.stdin.write(b"d")
+                    target_client.stdin.flush()
+                    wait_frames(count)
+                consumer.wait(timeout=7)
+                self.assertEqual(consumer.returncode, 0)
                 pixels = raw.read_bytes()
                 width, height = ready["width"], ready["height"]
                 frame_bytes = width * height * 4
@@ -123,6 +158,59 @@ class WindowCapture(unittest.TestCase):
                     blue = start + ((height * 3 // 4) * width + width // 2) * 4
                     self.assertEqual(pixels[red:red + 3], bytes([0, 0, 255]))
                     self.assertEqual(pixels[blue:blue + 3], bytes([255, 0, 0]))
+                call("workspace", "1")
+                # A new consumer must receive an initial image even when the
+                # source has stayed unchanged since the previous consumer left.
+                original_command = command.copy()
+                raw = root / "reconnected.bgrx"
+                if transport:
+                    command[3] = "1"
+                    command[-1] = str(raw)
+                else:
+                    command[command.index("num-buffers=15")] = "num-buffers=1"
+                    command[-1] = f"location={raw}"
+                consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
+                processes.append(consumer)
+                wait_frames(1)
+                consumer.wait(timeout=7)
+                self.assertEqual(consumer.returncode, 0)
+                self.assertEqual(raw.stat().st_size, frame_bytes)
+                command = original_command
+                # Resize while the consumer is disconnected, then reconnect to
+                # the same node. Retired pool buffers must never be queued again.
+                resize_width, resize_height = ((481, 321) if transport else (480, 320))
+                target_client.stdin.write(b"o" if transport else b"r")
+                target_client.stdin.flush()
+                deadline = time.monotonic() + 7
+                while not any(w["id"] == target["id"] and w["width"] == resize_width for w in json.loads(call("windows"))):
+                    if time.monotonic() >= deadline:
+                        self.fail("Window did not resize")
+                    time.sleep(.02)
+                raw = root / "resized.bgrx"
+                if transport:
+                    command[-1] = str(raw)
+                else:
+                    command[-1] = f"location={raw}"
+                consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
+                processes.append(consumer)
+                width, height = round(resize_width * scale), round(resize_height * scale)
+                frame_bytes = width * height * 4
+                wait_frames(1)
+                time.sleep(.35)
+                self.assertEqual(raw.stat().st_size, frame_bytes, "paused resize left a stale frame in the stream")
+                for count in range(2, 16):
+                    target_client.stdin.write(b"d")
+                    target_client.stdin.flush()
+                    wait_frames(count)
+                consumer.wait(timeout=7)
+                self.assertEqual(consumer.returncode, 0)
+                resized = raw.read_bytes()
+                self.assertEqual(len(resized), frame_bytes * 15)
+                for index in (0, 14):
+                    red = index * frame_bytes + ((height // 4) * width + width // 2) * 4
+                    blue = index * frame_bytes + ((height * 3 // 4) * width + width // 2) * 4
+                    self.assertEqual(resized[red:red + 3], bytes([0, 0, 255]))
+                    self.assertEqual(resized[blue:blue + 3], bytes([255, 0, 0]))
                 # A live rule change revokes the window stream and synchronous
                 # window screenshot route, while normal display mapping remains.
                 config.write_text('window-rule app-id="ferese.test.window-capture" floating=#true\n'
@@ -135,12 +223,12 @@ class WindowCapture(unittest.TestCase):
                                           env=ipc_environment(env), capture_output=True, timeout=10)
                 self.assertNotEqual(rejected.returncode, 0)
                 self.assertIn(b"protected", rejected.stderr)
-                self.assertTrue(next(window for window in windows(2) if window["id"] == target["id"])["mapped"])
+                self.assertTrue(next(window for window in windows(2, width=None) if window["id"] == target["id"])["mapped"])
                 with Image.open(io.BytesIO(call("screenshot-window", str(cover["id"])))) as image:
                     self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 2)), (0, 255, 0))
                 os.killpg(processes[2].pid, signal.SIGTERM)
                 processes[2].wait(timeout=5)
-                protected = windows(1)[0]
+                protected = windows(1, width=resize_width)[0]
                 self.assertTrue(protected["mapped"])
                 for args in [("screenshot",), ("screenshot", "--geometry",
                              f'{protected["x"]},{protected["y"]} {protected["width"]}x{protected["height"]}')]:
@@ -152,11 +240,103 @@ class WindowCapture(unittest.TestCase):
                 call("reload-config")
                 with Image.open(io.BytesIO(call("screenshot-window", str(target["id"])))) as image:
                     self.assertEqual(image.convert("RGB").getpixel((image.width // 2, image.height // 4)), (255, 0, 0))
+                # Updates behind an opaque private window still belong in the
+                # filtered capture, even when display occlusion hides them.
+                config.write_text('window-rule app-id="ferese.test.window-capture" floating=#true\n'
+                                  'window-rule title="Occluding window" block-out-from-screencasts=#true\n')
+                call("reload-config")
+                private_cover = subprocess.Popen([str(root / "client"), "cover", "interactive"],
+                                                 env=ipc_environment(env), stdin=subprocess.PIPE,
+                                                 stdout=log, stderr=log, start_new_session=True)
+                processes.append(private_cover)
+                pair = windows(2, width=None)
+                cover = next(window for window in pair if window["title"] == "Occluding window")
+                target = next(window for window in pair if window["id"] == target["id"])
+                marker_x = target["x"] + target["width"] / 2
+                marker_y = target["y"] + target["height"] / 2
+                self.assertLess(cover["x"], marker_x)
+                self.assertGreater(cover["x"] + cover["width"], marker_x)
+                self.assertLess(cover["y"], marker_y)
+                self.assertGreater(cover["y"] + cover["height"], marker_y)
+                # Monitor capture uses an independent filtered scene and the same
+                # buffer transport. It must also wait quietly between real updates.
+                # Adding the occluder and reloading rules can animate focus
+                # borders. Those are real damage, so establish a settled scene
+                # before asserting that an unchanged monitor produces no frames.
+                previous, stable_since = None, time.monotonic()
+                deadline = time.monotonic() + 7
+                while time.monotonic() < deadline:
+                    image = call("screenshot")
+                    if image != previous:
+                        previous, stable_since = image, time.monotonic()
+                    elif time.monotonic() - stable_since >= .35:
+                        break
+                    time.sleep(.04)
+                else:
+                    self.fail("Monitor did not settle before the damage-waiting check")
+                sources = json.loads(subprocess.check_output([str(PORTAL), "--sources"], env=stream_env, timeout=7))
+                worker = subprocess.Popen([str(PORTAL), "--stream", sources[0]["name"], "hidden"], env=stream_env, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=log, start_new_session=True)
+                processes.append(worker)
+                with selectors.DefaultSelector() as selector:
+                    selector.register(worker.stdout, selectors.EVENT_READ)
+                    self.assertTrue(selector.select(10), log.name)
+                ready = json.loads(worker.stdout.readline())
+                raw = root / "monitor.bgrx"
+                frame_bytes = ready["width"] * ready["height"] * 4
+                if transport:
+                    command = [str(root / "consumer"), str(ready["node"]), transport, "15", str(raw)]
+                else:
+                    command = ["gst-launch-1.0", "-q", "pipewiresrc", f'path={ready["node"]}', "num-buffers=15", "!",
+                               "video/x-raw,format=BGRx", "!", "filesink", "buffer-mode=unbuffered", f"location={raw}"]
+                consumer = subprocess.Popen(command, env=stream_env, stdout=log, stderr=log, start_new_session=True)
+                processes.append(consumer)
+                wait_frames(1)
+                # Wait for a real compositor report, rather than inferring GPU
+                # reuse merely from the absence of delivered duplicate frames.
+                deadline = time.monotonic() + 7
+                while time.monotonic() < deadline:
+                    reports = [line for line in (root / "compositor.log").read_text().splitlines()
+                               if "capture scene outcomes" in line and "include_cursor=false" in line]
+                    if any(int(match.group(1)) > 0 for line in reports
+                           if (match := re.search(r"scene_reuses=(\d+)", line))):
+                        break
+                    time.sleep(.05)
+                else:
+                    self.fail("Unchanged capture never reported reusing its filtered scene:\n"
+                              + (root / "compositor.log").read_text())
+                monitor_idle = raw.read_bytes()
+                self.assertEqual(len(monitor_idle), frame_bytes,
+                                 "unchanged monitor produced extra frames; first two images equal="
+                                 + str(monitor_idle[:frame_bytes] == monitor_idle[frame_bytes:2 * frame_bytes]))
+                for count in range(2, 16):
+                    target_client.stdin.write(b"d")
+                    target_client.stdin.flush()
+                    wait_frames(count)
+                consumer.wait(timeout=7)
+                self.assertEqual(consumer.returncode, 0)
+                monitor = raw.read_bytes()
+                self.assertEqual(len(monitor), frame_bytes * 15)
+                probe_x = round((cover["x"] + cover["width"] / 2) * scale)
+                probe_y = round((cover["y"] + cover["height"] / 4) * scale)
+                probe = (probe_y * ready["width"] + probe_x) * 4
+                for index in (0, 14):
+                    start = index * frame_bytes + probe
+                    self.assertNotEqual(monitor[start:start + 3], bytes([0, 255, 0]),
+                                        "private occluder leaked into the filtered capture")
+                for index in range(1, 15):
+                    self.assertNotEqual(monitor[(index - 1) * frame_bytes:index * frame_bytes], monitor[index * frame_bytes:(index + 1) * frame_bytes],
+                                        "monitor stream reused stale pixels after source damage")
+                worker.stdin.close()
+                self.assertEqual(worker.wait(timeout=7), 0)
             finally:
                 for process in reversed(processes):
                     if process.poll() is None:
                         os.killpg(process.pid, signal.SIGTERM)
                         process.wait(timeout=5)
+                    if process.stdin:
+                        process.stdin.close()
+                    if process.stdout and process.stdout is not log:
+                        process.stdout.close()
                 log.close()