Release layout slots for unbuffered XDG windows

compositor/src/handlers/compositor.rs

@@ -77,10 +77,16 @@ impl CompositorHandler for Ferese {
             };
 
             if let Some(window) = window {
-                if surface == &root {
-                    xdg_shell::apply_initial_window_rules(self, &window);
-                }
                 window.on_commit();
+                if surface == &root {
+                    if crate::state::window_has_buffer(&window) {
+                        xdg_shell::apply_initial_window_rules(self, &window);
+                    } else if self.window_ids.contains_key(&window) {
+                        self.remove_tiled_window(&window);
+                        self.space.map_element(window.clone(), (0, 0), false);
+                        self.restore_keyboard_focus();
+                    }
+                }
                 self.record_client_commit(&window);
             }
         }

compositor/src/state.rs

@@ -2876,7 +2876,7 @@ fn client_size(window: &Window) -> Option<ClientSize> {
     })
 }
 
-fn window_has_buffer(window: &Window) -> bool {
+pub(crate) fn window_has_buffer(window: &Window) -> bool {
     window.toplevel().is_some_and(|toplevel| {
         smithay::backend::renderer::utils::with_renderer_surface_state(
             toplevel.wl_surface(),

scripts/tests/fixtures/unmapped-toplevel.c

@@ -0,0 +1,130 @@
+#define _GNU_SOURCE
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <unistd.h>
+#include <wayland-client.h>
+#include "xdg-shell-client-protocol.h"
+
+static struct wl_compositor *compositor;
+static struct wl_shm *shm;
+static struct xdg_wm_base *wm;
+static struct wl_surface *surface;
+
+static void ping(void *data, struct xdg_wm_base *base, uint32_t serial) {
+    (void)data;
+    xdg_wm_base_pong(base, serial);
+}
+
+static const struct xdg_wm_base_listener wm_listener = {.ping = ping};
+
+static void global(void *data, struct wl_registry *registry, uint32_t name,
+                   const char *interface, uint32_t version) {
+    (void)data;
+    (void)version;
+    if (!strcmp(interface, "wl_compositor"))
+        compositor = wl_registry_bind(registry, name, &wl_compositor_interface, 4);
+    else if (!strcmp(interface, "wl_shm"))
+        shm = wl_registry_bind(registry, name, &wl_shm_interface, 1);
+    else if (!strcmp(interface, "xdg_wm_base")) {
+        wm = wl_registry_bind(registry, name, &xdg_wm_base_interface, 1);
+        xdg_wm_base_add_listener(wm, &wm_listener, NULL);
+    }
+}
+
+static void removed(void *data, struct wl_registry *registry, uint32_t name) {
+    (void)data;
+    (void)registry;
+    (void)name;
+}
+
+static const struct wl_registry_listener registry_listener = {global, removed};
+
+static void configure(void *data, struct xdg_surface *xdg, uint32_t serial) {
+    (void)data;
+    xdg_surface_ack_configure(xdg, serial);
+}
+
+static const struct xdg_surface_listener surface_listener = {.configure = configure};
+
+static void size(void *data, struct xdg_toplevel *top, int32_t width, int32_t height,
+                 struct wl_array *states) {
+    (void)data;
+    (void)top;
+    (void)width;
+    (void)height;
+    (void)states;
+}
+
+static void close_window(void *data, struct xdg_toplevel *top) {
+    (void)data;
+    (void)top;
+    exit(0);
+}
+
+static const struct xdg_toplevel_listener top_listener = {
+    .configure = size,
+    .close = close_window,
+};
+
+static void phase(struct wl_display *display, const char *name) {
+    if (wl_display_roundtrip(display) < 0)
+        exit(5);
+    puts(name);
+    fflush(stdout);
+    if (getchar() == EOF)
+        exit(6);
+}
+
+int main(void) {
+    struct wl_display *display = wl_display_connect(NULL);
+    if (!display)
+        return 1;
+    struct wl_registry *registry = wl_display_get_registry(display);
+    wl_registry_add_listener(registry, &registry_listener, NULL);
+    if (wl_display_roundtrip(display) < 0 || !compositor || !shm || !wm)
+        return 2;
+
+    int fd = memfd_create("unmapped-toplevel", MFD_CLOEXEC);
+    if (fd < 0 || ftruncate(fd, 64 * 64 * 4))
+        return 3;
+    uint32_t *pixels = mmap(NULL, 64 * 64 * 4, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (pixels == MAP_FAILED)
+        return 4;
+    for (size_t i = 0; i < 64 * 64; i++)
+        pixels[i] = 0xff3182ce;
+    struct wl_shm_pool *pool = wl_shm_create_pool(shm, fd, 64 * 64 * 4);
+    struct wl_buffer *buffer = wl_shm_pool_create_buffer(
+        pool, 0, 64, 64, 64 * 4, WL_SHM_FORMAT_XRGB8888);
+    wl_shm_pool_destroy(pool);
+    close(fd);
+
+    surface = wl_compositor_create_surface(compositor);
+    struct xdg_surface *xdg = xdg_wm_base_get_xdg_surface(wm, surface);
+    xdg_surface_add_listener(xdg, &surface_listener, NULL);
+    struct xdg_toplevel *top = xdg_surface_get_toplevel(xdg);
+    xdg_toplevel_add_listener(top, &top_listener, NULL);
+    xdg_toplevel_set_app_id(top, "ferese.test.unmapped");
+    xdg_toplevel_set_title(top, "Buffer lifecycle test");
+
+    wl_surface_commit(surface);
+    phase(display, "initial-empty");
+
+    xdg_surface_set_window_geometry(xdg, 0, 0, 64, 64);
+    wl_surface_attach(surface, buffer, 0, 0);
+    wl_surface_damage(surface, 0, 0, 64, 64);
+    wl_surface_commit(surface);
+    phase(display, "visible");
+
+    wl_surface_attach(surface, NULL, 0, 0);
+    wl_surface_commit(surface);
+    phase(display, "detached");
+
+    wl_surface_attach(surface, buffer, 0, 0);
+    wl_surface_damage(surface, 0, 0, 64, 64);
+    wl_surface_commit(surface);
+    phase(display, "remapped");
+    return 0;
+}

scripts/tests/test_unmapped_toplevel_isolated.py

@@ -0,0 +1,128 @@
+"""Opt-in XDG buffer lifecycle test for a disposable nested compositor.
+
+FERESE_TEST_UNMAPPED=1 python3 scripts/tests/test_unmapped_toplevel_isolated.py
+Requires cc, pkg-config, wayland-scanner, and wayland-protocols.
+"""
+
+import json
+import os
+from pathlib import Path
+import select
+import subprocess
+import tempfile
+import time
+import unittest
+
+
+@unittest.skipUnless(os.environ.get("FERESE_TEST_UNMAPPED") == "1", "opt-in nested test")
+class UnmappedToplevelTest(unittest.TestCase):
+    def test_only_buffered_toplevels_take_layout_space(self):
+        repo = Path(__file__).resolve().parents[2]
+        with tempfile.TemporaryDirectory(prefix="ferese-unmapped-test-") as temporary:
+            root = Path(temporary)
+            protocols = subprocess.check_output(
+                ["pkg-config", "--variable=pkgdatadir", "wayland-protocols"], text=True
+            ).strip()
+            xml = str(Path(protocols) / "stable/xdg-shell/xdg-shell.xml")
+            for kind, output in [
+                ("client-header", "xdg-shell-client-protocol.h"),
+                ("private-code", "xdg-shell-protocol.c"),
+            ]:
+                subprocess.run(["wayland-scanner", kind, xml, str(root / output)], check=True)
+            flags = subprocess.check_output(
+                ["pkg-config", "--cflags", "--libs", "wayland-client"], text=True
+            ).split()
+            subprocess.run(
+                [
+                    "cc", "-Wall", "-Wextra", "-I", str(root),
+                    str(repo / "scripts/tests/fixtures/unmapped-toplevel.c"),
+                    str(root / "xdg-shell-protocol.c"), "-o", str(root / "client"),
+                    *flags,
+                ],
+                check=True,
+            )
+
+            runtime = root / "runtime"
+            runtime.mkdir(mode=0o700)
+            config = root / "config/ferese"
+            config.mkdir(parents=True)
+            (config / "config.kdl").write_text("\n")
+            host_display = os.environ["WAYLAND_DISPLAY"]
+            if not host_display.startswith("/"):
+                host_display = str(Path(os.environ["XDG_RUNTIME_DIR"]) / host_display)
+            env = dict(
+                os.environ,
+                XDG_RUNTIME_DIR=str(runtime),
+                XDG_CONFIG_HOME=str(root / "config"),
+                WAYLAND_DISPLAY=host_display,
+            )
+            compositor_binary = repo / os.environ.get("FERESE_TEST_BINARY", "target/debug/ferese")
+            ctl_binary = repo / os.environ.get("FERESE_TEST_CTL", "target/debug/feresectl")
+            with (root / "compositor.log").open("w") as log:
+                compositor = subprocess.Popen(
+                    [str(compositor_binary), "--backend", "nested"],
+                    env=env,
+                    stdout=log,
+                    stderr=subprocess.STDOUT,
+                )
+                client = None
+                try:
+                    for _ in range(100):
+                        if compositor.poll() is not None:
+                            self.fail((root / "compositor.log").read_text())
+                        sockets = [
+                            path for path in runtime.glob("wayland-*")
+                            if not path.name.endswith(".lock")
+                        ]
+                        if sockets:
+                            break
+                        time.sleep(0.1)
+                    self.assertTrue(sockets, "nested Wayland socket did not appear")
+                    client = subprocess.Popen(
+                        [str(root / "client")],
+                        env=dict(env, WAYLAND_DISPLAY=str(sockets[0])),
+                        stdin=subprocess.PIPE,
+                        stdout=subprocess.PIPE,
+                        text=True,
+                    )
+
+                    for phase, visible in [
+                        ("initial-empty", False),
+                        ("visible", True),
+                        ("detached", False),
+                        ("remapped", True),
+                    ]:
+                        ready, _, _ = select.select([client.stdout], [], [], 10)
+                        self.assertTrue(ready, f"client did not reach {phase}")
+                        self.assertEqual(client.stdout.readline().strip(), phase)
+                        deadline = time.monotonic() + 3
+                        while True:
+                            result = subprocess.run(
+                                [str(ctl_binary), "get-focused-window"],
+                                env=env,
+                                capture_output=True,
+                                text=True,
+                                check=True,
+                            )
+                            focused = json.loads(result.stdout)
+                            if (focused is not None and focused.get("id") is not None) == visible:
+                                break
+                            if time.monotonic() >= deadline:
+                                self.fail(f"{phase}: unexpected focused window {focused}")
+                            time.sleep(0.05)
+                        client.stdin.write("\n")
+                        client.stdin.flush()
+                    self.assertEqual(client.wait(timeout=5), 0)
+                finally:
+                    if client and client.poll() is None:
+                        client.terminate()
+                        client.wait(timeout=5)
+                    if client:
+                        client.stdin.close()
+                        client.stdout.close()
+                    compositor.terminate()
+                    compositor.wait(timeout=5)
+
+
+if __name__ == "__main__":
+    unittest.main()