backend/wayland: implement the output layer API

The output layer API is implemented using subsurfaces. I chose to
implement this API in the Wayland backend before doing so in the DRM
backend, because it's way easier on Wayland. On DRM, one needs to figure
out how buffers can be mapped to KMS planes (libliftoff can help) and
perform atomic test-only commits (our current DRM backend isn't ready
for this).

backend/wayland/backend.c

@@ -208,6 +208,9 @@ static void registry_global(void *data, struct wl_registry *registry,
 	if (strcmp(iface, wl_compositor_interface.name) == 0) {
 		wl->compositor = wl_registry_bind(registry, name,
 			&wl_compositor_interface, 4);
+	} else if (strcmp(iface, wl_subcompositor_interface.name) == 0) {
+		wl->subcompositor = wl_registry_bind(registry, name,
+			&wl_subcompositor_interface, 1);
 	} else if (strcmp(iface, wl_seat_interface.name) == 0) {
 		struct wl_seat *wl_seat = wl_registry_bind(registry, name,
 			&wl_seat_interface, 5);

backend/wayland/output.c

@@ -13,6 +13,7 @@
 #include <wlr/interfaces/wlr_output.h>
 #include <wlr/render/wlr_renderer.h>
 #include <wlr/types/wlr_matrix.h>
+#include <wlr/util/box.h>
 #include <wlr/util/log.h>
 
 #include "backend/wayland.h"
@@ -250,6 +251,62 @@ static struct wlr_wl_buffer *get_or_create_wl_buffer(struct wlr_wl_backend *wl,
 	return create_wl_buffer(wl, wlr_buffer);
 }
 
+static void output_layer_pending_box(struct wlr_wl_output_layer *layer,
+		struct wlr_box *out) {
+	out->x = layer->base.current.x;
+	out->y = layer->base.current.y;
+	if (layer->base.pending.committed & WLR_OUTPUT_LAYER_STATE_POSITION) {
+		out->x = layer->base.pending.x;
+		out->y = layer->base.pending.y;
+	}
+
+	struct wlr_buffer *wlr_buffer = layer->base.current.buffer;
+	if (layer->base.pending.committed & WLR_OUTPUT_LAYER_STATE_BUFFER) {
+		wlr_buffer = layer->base.pending.buffer;
+	}
+
+	out->width = out->height = 0;
+	if (wlr_buffer != NULL) {
+		out->width = wlr_buffer->width;
+		out->height = wlr_buffer->height;
+	}
+}
+
+/* Update the output layers' "accepted" flag */
+static void output_test_layers(struct wlr_wl_output *output) {
+	struct wlr_output *wlr_output = &output->wlr_output;
+
+	// Iterate over layers from top to bottom. Reject layers that are under a
+	// rejected layer.
+	pixman_region32_t rejected_region;
+	pixman_region32_init(&rejected_region);
+	pixman_region32_t intersect;
+	pixman_region32_init(&intersect);
+	struct wlr_wl_output_layer *layer;
+	wl_list_for_each_reverse(layer, &wlr_output->pending.layers,
+			base.pending.link) {
+		struct wlr_box box = {0};
+		output_layer_pending_box(layer, &box);
+
+		pixman_region32_intersect_rect(&intersect, &rejected_region,
+			box.x, box.y, box.width, box.height);
+		bool accepted = !pixman_region32_not_empty(&intersect);
+
+		if ((layer->base.pending.committed & WLR_OUTPUT_LAYER_STATE_BUFFER) &&
+				accepted) {
+			accepted = test_buffer(output->backend, layer->base.pending.buffer);
+		}
+
+		if (!accepted) {
+			pixman_region32_union_rect(&rejected_region, &rejected_region,
+				box.x, box.y, box.width, box.height);
+		}
+		layer->base.accepted = accepted;
+	}
+	pixman_region32_fini(&intersect);
+	pixman_region32_fini(&rejected_region);
+}
+
 static bool output_test(struct wlr_output *wlr_output) {
 	struct wlr_wl_output *output =
 		get_wl_output_from_output(wlr_output);
@@ -271,6 +328,28 @@ static bool output_test(struct wlr_output *wlr_output) {
 		return false;
 	}
 
+	output_test_layers(output);
+
+	return true;
+}
+
+static bool output_layer_attach(struct wlr_wl_output_layer *layer,
+		struct wlr_buffer *wlr_buffer) {
+	struct wlr_wl_output *output =
+		get_wl_output_from_output(layer->base.output);
+
+	if (wlr_buffer == NULL) {
+		wl_surface_attach(layer->surface, NULL, 0, 0);
+		return true;
+	}
+
+	struct wlr_wl_buffer *buffer =
+		create_wl_buffer(output->backend, wlr_buffer);
+	if (buffer == NULL) {
+		return false;
+	}
+
+	wl_surface_attach(layer->surface, buffer->wl_buffer, 0, 0);
 	return true;
 }
 
@@ -291,6 +370,47 @@ static bool output_commit(struct wlr_output *wlr_output) {
 		}
 	}
 
+	// Update layer ordering
+	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_LAYERS) {
+		struct wlr_wl_output_layer *layer, *prev = NULL;
+		wl_list_for_each(layer, &wlr_output->pending.layers, base.pending.link) {
+			if (prev != NULL) {
+				wl_subsurface_place_above(layer->subsurface, prev->surface);
+			}
+
+			prev = layer;
+		}
+	}
+
+	output_test_layers(output);
+
+	struct wlr_wl_output_layer *layer;
+	wl_list_for_each(layer, &wlr_output->pending.layers, base.pending.link) {
+		if (!layer->base.accepted && layer->prev_accepted) {
+			wl_surface_attach(layer->surface, NULL, 0, 0);
+		}
+
+		if ((layer->base.pending.committed & WLR_OUTPUT_LAYER_STATE_BUFFER) &&
+				layer->base.accepted) {
+			if (!output_layer_attach(layer, layer->base.pending.buffer)) {
+				return false;
+			}
+		}
+
+		if (layer->base.pending.committed & WLR_OUTPUT_LAYER_STATE_POSITION) {
+			wl_subsurface_set_position(layer->subsurface,
+				layer->base.pending.x, layer->base.pending.y);
+		}
+
+		if (layer->base.pending.committed != 0 ||
+				layer->base.accepted != layer->prev_accepted) {
+			// TODO: make sure to commit the parent surface too
+			wl_surface_commit(layer->surface);
+		}
+
+		layer->prev_accepted = layer->base.accepted;
+	}
+
 	if (wlr_output->pending.committed & WLR_OUTPUT_STATE_BUFFER) {
 		struct wp_presentation_feedback *wp_feedback = NULL;
 		if (output->backend->presentation != NULL) {
@@ -457,6 +577,36 @@ static bool output_move_cursor(struct wlr_output *_output, int x, int y) {
 	return true;
 }
 
+static struct wlr_output_layer *output_create_layer(
+		struct wlr_output *wlr_output) {
+	struct wlr_wl_output *output = get_wl_output_from_output(wlr_output);
+	struct wlr_wl_backend *backend = output->backend;
+
+	struct wlr_wl_output_layer *layer = calloc(1, sizeof(*layer));
+	if (layer == NULL) {
+		return NULL;
+	}
+	wlr_output_layer_init(&layer->base, wlr_output);
+	layer->surface = wl_compositor_create_surface(backend->compositor);
+	layer->subsurface = wl_subcompositor_get_subsurface(backend->subcompositor,
+		layer->surface, output->surface);
+
+	struct wl_region *empty_region =
+		wl_compositor_create_region(backend->compositor);
+	wl_surface_set_input_region(layer->surface, empty_region);
+	wl_region_destroy(empty_region);
+
+	return &layer->base;
+}
+
+static void output_destroy_layer(struct wlr_output_layer *wlr_layer) {
+	struct wlr_wl_output_layer *layer =
+		(struct wlr_wl_output_layer *)wlr_layer;
+	wl_subsurface_destroy(layer->subsurface);
+	wl_surface_destroy(layer->surface);
+	free(layer);
+}
+
 static const struct wlr_output_impl output_impl = {
 	.destroy = output_destroy,
 	.test = output_test,
@@ -465,6 +615,8 @@ static const struct wlr_output_impl output_impl = {
 	.move_cursor = output_move_cursor,
 	.get_cursor_formats = output_get_formats,
 	.get_primary_formats = output_get_formats,
+	.create_layer = output_create_layer,
+	.destroy_layer = output_destroy_layer,
 };
 
 bool wlr_output_is_wl(struct wlr_output *wlr_output) {

include/backend/wayland.h

@@ -8,6 +8,7 @@
 
 #include <wlr/backend/wayland.h>
 #include <wlr/render/wlr_renderer.h>
+#include <wlr/types/wlr_output_layer.h>
 #include <wlr/types/wlr_pointer.h>
 #include <wlr/render/drm_format_set.h>
 
@@ -31,6 +32,7 @@ struct wlr_wl_backend {
 	struct wl_event_source *remote_display_src;
 	struct wl_registry *registry;
 	struct wl_compositor *compositor;
+	struct wl_subcompositor *subcompositor;
 	struct xdg_wm_base *xdg_wm_base;
 	struct zxdg_decoration_manager_v1 *zxdg_decoration_manager_v1;
 	struct zwp_pointer_gestures_v1 *zwp_pointer_gestures_v1;
@@ -84,6 +86,13 @@ struct wlr_wl_output {
 	} cursor;
 };
 
+struct wlr_wl_output_layer {
+	struct wlr_output_layer base;
+	struct wl_surface *surface;
+	struct wl_subsurface *subsurface;
+	bool prev_accepted;
+};
+
 struct wlr_wl_input_device {
 	struct wlr_input_device wlr_input_device;
 	uint32_t fingers;