backend/drm: set "max bpc" property based on pixel format

Since 1d581656c7 ("backend/drm: set "max bpc" to the max") we
set the "max bpc" property to the maximum value. The kernel driver
is supposed to clamp this value depending on hardware capabilities.

All kernel drivers lower the value depending on the GPU capabilities.
However, none of the drivers lower the value depending on the DP-MST
link capabilities. Thus, enabling a 4k@60Hz mode can fail on some
DP-MST setups due to the "max bpc" property.

Additionally, it's not a good idea to unconditionally set "max bpc"
to the max. A high bpc consumes more lanes and more clock speed,
which means higher power consumption and the busy lanes cannot be
used for something else (e.g. other data transfers on a USB-C cable).

For now, let's tie the "max bpc" to the pixel format of the buffer.
Introduce a heuristic to make "high bit-depth buffer" a synonym of
"I want the best quality".

This is not perfect: a "max bpc" higher than 8 might be desirable
for pixel formats with a color depth of 8 bits, for instance when
the color management KMS properties are used. But we don't really
support that yet, so let's leave this for later.

Closes: https://github.com/swaywm/sway/issues/7367
(cherry picked from commit d36dd96e8d)

backend/drm/atomic.c

@@ -1,4 +1,5 @@
 #define _POSIX_C_SOURCE 200809L
+#include <drm_fourcc.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <wlr/util/log.h>
@@ -142,6 +143,44 @@ static bool create_gamma_lut_blob(struct wlr_drm_backend *drm,
 	return true;
 }
 
+static uint64_t max_bpc_for_format(uint32_t format) {
+	switch (format) {
+	case DRM_FORMAT_XRGB2101010:
+	case DRM_FORMAT_ARGB2101010:
+	case DRM_FORMAT_XBGR2101010:
+	case DRM_FORMAT_ABGR2101010:
+		return 10;
+	case DRM_FORMAT_XBGR16161616F:
+	case DRM_FORMAT_ABGR16161616F:
+	case DRM_FORMAT_XBGR16161616:
+	case DRM_FORMAT_ABGR16161616:
+		return 16;
+	default:
+		return 8;
+	}
+}
+
+static uint64_t pick_max_bpc(struct wlr_drm_connector *conn, struct wlr_drm_fb *fb) {
+	if (fb == NULL) {
+		return 0;
+	}
+
+	uint32_t format = DRM_FORMAT_INVALID;
+	struct wlr_dmabuf_attributes attribs = {0};
+	if (wlr_buffer_get_dmabuf(fb->wlr_buf, &attribs)) {
+		format = attribs.format;
+	}
+
+	uint64_t target_bpc = max_bpc_for_format(format);
+	if (target_bpc < conn->max_bpc_bounds[0]) {
+		target_bpc = conn->max_bpc_bounds[0];
+	}
+	if (target_bpc > conn->max_bpc_bounds[1]) {
+		target_bpc = conn->max_bpc_bounds[1];
+	}
+	return target_bpc;
+}
+
 static void commit_blob(struct wlr_drm_backend *drm,
 		uint32_t *current, uint32_t next) {
 	if (*current == next) {
@@ -286,8 +325,8 @@ static bool atomic_crtc_commit(struct wlr_drm_connector *conn,
 		atomic_add(&atom, conn->id, conn->props.content_type,
 			DRM_MODE_CONTENT_TYPE_GRAPHICS);
 	}
-	if (modeset && active && conn->props.max_bpc != 0 && conn->max_bpc > 0) {
-		atomic_add(&atom, conn->id, conn->props.max_bpc, conn->max_bpc);
+	if (modeset && active && conn->props.max_bpc != 0 && conn->max_bpc_bounds[1] != 0) {
+		atomic_add(&atom, conn->id, conn->props.max_bpc, pick_max_bpc(conn, plane_get_next_fb(crtc->primary)));
 	}
 	atomic_add(&atom, crtc->id, crtc->props.mode_id, mode_id);
 	atomic_add(&atom, crtc->id, crtc->props.active, active);

backend/drm/drm.c

@@ -1242,10 +1242,10 @@ static void connect_drm_connector(struct wlr_drm_connector *wlr_conn,
 		wlr_conn->output.non_desktop = non_desktop;
 	}
 
-	wlr_conn->max_bpc = 0;
+	memset(wlr_conn->max_bpc_bounds, 0, sizeof(wlr_conn->max_bpc_bounds));
 	if (wlr_conn->props.max_bpc != 0) {
 		if (!introspect_drm_prop_range(drm->fd, wlr_conn->props.max_bpc,
-				NULL, &wlr_conn->max_bpc)) {
+				&wlr_conn->max_bpc_bounds[0], &wlr_conn->max_bpc_bounds[1])) {
 			wlr_log(WLR_ERROR, "Failed to introspect 'max bpc' property");
 		}
 	}

include/backend/drm/drm.h

@@ -108,7 +108,7 @@ struct wlr_drm_connector {
 	char name[24];
 	drmModeConnection status;
 	uint32_t id;
-	uint64_t max_bpc;
+	uint64_t max_bpc_bounds[2];
 	struct wlr_drm_lease *lease;
 
 	struct wlr_drm_crtc *crtc;