chase wlroots: Add support for HDR10 output

v2: Switch XRGB to XBGR
v3: Rewrite HDR mode checking and setting
v4: Restructure HDR support detection
v5: Fix code style
v6: Fix old style function declaration
v7: This function should be declared static
v8: This helper function can accept a const struct on input
v9: Rebase now that 0.20 is merged

include/config/rcxml.h

@@ -36,6 +36,17 @@ enum tearing_mode {
 	LAB_TEARING_FULLSCREEN_FORCED,
 };
 
+enum hdr_mode {
+	LAB_HDR_DISABLED = 0,
+	LAB_HDR_ENABLED,
+};
+
+enum render_bit_depth {
+	LAB_RENDER_BIT_DEPTH_DEFAULT = 0,
+	LAB_RENDER_BIT_DEPTH_8,
+	LAB_RENDER_BIT_DEPTH_10,
+};
+
 enum tiling_events_mode {
 	LAB_TILING_EVENTS_NEVER = 0,
 	LAB_TILING_EVENTS_REGION = 1 << 0,
@@ -74,6 +85,7 @@ struct rcxml {
 	int gap;
 	enum adaptive_sync_mode adaptive_sync;
 	enum tearing_mode allow_tearing;
+	enum hdr_mode hdr;
 	bool auto_enable_outputs;
 	bool reuse_output_mode;
 	bool xwayland_persistence;

include/output.h

@@ -71,6 +71,8 @@ struct wlr_box output_usable_area_in_layout_coords(struct output *output);
 void handle_output_power_manager_set_mode(struct wl_listener *listener,
 	void *data);
 void output_enable_adaptive_sync(struct output *output, bool enabled);
+void output_enable_hdr(struct output *output, struct wlr_output_state *os,
+	bool enabled);
 
 /**
  * Notifies whether a fullscreen view is displayed on the given output.

src/config/rcxml.c

@@ -1032,6 +1032,16 @@ set_tearing_mode(const char *str, enum tearing_mode *variable)
 	}
 }
 
+static void
+set_hdr_mode(const char *str, enum hdr_mode *variable)
+{
+	if (parse_bool(str, -1) == 1) {
+		*variable = LAB_HDR_ENABLED;
+	} else {
+		*variable = LAB_HDR_DISABLED;
+	}
+}
+
 /* Returns true if the node's children should also be traversed */
 static bool
 entry(xmlNode *node, char *nodename, char *content)
@@ -1096,6 +1106,8 @@ entry(xmlNode *node, char *nodename, char *content)
 		set_adaptive_sync_mode(content, &rc.adaptive_sync);
 	} else if (!strcasecmp(nodename, "allowTearing.core")) {
 		set_tearing_mode(content, &rc.allow_tearing);
+	} else if (!strcasecmp(nodename, "Hdr.core")) {
+		set_hdr_mode(content, &rc.hdr);
 	} else if (!strcasecmp(nodename, "autoEnableOutputs.core")) {
 		set_bool(content, &rc.auto_enable_outputs);
 	} else if (!strcasecmp(nodename, "reuseOutputMode.core")) {
@@ -1457,6 +1469,7 @@ rcxml_init(void)
 	rc.gap = 0;
 	rc.adaptive_sync = LAB_ADAPTIVE_SYNC_DISABLED;
 	rc.allow_tearing = LAB_TEARING_DISABLED;
+	rc.hdr = LAB_HDR_DISABLED;
 	rc.auto_enable_outputs = true;
 	rc.reuse_output_mode = false;
 	rc.xwayland_persistence = false;

src/output.c

@@ -9,6 +9,7 @@
 #define _POSIX_C_SOURCE 200809L
 #include "output.h"
 #include <assert.h>
+#include <drm_fourcc.h>
 #include <strings.h>
 #include <wlr/backend/wayland.h>
 #include <wlr/config.h>
@@ -51,6 +52,45 @@
 	#include <wlr/backend/session.h>
 #endif
 
+static enum render_bit_depth
+bit_depth_from_format(uint32_t render_format)
+{
+	if (render_format == DRM_FORMAT_XRGB2101010 || render_format == DRM_FORMAT_XBGR2101010) {
+		return LAB_RENDER_BIT_DEPTH_10;
+	} else if (render_format == DRM_FORMAT_XRGB8888 || render_format == DRM_FORMAT_ARGB8888 ||
+		render_format == DRM_FORMAT_XBGR8888 || render_format == DRM_FORMAT_ABGR8888) {
+		return LAB_RENDER_BIT_DEPTH_8;
+	}
+	return LAB_RENDER_BIT_DEPTH_DEFAULT;
+}
+
+static enum render_bit_depth
+get_config_render_bit_depth(void)
+{
+	if (rc.hdr == LAB_HDR_ENABLED) {
+		return LAB_RENDER_BIT_DEPTH_10;
+	}
+	return LAB_RENDER_BIT_DEPTH_8;
+}
+
+static void
+output_state_setup_hdr(struct output *output)
+{
+	uint32_t render_format = output->wlr_output->render_format;
+	enum render_bit_depth render_bit_depth = get_config_render_bit_depth();
+	if (render_bit_depth == LAB_RENDER_BIT_DEPTH_10 &&
+		bit_depth_from_format(render_format) == render_bit_depth) {
+		// 10-bit was set successfully before, try to save some tests by reusing the format
+		wlr_output_state_set_render_format(&output->pending, render_format);
+	} else if (render_bit_depth == LAB_RENDER_BIT_DEPTH_10) {
+		wlr_output_state_set_render_format(&output->pending, DRM_FORMAT_XBGR2101010);
+	} else {
+		wlr_output_state_set_render_format(&output->pending, DRM_FORMAT_XBGR8888);
+	}
+	bool hdr = rc.hdr == LAB_HDR_ENABLED;
+	output_enable_hdr(output, &output->pending, hdr);
+}
+
 bool
 output_get_tearing_allowance(struct output *output)
 {
@@ -371,6 +411,8 @@ configure_new_output(struct output *output)
 		output_enable_adaptive_sync(output, true);
 	}
 
+	output_state_setup_hdr(output);
+
 	output_state_commit(output);
 
 	wlr_output_effective_resolution(wlr_output,
@@ -653,6 +695,7 @@ output_config_apply(struct wlr_output_configuration_v1 *config)
 			wlr_output_state_set_transform(os, head->state.transform);
 			output_enable_adaptive_sync(output,
 				head->state.adaptive_sync_enabled);
+			output_state_setup_hdr(output);
 		}
 		if (!output_state_commit(output)) {
 			/*
@@ -1136,3 +1179,48 @@ output_set_has_fullscreen_view(struct output *output, bool has_fullscreen_view)
 	output_enable_adaptive_sync(output, has_fullscreen_view);
 	output_state_commit(output);
 }
+
+static bool
+output_supports_hdr(const struct wlr_output *output, const char **unsupported_reason_ptr)
+{
+	const char *unsupported_reason = NULL;
+	if (!(output->supported_primaries & WLR_COLOR_NAMED_PRIMARIES_BT2020)) {
+		unsupported_reason = "BT2020 primaries not supported by output";
+	} else if (!(output->supported_transfer_functions &
+		WLR_COLOR_TRANSFER_FUNCTION_ST2084_PQ)) {
+		unsupported_reason = "PQ transfer function not supported by output";
+	} else if (!server.renderer->features.output_color_transform) {
+		unsupported_reason = "renderer doesn't support output color transforms";
+	}
+	if (unsupported_reason_ptr) {
+		*unsupported_reason_ptr = unsupported_reason;
+	}
+	return !unsupported_reason;
+}
+
+void
+output_enable_hdr(struct output *output, struct wlr_output_state *os, bool enabled)
+{
+	const char *unsupported_reason = NULL;
+	if (enabled && !output_supports_hdr(output->wlr_output, &unsupported_reason)) {
+		wlr_log(WLR_INFO, "Cannot enable HDR on output %s: %s",
+			output->wlr_output->name, unsupported_reason);
+		enabled = false;
+	}
+
+	if (!enabled) {
+		if (output->wlr_output->supported_primaries != 0 ||
+			output->wlr_output->supported_transfer_functions != 0) {
+			wlr_log(WLR_DEBUG, "Disabling HDR on output %s", output->wlr_output->name);
+			wlr_output_state_set_image_description(os, NULL);
+		}
+		return;
+	}
+
+	wlr_log(WLR_DEBUG, "Enabling HDR on output %s", output->wlr_output->name);
+	const struct wlr_output_image_description image_desc = {
+		.primaries = WLR_COLOR_NAMED_PRIMARIES_BT2020,
+		.transfer_function = WLR_COLOR_TRANSFER_FUNCTION_ST2084_PQ,
+	};
+	wlr_output_state_set_image_description(os, &image_desc);
+}