chase wlroots: Add support for HDR10 output

v2: Switch XRGB to XBGR
v3: Rewrite HDR mode checking and setting
v4: Restructure HDR support detection
v5: Fix code style
v6: Fix old style function declaration
v7: This function should be declared static
v8: This helper function can accept a const struct on input
v9: Rebase now that 0.20 is merged
v10: Rewrite with multiple color format attempts
v11: Add in the parts that accidentally got left in my
     original color-management-v1 patch
v12: Add missing function prototype

include/config/rcxml.h

@@ -36,6 +36,17 @@ enum tearing_mode {
 	LAB_TEARING_FULLSCREEN_FORCED,
 };
 
+enum hdr_mode {
+	LAB_HDR_DISABLED = 0,
+	LAB_HDR_ENABLED,
+};
+
+enum render_bit_depth {
+	LAB_RENDER_BIT_DEPTH_DEFAULT = 0,
+	LAB_RENDER_BIT_DEPTH_8,
+	LAB_RENDER_BIT_DEPTH_10,
+};
+
 enum tiling_events_mode {
 	LAB_TILING_EVENTS_NEVER = 0,
 	LAB_TILING_EVENTS_REGION = 1 << 0,
@@ -74,6 +85,7 @@ struct rcxml {
 	int gap;
 	enum adaptive_sync_mode adaptive_sync;
 	enum tearing_mode allow_tearing;
+	enum hdr_mode hdr;
 	bool auto_enable_outputs;
 	bool reuse_output_mode;
 	uint32_t allowed_interfaces;

include/output.h

@@ -71,6 +71,9 @@ struct wlr_box output_usable_area_in_layout_coords(struct output *output);
 void handle_output_power_manager_set_mode(struct wl_listener *listener,
 	void *data);
 void output_enable_adaptive_sync(struct output *output, bool enabled);
+void output_enable_hdr(struct output *output, struct wlr_output_state *os,
+	bool enabled, bool silent);
+void output_state_setup_hdr(struct output *output, bool silent);
 
 /**
  * Notifies whether a fullscreen view is displayed on the given output.

src/config/rcxml.c

@@ -1070,6 +1070,16 @@ set_tearing_mode(const char *str, enum tearing_mode *variable)
 	}
 }
 
+static void
+set_hdr_mode(const char *str, enum hdr_mode *variable)
+{
+	if (parse_bool(str, -1) == 1) {
+		*variable = LAB_HDR_ENABLED;
+	} else {
+		*variable = LAB_HDR_DISABLED;
+	}
+}
+
 /* Returns true if the node's children should also be traversed */
 static bool
 entry(xmlNode *node, char *nodename, char *content)
@@ -1134,6 +1144,8 @@ entry(xmlNode *node, char *nodename, char *content)
 		set_adaptive_sync_mode(content, &rc.adaptive_sync);
 	} else if (!strcasecmp(nodename, "allowTearing.core")) {
 		set_tearing_mode(content, &rc.allow_tearing);
+	} else if (!strcasecmp(nodename, "Hdr.core")) {
+		set_hdr_mode(content, &rc.hdr);
 	} else if (!strcasecmp(nodename, "autoEnableOutputs.core")) {
 		set_bool(content, &rc.auto_enable_outputs);
 	} else if (!strcasecmp(nodename, "reuseOutputMode.core")) {
@@ -1505,6 +1517,7 @@ rcxml_init(void)
 	rc.gap = 0;
 	rc.adaptive_sync = LAB_ADAPTIVE_SYNC_DISABLED;
 	rc.allow_tearing = LAB_TEARING_DISABLED;
+	rc.hdr = LAB_HDR_DISABLED;
 	rc.auto_enable_outputs = true;
 	rc.reuse_output_mode = false;
 	rc.allowed_interfaces = UINT32_MAX;

src/output-state.c

@@ -24,6 +24,10 @@ output_state_init(struct output *output)
 			backup_config, output->wlr_output);
 
 	wlr_output_head_v1_state_apply(&backup_head->state, &output->pending);
+
+	/* This must be applied outside of config */
+	output_state_setup_hdr(output, true);
+
 	wlr_output_configuration_v1_destroy(backup_config);
 }
 
@@ -35,6 +39,7 @@ output_state_commit(struct output *output)
 	if (committed) {
 		wlr_output_state_finish(&output->pending);
 		wlr_output_state_init(&output->pending);
+		output_state_setup_hdr(output, true); /* So must this */
 	} else {
 		wlr_log(WLR_ERROR, "Failed to commit frame");
 	}

src/output.c

@@ -9,6 +9,7 @@
 #define _POSIX_C_SOURCE 200809L
 #include "output.h"
 #include <assert.h>
+#include <drm_fourcc.h>
 #include <strings.h>
 #include <wlr/backend/wayland.h>
 #include <wlr/config.h>
@@ -51,6 +52,126 @@
 	#include <wlr/backend/session.h>
 #endif
 
+static uint32_t output_formats_8bit[] = {
+	/* 32 bpp RGB with 8 bit component width */
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+
+	/* 24 bpp RGB */
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+};
+
+static const size_t output_formats_8bit_count = sizeof(output_formats_8bit) /
+	sizeof(output_formats_8bit[0]);
+
+uint32_t output_formats_10bit[] = {
+	/* 32 bpp RGB with 10 bit component width */
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_RGBX1010102,
+	DRM_FORMAT_BGRX1010102,
+
+	DRM_FORMAT_ARGB2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_RGBA1010102,
+	DRM_FORMAT_BGRA1010102,
+};
+
+static const size_t output_formats_10bit_count = sizeof(output_formats_10bit) /
+	sizeof(output_formats_10bit[0]);
+
+static bool
+output_set_render_format(struct output *output, uint32_t candidates[], size_t count)
+{
+	for (size_t i = 0; i < count; i++) {
+		wlr_output_state_set_render_format(&output->pending, candidates[i]);
+		if (wlr_output_test_state(output->wlr_output, &output->pending)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static bool
+output_format_in_candidates(uint32_t render_format, uint32_t candidates[], size_t count)
+{
+	for (size_t i = 0; i < count; i++) {
+		if (candidates[i] == render_format) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static enum render_bit_depth
+bit_depth_from_format(uint32_t render_format)
+{
+	if (output_format_in_candidates(render_format, output_formats_10bit,
+			output_formats_10bit_count)) {
+		return LAB_RENDER_BIT_DEPTH_10;
+	} else if (output_format_in_candidates(render_format, output_formats_8bit,
+			output_formats_8bit_count)) {
+		return LAB_RENDER_BIT_DEPTH_8;
+	}
+	return LAB_RENDER_BIT_DEPTH_DEFAULT;
+}
+
+static enum render_bit_depth
+get_config_render_bit_depth(void)
+{
+	if (rc.hdr == LAB_HDR_ENABLED) {
+		return LAB_RENDER_BIT_DEPTH_10;
+	}
+	return LAB_RENDER_BIT_DEPTH_8;
+}
+
+void
+output_state_setup_hdr(struct output *output, bool silent)
+{
+	uint32_t render_format = output->wlr_output->render_format;
+	bool hdr_succeeded = false;
+	enum render_bit_depth render_bit_depth = get_config_render_bit_depth();
+	if (render_bit_depth == LAB_RENDER_BIT_DEPTH_10 &&
+		bit_depth_from_format(render_format) == render_bit_depth) {
+		/* 10-bit was set successfully before, try to save some
+		 * tests by reusing the format
+		 */
+		wlr_output_state_set_render_format(&output->pending, render_format);
+		hdr_succeeded = true;
+	} else if (render_bit_depth == LAB_RENDER_BIT_DEPTH_10) {
+		hdr_succeeded = output_set_render_format(output, output_formats_10bit,
+			output_formats_10bit_count);
+		if (!hdr_succeeded) {
+			if (!silent) {
+				wlr_log(WLR_INFO, "No 10 bit color formats"
+					" supported, HDR disabled.");
+			}
+			if (!output_set_render_format(output, output_formats_8bit,
+					output_formats_8bit_count)) {
+				if (!silent) {
+					wlr_log(WLR_ERROR, "No 8 bit color formats"
+						" supported either!");
+				}
+			}
+		}
+	} else {
+		if (!output_set_render_format(output, output_formats_8bit,
+				output_formats_8bit_count) && !silent) {
+			wlr_log(WLR_ERROR, "No 8 bit color formats supported!");
+		}
+	}
+	bool hdr = hdr_succeeded && (rc.hdr == LAB_HDR_ENABLED);
+	output_enable_hdr(output, &output->pending, hdr, silent);
+}
+
 bool
 output_get_tearing_allowance(struct output *output)
 {
@@ -371,6 +492,8 @@ configure_new_output(struct output *output)
 		output_enable_adaptive_sync(output, true);
 	}
 
+	output_state_setup_hdr(output, false);
+
 	output_state_commit(output);
 
 	wlr_output_effective_resolution(wlr_output,
@@ -653,6 +776,7 @@ output_config_apply(struct wlr_output_configuration_v1 *config)
 			wlr_output_state_set_transform(os, head->state.transform);
 			output_enable_adaptive_sync(output,
 				head->state.adaptive_sync_enabled);
+			output_state_setup_hdr(output, false);
 		}
 		if (!output_state_commit(output)) {
 			/*
@@ -1136,3 +1260,57 @@ output_set_has_fullscreen_view(struct output *output, bool has_fullscreen_view)
 	output_enable_adaptive_sync(output, has_fullscreen_view);
 	output_state_commit(output);
 }
+
+static bool
+output_supports_hdr(const struct wlr_output *output, const char **unsupported_reason_ptr)
+{
+	const char *unsupported_reason = NULL;
+	if (!(output->supported_primaries & WLR_COLOR_NAMED_PRIMARIES_BT2020)) {
+		unsupported_reason = "BT2020 primaries not supported by output";
+	} else if (!(output->supported_transfer_functions &
+		WLR_COLOR_TRANSFER_FUNCTION_ST2084_PQ)) {
+		unsupported_reason = "PQ transfer function not supported by output";
+	} else if (!server.renderer->features.output_color_transform) {
+		unsupported_reason = "renderer doesn't support output color transforms";
+	}
+	if (unsupported_reason_ptr) {
+		*unsupported_reason_ptr = unsupported_reason;
+	}
+	return !unsupported_reason;
+}
+
+void
+output_enable_hdr(struct output *output, struct wlr_output_state *os,
+	bool enabled, bool silent)
+{
+	const char *unsupported_reason = NULL;
+	if (enabled && !output_supports_hdr(output->wlr_output, &unsupported_reason)) {
+		if (!silent) {
+			wlr_log(WLR_INFO, "Cannot enable HDR on output %s: %s",
+				output->wlr_output->name, unsupported_reason);
+		}
+		enabled = false;
+	}
+
+	if (!enabled) {
+		if (output->wlr_output->supported_primaries != 0 ||
+			output->wlr_output->supported_transfer_functions != 0) {
+			if (!silent) {
+				wlr_log(WLR_DEBUG, "Disabling HDR on output %s",
+					output->wlr_output->name);
+			}
+			wlr_output_state_set_image_description(os, NULL);
+		}
+		return;
+	}
+
+	if (!silent) {
+		wlr_log(WLR_DEBUG, "Enabling HDR on output %s",
+			output->wlr_output->name);
+	}
+	const struct wlr_output_image_description image_desc = {
+		.primaries = WLR_COLOR_NAMED_PRIMARIES_BT2020,
+		.transfer_function = WLR_COLOR_TRANSFER_FUNCTION_ST2084_PQ,
+	};
+	wlr_output_state_set_image_description(os, &image_desc);
+}