chase wlroots: Add support for HDR10 output

v2: Switch XRGB to XBGR
v3: Rewrite HDR mode checking and setting
v4: Restructure HDR support detection
v5: Fix code style
v6: Fix old style function declaration
v7: This function should be declared static
v8: This helper function can accept a const struct on input
v9: Rebase now that 0.20 is merged
v10: Rewrite with multiple color format attempts
v11: Add in the parts that accidentally got left in my
     original color-management-v1 patch
v12: Add missing function prototype
v13: Apply suggested changes
v14: Changed HDR application setup in new output
v15: Rewrite configure_new_output to use lab_wlr_scene_output_commit
v16: Fixed application of HDR on external mode or output config change
v17: Fixed it for real this time instead of crashing
v18: Moved the effective resolution collection, plus one style change.

src/config/rcxml.c

@@ -1080,6 +1080,16 @@ set_tearing_mode(const char *str, enum tearing_mode *variable)
 	}
 }
 
+static void
+set_hdr_mode(const char *str, enum render_bit_depth *variable)
+{
+	if (parse_bool(str, -1) == 1) {
+		*variable = LAB_RENDER_BIT_DEPTH_10;
+	} else {
+		*variable = LAB_RENDER_BIT_DEPTH_8;
+	}
+}
+
 /* Returns true if the node's children should also be traversed */
 static bool
 entry(xmlNode *node, char *nodename, char *content)
@@ -1144,6 +1154,8 @@ entry(xmlNode *node, char *nodename, char *content)
 		set_adaptive_sync_mode(content, &rc.adaptive_sync);
 	} else if (!strcasecmp(nodename, "allowTearing.core")) {
 		set_tearing_mode(content, &rc.allow_tearing);
+	} else if (!strcasecmp(nodename, "Hdr.core")) {
+		set_hdr_mode(content, &rc.target_render_depth);
 	} else if (!strcasecmp(nodename, "autoEnableOutputs.core")) {
 		set_bool(content, &rc.auto_enable_outputs);
 	} else if (!strcasecmp(nodename, "reuseOutputMode.core")) {
@@ -1518,6 +1530,7 @@ rcxml_init(void)
 	rc.gap = 0;
 	rc.adaptive_sync = LAB_ADAPTIVE_SYNC_DISABLED;
 	rc.allow_tearing = LAB_TEARING_DISABLED;
+	rc.target_render_depth = LAB_RENDER_BIT_DEPTH_DEFAULT;
 	rc.auto_enable_outputs = true;
 	rc.reuse_output_mode = false;
 	rc.allowed_interfaces = UINT32_MAX;

src/output-state.c

@@ -24,6 +24,7 @@ output_state_init(struct output *output)
 			backup_config, output->wlr_output);
 
 	wlr_output_head_v1_state_apply(&backup_head->state, &output->pending);
+
 	wlr_output_configuration_v1_destroy(backup_config);
 }
 

src/output.c

@@ -9,6 +9,7 @@
 #define _POSIX_C_SOURCE 200809L
 #include "output.h"
 #include <assert.h>
+#include <drm_fourcc.h>
 #include <strings.h>
 #include <wlr/backend/wayland.h>
 #include <wlr/config.h>
@@ -51,6 +52,150 @@
 	#include <wlr/backend/session.h>
 #endif
 
+static uint32_t output_formats_8bit[] = {
+	/* 32 bpp RGB with 8 bit component width */
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_XBGR8888,
+	DRM_FORMAT_RGBX8888,
+	DRM_FORMAT_BGRX8888,
+
+	DRM_FORMAT_ARGB8888,
+	DRM_FORMAT_ABGR8888,
+	DRM_FORMAT_RGBA8888,
+	DRM_FORMAT_BGRA8888,
+
+	/* 24 bpp RGB */
+	DRM_FORMAT_RGB888,
+	DRM_FORMAT_BGR888,
+};
+
+uint32_t output_formats_10bit[] = {
+	/* 32 bpp RGB with 10 bit component width */
+	DRM_FORMAT_XRGB2101010,
+	DRM_FORMAT_XBGR2101010,
+	DRM_FORMAT_RGBX1010102,
+	DRM_FORMAT_BGRX1010102,
+
+	DRM_FORMAT_ARGB2101010,
+	DRM_FORMAT_ABGR2101010,
+	DRM_FORMAT_RGBA1010102,
+	DRM_FORMAT_BGRA1010102,
+};
+
+static bool
+output_set_render_format(struct output *output, uint32_t candidates[], size_t count)
+{
+	for (size_t i = 0; i < count; i++) {
+		wlr_output_state_set_render_format(&output->pending, candidates[i]);
+		if (wlr_output_test_state(output->wlr_output, &output->pending)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static bool
+output_format_in_candidates(uint32_t render_format, uint32_t candidates[], size_t count)
+{
+	for (size_t i = 0; i < count; i++) {
+		if (candidates[i] == render_format) {
+			return true;
+		}
+	}
+	return false;
+}
+
+static enum render_bit_depth
+bit_depth_from_format(uint32_t render_format)
+{
+	if (output_format_in_candidates(render_format, output_formats_10bit,
+			ARRAY_SIZE(output_formats_10bit))) {
+		return LAB_RENDER_BIT_DEPTH_10;
+	} else if (output_format_in_candidates(render_format, output_formats_8bit,
+			ARRAY_SIZE(output_formats_8bit))) {
+		return LAB_RENDER_BIT_DEPTH_8;
+	}
+	return LAB_RENDER_BIT_DEPTH_DEFAULT;
+}
+
+static enum render_bit_depth
+get_config_render_bit_depth(void)
+{
+	return rc.target_render_depth;
+}
+
+static bool
+output_supports_hdr(const struct wlr_output *output, const char **unsupported_reason_ptr)
+{
+	const char *unsupported_reason = NULL;
+	if (!(output->supported_primaries & WLR_COLOR_NAMED_PRIMARIES_BT2020)) {
+		unsupported_reason = "BT2020 primaries not supported by output";
+	} else if (!(output->supported_transfer_functions &
+		WLR_COLOR_TRANSFER_FUNCTION_ST2084_PQ)) {
+		unsupported_reason = "PQ transfer function not supported by output";
+	} else if (!server.renderer->features.output_color_transform) {
+		unsupported_reason = "renderer doesn't support output color transforms";
+	}
+	if (unsupported_reason_ptr) {
+		*unsupported_reason_ptr = unsupported_reason;
+	}
+	return !unsupported_reason;
+}
+
+void
+output_state_setup_hdr(struct output *output, bool silent)
+{
+	uint32_t render_format = output->wlr_output->render_format;
+	const char *unsupported_reason = NULL;
+	bool hdr_supported = output_supports_hdr(output->wlr_output,
+		&unsupported_reason);
+	bool hdr_succeeded = false;
+
+	enum render_bit_depth render_bit_depth = get_config_render_bit_depth();
+	if (render_bit_depth == LAB_RENDER_BIT_DEPTH_DEFAULT) {
+		render_bit_depth = bit_depth_from_format(render_format);
+	}
+
+	if (!hdr_supported && render_bit_depth == LAB_RENDER_BIT_DEPTH_10) {
+		if (!silent) {
+			wlr_log(WLR_INFO, "Cannot enable HDR on output %s: %s",
+				output->wlr_output->name, unsupported_reason);
+		}
+		render_bit_depth = LAB_RENDER_BIT_DEPTH_8;
+	}
+
+	if (render_bit_depth == LAB_RENDER_BIT_DEPTH_10 &&
+			bit_depth_from_format(render_format) == render_bit_depth) {
+		/* 10-bit was set successfully before, try to save some
+		 * tests by reusing the format
+		 */
+		hdr_succeeded = true;
+	} else if (render_bit_depth == LAB_RENDER_BIT_DEPTH_10) {
+		hdr_succeeded = output_set_render_format(output, output_formats_10bit,
+			ARRAY_SIZE(output_formats_10bit));
+		if (!hdr_succeeded) {
+			if (!silent) {
+				wlr_log(WLR_INFO, "No 10 bit color formats"
+					" supported, HDR disabled.");
+			}
+			if (!output_set_render_format(output, output_formats_8bit,
+					ARRAY_SIZE(output_formats_8bit))) {
+				if (!silent) {
+					wlr_log(WLR_ERROR, "No 8 bit color formats"
+						" supported either!");
+				}
+			}
+		}
+	} else {
+		if (!output_set_render_format(output, output_formats_8bit,
+				ARRAY_SIZE(output_formats_8bit)) && !silent) {
+			wlr_log(WLR_ERROR, "No 8 bit color formats supported!");
+		}
+	}
+
+	output_enable_hdr(output, &output->pending, hdr_succeeded, silent);
+}
+
 bool
 output_get_tearing_allowance(struct output *output)
 {
@@ -371,10 +516,7 @@ configure_new_output(struct output *output)
 		output_enable_adaptive_sync(output, true);
 	}
 
-	output_state_commit(output);
-
-	wlr_output_effective_resolution(wlr_output,
-		&output->usable_area.width, &output->usable_area.height);
+	output_state_setup_hdr(output, false);
 
 	/*
 	 * Wait until wlr_output_layout_add_auto() returns before
@@ -384,6 +526,19 @@ configure_new_output(struct output *output)
 	server.pending_output_layout_change++;
 	add_output_to_layout(output);
 	server.pending_output_layout_change--;
+
+	/*
+	 * Commit the output this way instead, HDR needs a buffer, and
+	 * this commit must be called after the output is added to the
+	 * layout above.
+	 */
+	lab_wlr_scene_output_commit(output->scene_output, &output->pending);
+
+	/*
+	 * Collect the effective resolution after the final commit.
+	 */
+	wlr_output_effective_resolution(wlr_output,
+		&output->usable_area.width, &output->usable_area.height);
 }
 
 static uint64_t
@@ -653,6 +808,7 @@ output_config_apply(struct wlr_output_configuration_v1 *config)
 			wlr_output_state_set_transform(os, head->state.transform);
 			output_enable_adaptive_sync(output,
 				head->state.adaptive_sync_enabled);
+			output_state_setup_hdr(output, false);
 		}
 		if (!output_state_commit(output)) {
 			/*
@@ -666,6 +822,19 @@ output_config_apply(struct wlr_output_configuration_v1 *config)
 			break;
 		}
 
+		if (output_enabled) {
+			/*
+			 * The above commit was likely made without an image
+			 * buffer attached. This will break applying HDR color
+			 * transformation, since image descriptions must be
+			 * committed with a buffer attached. Queue the HDR mode
+			 * again if output is enabled, but make it silent,
+			 * since it would have emitted messages already when
+			 * called above.
+			 */
+			output_state_setup_hdr(output, true);
+		}
+
 		/*
 		 * Add or remove output from layout only if the commit went
 		 * through. Note that at startup, the output may have already
@@ -1136,3 +1305,34 @@ output_set_has_fullscreen_view(struct output *output, bool has_fullscreen_view)
 	output_enable_adaptive_sync(output, has_fullscreen_view);
 	output_state_commit(output);
 }
+
+void
+output_enable_hdr(struct output *output, struct wlr_output_state *os,
+	bool enabled, bool silent)
+{
+	if (enabled && !output_supports_hdr(output->wlr_output, NULL)) {
+		enabled = false;
+	}
+
+	if (!enabled) {
+		if (output->wlr_output->supported_primaries != 0 ||
+				output->wlr_output->supported_transfer_functions != 0) {
+			if (!silent) {
+				wlr_log(WLR_DEBUG, "Disabling HDR on output %s",
+					output->wlr_output->name);
+			}
+			wlr_output_state_set_image_description(os, NULL);
+		}
+		return;
+	}
+
+	if (!silent) {
+		wlr_log(WLR_DEBUG, "Enabling HDR on output %s",
+			output->wlr_output->name);
+	}
+	const struct wlr_output_image_description image_desc = {
+		.primaries = WLR_COLOR_NAMED_PRIMARIES_BT2020,
+		.transfer_function = WLR_COLOR_TRANSFER_FUNCTION_ST2084_PQ,
+	};
+	wlr_output_state_set_image_description(os, &image_desc);
+}