render/vulkan: add color transformation matrix

include/render/vulkan.h

@@ -335,6 +335,7 @@ struct wlr_vk_vert_pcr_data {
 };
 
 struct wlr_vk_frag_output_pcr_data {
+	float matrix[4][4]; // only a 3x3 subset is used
 	float lut_3d_offset;
 	float lut_3d_scale;
 };

render/vulkan/pass.c

@@ -178,6 +178,7 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 			.uv_off = { 0, 0 },
 			.uv_size = { 1, 1 },
 		};
+		mat3_to_mat4(final_matrix, vert_pcr_data.mat4);
 
 		struct wlr_vk_color_transform *transform = NULL;
 		size_t dim = 1;
@@ -188,10 +189,14 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 		}
 
 		struct wlr_vk_frag_output_pcr_data frag_pcr_data = {
+			.matrix = {
+				{1, 0, 0},
+				{0, 1, 0},
+				{0, 0, 1},
+			},
 			.lut_3d_offset = 0.5f / dim,
 			.lut_3d_scale = (float)(dim - 1) / dim,
 		};
-		mat3_to_mat4(final_matrix, vert_pcr_data.mat4);
 
 		if (pass->color_transform) {
 			bind_pipeline(pass, render_buffer->plain.render_setup->output_pipe_lut3d);

render/vulkan/shaders/output.frag

@@ -8,8 +8,9 @@ layout(location = 0) in vec2 uv;
 layout(location = 0) out vec4 out_color;
 
 /* struct wlr_vk_frag_output_pcr_data */
-layout(push_constant) uniform UBO {
+layout(push_constant, row_major) uniform UBO {
-	layout(offset = 80) float lut_3d_offset;
+	layout(offset = 80) mat4 matrix;
+	float lut_3d_offset;
 	float lut_3d_scale;
 } data;
 
@@ -43,6 +44,8 @@ void main() {
 		rgb = in_color.rgb / alpha;
 	}
 
+	rgb = mat3(data.matrix) * rgb;
+
 	if (OUTPUT_TRANSFORM == OUTPUT_TRANSFORM_LUT_3D) {
 		// Apply 3D LUT
 		vec3 pos = data.lut_3d_offset + rgb * data.lut_3d_scale;