render/vulkan: Use shader variants for different output transforms

include/render/vulkan.h

@@ -268,7 +268,8 @@ struct wlr_vk_renderer {
 	VkShaderModule vert_module;
 	VkShaderModule tex_frag_module;
 	VkShaderModule quad_frag_module;
-	VkShaderModule output_module;
+	VkShaderModule output_module_srgb;
+	VkShaderModule output_module_3d_lut;
 
 	struct wl_list pipeline_layouts; // struct wlr_vk_pipeline_layout.link
 

render/vulkan/pass.c

@@ -117,44 +117,41 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 			.uv_size = { 1, 1 },
 		};
 
-		size_t dim = 1;
-		if (pass->color_transform && pass->color_transform->type == COLOR_TRANSFORM_LUT_3D) {
-			struct wlr_color_transform_lut3d *lut3d =
-				wlr_color_transform_lut3d_from_base(pass->color_transform);
-			dim = lut3d->dim_len;
-		}
-
-		struct wlr_vk_frag_output_pcr_data frag_pcr_data = {
-			.lut_3d_offset = 0.5f / dim,
-			.lut_3d_scale = (float)(dim - 1) / dim,
-		};
 		mat3_to_mat4(final_matrix, vert_pcr_data.mat4);
-
-		if (pass->color_transform) {
-			bind_pipeline(pass, render_buffer->plain.render_setup->output_pipe_lut3d);
-		} else {
-			bind_pipeline(pass, render_buffer->plain.render_setup->output_pipe_srgb);
-		}
 		vkCmdPushConstants(render_cb->vk, renderer->output_pipe_layout,
 			VK_SHADER_STAGE_VERTEX_BIT, 0, sizeof(vert_pcr_data), &vert_pcr_data);
-		vkCmdPushConstants(render_cb->vk, renderer->output_pipe_layout,
-			VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(vert_pcr_data),
-			sizeof(frag_pcr_data), &frag_pcr_data);
 
-		VkDescriptorSet lut_ds;
+		VkDescriptorSet ds[2];
-		if (pass->color_transform && pass->color_transform->type == COLOR_TRANSFORM_LUT_3D) {
+		size_t ds_len = 0;
-			struct wlr_vk_color_transform *transform =
+		ds[ds_len++] = render_buffer->plain.blend_descriptor_set;
-				get_color_transform(pass->color_transform, renderer);
+
-			assert(transform);
+		VkPipeline pl = render_buffer->plain.render_setup->output_pipe_srgb;
-			lut_ds = transform->lut_3d.ds;
+		if (pass->color_transform) {
-		} else {
+			switch (pass->color_transform->type){
-			lut_ds = renderer->output_ds_lut3d_dummy;
+			case COLOR_TRANSFORM_LUT_3D: {
+				pl = render_buffer->plain.render_setup->output_pipe_lut3d;
+				struct wlr_vk_color_transform *transform =
+					get_color_transform(pass->color_transform, renderer);
+				ds[ds_len++] = transform->lut_3d.ds;
+
+				struct wlr_color_transform_lut3d *lut =
+					wlr_color_transform_lut3d_from_base(pass->color_transform);
+				struct wlr_vk_frag_output_pcr_data frag_pcr_data = {
+					.lut_3d_offset = 0.5f / lut->dim_len,
+					.lut_3d_scale = (float)(lut->dim_len - 1) / lut->dim_len,
+				};
+				vkCmdPushConstants(render_cb->vk, renderer->output_pipe_layout,
+					VK_SHADER_STAGE_FRAGMENT_BIT, sizeof(vert_pcr_data),
+					sizeof(frag_pcr_data), &frag_pcr_data);
+
+				break;
+			}
+			case COLOR_TRANSFORM_SRGB:
+				break;
+			}
 		}
-		VkDescriptorSet ds[] = {
+
-			render_buffer->plain.blend_descriptor_set, // set 0
+		bind_pipeline(pass, pl);
-			lut_ds, // set 1
-		};
-		size_t ds_len = sizeof(ds) / sizeof(ds[0]);
 		vkCmdBindDescriptorSets(render_cb->vk,
 			VK_PIPELINE_BIND_POINT_GRAPHICS, renderer->output_pipe_layout,
 			0, ds_len, ds, 0, NULL);

render/vulkan/renderer.c

@@ -24,7 +24,8 @@
 #include "render/vulkan/shaders/common.vert.h"
 #include "render/vulkan/shaders/texture.frag.h"
 #include "render/vulkan/shaders/quad.frag.h"
-#include "render/vulkan/shaders/output.frag.h"
+#include "render/vulkan/shaders/output_inverse_rgb.frag.h"
+#include "render/vulkan/shaders/output_lut_3d.frag.h"
 #include "types/wlr_buffer.h"
 #include "types/wlr_matrix.h"
 
@@ -1117,7 +1118,8 @@ static void vulkan_destroy(struct wlr_renderer *wlr_renderer) {
 	vkDestroyShaderModule(dev->dev, renderer->vert_module, NULL);
 	vkDestroyShaderModule(dev->dev, renderer->tex_frag_module, NULL);
 	vkDestroyShaderModule(dev->dev, renderer->quad_frag_module, NULL);
-	vkDestroyShaderModule(dev->dev, renderer->output_module, NULL);
+	vkDestroyShaderModule(dev->dev, renderer->output_module_srgb, NULL);
+	vkDestroyShaderModule(dev->dev, renderer->output_module_3d_lut, NULL);
 
 	struct wlr_vk_pipeline_layout *pipeline_layout, *pipeline_layout_tmp;
 	wl_list_for_each_safe(pipeline_layout, pipeline_layout_tmp,
@@ -1797,18 +1799,15 @@ static bool init_blend_to_output_pipeline(struct wlr_vk_renderer *renderer,
 	VkResult res;
 	VkDevice dev = renderer->dev->dev;
 
-	uint32_t output_transform_type = transform;
+	VkShaderModule output_module;
-	VkSpecializationMapEntry spec_entry = {
+	switch(transform) {
-		.constantID = 0,
+	case WLR_VK_OUTPUT_TRANSFORM_INVERSE_SRGB:
-		.offset = 0,
+		output_module = renderer->output_module_srgb;
-		.size = sizeof(uint32_t),
+		break;
-	};
+	case WLR_VK_OUTPUT_TRANSFORM_LUT3D:
-	VkSpecializationInfo specialization = {
+		output_module = renderer->output_module_3d_lut;
-		.mapEntryCount = 1,
+		break;
-		.pMapEntries = &spec_entry,
+	}
-		.dataSize = sizeof(uint32_t),
-		.pData = &output_transform_type,
-	};
 
 	VkPipelineShaderStageCreateInfo tex_stages[2] = {
 		{
@@ -1820,9 +1819,8 @@ static bool init_blend_to_output_pipeline(struct wlr_vk_renderer *renderer,
 		{
 			.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO,
 			.stage = VK_SHADER_STAGE_FRAGMENT_BIT,
-			.module = renderer->output_module,
+			.module = output_module,
 			.pName = "main",
-			.pSpecializationInfo = &specialization,
 		},
 	};
 
@@ -2140,12 +2138,25 @@ static bool init_static_render_data(struct wlr_vk_renderer *renderer) {
 
 	sinfo = (VkShaderModuleCreateInfo){
 		.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
-		.codeSize = sizeof(output_frag_data),
+		.codeSize = sizeof(output_inverse_rgb_data),
-		.pCode = output_frag_data,
+		.pCode = output_inverse_rgb_data,
 	};
-	res = vkCreateShaderModule(dev, &sinfo, NULL, &renderer->output_module);
+
+	res = vkCreateShaderModule(dev, &sinfo, NULL, &renderer->output_module_srgb);
 	if (res != VK_SUCCESS) {
-		wlr_vk_error("Failed to create blend->output fragment shader module", res);
+		wlr_vk_error("Failed to create blend->output fragment shader module for srgb", res);
+		return false;
+	}
+
+	sinfo = (VkShaderModuleCreateInfo){
+		.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO,
+		.codeSize = sizeof(output_lut_3d_data),
+		.pCode = output_lut_3d_data,
+	};
+
+	res = vkCreateShaderModule(dev, &sinfo, NULL, &renderer->output_module_3d_lut);
+	if (res != VK_SUCCESS) {
+		wlr_vk_error("Failed to create blend->output fragment shader module for 3d lut", res);
 		return false;
 	}
 

render/vulkan/shaders/meson.build

@@ -2,7 +2,6 @@ vulkan_shaders_src = [
 	'common.vert',
 	'texture.frag',
 	'quad.frag',
-	'output.frag',
 ]
 
 vulkan_shaders = []
@@ -21,4 +20,24 @@ foreach shader : vulkan_shaders_src
 	vulkan_shaders += [header]
 endforeach
 
+vulkan_shader_output_color_transforms = {
+	'inverse_rgb': '0',
+	'lut_3d': '1',
+}
+
+foreach name, ident : vulkan_shader_output_color_transforms
+	name = 'output_' + name
+	args = [glslang, '-DOUTPUT_TRANSFORM=' + ident, '-V', '@INPUT@', '-o', '@OUTPUT@', '--vn', name + '_data']
+	if glslang_version.version_compare('>=11.0.0')
+		args += '--quiet'
+	endif
+	header = custom_target(
+		name + '.frag_spv',
+		output: name + '.frag.h',
+		input: 'output.frag',
+		command: args)
+
+	vulkan_shaders += [header]
+endforeach
+
 wlr_files += vulkan_shaders

render/vulkan/shaders/output.frag

@@ -2,22 +2,22 @@
 
 layout (input_attachment_index = 0, set = 0, binding = 0) uniform subpassInput in_color;
 
-layout(set = 1, binding = 0) uniform sampler3D lut_3d;
+// Matches enum wlr_vk_output_transform
+#define OUTPUT_TRANSFORM_INVERSE_SRGB 0
+#define OUTPUT_TRANSFORM_LUT_3D 1
 
 layout(location = 0) in vec2 uv;
 layout(location = 0) out vec4 out_color;
 
-/* struct wlr_vk_frag_output_pcr_data */
+#if OUTPUT_TRANSFORM == OUTPUT_TRANSFORM_LUT_3D
-layout(push_constant) uniform UBO {
+	layout(set = 1, binding = 0) uniform sampler3D lut_3d;
-	layout(offset = 80) float lut_3d_offset;
-	float lut_3d_scale;
-} data;
 
-layout (constant_id = 0) const int OUTPUT_TRANSFORM = 0;
+	/* struct wlr_vk_frag_output_pcr_data */
-
+	layout(push_constant) uniform UBO {
-// Matches enum wlr_vk_output_transform
+		layout(offset = 80) float lut_3d_offset;
-#define OUTPUT_TRANSFORM_INVERSE_SRGB 0
+		float lut_3d_scale;
-#define OUTPUT_TRANSFORM_LUT_3D 1
+	} data;
+#endif
 
 float linear_channel_to_srgb(float x) {
 	return max(min(x * 12.92, 0.04045), 1.055 * pow(x, 1. / 2.4) - 0.055);
@@ -33,17 +33,19 @@ void main() {
 	// Convert from pre-multiplied alpha to straight alpha
 	vec3 rgb = val.rgb / val.a;
 
-	if (OUTPUT_TRANSFORM == OUTPUT_TRANSFORM_LUT_3D) {
+#if OUTPUT_TRANSFORM == OUTPUT_TRANSFORM_LUT_3D
-		// Apply 3D LUT
+	// Apply 3D LUT
-		vec3 pos = data.lut_3d_offset + rgb * data.lut_3d_scale;
+	vec3 pos = data.lut_3d_offset + rgb * data.lut_3d_scale;
-		rgb = texture(lut_3d, pos).rgb;
+	rgb = texture(lut_3d, pos).rgb;
-	} else { // OUTPUT_TRANSFORM_INVERSE_SRGB
+#endif
-		rgb = vec3(
+
-			linear_channel_to_srgb(rgb.r),
+#if OUTPUT_TRANSFORM == OUTPUT_TRANSFORM_INVERSE_SRGB
-			linear_channel_to_srgb(rgb.g),
+	rgb = vec3(
-			linear_channel_to_srgb(rgb.b)
+		linear_channel_to_srgb(rgb.r),
-		);
+		linear_channel_to_srgb(rgb.g),
-	}
+		linear_channel_to_srgb(rgb.b)
+	);
+#endif
 
 	// Back to pre-multiplied alpha
 	out_color = vec4(rgb * val.a, val.a);