render/vulkan: clear blend image on first use

The two-pass blend image is created with VK_IMAGE_LAYOUT_UNDEFINED, so
on its first use loadOp=LOAD loads uninitialized memory. This oughtn't
be an issue, as we render onto it before we read it. These renders are
blends, so even opaque content is rendered with reference to an uninit
dst. This too ought to be fine: src*1 + dst*0 = src for all finite dst.

But the blend image pixfmt is VK_FORMAT_R16G16B16A16_SFLOAT, so uninit
pixels can be NaN, inf, or -inf, and now src*1 + dst*0 = NaN/inf/-inf.
This is bad enough assuming the uninitialized blend image holds random
bytes (2048/65536 values are not finite), even worse on any driver/GPU
with a framebuffer compression scheme that so happens to reliably read
NaNs from any uninitialized compressed image...

Most Mesa drivers happen not to do this perfectly valid thing, so this
is only reliably a problem (afaict) for honeykrisp i.e. AGX i.e. Asahi
Linux i.e. Apple Silicon, where after an upgrade to wlroots 0.20, sway
renders a black screen forever, unless you get quite lucky spamming VT
switches, in which case there's flickery garbage on exactly one of the
two swapchain buffers.

The blend image persists across frames, so it suffices to clear before
first real use. Rather than clear by hand, make a loadOp=CLEAR variant
of the render pass and use it for that first frame only. Adding a pass
sounds heavy, but render pass compatibility ignores loadOp and layouts
such that the new pass reuses the pipelines and framebuffer, and costs
one VkRenderPass object but not the usual pipeline/shader (re)compile.

include/render/vulkan.h

@@ -212,6 +212,7 @@ struct wlr_vk_render_format_setup {
 	bool use_blending_buffer;
 	bool use_srgb;
 	VkRenderPass render_pass;
+	VkRenderPass render_pass_clear;
 
 	VkPipeline output_pipe_identity;
 	VkPipeline output_pipe_srgb;

render/vulkan/pass.c

@@ -429,33 +429,32 @@ static bool render_pass_submit(struct wlr_render_pass *wlr_pass) {
 	}
 
 	if (pass->two_pass) {
-		// The render pass changes the blend image layout from
+		// On the first frame the clear render pass transitions the blend
-		// color attachment to read only, so on each frame, before
+		// image from undefined and we just mark it transitioned. On every
-		// the render pass starts, we change it back
+		// frame after, the previous frame left it read-only, so we change
-		VkImageLayout blend_src_layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+		// it back to a color attachment before the render pass starts
-		if (!render_buffer->two_pass.blend_transitioned) {
+		if (render_buffer->two_pass.blend_transitioned) {
-			blend_src_layout = VK_IMAGE_LAYOUT_UNDEFINED;
+			VkImageMemoryBarrier blend_acq_barrier = {
+				.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
+				.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+				.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
+				.image = render_buffer->two_pass.blend_image,
+				.oldLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
+				.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+				.srcAccessMask = VK_ACCESS_SHADER_READ_BIT,
+				.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
+				.subresourceRange = {
+					.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
+					.layerCount = 1,
+					.levelCount = 1,
+				},
+			};
+			vkCmdPipelineBarrier(stage_cb->vk, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+				VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+				0, 0, NULL, 0, NULL, 1, &blend_acq_barrier);
+		} else {
 			render_buffer->two_pass.blend_transitioned = true;
 		}
-
-		VkImageMemoryBarrier blend_acq_barrier = {
-			.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
-			.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
-			.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
-			.image = render_buffer->two_pass.blend_image,
-			.oldLayout = blend_src_layout,
-			.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
-			.srcAccessMask = VK_ACCESS_SHADER_READ_BIT,
-			.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
-			.subresourceRange = {
-				.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
-				.layerCount = 1,
-				.levelCount = 1,
-			},
-		};
-		vkCmdPipelineBarrier(stage_cb->vk, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
-			VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
-			0, 0, NULL, 0, NULL, 1, &blend_acq_barrier);
 	}
 
 	// acquire render buffer before rendering
@@ -1334,11 +1333,15 @@ struct wlr_vk_render_pass *vulkan_begin_render_pass(struct wlr_vk_renderer *rend
 	int height = buffer->wlr_buffer->height;
 	VkRect2D rect = { .extent = { width, height } };
 
+	bool blend_first_use = pass->two_pass && !buffer->two_pass.blend_transitioned;
+	VkClearValue clear_value = {0};
 	VkRenderPassBeginInfo rp_info = {
 		.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO,
 		.renderArea = rect,
-		.clearValueCount = 0,
+		.clearValueCount = blend_first_use ? 1 : 0,
-		.renderPass = render_setup->render_pass,
+		.pClearValues = blend_first_use ? &clear_value : NULL,
+		.renderPass = blend_first_use ?
+			render_setup->render_pass_clear : render_setup->render_pass,
 		.framebuffer = buffer_out->framebuffer,
 	};
 	vkCmdBeginRenderPass(cb->vk, &rp_info, VK_SUBPASS_CONTENTS_INLINE);

render/vulkan/renderer.c

@@ -169,6 +169,7 @@ static void destroy_render_format_setup(struct wlr_vk_renderer *renderer,
 
 	VkDevice dev = renderer->dev->dev;
 	vkDestroyRenderPass(dev, setup->render_pass, NULL);
+	vkDestroyRenderPass(dev, setup->render_pass_clear, NULL);
 	vkDestroyPipeline(dev, setup->output_pipe_identity, NULL);
 	vkDestroyPipeline(dev, setup->output_pipe_srgb, NULL);
 	vkDestroyPipeline(dev, setup->output_pipe_pq, NULL);
@@ -2585,6 +2586,14 @@ static struct wlr_vk_render_format_setup *find_or_create_render_setup(
 			goto error;
 		}
 
+		attachments[0].loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR;
+		attachments[0].initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
+		res = vkCreateRenderPass(dev, &rp_info, NULL, &setup->render_pass_clear);
+		if (res != VK_SUCCESS) {
+			wlr_vk_error("Failed to create 2-step clear render pass", res);
+			goto error;
+		}
+
 		// this is only well defined if render pass has a 2nd subpass
 		if (!init_blend_to_output_pipeline(
 				renderer, setup->render_pass, renderer->output_pipe_layout,