render/vulkan: Refactor image usages for modifiers

Makes this much more extensible so we could add a storage type in future for compute.

Signed-off-by: Joshua Ashton <joshua@froggi.es>

include/render/vulkan.h

@@ -37,6 +37,12 @@ struct wlr_vk_instance *vulkan_instance_create(size_t ext_count,
 	const char **exts, bool debug);
 void vulkan_instance_destroy(struct wlr_vk_instance *ini);
 
+enum wlr_vk_image_usage {
+	WLR_VK_IMAGE_USAGE_RENDER,
+	WLR_VK_IMAGE_USAGE_SAMPLED,
+	WLR_VK_IMAGE_USAGE_COUNT,
+};
+
 // Logical vulkan device state.
 // Ownership can be shared by multiple renderers, reference counted
 // with `renderers`.
@@ -62,8 +68,7 @@ struct wlr_vk_device {
 
 	uint32_t format_prop_count;
 	struct wlr_vk_format_props *format_props;
-	struct wlr_drm_format_set dmabuf_render_formats;
+	struct wlr_drm_format_set dmabuf_formats[WLR_VK_IMAGE_USAGE_COUNT];
-	struct wlr_drm_format_set dmabuf_texture_formats;
 
 	// supported formats for textures (contains only those formats
 	// that support everything we need for textures)
@@ -106,22 +111,21 @@ struct wlr_vk_format_modifier_props {
 	bool export_imported;
 };
 
+VkImageUsageFlags wlr_vk_image_usage_to_vk(enum wlr_vk_image_usage usage);
+
 struct wlr_vk_format_props {
 	struct wlr_vk_format format;
 	VkExtent2D max_extent; // relevant if not created as dma_buf
 	VkFormatFeatureFlags features; // relevant if not created as dma_buf
 
-	uint32_t render_mod_count;
+	uint32_t mod_count[WLR_VK_IMAGE_USAGE_COUNT];
-	struct wlr_vk_format_modifier_props *render_mods;
+	struct wlr_vk_format_modifier_props *mods[WLR_VK_IMAGE_USAGE_COUNT];
-
-	uint32_t texture_mod_count;
-	struct wlr_vk_format_modifier_props *texture_mods;
 };
 
 void vulkan_format_props_query(struct wlr_vk_device *dev,
 	const struct wlr_vk_format *format);
 struct wlr_vk_format_modifier_props *vulkan_format_props_find_modifier(
-	struct wlr_vk_format_props *props, uint64_t mod, bool render);
+	struct wlr_vk_format_props *props, uint64_t mod, enum wlr_vk_image_usage usage);
 void vulkan_format_props_finish(struct wlr_vk_format_props *props);
 
 // For each format we want to render, we need a separate renderpass
@@ -260,7 +264,7 @@ struct wlr_vk_texture *vulkan_get_texture(struct wlr_texture *wlr_texture);
 VkImage vulkan_import_dmabuf(struct wlr_vk_renderer *renderer,
 	const struct wlr_dmabuf_attributes *attribs,
 	VkDeviceMemory mems[static WLR_DMABUF_MAX_PLANES], uint32_t *n_mems,
-	bool for_render);
+	enum wlr_vk_image_usage usage);
 struct wlr_texture *vulkan_texture_from_buffer(
 	struct wlr_renderer *wlr_renderer, struct wlr_buffer *buffer);
 void vulkan_texture_destroy(struct wlr_vk_texture *texture);

render/vulkan/pixel_format.c

@@ -38,13 +38,9 @@ const struct wlr_vk_format *vulkan_get_format_from_drm(uint32_t drm_format) {
 	return NULL;
 }
 
-static const VkImageUsageFlags render_usage =
-	VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT;
 static const VkImageUsageFlags tex_usage =
 	VK_IMAGE_USAGE_SAMPLED_BIT |
 	VK_IMAGE_USAGE_TRANSFER_DST_BIT;
-static const VkImageUsageFlags dma_tex_usage =
-	VK_IMAGE_USAGE_SAMPLED_BIT;
 
 static const VkFormatFeatureFlags tex_features =
 	VK_FORMAT_FEATURE_TRANSFER_SRC_BIT |
@@ -53,14 +49,31 @@ static const VkFormatFeatureFlags tex_features =
 	// NOTE: we don't strictly require this, we could create a NEAREST
 	// sampler for formats that need it, in case this ever makes problems.
 	VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT;
-static const VkFormatFeatureFlags render_features =
+
-	VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT |
+static const VkImageUsageFlags mod_set_usage[WLR_VK_IMAGE_USAGE_COUNT] = {
-	VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT;
+	[WLR_VK_IMAGE_USAGE_RENDER] = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT,
-static const VkFormatFeatureFlags dma_tex_features =
+	[WLR_VK_IMAGE_USAGE_SAMPLED] = VK_IMAGE_USAGE_SAMPLED_BIT,
-	VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT |
+};
-	// NOTE: we don't strictly require this, we could create a NEAREST
+
-	// sampler for formats that need it, in case this ever makes problems.
+static const VkFormatFeatureFlags mod_set_features[WLR_VK_IMAGE_USAGE_COUNT] = {
-	VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT;
+	[WLR_VK_IMAGE_USAGE_RENDER] = VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT |
+		VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BLEND_BIT,
+
+	[WLR_VK_IMAGE_USAGE_SAMPLED] = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT |
+		VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT,
+};
+
+VkImageUsageFlags wlr_vk_image_usage_to_vk(enum wlr_vk_image_usage usage) {
+	return mod_set_usage[usage];
+}
+
+static const char *wlr_vk_image_usage_to_str(enum wlr_vk_image_usage usage) {
+	switch (usage) {
+		case WLR_VK_IMAGE_USAGE_RENDER:  return "render";
+		case WLR_VK_IMAGE_USAGE_SAMPLED: return "sampled";
+		default:                         return "unknown";
+	}
+}
 
 static bool query_modifier_support(struct wlr_vk_device *dev,
 		struct wlr_vk_format_props *props, size_t modifier_count,
@@ -88,21 +101,17 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 	vkGetPhysicalDeviceFormatProperties2(dev->phdev,
 		props->format.vk_format, &fmtp);
 
-	props->render_mods = calloc(modp.drmFormatModifierCount,
+	for (int i = 0; i < WLR_VK_IMAGE_USAGE_COUNT; i++) {
-		sizeof(*props->render_mods));
+		props->mods[i] = calloc(modp.drmFormatModifierCount,
-	if (!props->render_mods) {
+			sizeof(*props->mods[i]));
+		if (!props->mods[i]) {
 			wlr_log_errno(WLR_ERROR, "Allocation failed");
+			for (int j = 0; j < i; j++) {
+				free(props->mods[j]);
+			}
 			free(modp.pDrmFormatModifierProperties);
 			return false;
 		}
-
-	props->texture_mods = calloc(modp.drmFormatModifierCount,
-		sizeof(*props->texture_mods));
-	if (!props->texture_mods) {
-		wlr_log_errno(WLR_ERROR, "Allocation failed");
-		free(modp.pDrmFormatModifierProperties);
-		free(props->render_mods);
-		return false;
 	}
 
 	// detailed check
@@ -138,9 +147,10 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 			m.drmFormatModifier, m.drmFormatModifierTilingFeatures,
 			m.drmFormatModifierPlaneCount);
 
+		for (int i = 0; i < WLR_VK_IMAGE_USAGE_COUNT; i++) {
 			// check that specific modifier for render usage
-		if ((m.drmFormatModifierTilingFeatures & render_features) == render_features) {
+			if ((m.drmFormatModifierTilingFeatures & mod_set_features[i]) == mod_set_features[i]) {
-			fmti.usage = render_usage;
+				fmti.usage = mod_set_usage[i];
 
 				modi.drmFormatModifier = m.drmFormatModifier;
 				res = vkGetPhysicalDeviceImageFormatProperties2(
@@ -151,71 +161,32 @@ static bool query_modifier_support(struct wlr_vk_device *dev,
 							res);
 					}
 
-				wlr_log(WLR_DEBUG, "    >> rendering: format not supported");
+					wlr_log(WLR_DEBUG, "    >> %s: format not supported", wlr_vk_image_usage_to_str(i));
 				} else if (emp->externalMemoryFeatures &
 						VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT) {
-				unsigned c = props->render_mod_count;
+					unsigned c = props->mod_count[i];
 					VkExtent3D me = ifmtp.imageFormatProperties.maxExtent;
 					VkExternalMemoryProperties emp = efmtp.externalMemoryProperties;
-				props->render_mods[c].props = m;
+					props->mods[i][c].props = m;
-				props->render_mods[c].max_extent.width = me.width;
+					props->mods[i][c].max_extent.width = me.width;
-				props->render_mods[c].max_extent.height = me.height;
+					props->mods[i][c].max_extent.height = me.height;
-				props->render_mods[c].dmabuf_flags = emp.externalMemoryFeatures;
+					props->mods[i][c].dmabuf_flags = emp.externalMemoryFeatures;
-				props->render_mods[c].export_imported =
+					props->mods[i][c].export_imported =
 						(emp.exportFromImportedHandleTypes &
 						VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
-				++props->render_mod_count;
+					++props->mod_count[i];
 
 					found = true;
-				wlr_drm_format_set_add(&dev->dmabuf_render_formats,
+					wlr_drm_format_set_add(&dev->dmabuf_formats[i],
 						props->format.drm_format, m.drmFormatModifier);
 
-				wlr_log(WLR_DEBUG, "    >> rendering: supported");
+					wlr_log(WLR_DEBUG, "    >> %s: supported", wlr_vk_image_usage_to_str(i));
 				} else {
-				wlr_log(WLR_DEBUG, "    >> rendering: importing not supported");
+					wlr_log(WLR_DEBUG, "    >> %s: importing not supported", wlr_vk_image_usage_to_str(i));
 				}
 			} else {
-			wlr_log(WLR_DEBUG, "    >> rendering: format features not supported");
+				wlr_log(WLR_DEBUG, "    >> %s: format features not supported", wlr_vk_image_usage_to_str(i));
 			}
-
-		// check that specific modifier for texture usage
-		if ((m.drmFormatModifierTilingFeatures & dma_tex_features) == dma_tex_features) {
-			fmti.usage = dma_tex_usage;
-
-			modi.drmFormatModifier = m.drmFormatModifier;
-			res = vkGetPhysicalDeviceImageFormatProperties2(
-				dev->phdev, &fmti, &ifmtp);
-			if (res != VK_SUCCESS) {
-				if (res != VK_ERROR_FORMAT_NOT_SUPPORTED) {
-					wlr_vk_error("vkGetPhysicalDeviceImageFormatProperties2",
-						res);
-				}
-
-				wlr_log(WLR_DEBUG, "    >> dmatex: format not supported");
-			} else if (emp->externalMemoryFeatures &
-					VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT) {
-				unsigned c = props->texture_mod_count;
-				VkExtent3D me = ifmtp.imageFormatProperties.maxExtent;
-				VkExternalMemoryProperties emp = efmtp.externalMemoryProperties;
-				props->texture_mods[c].props = m;
-				props->texture_mods[c].max_extent.width = me.width;
-				props->texture_mods[c].max_extent.height = me.height;
-				props->texture_mods[c].dmabuf_flags = emp.externalMemoryFeatures;
-				props->texture_mods[c].export_imported =
-					(emp.exportFromImportedHandleTypes &
-					 VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
-				++props->texture_mod_count;
-
-				found = true;
-				wlr_drm_format_set_add(&dev->dmabuf_texture_formats,
-					props->format.drm_format, m.drmFormatModifier);
-
-				wlr_log(WLR_DEBUG, "    >> dmatex: supported");
-			} else {
-				wlr_log(WLR_DEBUG, "    >> dmatex: importing not supported");
-			}
-		} else {
-			wlr_log(WLR_DEBUG, "    >> dmatex: format features not supported");
 		}
 	}
 
@@ -300,23 +271,16 @@ void vulkan_format_props_query(struct wlr_vk_device *dev,
 }
 
 void vulkan_format_props_finish(struct wlr_vk_format_props *props) {
-	free(props->texture_mods);
+	for (int i = 0; i < WLR_VK_IMAGE_USAGE_COUNT; i++) {
-	free(props->render_mods);
+		free(props->mods[i]);
+	}
 }
 
 struct wlr_vk_format_modifier_props *vulkan_format_props_find_modifier(
-		struct wlr_vk_format_props *props, uint64_t mod, bool render) {
+		struct wlr_vk_format_props *props, uint64_t mod, enum wlr_vk_image_usage usage) {
-	if (render) {
+	for (unsigned i = 0u; i < props->mod_count[usage]; ++i) {
-		for (unsigned i = 0u; i < props->render_mod_count; ++i) {
+		if (props->mods[usage][i].props.drmFormatModifier == mod) {
-			if (props->render_mods[i].props.drmFormatModifier == mod) {
+			return &props->mods[usage][i];
-				return &props->render_mods[i];
-			}
-		}
-	} else {
-		for (unsigned i = 0u; i < props->texture_mod_count; ++i) {
-			if (props->texture_mods[i].props.drmFormatModifier == mod) {
-				return &props->texture_mods[i];
-			}
 		}
 	}
 

render/vulkan/renderer.c

@@ -443,7 +443,7 @@ static struct wlr_vk_render_buffer *create_render_buffer(
 	}
 
 	buffer->image = vulkan_import_dmabuf(renderer, &dmabuf,
-		buffer->memories, &buffer->mem_count, true);
+		buffer->memories, &buffer->mem_count, WLR_VK_IMAGE_USAGE_RENDER);
 	if (!buffer->image) {
 		goto error_buffer;
 	}
@@ -900,13 +900,13 @@ static void vulkan_render_quad_with_matrix(struct wlr_renderer *wlr_renderer,
 static const struct wlr_drm_format_set *vulkan_get_dmabuf_texture_formats(
 		struct wlr_renderer *wlr_renderer) {
 	struct wlr_vk_renderer *renderer = vulkan_get_renderer(wlr_renderer);
-	return &renderer->dev->dmabuf_texture_formats;
+	return &renderer->dev->dmabuf_formats[WLR_VK_IMAGE_USAGE_SAMPLED];
 }
 
 static const struct wlr_drm_format_set *vulkan_get_render_formats(
 		struct wlr_renderer *wlr_renderer) {
 	struct wlr_vk_renderer *renderer = vulkan_get_renderer(wlr_renderer);
-	return &renderer->dev->dmabuf_render_formats;
+	return &renderer->dev->dmabuf_formats[WLR_VK_IMAGE_USAGE_RENDER];
 }
 
 static uint32_t vulkan_preferred_read_format(

render/vulkan/texture.c

@@ -378,7 +378,7 @@ static bool is_dmabuf_disjoint(const struct wlr_dmabuf_attributes *attribs) {
 VkImage vulkan_import_dmabuf(struct wlr_vk_renderer *renderer,
 		const struct wlr_dmabuf_attributes *attribs,
 		VkDeviceMemory mems[static WLR_DMABUF_MAX_PLANES], uint32_t *n_mems,
-		bool for_render) {
+		enum wlr_vk_image_usage usage) {
 	VkResult res;
 	VkDevice dev = renderer->dev->dev;
 	*n_mems = 0u;
@@ -398,7 +398,7 @@ VkImage vulkan_import_dmabuf(struct wlr_vk_renderer *renderer,
 	uint32_t plane_count = attribs->n_planes;
 	assert(plane_count < WLR_DMABUF_MAX_PLANES);
 	struct wlr_vk_format_modifier_props *mod =
-		vulkan_format_props_find_modifier(fmt, attribs->modifier, for_render);
+		vulkan_format_props_find_modifier(fmt, attribs->modifier, usage);
 	if (!mod || !(mod->dmabuf_flags & VK_EXTERNAL_MEMORY_FEATURE_IMPORTABLE_BIT)) {
 		wlr_log(WLR_ERROR, "Format %"PRIx32" (%.4s) can't be used with modifier "
 			"%"PRIx64, attribs->format, (const char*) &attribs->format,
@@ -440,9 +440,7 @@ VkImage vulkan_import_dmabuf(struct wlr_vk_renderer *renderer,
 	img_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
 	img_info.initialLayout = VK_IMAGE_LAYOUT_PREINITIALIZED;
 	img_info.extent = (VkExtent3D) { attribs->width, attribs->height, 1 };
-	img_info.usage = for_render ?
+	img_info.usage = wlr_vk_image_usage_to_vk(usage);
-		VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT :
-		VK_IMAGE_USAGE_SAMPLED_BIT;
 	if (disjoint) {
 		img_info.flags = VK_IMAGE_CREATE_DISJOINT_BIT;
 	}
@@ -600,7 +598,7 @@ static struct wlr_texture *vulkan_texture_from_dmabuf(struct wlr_renderer *wlr_r
 
 	texture->format = &fmt->format;
 	texture->image = vulkan_import_dmabuf(renderer, attribs,
-		texture->memories, &texture->mem_count, false);
+		texture->memories, &texture->mem_count, WLR_VK_IMAGE_USAGE_SAMPLED);
 	if (!texture->image) {
 		goto error;
 	}

render/vulkan/vulkan.c

@@ -536,8 +536,9 @@ void vulkan_device_destroy(struct wlr_vk_device *dev) {
 		close(dev->drm_fd);
 	}
 
-	wlr_drm_format_set_finish(&dev->dmabuf_render_formats);
+	for (int i = 0; i < WLR_VK_IMAGE_USAGE_COUNT; i++) {
-	wlr_drm_format_set_finish(&dev->dmabuf_texture_formats);
+		wlr_drm_format_set_finish(&dev->dmabuf_formats[i]);
+	}
 
 	for (unsigned i = 0u; i < dev->format_prop_count; ++i) {
 		vulkan_format_props_finish(&dev->format_props[i]);