WIP Introduce WLR_GBM_EXPLICIT_UPGRADE

When the `WLR_GBM_EXPLICIT_UPGRADE` flag is set to 1, the GBM allocator
will attempt to "upgrade" buffers allocated with implicit modifiers to
be managed with explicit modifiers.

This is useful to enable the conservative modifier fallback for the
Vulkan renderer on drivers/hardware that fully support implicit and
explicit modifiers, including being able to report the modifier of an
implicitly allocated BO.

docs/env_vars.md

@@ -18,6 +18,9 @@ wlroots reads these environment variables
   and Vulkan
 * *WLR_EGL_NO_MODIFIERS*: set to 1 to disable format modifiers in EGL, this can
   be used to understand and work around driver bugs.
+* *WLR_GBM_EXPLICIT_UPGRADE*: set to 1 to attempt to "upgrade" buffers
+  allocated with implicit modifiers to explicit modifiers. May help when a
+  conservative modifier is required, but may not work with all drivers.
 
 ## DRM backend
 

include/render/allocator/gbm.h

@@ -22,6 +22,10 @@ struct wlr_gbm_allocator {
 	struct gbm_device *gbm_device;
 
 	struct wl_list buffers; // wlr_gbm_buffer.link
+
+	struct {
+		bool explicit_upgrade;
+	} WLR_PRIVATE;
 };
 
 /**

render/allocator/gbm.c

@@ -11,6 +11,7 @@
 
 #include "render/allocator/gbm.h"
 #include "render/drm_format_set.h"
+#include "util/env.h"
 
 static const struct wlr_buffer_impl buffer_impl;
 
@@ -90,6 +91,10 @@ static struct wlr_gbm_buffer *create_buffer(struct wlr_gbm_allocator *alloc,
 		return NULL;
 	}
 
+	if (!has_modifier && alloc->explicit_upgrade) {
+		fallback_modifier = gbm_bo_get_modifier(bo);
+	}
+
 	struct wlr_gbm_buffer *buffer = calloc(1, sizeof(*buffer));
 	if (buffer == NULL) {
 		gbm_bo_destroy(bo);
@@ -174,6 +179,7 @@ struct wlr_allocator *wlr_gbm_allocator_create(int fd) {
 		return NULL;
 	}
 	wlr_allocator_init(&alloc->base, &allocator_impl, WLR_BUFFER_CAP_DMABUF);
+	alloc->explicit_upgrade = env_parse_bool("WLR_GBM_EXPLICIT_UPGRADE");
 
 	alloc->fd = fd;
 	wl_list_init(&alloc->buffers);

types/wlr_output_swapchain_manager.c

@@ -9,6 +9,7 @@
 #include <wlr/util/log.h>
 #include "render/drm_format_set.h"
 #include "types/wlr_output.h"
+#include "util/env.h"
 
 struct wlr_output_swapchain_manager_output {
 	struct wlr_output *output;
@@ -22,6 +23,9 @@ struct wlr_output_swapchain_manager_output {
 	// wlr_output_swapchain_manager_apply() is called. Can be either a pointer
 	// to the newly allocated swapchain, or the old swapchain, or NULL.
 	struct wlr_swapchain *pending_swapchain;
+	// Whether we will trust the allocator to upgrade implicit allocations to
+	// use explicit modifiers.
+	bool explicit_upgrade;
 };
 
 void wlr_output_swapchain_manager_init(struct wlr_output_swapchain_manager *manager,
@@ -55,6 +59,7 @@ static struct wlr_output_swapchain_manager_output *manager_get_or_add_output(
 	}
 	*manager_output = (struct wlr_output_swapchain_manager_output){
 		.output = output,
+		.explicit_upgrade = env_parse_bool("WLR_GBM_EXPLICIT_UPGRADE"),
 	};
 	return manager_output;
 }
@@ -122,7 +127,9 @@ static bool manager_output_prepare(struct wlr_output_swapchain_manager_output *m
 	}
 
 	if (!explicit_modifiers && (format.len != 1 || format.modifiers[0] != DRM_FORMAT_MOD_LINEAR)) {
-		if (!wlr_drm_format_has(&format, DRM_FORMAT_MOD_INVALID)) {
+		if (!wlr_drm_format_has(&format, DRM_FORMAT_MOD_INVALID) &&
+				!(wlr_drm_format_set_has(display_formats, format.format, DRM_FORMAT_MOD_INVALID) &&
+				manager_output->explicit_upgrade)) {
 			wlr_log(WLR_DEBUG, "Implicit modifiers not supported");
 			wlr_drm_format_finish(&format);
 			return false;