backends: implement custom EGL and renderer initialization

Compositors now have more control over how the backend creates its
renderer. Currently all backends create an EGL/GLES2 renderer, so
the necessary attributes for creating the context are passed to a
user-provided callback function. It is responsible for initializing
provided wlr_egl and to return a renderer. On fail, return 0.

Fixes #987

backend/backend.c

@@ -72,8 +72,9 @@ static size_t parse_outputs_env(const char *name) {
 	return outputs;
 }
 
-static struct wlr_backend *attempt_wl_backend(struct wl_display *display) {
-	struct wlr_backend *backend = wlr_wl_backend_create(display, NULL);
+static struct wlr_backend *attempt_wl_backend(struct wl_display *display,
+		wlr_renderer_create_func_t create_renderer_func) {
+	struct wlr_backend *backend = wlr_wl_backend_create(display, NULL, create_renderer_func);
 	if (backend == NULL) {
 		return NULL;
 	}
@@ -88,8 +89,8 @@ static struct wlr_backend *attempt_wl_backend(struct wl_display *display) {
 
 #ifdef WLR_HAS_X11_BACKEND
 static struct wlr_backend *attempt_x11_backend(struct wl_display *display,
-		const char *x11_display) {
-	struct wlr_backend *backend = wlr_x11_backend_create(display, x11_display);
+		const char *x11_display, wlr_renderer_create_func_t create_renderer_func) {
+	struct wlr_backend *backend = wlr_x11_backend_create(display, x11_display, create_renderer_func);
 	if (backend == NULL) {
 		return NULL;
 	}
@@ -104,8 +105,8 @@ static struct wlr_backend *attempt_x11_backend(struct wl_display *display,
 #endif
 
 static struct wlr_backend *attempt_headless_backend(
-		struct wl_display *display) {
-	struct wlr_backend *backend = wlr_headless_backend_create(display);
+		struct wl_display *display, wlr_renderer_create_func_t create_renderer_func) {
+	struct wlr_backend *backend = wlr_headless_backend_create(display, create_renderer_func);
 	if (backend == NULL) {
 		return NULL;
 	}
@@ -119,7 +120,8 @@ static struct wlr_backend *attempt_headless_backend(
 }
 
 static struct wlr_backend *attempt_drm_backend(struct wl_display *display,
-		struct wlr_backend *backend, struct wlr_session *session) {
+		struct wlr_backend *backend, struct wlr_session *session,
+		wlr_renderer_create_func_t create_renderer_func) {
 	int gpus[8];
 	size_t num_gpus = wlr_session_find_gpus(session, 8, gpus);
 	struct wlr_backend *primary_drm = NULL;
@@ -127,7 +129,7 @@ static struct wlr_backend *attempt_drm_backend(struct wl_display *display,
 
 	for (size_t i = 0; i < num_gpus; ++i) {
 		struct wlr_backend *drm = wlr_drm_backend_create(display, session,
-			gpus[i], primary_drm);
+			gpus[i], primary_drm, create_renderer_func);
 		if (!drm) {
 			wlr_log(L_ERROR, "Failed to open DRM device %d", gpus[i]);
 			continue;
@@ -145,15 +147,15 @@ static struct wlr_backend *attempt_drm_backend(struct wl_display *display,
 
 static struct wlr_backend *attempt_backend_by_name(struct wl_display *display,
 		struct wlr_backend *backend, struct wlr_session **session,
-		const char *name) {
+		const char *name, wlr_renderer_create_func_t create_renderer_func) {
 	if (strcmp(name, "wayland") == 0) {
-		return attempt_wl_backend(display);
+		return attempt_wl_backend(display, create_renderer_func);
 #ifdef WLR_HAS_X11_BACKEND
 	} else if (strcmp(name, "x11") == 0) {
-		return attempt_x11_backend(display, NULL);
+		return attempt_x11_backend(display, NULL, create_renderer_func);
 #endif
 	} else if (strcmp(name, "headless") == 0) {
-		return attempt_headless_backend(display);
+		return attempt_headless_backend(display, create_renderer_func);
 	} else if (strcmp(name, "drm") == 0 || strcmp(name, "libinput") == 0) {
 		// DRM and libinput need a session
 		*session = wlr_session_create(display);
@@ -165,7 +167,7 @@ static struct wlr_backend *attempt_backend_by_name(struct wl_display *display,
 		if (strcmp(name, "libinput") == 0) {
 			return wlr_libinput_backend_create(display, *session);
 		} else {
-			return attempt_drm_backend(display, backend, *session);
+			return attempt_drm_backend(display, backend, *session, create_renderer_func);
 		}
 	}
 
@@ -173,7 +175,8 @@ static struct wlr_backend *attempt_backend_by_name(struct wl_display *display,
 	return NULL;
 }
 
-struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
+struct wlr_backend *wlr_backend_autocreate(struct wl_display *display,
+		wlr_renderer_create_func_t create_renderer_func) {
 	struct wlr_backend *backend = wlr_multi_backend_create(display);
 	if (!backend) {
 		wlr_log(L_ERROR, "could not allocate multibackend");
@@ -195,7 +198,7 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 		char *name = strtok_r(names, ",", &saveptr);
 		while (name != NULL) {
 			struct wlr_backend *subbackend =
-				attempt_backend_by_name(display, backend, &session, name);
+				attempt_backend_by_name(display, backend, &session, name, create_renderer_func);
 			if (subbackend == NULL) {
 				wlr_log(L_ERROR, "failed to start backend '%s'", name);
 				wlr_backend_destroy(backend);
@@ -218,7 +221,8 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 
 	if (getenv("WAYLAND_DISPLAY") || getenv("_WAYLAND_DISPLAY") ||
 			getenv("WAYLAND_SOCKET")) {
-		struct wlr_backend *wl_backend = attempt_wl_backend(display);
+		struct wlr_backend *wl_backend = attempt_wl_backend(display,
+			create_renderer_func);
 		if (wl_backend) {
 			wlr_multi_backend_add(backend, wl_backend);
 			return backend;
@@ -229,7 +233,7 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 	const char *x11_display = getenv("DISPLAY");
 	if (x11_display) {
 		struct wlr_backend *x11_backend =
-			attempt_x11_backend(display, x11_display);
+			attempt_x11_backend(display, x11_display, create_renderer_func);
 		if (x11_backend) {
 			wlr_multi_backend_add(backend, x11_backend);
 			return backend;
@@ -255,7 +259,7 @@ struct wlr_backend *wlr_backend_autocreate(struct wl_display *display) {
 	wlr_multi_backend_add(backend, libinput);
 
 	struct wlr_backend *primary_drm =
-		attempt_drm_backend(display, backend, session);
+		attempt_drm_backend(display, backend, session, create_renderer_func);
 	if (!primary_drm) {
 		wlr_log(L_ERROR, "Failed to open any DRM device");
 		wlr_backend_destroy(libinput);

backend/drm/backend.c

@@ -114,7 +114,8 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 }
 
 struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
-		struct wlr_session *session, int gpu_fd, struct wlr_backend *parent) {
+		struct wlr_session *session, int gpu_fd, struct wlr_backend *parent,
+		wlr_renderer_create_func_t create_renderer_func) {
 	assert(display && session && gpu_fd >= 0);
 	assert(!parent || wlr_backend_is_drm(parent));
 
@@ -161,7 +162,7 @@ struct wlr_backend *wlr_drm_backend_create(struct wl_display *display,
 		goto error_event;
 	}
 
-	if (!init_drm_renderer(drm, &drm->renderer)) {
+	if (!init_drm_renderer(drm, &drm->renderer, create_renderer_func)) {
 		wlr_log(L_ERROR, "Failed to initialize renderer");
 		goto error_event;
 	}

backend/drm/renderer.c

@@ -19,29 +19,28 @@
 #endif
 
 bool init_drm_renderer(struct wlr_drm_backend *drm,
-		struct wlr_drm_renderer *renderer) {
+		struct wlr_drm_renderer *renderer, wlr_renderer_create_func_t create_renderer_func) {
 	renderer->gbm = gbm_create_device(drm->fd);
 	if (!renderer->gbm) {
 		wlr_log(L_ERROR, "Failed to create GBM device");
 		return false;
 	}
 
-	if (!wlr_egl_init(&renderer->egl, EGL_PLATFORM_GBM_MESA, renderer->gbm,
-			NULL, GBM_FORMAT_ARGB8888)) {
-		goto error_gbm;
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
 	}
 
-	renderer->wlr_rend = wlr_gles2_renderer_create(&renderer->egl);
+	renderer->wlr_rend = create_renderer_func(&renderer->egl,
+		EGL_PLATFORM_GBM_MESA, renderer->gbm, NULL, GBM_FORMAT_ARGB8888);
+
 	if (!renderer->wlr_rend) {
-		wlr_log(L_ERROR, "Failed to create WLR renderer");
-		goto error_egl;
+		wlr_log(L_ERROR, "Failed to create EGL/WLR renderer");
+		goto error_gbm;
 	}
 
 	renderer->fd = drm->fd;
 	return true;
 
-error_egl:
-	wlr_egl_finish(&renderer->egl);
 error_gbm:
 	gbm_device_destroy(renderer->gbm);
 	return false;

backend/headless/backend.c

@@ -78,7 +78,8 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 	backend_destroy(&backend->backend);
 }
 
-struct wlr_backend *wlr_headless_backend_create(struct wl_display *display) {
+struct wlr_backend *wlr_headless_backend_create(struct wl_display *display,
+		wlr_renderer_create_func_t create_renderer_func) {
 	wlr_log(L_INFO, "Creating headless backend");
 
 	struct wlr_headless_backend *backend =
@@ -101,16 +102,18 @@ struct wlr_backend *wlr_headless_backend_create(struct wl_display *display) {
 		EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
 		EGL_NONE,
 	};
-	bool ok = wlr_egl_init(&backend->egl, EGL_PLATFORM_SURFACELESS_MESA, NULL,
-		(EGLint *)config_attribs, 0);
-	if (!ok) {
-		free(backend);
-		return NULL;
+
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
 	}
 
-	backend->renderer = wlr_gles2_renderer_create(&backend->egl);
-	if (backend->renderer == NULL) {
+	backend->renderer = create_renderer_func(&backend->egl, EGL_PLATFORM_SURFACELESS_MESA,
+		NULL, (EGLint*)config_attribs, 0);
+
+	if (!backend->renderer) {
 		wlr_log(L_ERROR, "Failed to create renderer");
+		free(backend);
+		return NULL;
 	}
 
 	backend->display_destroy.notify = handle_display_destroy;

backend/wayland/backend.c

@@ -139,7 +139,7 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 }
 
 struct wlr_backend *wlr_wl_backend_create(struct wl_display *display,
-		const char *remote) {
+		const char *remote, wlr_renderer_create_func_t create_renderer_func) {
 	wlr_log(L_INFO, "Creating wayland backend");
 
 	struct wlr_wl_backend *backend = calloc(1, sizeof(struct wlr_wl_backend));
@@ -174,14 +174,14 @@ struct wlr_backend *wlr_wl_backend_create(struct wl_display *display,
 		EGL_ALPHA_SIZE, 1,
 		EGL_NONE,
 	};
-	if (!wlr_egl_init(&backend->egl, EGL_PLATFORM_WAYLAND_EXT,
-			backend->remote_display, config_attribs, WL_SHM_FORMAT_ARGB8888)) {
-		wlr_log(L_ERROR, "Could not initialize EGL");
-		goto error_egl;
-	}
-	wlr_egl_bind_display(&backend->egl, backend->local_display);
 
-	backend->renderer = wlr_gles2_renderer_create(&backend->egl);
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
+	}
+
+	backend->renderer = create_renderer_func(&backend->egl, EGL_PLATFORM_WAYLAND_EXT,
+		backend->remote_display, config_attribs, WL_SHM_FORMAT_ARGB8888);
+
 	if (backend->renderer == NULL) {
 		wlr_log(L_ERROR, "Could not create renderer");
 		goto error_renderer;
@@ -193,8 +193,6 @@ struct wlr_backend *wlr_wl_backend_create(struct wl_display *display,
 	return &backend->backend;
 
 error_renderer:
-	wlr_egl_finish(&backend->egl);
-error_egl:
 	wl_registry_destroy(backend->registry);
 error_registry:
 	wl_display_disconnect(backend->remote_display);

backend/x11/backend.c

@@ -232,7 +232,7 @@ static void handle_display_destroy(struct wl_listener *listener, void *data) {
 }
 
 struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
-		const char *x11_display) {
+		const char *x11_display, wlr_renderer_create_func_t create_renderer_func) {
 	struct wlr_x11_backend *x11 = calloc(1, sizeof(*x11));
 	if (!x11) {
 		return NULL;
@@ -267,15 +267,16 @@ struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
 
 	x11->screen = xcb_setup_roots_iterator(xcb_get_setup(x11->xcb_conn)).data;
 
-	if (!wlr_egl_init(&x11->egl, EGL_PLATFORM_X11_KHR, x11->xlib_conn, NULL,
-			x11->screen->root_visual)) {
-		goto error_event;
+	if (!create_renderer_func) {
+		create_renderer_func = wlr_renderer_autocreate;
 	}
 
-	x11->renderer = wlr_gles2_renderer_create(&x11->egl);
+	x11->renderer = create_renderer_func(&x11->egl, EGL_PLATFORM_X11_KHR,
+		x11->xlib_conn, NULL, x11->screen->root_visual);
+
 	if (x11->renderer == NULL) {
 		wlr_log(L_ERROR, "Failed to create renderer");
-		goto error_egl;
+		goto error_event;
 	}
 
 	wlr_input_device_init(&x11->keyboard_dev, WLR_INPUT_DEVICE_KEYBOARD,
@@ -288,8 +289,6 @@ struct wlr_backend *wlr_x11_backend_create(struct wl_display *display,
 
 	return &x11->backend;
 
-error_egl:
-	wlr_egl_finish(&x11->egl);
 error_event:
 	wl_event_source_remove(x11->event_source);
 error_x11: