diff --git a/src/config/session.c b/src/config/session.c
index 9a05d255..3ba7913b 100644
--- a/src/config/session.c
+++ b/src/config/session.c
@@ -21,7 +21,6 @@
 #include "labwc.h"
 
 static const char *const env_vars[] = {
-	"DISPLAY",
 	"WAYLAND_DISPLAY",
 	"XDG_CURRENT_DESKTOP",
 	"XCURSOR_SIZE",
@@ -208,6 +207,21 @@ should_update_activation(void)
 	return have_drm;
 }
 
+static void
+execute_update(const char *env_keys, const char *env_unset_keys, bool initialize)
+{
+	char *cmd =
+		strdup_printf("dbus-update-activation-environment %s",
+			initialize ? env_keys : env_unset_keys);
+	spawn_async_no_shell(cmd);
+	free(cmd);
+
+	cmd = strdup_printf("systemctl --user %s %s",
+		initialize ? "import-environment" : "unset-environment", env_keys);
+	spawn_async_no_shell(cmd);
+	free(cmd);
+}
+
 static void
 update_activation_env(bool initialize)
 {
@@ -227,19 +241,18 @@ update_activation_env(bool initialize)
 	char *env_keys = str_join(env_vars, "%s", " ");
 	char *env_unset_keys = initialize ? NULL : str_join(env_vars, "%s=", " ");
 
-	char *cmd =
-		strdup_printf("dbus-update-activation-environment %s",
-			initialize ? env_keys : env_unset_keys);
-	spawn_async_no_shell(cmd);
-	free(cmd);
-
-	cmd = strdup_printf("systemctl --user %s %s",
-		initialize ? "import-environment" : "unset-environment", env_keys);
-	spawn_async_no_shell(cmd);
-	free(cmd);
+	execute_update(env_keys, env_unset_keys, initialize);
 
 	free(env_keys);
 	free(env_unset_keys);
+
+#if HAVE_XWAYLAND
+	if (server.xwayland) {
+		// DISPLAY is only set if xwayland was initialized successfully,
+		// so we only update the env in that case
+		execute_update("DISPLAY", "DISPLAY=", initialize);
+	}
+#endif
 }
 
 void
diff --git a/src/input/cursor.c b/src/input/cursor.c
index a7cba87d..315898b7 100644
--- a/src/input/cursor.c
+++ b/src/input/cursor.c
@@ -1166,7 +1166,7 @@ cursor_process_button_press(struct seat *seat, uint32_t button, uint32_t time_ms
 		if (layer && layer->current.keyboard_interactive) {
 			layer_try_set_focus(seat, layer);
 		}
-#ifdef HAVE_XWAYLAND
+#if HAVE_XWAYLAND
 	} else if (ctx.type == LAB_NODE_UNMANAGED) {
 		desktop_focus_view_or_surface(seat, NULL, ctx.surface,
 			/*raise*/ false);
diff --git a/src/server.c b/src/server.c
index c5ca9fff..2e3ece02 100644
--- a/src/server.c
+++ b/src/server.c
@@ -583,6 +583,8 @@ server_init(void)
 	server.workspace_tree = lab_wlr_scene_tree_create(&server.scene->tree);
 	server.xdg_popup_tree = lab_wlr_scene_tree_create(&server.scene->tree);
 #if HAVE_XWAYLAND
+	// Creating/setting this is harmless when xwayland support is built-in
+	// but xwayland could not be successfully started.
 	server.unmanaged_tree = lab_wlr_scene_tree_create(&server.scene->tree);
 #endif
 	server.menu_tree = lab_wlr_scene_tree_create(&server.scene->tree);
diff --git a/src/view.c b/src/view.c
index c44a2a6d..1d43fcfa 100644
--- a/src/view.c
+++ b/src/view.c
@@ -54,6 +54,8 @@ view_from_wlr_surface(struct wlr_surface *surface)
 		return xdg_surface->data;
 	}
 #if HAVE_XWAYLAND
+	// Doing this is harmless even in the case that xwayland could not be
+	// successfully started.
 	struct wlr_xwayland_surface *xsurface =
 		wlr_xwayland_surface_try_from_wlr_surface(surface);
 	if (xsurface) {
diff --git a/src/xwayland.c b/src/xwayland.c
index 7e033dfc..cfe7d691 100644
--- a/src/xwayland.c
+++ b/src/xwayland.c
@@ -1218,8 +1218,9 @@ xwayland_server_init(struct wlr_compositor *compositor)
 		wlr_xwayland_create(server.wl_display,
 			compositor, /* lazy */ !rc.xwayland_persistence);
 	if (!server.xwayland) {
-		wlr_log(WLR_ERROR, "cannot create xwayland server");
-		exit(EXIT_FAILURE);
+		wlr_log(WLR_ERROR, "failed to create xwayland server, continuing without");
+		unsetenv("DISPLAY");
+		return;
 	}
 	server.xwayland_new_surface.notify = handle_new_surface;
 	wl_signal_add(&server.xwayland->events.new_surface,
@@ -1308,6 +1309,11 @@ void
 xwayland_server_finish(void)
 {
 	struct wlr_xwayland *xwayland = server.xwayland;
+
+	if (!xwayland) {
+		return;
+	}
+
 	wl_list_remove(&server.xwayland_new_surface.link);
 	wl_list_remove(&server.xwayland_server_ready.link);
 	wl_list_remove(&server.xwayland_xwm_ready.link);