diff --git a/backend/drm/drm.c b/backend/drm/drm.c
index f34c9a085..1d85e2d2e 100644
--- a/backend/drm/drm.c
+++ b/backend/drm/drm.c
@@ -419,7 +419,6 @@ static void layer_handle_addon_destroy(struct wlr_addon *addon) {
 #if HAVE_LIBLIFTOFF
 	liftoff_layer_destroy(layer->liftoff);
 #endif
-	drm_fb_clear(&layer->pending_fb);
 	drm_fb_clear(&layer->queued_fb);
 	drm_fb_clear(&layer->current_fb);
 	free(layer->candidate_planes);
@@ -554,6 +553,7 @@ static void drm_connector_set_pending_page_flip(struct wlr_drm_connector *conn,
 static void drm_connector_apply_commit(const struct wlr_drm_connector_state *state,
 		struct wlr_drm_page_flip *page_flip) {
 	struct wlr_drm_connector *conn = state->connector;
+	struct wlr_drm_backend *drm = conn->backend;
 	struct wlr_drm_crtc *crtc = conn->crtc;
 
 	drm_fb_copy(&crtc->primary->queued_fb, state->primary_fb);
@@ -563,9 +563,13 @@ static void drm_connector_apply_commit(const struct wlr_drm_connector_state *sta
 	}
 	drm_fb_clear(&conn->cursor_pending_fb);
 
-	struct wlr_drm_layer *layer;
-	wl_list_for_each(layer, &crtc->layers, link) {
-		drm_fb_move(&layer->queued_fb, &layer->pending_fb);
+	if (crtc->liftoff) {
+		for (size_t i = 0; i < state->base->layers_len; i++) {
+			struct wlr_output_layer_state *layer_state = &state->base->layers[i];
+			struct wlr_drm_layer *layer = get_or_create_layer(drm, crtc, layer_state->layer);
+			assert(layer != NULL);
+			drm_fb_copy(&layer->queued_fb, state->layer_fbs[i]);
+		}
 	}
 
 	drm_connector_set_pending_page_flip(conn, page_flip);
@@ -584,22 +588,6 @@ static void drm_connector_apply_commit(const struct wlr_drm_connector_state *sta
 	}
 }
 
-static void drm_connector_rollback_commit(const struct wlr_drm_connector_state *state) {
-	struct wlr_drm_crtc *crtc = state->connector->crtc;
-
-	// The set_cursor() hook is a bit special: it's not really synchronized
-	// to commit() or test(). Once set_cursor() returns true, the new
-	// cursor is effectively committed. So don't roll it back here, or we
-	// risk ending up in a state where we don't have a cursor FB but
-	// wlr_drm_connector.cursor_enabled is true.
-	// TODO: fix our output interface to avoid this issue.
-
-	struct wlr_drm_layer *layer;
-	wl_list_for_each(layer, &crtc->layers, link) {
-		drm_fb_clear(&layer->pending_fb);
-	}
-}
-
 static bool drm_commit(struct wlr_drm_backend *drm,
 		const struct wlr_drm_device_state *state,
 		uint32_t flags, bool test_only) {
@@ -621,15 +609,18 @@ static bool drm_commit(struct wlr_drm_backend *drm,
 			drm_connector_apply_commit(&state->connectors[i], page_flip);
 		}
 	} else {
-		for (size_t i = 0; i < state->connectors_len; i++) {
-			drm_connector_rollback_commit(&state->connectors[i]);
-		}
+		// The set_cursor() hook is a bit special: it's not really synchronized
+		// to commit() or test(). Once set_cursor() returns true, the new
+		// cursor is effectively committed. So don't roll it back here, or we
+		// risk ending up in a state where we don't have a cursor FB but
+		// wlr_drm_connector.cursor_enabled is true.
+		// TODO: fix our output interface to avoid this issue.
 		drm_page_flip_destroy(page_flip);
 	}
 	return ok;
 }
 
-static void drm_connector_state_init(struct wlr_drm_connector_state *state,
+static bool drm_connector_state_init(struct wlr_drm_connector_state *state,
 		struct wlr_drm_connector *conn,
 		const struct wlr_output_state *base) {
 	*state = (struct wlr_drm_connector_state){
@@ -667,6 +658,13 @@ static void drm_connector_state_init(struct wlr_drm_connector_state *state,
 		state->mode.type = DRM_MODE_TYPE_USERDEF;
 	}
 
+	if (base->layers_len > 0) {
+		state->layer_fbs = calloc(base->layers_len, sizeof(state->layer_fbs[0]));
+		if (state->layer_fbs == NULL) {
+			return false;
+		}
+	}
+
 	if (output_pending_enabled(&conn->output, base)) {
 		// The CRTC must be set up before this function is called
 		assert(conn->crtc != NULL);
@@ -691,13 +689,32 @@ static void drm_connector_state_init(struct wlr_drm_connector_state *state,
 				state->cursor_fb = drm_fb_lock(cursor->current_fb);
 			}
 		}
+
+		if (!conn->backend->parent && conn->crtc->liftoff) {
+			for (size_t i = 0; i < base->layers_len; i++) {
+				struct wlr_output_layer_state *layer_state = &base->layers[i];
+				if (layer_state->buffer != NULL) {
+					drm_fb_import(&state->layer_fbs[i], conn->backend, layer_state->buffer, NULL);
+				}
+
+				if (get_or_create_layer(conn->backend, conn->crtc, layer_state->layer) == NULL) {
+					return false;
+				}
+			}
+		}
 	}
+
+	return true;
 }
 
 static void drm_connector_state_finish(struct wlr_drm_connector_state *state) {
 	drm_fb_clear(&state->primary_fb);
 	drm_fb_clear(&state->cursor_fb);
 	wlr_drm_syncobj_timeline_unref(state->wait_timeline);
+	for (size_t i = 0; i < state->base->layers_len; i++) {
+		drm_fb_clear(&state->layer_fbs[i]);
+	}
+	free(state->layer_fbs);
 }
 
 static bool drm_connector_state_update_primary_fb(struct wlr_drm_connector *conn,
@@ -770,39 +787,6 @@ static bool drm_connector_state_update_primary_fb(struct wlr_drm_connector *conn
 	return true;
 }
 
-static bool drm_connector_set_pending_layer_fbs(struct wlr_drm_connector *conn,
-		const struct wlr_output_state *state) {
-	struct wlr_drm_backend *drm = conn->backend;
-
-	struct wlr_drm_crtc *crtc = conn->crtc;
-	if (!crtc || drm->parent) {
-		return false;
-	}
-
-	if (!crtc->liftoff) {
-		return true; // libliftoff is disabled
-	}
-
-	assert(state->committed & WLR_OUTPUT_STATE_LAYERS);
-
-	for (size_t i = 0; i < state->layers_len; i++) {
-		struct wlr_output_layer_state *layer_state = &state->layers[i];
-		struct wlr_drm_layer *layer =
-			get_or_create_layer(drm, crtc, layer_state->layer);
-		if (!layer) {
-			return false;
-		}
-
-		if (layer_state->buffer != NULL) {
-			drm_fb_import(&layer->pending_fb, drm, layer_state->buffer, NULL);
-		} else {
-			drm_fb_clear(&layer->pending_fb);
-		}
-	}
-
-	return true;
-}
-
 static bool drm_connector_alloc_crtc(struct wlr_drm_connector *conn);
 
 static bool drm_connector_prepare(struct wlr_drm_connector_state *conn_state, bool test_only) {
@@ -850,11 +834,6 @@ static bool drm_connector_prepare(struct wlr_drm_connector_state *conn_state, bo
 			return false;
 		}
 	}
-	if (state->committed & WLR_OUTPUT_STATE_LAYERS) {
-		if (!drm_connector_set_pending_layer_fbs(conn, conn_state->base)) {
-			return false;
-		}
-	}
 
 	if (conn_state->active && !conn_state->primary_fb) {
 		wlr_drm_conn_log(conn, WLR_DEBUG,
@@ -886,7 +865,9 @@ static bool drm_connector_commit_state(struct wlr_drm_connector *conn,
 
 	bool ok = false;
 	struct wlr_drm_connector_state pending = {0};
-	drm_connector_state_init(&pending, conn, state);
+	if (!drm_connector_state_init(&pending, conn, state)) {
+		return false;
+	}
 	struct wlr_drm_device_state pending_dev = {
 		.modeset = state->allow_reconfiguration,
 		// The wlr_output API requires non-modeset commits with a new buffer to
@@ -1884,7 +1865,9 @@ bool commit_drm_device(struct wlr_drm_backend *drm,
 		}
 
 		struct wlr_drm_connector_state *conn_state = &conn_states[conn_states_len];
-		drm_connector_state_init(conn_state, conn, &output_state->base);
+		if (!drm_connector_state_init(conn_state, conn, &output_state->base)) {
+			goto out;
+		}
 		conn_states_len++;
 
 		if (!drm_connector_prepare(conn_state, test_only)) {
diff --git a/backend/drm/libliftoff.c b/backend/drm/libliftoff.c
index 67b1ca527..adc658deb 100644
--- a/backend/drm/libliftoff.c
+++ b/backend/drm/libliftoff.c
@@ -180,8 +180,8 @@ static uint64_t to_fp16(double v) {
 }
 
 static bool set_layer_props(struct wlr_drm_backend *drm,
-		const struct wlr_output_layer_state *state, uint64_t zpos,
-		struct wl_array *fb_damage_clips_arr) {
+		const struct wlr_output_layer_state *state, struct wlr_drm_fb *fb,
+		uint64_t zpos, struct wl_array *fb_damage_clips_arr) {
 	struct wlr_drm_layer *layer = get_drm_layer(drm, state->layer);
 
 	uint32_t width = 0, height = 0;
@@ -190,7 +190,6 @@ static bool set_layer_props(struct wlr_drm_backend *drm,
 		height = state->buffer->height;
 	}
 
-	struct wlr_drm_fb *fb = layer->pending_fb;
 	int ret = 0;
 	if (state->buffer == NULL) {
 		ret = liftoff_layer_set_property(layer->liftoff, "FB_ID", 0);
@@ -358,12 +357,10 @@ static bool add_connector(drmModeAtomicReq *req,
 				(uintptr_t)&state->out_fence_fd);
 		}
 
-		if (state->base->committed & WLR_OUTPUT_STATE_LAYERS) {
-			for (size_t i = 0; i < state->base->layers_len; i++) {
-				const struct wlr_output_layer_state *layer_state = &state->base->layers[i];
-				ok = ok && set_layer_props(drm, layer_state, i + 1,
-					fb_damage_clips_arr);
-			}
+		for (size_t i = 0; i < state->base->layers_len; i++) {
+			const struct wlr_output_layer_state *layer_state = &state->base->layers[i];
+			ok = ok && set_layer_props(drm, layer_state,
+				state->layer_fbs[i], i + 1, fb_damage_clips_arr);
 		}
 
 		if (crtc->cursor) {
diff --git a/include/backend/drm/drm.h b/include/backend/drm/drm.h
index 9409b7b10..8f0132da4 100644
--- a/include/backend/drm/drm.h
+++ b/include/backend/drm/drm.h
@@ -52,8 +52,6 @@ struct wlr_drm_layer {
 	struct wlr_addon addon; // wlr_output_layer.addons
 	struct wl_list link; // wlr_drm_crtc.layers
 
-	/* Buffer to be submitted to the kernel on the next page-flip */
-	struct wlr_drm_fb *pending_fb;
 	/* Buffer submitted to the kernel, will be presented on next vblank */
 	struct wlr_drm_fb *queued_fb;
 	/* Buffer currently displayed on screen */
@@ -148,6 +146,7 @@ struct wlr_drm_connector_state {
 	struct wlr_drm_fb *primary_fb;
 	struct wlr_drm_viewport primary_viewport;
 	struct wlr_drm_fb *cursor_fb;
+	struct wlr_drm_fb **layer_fbs; // same length as base.layers_len
 
 	struct wlr_drm_syncobj_timeline *wait_timeline;
 	uint64_t wait_point;