diff --git a/types/output/cursor.c b/types/output/cursor.c
index b3ec152ce..70647afb7 100644
--- a/types/output/cursor.c
+++ b/types/output/cursor.c
@@ -288,13 +288,7 @@ static struct wlr_buffer *render_cursor_buffer(struct wlr_output_cursor *cursor)
 static bool output_cursor_attempt_hardware(struct wlr_output_cursor *cursor) {
 	struct wlr_output *output = cursor->output;
 
-	if (!output->impl->set_cursor ||
-			output->software_cursor_locks > 0) {
-		return false;
-	}
-
-	struct wlr_output_cursor *hwcur = output->hardware_cursor;
-	if (hwcur != NULL && hwcur != cursor) {
+	if (!output->impl->set_cursor || output->software_cursor_locks > 0) {
 		return false;
 	}
 
@@ -422,12 +416,15 @@ bool output_cursor_set_texture(struct wlr_output_cursor *cursor,
 		wl_list_init(&cursor->renderer_destroy.link);
 	}
 
-	if (output_cursor_attempt_hardware(cursor)) {
-		return true;
+	if (output->hardware_cursor == NULL || output->hardware_cursor == cursor) {
+		if (output_cursor_attempt_hardware(cursor)) {
+			return true;
+		}
+
+		wlr_log(WLR_DEBUG, "Falling back to software cursor on output '%s'", output->name);
+		output_disable_hardware_cursor(output);
 	}
 
-	wlr_log(WLR_DEBUG, "Falling back to software cursor on output '%s'", output->name);
-	output_disable_hardware_cursor(output);
 	output_cursor_damage_whole(cursor);
 	return true;
 }