diff --git a/src/wayland-server.c b/src/wayland-server.c
index 82ed3260..df9bd07f 100644
--- a/src/wayland-server.c
+++ b/src/wayland-server.c
@@ -750,7 +750,7 @@ wl_seat_set_keyboard(struct wl_seat *seat, struct wl_keyboard *keyboard)
 {
 	if (keyboard && (seat->keyboard || keyboard->seat))
 		return; /* XXX: error? */
-	if (!keyboard && seat->keyboard)
+	if (!keyboard && !seat->keyboard)
 		return;
 
 	seat->keyboard = keyboard;