diff --git a/wayland-system-compositor.c b/wayland-system-compositor.c
index 5e0a477e..4bc6e572 100644
--- a/wayland-system-compositor.c
+++ b/wayland-system-compositor.c
@@ -973,9 +973,8 @@ notify_button(struct wlsc_input_device *device,
 
 	surface = pick_surface(device, &sx, &sy);
 	if (surface) {
-		wlsc_surface_raise(surface);
-
 		if (state) {
+			wlsc_surface_raise(surface);
 			device->grab++;
 			device->grab_surface = surface;
 			wlsc_input_device_set_keyboard_focus(device,