diff --git a/src/cursor.c b/src/cursor.c
index b576b5c5..f2c923a4 100644
--- a/src/cursor.c
+++ b/src/cursor.c
@@ -592,8 +592,12 @@ cursor_button(struct wl_listener *listener, void *data)
 
 	/* Determine closest resize edges in case action is Resize */
 	resize_edges = ssd_resize_edges(view_area);
-	resize_edges |= server->seat.cursor->x < view->x + view->w / 2 ? WLR_EDGE_LEFT : WLR_EDGE_RIGHT;
-	resize_edges |= server->seat.cursor->y < view->y + view->h / 2 ? WLR_EDGE_TOP : WLR_EDGE_BOTTOM;
+	if (!resize_edges) {
+		resize_edges |= server->seat.cursor->x < view->x + view->w / 2
+			? WLR_EDGE_LEFT : WLR_EDGE_RIGHT;
+		resize_edges |= server->seat.cursor->y < view->y + view->h / 2
+			? WLR_EDGE_TOP : WLR_EDGE_BOTTOM;
+	}
 
 mousebindings:
 	if (event->state == WLR_BUTTON_RELEASED) {