diff --git a/include/labwc.h b/include/labwc.h
index 8dae3cf9..e422d4b8 100644
--- a/include/labwc.h
+++ b/include/labwc.h
@@ -103,19 +103,22 @@ struct seat {
 
 	/**
 	 * pressed view/surface/node will usually be NULL and is only set on
-	 * button press while the mouse is over a surface and reset to NULL on
-	 * button release.
+	 * button press while the mouse is over a view or surface, and reset
+	 * to NULL on button release.
 	 * It is used to send cursor motion events to a surface even though
 	 * the cursor has left the surface in the meantime.
 	 *
 	 * This allows to keep dragging a scrollbar or selecting text even
 	 * when moving outside of the window.
+	 *
+	 * Both (view && !surface) and (surface && !view) are possible.
 	 */
 	struct {
 		struct view *view;
 		struct wlr_scene_node *node;
 		struct wlr_surface *surface;
 		struct wlr_surface *toplevel;
+		uint32_t resize_edges;
 	} pressed;
 
 	struct wl_client *active_client_while_inhibited;
@@ -569,7 +572,7 @@ void seat_focus_surface(struct seat *seat, struct wlr_surface *surface);
 void seat_set_focus_layer(struct seat *seat, struct wlr_layer_surface_v1 *layer);
 void seat_set_pressed(struct seat *seat, struct view *view,
 	struct wlr_scene_node *node, struct wlr_surface *surface,
-	struct wlr_surface *toplevel);
+	struct wlr_surface *toplevel, uint32_t resize_edges);
 void seat_reset_pressed(struct seat *seat);
 
 void interactive_begin(struct view *view, enum input_mode mode,
diff --git a/src/action.c b/src/action.c
index d6782c02..9eb81cf7 100644
--- a/src/action.c
+++ b/src/action.c
@@ -166,9 +166,23 @@ show_menu(struct server *server, struct view *view, const char *menu_name)
 }
 
 static struct view *
-activator_or_focused_view(struct view *activator, struct server *server)
+view_for_action(struct view *activator, struct server *server,
+	struct action *action)
 {
-	return activator ? activator : desktop_focused_view(server);
+	/* View is explicitly specified for mousebinds */
+	if (activator) {
+		return activator;
+	}
+
+	/* Select view based on action type for keybinds */
+	switch (action->type) {
+	case ACTION_TYPE_FOCUS:
+	case ACTION_TYPE_MOVE:
+	case ACTION_TYPE_RESIZE:
+		return get_cursor_context(server).view;
+	default:
+		return desktop_focused_view(server);
+	}
 }
 
 void
@@ -194,7 +208,7 @@ actions_run(struct view *activator, struct server *server,
 		 * Refetch view because it may have been changed due to the
 		 * previous action
 		 */
-		view = activator_or_focused_view(activator, server);
+		view = view_for_action(activator, server, action);
 
 		switch (action->type) {
 		case ACTION_TYPE_CLOSE:
@@ -275,7 +289,6 @@ actions_run(struct view *activator, struct server *server,
 			}
 			break;
 		case ACTION_TYPE_FOCUS:
-			view = get_cursor_context(server).view;
 			if (view) {
 				desktop_focus_and_activate_view(&server->seat, view);
 			}
@@ -286,7 +299,6 @@ actions_run(struct view *activator, struct server *server,
 			}
 			break;
 		case ACTION_TYPE_MOVE:
-			view = get_cursor_context(server).view;
 			if (view) {
 				interactive_begin(view, LAB_INPUT_STATE_MOVE, 0);
 			}
@@ -297,7 +309,6 @@ actions_run(struct view *activator, struct server *server,
 			}
 			break;
 		case ACTION_TYPE_RESIZE:
-			view = get_cursor_context(server).view;
 			if (view) {
 				interactive_begin(view, LAB_INPUT_STATE_RESIZE,
 					resize_edges);
diff --git a/src/cursor.c b/src/cursor.c
index c58bd52f..990b312b 100644
--- a/src/cursor.c
+++ b/src/cursor.c
@@ -255,8 +255,10 @@ update_pressed_surface(struct seat *seat, struct cursor_context *ctx)
 	if (seat->pressed.surface && ctx->surface != seat->pressed.surface) {
 		struct wlr_surface *toplevel = get_toplevel(ctx->surface);
 		if (toplevel && toplevel == seat->pressed.toplevel) {
-			seat_set_pressed(seat, ctx->view, ctx->node,
-				ctx->surface, toplevel);
+			/* No need to recompute resize edges here */
+			seat_set_pressed(seat, ctx->view,
+				ctx->node, ctx->surface, toplevel,
+				seat->pressed.resize_edges);
 			return true;
 		}
 	}
@@ -424,12 +426,15 @@ process_cursor_motion(struct server *server, uint32_t time)
 	wl_list_for_each(mousebind, &rc.mousebinds, link) {
 		if (mousebind->mouse_event == MOUSE_ACTION_DRAG
 				&& mousebind->pressed_in_context) {
-			/* Find closest resize edges in case action is Resize */
-			uint32_t resize_edges =
-				determine_resize_edges(seat->cursor, &ctx);
-
+			/*
+			 * Use view and resize edges from the press
+			 * event (not the motion event) to prevent
+			 * moving/resizing the wrong view
+			 */
 			mousebind->pressed_in_context = false;
-			actions_run(NULL, server, &mousebind->actions, resize_edges);
+			actions_run(seat->pressed.view,
+				server, &mousebind->actions,
+				seat->pressed.resize_edges);
 		}
 	}
 
@@ -825,9 +830,13 @@ cursor_button(struct wl_listener *listener, void *data)
 	}
 
 	/* Handle _press */
-	if (ctx.surface) {
+	/* Determine closest resize edges in case action is Resize */
+	resize_edges = determine_resize_edges(seat->cursor, &ctx);
+
+	if (ctx.view || ctx.surface) {
+		/* Store resize edges for later action processing */
 		seat_set_pressed(seat, ctx.view, ctx.node, ctx.surface,
-			get_toplevel(ctx.surface));
+			get_toplevel(ctx.surface), resize_edges);
 	}
 
 	if (server->input_mode == LAB_INPUT_STATE_MENU) {
@@ -850,9 +859,6 @@ cursor_button(struct wl_listener *listener, void *data)
 		}
 	}
 
-	/* Determine closest resize edges in case action is Resize */
-	resize_edges = determine_resize_edges(seat->cursor, &ctx);
-
 mousebindings:
 	if (event->state == WLR_BUTTON_RELEASED) {
 		triggered_frame_binding |= handle_release_mousebinding(ctx.view,
diff --git a/src/seat.c b/src/seat.c
index 16aaa256..70a240e8 100644
--- a/src/seat.c
+++ b/src/seat.c
@@ -384,27 +384,34 @@ pressed_surface_destroy(struct wl_listener *listener, void *data)
 void
 seat_set_pressed(struct seat *seat, struct view *view,
 	struct wlr_scene_node *node, struct wlr_surface *surface,
-	struct wlr_surface *toplevel)
+	struct wlr_surface *toplevel, uint32_t resize_edges)
 {
-	assert(surface);
+	assert(view || surface);
 	seat_reset_pressed(seat);
 
 	seat->pressed.view = view;
 	seat->pressed.node = node;
 	seat->pressed.surface = surface;
 	seat->pressed.toplevel = toplevel;
-	seat->pressed_surface_destroy.notify = pressed_surface_destroy;
-	wl_signal_add(&surface->events.destroy, &seat->pressed_surface_destroy);
+	seat->pressed.resize_edges = resize_edges;
+
+	if (surface) {
+		seat->pressed_surface_destroy.notify = pressed_surface_destroy;
+		wl_signal_add(&surface->events.destroy,
+			&seat->pressed_surface_destroy);
+	}
 }
 
 void
 seat_reset_pressed(struct seat *seat)
 {
 	if (seat->pressed.surface) {
-		seat->pressed.view = NULL;
-		seat->pressed.node = NULL;
-		seat->pressed.surface = NULL;
-		seat->pressed.toplevel = NULL;
 		wl_list_remove(&seat->pressed_surface_destroy.link);
 	}
+
+	seat->pressed.view = NULL;
+	seat->pressed.node = NULL;
+	seat->pressed.surface = NULL;
+	seat->pressed.toplevel = NULL;
+	seat->pressed.resize_edges = 0;
 }
diff --git a/src/view.c b/src/view.c
index 2976a087..615b91d7 100644
--- a/src/view.c
+++ b/src/view.c
@@ -820,6 +820,10 @@ view_destroy(struct view *view)
 		need_cursor_update = true;
 	}
 
+	if (server->seat.pressed.view == view) {
+		seat_reset_pressed(&server->seat);
+	}
+
 	osd_on_view_destroy(view);
 
 	if (view->scene_tree) {