diff --git a/include/sway/scene_descriptor.h b/include/sway/scene_descriptor.h
index de54d133a..e69d3f67b 100644
--- a/include/sway/scene_descriptor.h
+++ b/include/sway/scene_descriptor.h
@@ -4,6 +4,8 @@
 
 enum sway_scene_descriptor_type {
 	SWAY_SCENE_DESC_BUFFER_TIMER,
+	SWAY_SCENE_DESC_NON_INTERACTIVE,
+	SWAY_SCENE_DESC_CONTAINER,
 	SWAY_SCENE_DESC_DRAG_ICON,
 };
 
diff --git a/sway/input/cursor.c b/sway/input/cursor.c
index 8db513e37..b389fbbea 100644
--- a/sway/input/cursor.c
+++ b/sway/input/cursor.c
@@ -37,165 +37,75 @@ static uint32_t get_current_time_msec(void) {
 	return now.tv_sec * 1000 + now.tv_nsec / 1000000;
 }
 
-static struct wlr_surface *layer_surface_at(struct sway_output *output,
-		struct wl_list *layer, double ox, double oy, double *sx, double *sy) {
-	struct sway_layer_surface *sway_layer;
-	wl_list_for_each_reverse(sway_layer, layer, link) {
-		double _sx = ox - sway_layer->geo.x;
-		double _sy = oy - sway_layer->geo.y;
-		struct wlr_surface *sub = wlr_layer_surface_v1_surface_at(
-			sway_layer->layer_surface, _sx, _sy, sx, sy);
-		if (sub) {
-			return sub;
-		}
-	}
-	return NULL;
-}
-
-static bool surface_is_xdg_popup(struct wlr_surface *surface) {
-    if (wlr_surface_is_xdg_surface(surface)) {
-        struct wlr_xdg_surface *xdg_surface =
-            wlr_xdg_surface_from_wlr_surface(surface);
-        return xdg_surface->role == WLR_XDG_SURFACE_ROLE_POPUP;
-    }
-    return false;
-}
-
-static struct wlr_surface *layer_surface_popup_at(struct sway_output *output,
-		struct wl_list *layer, double ox, double oy, double *sx, double *sy) {
-	struct sway_layer_surface *sway_layer;
-	wl_list_for_each_reverse(sway_layer, layer, link) {
-		double _sx = ox - sway_layer->geo.x;
-		double _sy = oy - sway_layer->geo.y;
-		struct wlr_surface *sub = wlr_layer_surface_v1_surface_at(
-			sway_layer->layer_surface, _sx, _sy, sx, sy);
-		if (sub && surface_is_xdg_popup(sub)) {
-			return sub;
-		}
-	}
-	return NULL;
-}
-
 /**
  * Returns the node at the cursor's position. If there is a surface at that
  * location, it is stored in **surface (it may not be a view).
  */
 struct sway_node *node_at_coords(
 		struct sway_seat *seat, double lx, double ly,
-		struct wlr_surface **surface, double *sx, double *sy) {
-	// find the output the cursor is on
+		struct wlr_surface **_surface, double *sx, double *sy) {
+	struct wlr_scene_node *scene_node = NULL;
+	size_t num_layers = sizeof(root->layers) / sizeof(struct wlr_scene_tree *);
+	for (int i = num_layers - 1; i >= 0 && !scene_node; i--) {
+		struct wlr_scene_tree *layer = ((struct wlr_scene_tree **) &root->layers)[i];
+		struct sway_scene_descriptor *desc = layer->node.data;
+
+		if (!desc || desc->type != SWAY_SCENE_DESC_NON_INTERACTIVE) {
+			scene_node = wlr_scene_node_at(&layer->node, lx, ly, sx, sy);
+		}
+	}
+
+	if (scene_node) {
+		// determine what wlr_surface we clicked on
+		if (scene_node->type == WLR_SCENE_NODE_BUFFER) {
+			struct wlr_scene_buffer *scene_buffer =
+				wlr_scene_buffer_from_node(scene_node);
+			struct wlr_scene_surface *scene_surface =
+				wlr_scene_surface_from_buffer(scene_buffer);
+
+			if (scene_surface) {
+				*_surface = scene_surface->surface;
+			}
+		}
+
+		// determine what container we clicked on
+		struct wlr_scene_node *current = scene_node;
+		while (true) {
+			if (current->data) {
+				struct sway_scene_descriptor *desc = current->data;
+
+				if (desc->type == SWAY_SCENE_DESC_CONTAINER) {
+					struct sway_container *con = desc->data;
+					return &con->node;
+				}
+			}
+
+			if (!current->parent) {
+				break;
+			}
+
+			current = &current->parent->node;
+		}
+	}
+
+	// if we aren't on a container, determine what workspace we are on
 	struct wlr_output *wlr_output = wlr_output_layout_output_at(
 			root->output_layout, lx, ly);
 	if (wlr_output == NULL) {
 		return NULL;
 	}
+
 	struct sway_output *output = wlr_output->data;
 	if (!output || !output->enabled) {
 		// output is being destroyed or is being enabled
 		return NULL;
 	}
-	double ox = lx, oy = ly;
-	wlr_output_layout_output_coords(root->output_layout, wlr_output, &ox, &oy);
 
-	// layer surfaces on the overlay layer are rendered on top
-	if ((*surface = layer_surface_at(output,
-				&output->shell_layers[ZWLR_LAYER_SHELL_V1_LAYER_OVERLAY],
-				ox, oy, sx, sy))) {
-		return NULL;
-	}
-
-	// check for unmanaged views
-#if HAVE_XWAYLAND
-	struct wl_list *unmanaged = &root->xwayland_unmanaged;
-	struct sway_xwayland_unmanaged *unmanaged_surface;
-	wl_list_for_each_reverse(unmanaged_surface, unmanaged, link) {
-		struct wlr_xwayland_surface *xsurface =
-			unmanaged_surface->wlr_xwayland_surface;
-
-		double _sx = lx - unmanaged_surface->lx;
-		double _sy = ly - unmanaged_surface->ly;
-		if (wlr_surface_point_accepts_input(xsurface->surface, _sx, _sy)) {
-			*surface = xsurface->surface;
-			*sx = _sx;
-			*sy = _sy;
-			return NULL;
-		}
-	}
-#endif
-
-	if (root->fullscreen_global) {
-		// Try fullscreen container
-		struct sway_container *con = tiling_container_at(
-				&root->fullscreen_global->node, lx, ly, surface, sx, sy);
-		if (con) {
-			return &con->node;
-		}
-		return NULL;
-	}
-
-	// find the focused workspace on the output for this seat
 	struct sway_workspace *ws = output_get_active_workspace(output);
 	if (!ws) {
 		return NULL;
 	}
 
-	if (ws->fullscreen) {
-		// Try transient containers
-		for (int i = 0; i < ws->floating->length; ++i) {
-			struct sway_container *floater = ws->floating->items[i];
-			if (container_is_transient_for(floater, ws->fullscreen)) {
-				struct sway_container *con = tiling_container_at(
-						&floater->node, lx, ly, surface, sx, sy);
-				if (con) {
-					return &con->node;
-				}
-			}
-		}
-		// Try fullscreen container
-		struct sway_container *con =
-			tiling_container_at(&ws->fullscreen->node, lx, ly, surface, sx, sy);
-		if (con) {
-			return &con->node;
-		}
-		return NULL;
-	}
-	if ((*surface = layer_surface_popup_at(output,
-				&output->shell_layers[ZWLR_LAYER_SHELL_V1_LAYER_TOP],
-				ox, oy, sx, sy))) {
-		return NULL;
-	}
-	if ((*surface = layer_surface_popup_at(output,
-				&output->shell_layers[ZWLR_LAYER_SHELL_V1_LAYER_BOTTOM],
-				ox, oy, sx, sy))) {
-		return NULL;
-	}
-	if ((*surface = layer_surface_popup_at(output,
-				&output->shell_layers[ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND],
-				ox, oy, sx, sy))) {
-		return NULL;
-	}
-	if ((*surface = layer_surface_at(output,
-				&output->shell_layers[ZWLR_LAYER_SHELL_V1_LAYER_TOP],
-				ox, oy, sx, sy))) {
-		return NULL;
-	}
-
-	struct sway_container *c;
-	if ((c = container_at(ws, lx, ly, surface, sx, sy))) {
-		return &c->node;
-	}
-
-	if ((*surface = layer_surface_at(output,
-				&output->shell_layers[ZWLR_LAYER_SHELL_V1_LAYER_BOTTOM],
-				ox, oy, sx, sy))) {
-		return NULL;
-	}
-	if ((*surface = layer_surface_at(output,
-				&output->shell_layers[ZWLR_LAYER_SHELL_V1_LAYER_BACKGROUND],
-				ox, oy, sx, sy))) {
-		return NULL;
-	}
-
 	return &ws->node;
 }
 
diff --git a/sway/tree/container.c b/sway/tree/container.c
index d75b2d16d..65a5c9b38 100644
--- a/sway/tree/container.c
+++ b/sway/tree/container.c
@@ -20,6 +20,7 @@
 #include "sway/input/input-manager.h"
 #include "sway/input/seat.h"
 #include "sway/ipc-server.h"
+#include "sway/scene_descriptor.h"
 #include "sway/output.h"
 #include "sway/server.h"
 #include "sway/tree/arrange.h"
@@ -88,6 +89,21 @@ struct sway_container *container_create(struct sway_view *view) {
 		c->current.children = create_list();
 	}
 
+	scene_descriptor_assign(c->scene_node, SWAY_SCENE_DESC_CONTAINER, c);
+	if (!c->scene_node->data) {
+		alloc_failure = true;
+	}
+
+	// also assign the scene descriptor to the view. On fullscreen clients
+	// the container will not be part of the scene graph, only the view itself
+	// So to not confuse input logic, we need to assign this.
+	if (view) {
+		scene_descriptor_assign(view->scene_node, SWAY_SCENE_DESC_CONTAINER, c);
+		if (!view->scene_node->data) {
+			alloc_failure = true;
+		}
+	}
+
 	if (alloc_failure) {
 		wlr_scene_node_destroy(&c->scene_tree->node);
 		free(c);
diff --git a/sway/tree/root.c b/sway/tree/root.c
index faa6a478e..666d5103c 100644
--- a/sway/tree/root.c
+++ b/sway/tree/root.c
@@ -8,6 +8,7 @@
 #include "sway/input/seat.h"
 #include "sway/ipc-server.h"
 #include "sway/output.h"
+#include "sway/scene_descriptor.h"
 #include "sway/tree/arrange.h"
 #include "sway/tree/container.h"
 #include "sway/tree/root.h"
@@ -50,6 +51,12 @@ struct sway_root *root_create(void) {
 			alloc_scene_tree(&root_scene->tree, &alloc_failure);
 	}
 
+	scene_descriptor_assign(&root->layers.seat->node,
+		SWAY_SCENE_DESC_NON_INTERACTIVE, NULL);
+	if (!root->layers.seat->node.data) {
+		alloc_failure = true;
+	}
+
 	if (alloc_failure) {
 		wlr_scene_node_destroy(&root_scene->tree.node);
 		free(root);