diff --git a/src/pipewire/context.c b/src/pipewire/context.c
index 55f3eba5a..445485190 100644
--- a/src/pipewire/context.c
+++ b/src/pipewire/context.c
@@ -1191,24 +1191,23 @@ again:
 					context, n, n->name, n->active, n->want_driver, target);
 
 			t = n->want_driver && n->active ? target : NULL;
-			if (t != NULL && !n->always_process) {
-				/* first do a check without moving the node to
-				 * the target driver to see if we are passive */
-				collect_nodes(context, n);
-				if (n->passive)
-					t = NULL;
-			}
-			pw_impl_node_set_driver(n, t);
-			if (t == NULL) {
-				/* no driver, make sure the node stops */
-				ensure_state(n, false);
-			}
-			else {
+			if (t != NULL) {
+				pw_impl_node_set_driver(n, t);
 				/* we configured a driver, move all linked
 				 * nodes to it as well */
 				if (n->always_process)
 					t->passive = false;
 				collect_nodes(context, n);
+				if (!n->always_process && n->passive)
+					/* nothing active and node doesn't need to be
+					 * scheduled, remove from driver again and
+					 * stop */
+					t = NULL;
+			}
+			if (t == NULL) {
+				/* no driver, make sure the node stops */
+				pw_impl_node_set_driver(n, NULL);
+				ensure_state(n, false);
 			}
 		}
 		n->visited = false;