diff --git a/src/pipewire/context.c b/src/pipewire/context.c
index 6045d0463..200c7f6b6 100644
--- a/src/pipewire/context.c
+++ b/src/pipewire/context.c
@@ -1198,7 +1198,11 @@ again:
 		/* collect quantum and rate */
 		spa_list_for_each(s, &n->follower_list, follower_link) {
 
-			if (s->info.state > PW_NODE_STATE_SUSPENDED) {
+			if (!s->moved) {
+				/* We only try to enforce the lock flags for nodes that
+				 * are not recently moved between drivers. The nodes that
+				 * are moved should try to enforce their quantum on the
+				 * new driver. */
 				lock_quantum |= s->lock_quantum;
 				lock_rate |= s->lock_rate;
 			}
@@ -1236,6 +1240,8 @@ again:
 
 			if (s->active)
 				running = !n->passive;
+
+			s->moved = false;
 		}
 
 		if (force_quantum)
diff --git a/src/pipewire/impl-node.c b/src/pipewire/impl-node.c
index 2dbd05d3e..93bb8e832 100644
--- a/src/pipewire/impl-node.c
+++ b/src/pipewire/impl-node.c
@@ -842,6 +842,7 @@ int pw_impl_node_set_driver(struct pw_impl_node *node, struct pw_impl_node *driv
 			old->name, old->info.id, driver->name, driver->info.id);
 
 	node->driver_node = driver;
+	node->moved = true;
 
 	pw_loop_invoke(node->data_loop,
 		       do_move_nodes, SPA_ID_INVALID, &driver, sizeof(struct pw_impl_node *),
diff --git a/src/pipewire/private.h b/src/pipewire/private.h
index 6e6662a98..5253d5eb9 100644
--- a/src/pipewire/private.h
+++ b/src/pipewire/private.h
@@ -700,6 +700,7 @@ struct pw_impl_node {
 	unsigned int lock_rate:1;	/**< don't change graph rate */
 	unsigned int transport_sync:1;	/**< supports transport sync */
 	unsigned int current_pending:1;	/**< a quantum/rate update is pending */
+	unsigned int moved:1;		/**< the node was moved drivers */
 
 	uint32_t port_user_data_size;	/**< extra size for port user data */