diff --git a/src/pipewire/context.c b/src/pipewire/context.c
index 2231686dd..2d6f0ebc5 100644
--- a/src/pipewire/context.c
+++ b/src/pipewire/context.c
@@ -58,7 +58,7 @@ struct impl {
 	struct pw_context this;
 	struct spa_handle *dbus_handle;
 	struct spa_plugin_loader plugin_loader;
-	unsigned int recalc:1;
+	int recalc;
 	unsigned int recalc_pending:1;
 
 	uint32_t cpu_count;
@@ -1445,13 +1445,13 @@ int pw_context_recalc_graph(struct pw_context *context, const char *reason)
 
 	pw_log_info("%p: busy:%d reason:%s", context, impl->recalc, reason);
 
-	if (impl->recalc) {
+	if (impl->recalc > 0) {
 		impl->recalc_pending = true;
 		return -EBUSY;
 	}
 
 again:
-	impl->recalc = true;
+	impl->recalc++;
 	freewheel = false;
 
 	/* clean up the flags first */
@@ -1832,8 +1832,7 @@ again:
 		/* now that all the followers are ready, start the driver */
 		ensure_state(n, running);
 	}
-	impl->recalc = false;
-	if (impl->recalc_pending) {
+	if (--impl->recalc == 0 && impl->recalc_pending) {
 		impl->recalc_pending = false;
 		goto again;
 	}
@@ -1841,6 +1840,23 @@ again:
 	return 0;
 }
 
+int pw_context_freeze_recalc_graph(struct pw_context *context)
+{
+	struct impl *impl = SPA_CONTAINER_OF(context, struct impl, this);
+	impl->recalc++;
+	return 0;
+}
+
+int pw_context_thaw_recalc_graph(struct pw_context *context, const char *reason)
+{
+	struct impl *impl = SPA_CONTAINER_OF(context, struct impl, this);
+	if (--impl->recalc == 0 && impl->recalc_pending) {
+		impl->recalc_pending = false;
+		pw_context_recalc_graph(context, reason);
+	}
+	return 0;
+}
+
 SPA_EXPORT
 int pw_context_add_spa_lib(struct pw_context *context,
 		const char *factory_regexp, const char *lib)
diff --git a/src/pipewire/impl-node.c b/src/pipewire/impl-node.c
index f209385ab..2fae3af0c 100644
--- a/src/pipewire/impl-node.c
+++ b/src/pipewire/impl-node.c
@@ -286,6 +286,8 @@ static void node_deactivate(struct pw_impl_node *this)
 	/* make sure the node doesn't get woken up while not active */
 	remove_node_from_graph(this);
 
+	pw_context_freeze_recalc_graph(this->context);
+
 	spa_list_for_each(port, &this->input_ports, link) {
 		spa_list_for_each(link, &port->links, input_link)
 			pw_impl_link_deactivate(link);
@@ -294,6 +296,7 @@ static void node_deactivate(struct pw_impl_node *this)
 		spa_list_for_each(link, &port->links, output_link)
 			pw_impl_link_deactivate(link);
 	}
+	pw_context_thaw_recalc_graph(this->context, "node deactivate");
 }
 
 static int idle_node(struct pw_impl_node *this)
@@ -327,6 +330,8 @@ static void node_activate(struct pw_impl_node *this)
 	struct pw_impl_port *port;
 
 	pw_log_debug("%p: activate", this);
+	pw_context_freeze_recalc_graph(this->context);
+
 	spa_list_for_each(port, &this->output_ports, link) {
 		struct pw_impl_link *link;
 		spa_list_for_each(link, &port->links, output_link)
@@ -337,6 +342,7 @@ static void node_activate(struct pw_impl_node *this)
 		spa_list_for_each(link, &port->links, input_link)
 			pw_impl_link_activate(link);
 	}
+	pw_context_thaw_recalc_graph(this->context, "node activate");
 }
 
 static int start_node(struct pw_impl_node *this)
@@ -537,6 +543,8 @@ static int suspend_node(struct pw_impl_node *this)
 	if (this->info.state > 0 && this->info.state <= PW_NODE_STATE_SUSPENDED)
 		return 0;
 
+	pw_context_freeze_recalc_graph(this->context);
+
 	node_deactivate(this);
 
 	pw_log_debug("%p: suspend node driving:%d driver:%d prepared:%d", this,
@@ -555,6 +563,8 @@ static int suspend_node(struct pw_impl_node *this)
 	spa_list_for_each(p, &this->output_ports, link)
 		pw_impl_port_suspend(p);
 
+	pw_context_thaw_recalc_graph(this->context, "node suspend");
+
 	node_update_state(this, PW_NODE_STATE_SUSPENDED, 0, NULL);
 
 	return res;
@@ -2390,15 +2400,15 @@ void pw_impl_node_destroy(struct pw_impl_node *node)
 	struct pw_impl_port *port;
 	struct pw_impl_node *follower;
 	struct pw_context *context = node->context;
-	bool active, had_driver;
 
-	active = node->active;
 	node->active = false;
 	node->runnable = false;
 
 	pw_log_debug("%p: destroy", impl);
 	pw_log_info("(%s-%u) destroy", node->name, node->info.id);
 
+	pw_context_freeze_recalc_graph(context);
+
 	node_deactivate(node);
 
 	suspend_node(node);
@@ -2406,7 +2416,6 @@ void pw_impl_node_destroy(struct pw_impl_node *node)
 	pw_impl_node_emit_destroy(node);
 
 	pw_log_debug("%p: driver node %p", impl, node->driver_node);
-	had_driver = node != node->driver_node;
 
 	/* remove ourself as a follower from the driver node */
 	spa_list_remove(&node->follower_link);
@@ -2440,10 +2449,7 @@ void pw_impl_node_destroy(struct pw_impl_node *node)
 		spa_hook_remove(&node->global_listener);
 		pw_global_destroy(node->global);
 	}
-
-	if (active || had_driver)
-		pw_context_recalc_graph(context,
-				"active node destroy");
+	pw_context_thaw_recalc_graph(context, "node destroy");
 
 	pw_log_debug("%p: free", node);
 	pw_impl_node_emit_free(node);
diff --git a/src/pipewire/impl-port.c b/src/pipewire/impl-port.c
index 3b33f5600..84250cfea 100644
--- a/src/pipewire/impl-port.c
+++ b/src/pipewire/impl-port.c
@@ -1507,7 +1507,10 @@ static int do_destroy_link(void *data, struct pw_impl_link *link)
 
 void pw_impl_port_unlink(struct pw_impl_port *port)
 {
+	struct pw_context *context = port->node->context;
+	pw_context_freeze_recalc_graph(context);
 	pw_impl_port_for_each_link(port, do_destroy_link, port);
+	pw_context_thaw_recalc_graph(context, "port unlink");
 }
 
 static int do_remove_port(struct spa_loop *loop,
diff --git a/src/pipewire/private.h b/src/pipewire/private.h
index a29c1c3db..18e36b73c 100644
--- a/src/pipewire/private.h
+++ b/src/pipewire/private.h
@@ -1274,6 +1274,8 @@ int pw_proxy_init(struct pw_proxy *proxy, struct pw_core *core, const char *type
 void pw_proxy_remove(struct pw_proxy *proxy);
 
 int pw_context_recalc_graph(struct pw_context *context, const char *reason);
+int pw_context_freeze_recalc_graph(struct pw_context *context);
+int pw_context_thaw_recalc_graph(struct pw_context *context, const char *reason);
 
 void pw_impl_port_update_info(struct pw_impl_port *port, const struct spa_port_info *info);