diff --git a/src/pipewire/context.c b/src/pipewire/context.c
index c4ecaa004..422dc1071 100644
--- a/src/pipewire/context.c
+++ b/src/pipewire/context.c
@@ -56,7 +56,7 @@ struct impl {
 	struct pw_context this;
 	struct spa_handle *dbus_handle;
 	struct spa_plugin_loader plugin_loader;
-	unsigned int recalc:1;
+	int recalc;
 	unsigned int recalc_pending:1;
 
 	uint32_t cpu_count;
@@ -991,24 +991,40 @@ int pw_context_recalc_graph(struct pw_context *context, const char *reason)
 
 	pw_log_info("%p: busy:%d reason:%s", context, impl->recalc, reason);
 
-	if (impl->recalc) {
+	if (impl->recalc > 0) {
 		impl->recalc_pending = true;
 		return -EBUSY;
 	}
 
 again:
-	impl->recalc = true;
+	impl->recalc++;
 
 	pw_context_emit_recalc_graph(context);
 
-	impl->recalc = false;
-	if (impl->recalc_pending) {
+	if (--impl->recalc == 0 && impl->recalc_pending) {
 		impl->recalc_pending = false;
 		goto again;
 	}
 	return 0;
 }
 
+int pw_context_freeze_recalc_graph(struct pw_context *context)
+{
+	struct impl *impl = SPA_CONTAINER_OF(context, struct impl, this);
+	impl->recalc++;
+	return 0;
+}
+
+int pw_context_thaw_recalc_graph(struct pw_context *context, const char *reason)
+{
+	struct impl *impl = SPA_CONTAINER_OF(context, struct impl, this);
+	if (--impl->recalc == 0 && impl->recalc_pending) {
+		impl->recalc_pending = false;
+		pw_context_recalc_graph(context, reason);
+	}
+	return 0;
+}
+
 SPA_EXPORT
 int pw_context_add_spa_lib(struct pw_context *context,
 		const char *factory_regexp, const char *lib)
diff --git a/src/pipewire/impl-node.c b/src/pipewire/impl-node.c
index b4c4d3d97..6a73bb139 100644
--- a/src/pipewire/impl-node.c
+++ b/src/pipewire/impl-node.c
@@ -288,6 +288,8 @@ static void node_deactivate(struct pw_impl_node *this)
 	/* make sure the node doesn't get woken up while not active */
 	remove_node_from_graph(this);
 
+	pw_context_freeze_recalc_graph(this->context);
+
 	spa_list_for_each(port, &this->input_ports, link) {
 		spa_list_for_each(link, &port->links, input_link)
 			pw_impl_link_deactivate(link);
@@ -296,6 +298,7 @@ static void node_deactivate(struct pw_impl_node *this)
 		spa_list_for_each(link, &port->links, output_link)
 			pw_impl_link_deactivate(link);
 	}
+	pw_context_thaw_recalc_graph(this->context, "node deactivate");
 }
 
 static int idle_node(struct pw_impl_node *this)
@@ -329,6 +332,8 @@ static void node_activate(struct pw_impl_node *this)
 	struct pw_impl_port *port;
 
 	pw_log_debug("%p: activate", this);
+	pw_context_freeze_recalc_graph(this->context);
+
 	spa_list_for_each(port, &this->output_ports, link) {
 		struct pw_impl_link *link;
 		spa_list_for_each(link, &port->links, output_link)
@@ -339,6 +344,7 @@ static void node_activate(struct pw_impl_node *this)
 		spa_list_for_each(link, &port->links, input_link)
 			pw_impl_link_activate(link);
 	}
+	pw_context_thaw_recalc_graph(this->context, "node activate");
 }
 
 static int start_node(struct pw_impl_node *this)
@@ -541,6 +547,8 @@ static int suspend_node(struct pw_impl_node *this)
 	    (this->info.state == PW_NODE_STATE_SUSPENDED && impl->pending_state == PW_NODE_STATE_SUSPENDED))
 		return 0;
 
+	pw_context_freeze_recalc_graph(this->context);
+
 	node_deactivate(this);
 
 	pw_log_debug("%p: suspend node driving:%d driver:%d prepared:%d", this,
@@ -559,6 +567,8 @@ static int suspend_node(struct pw_impl_node *this)
 	spa_list_for_each(p, &this->output_ports, link)
 		pw_impl_port_suspend(p);
 
+	pw_context_thaw_recalc_graph(this->context, "node suspend");
+
 	node_update_state(this, PW_NODE_STATE_SUSPENDED, 0, NULL);
 
 	return res;
@@ -2408,15 +2418,15 @@ void pw_impl_node_destroy(struct pw_impl_node *node)
 	struct pw_impl_port *port;
 	struct pw_impl_node *follower;
 	struct pw_context *context = node->context;
-	bool active, had_driver;
 
-	active = node->active;
 	node->active = false;
 	node->runnable = false;
 
 	pw_log_debug("%p: destroy", impl);
 	pw_log_info("(%s-%u) destroy", node->name, node->info.id);
 
+	pw_context_freeze_recalc_graph(context);
+
 	node_deactivate(node);
 
 	suspend_node(node);
@@ -2424,7 +2434,6 @@ void pw_impl_node_destroy(struct pw_impl_node *node)
 	pw_impl_node_emit_destroy(node);
 
 	pw_log_debug("%p: driver node %p", impl, node->driver_node);
-	had_driver = node != node->driver_node;
 
 	/* remove ourself as a follower from the driver node */
 	spa_list_remove(&node->follower_link);
@@ -2458,10 +2467,7 @@ void pw_impl_node_destroy(struct pw_impl_node *node)
 		spa_hook_remove(&node->global_listener);
 		pw_global_destroy(node->global);
 	}
-
-	if (active || had_driver)
-		pw_context_recalc_graph(context,
-				"active node destroy");
+	pw_context_thaw_recalc_graph(context, "node destroy");
 
 	pw_log_debug("%p: free", node);
 	pw_impl_node_emit_free(node);
diff --git a/src/pipewire/impl-port.c b/src/pipewire/impl-port.c
index a65d6d2b3..2b1e9ac85 100644
--- a/src/pipewire/impl-port.c
+++ b/src/pipewire/impl-port.c
@@ -1515,7 +1515,10 @@ static int do_destroy_link(void *data, struct pw_impl_link *link)
 
 void pw_impl_port_unlink(struct pw_impl_port *port)
 {
+	struct pw_context *context = port->node->context;
+	pw_context_freeze_recalc_graph(context);
 	pw_impl_port_for_each_link(port, do_destroy_link, port);
+	pw_context_thaw_recalc_graph(context, "port unlink");
 }
 
 static int do_remove_port(struct spa_loop *loop,
diff --git a/src/pipewire/private.h b/src/pipewire/private.h
index bb810fd31..c1914c71c 100644
--- a/src/pipewire/private.h
+++ b/src/pipewire/private.h
@@ -1306,6 +1306,8 @@ int pw_proxy_init(struct pw_proxy *proxy, struct pw_core *core, const char *type
 void pw_proxy_remove(struct pw_proxy *proxy);
 
 int pw_context_recalc_graph(struct pw_context *context, const char *reason);
+int pw_context_freeze_recalc_graph(struct pw_context *context);
+int pw_context_thaw_recalc_graph(struct pw_context *context, const char *reason);
 
 void pw_impl_port_update_info(struct pw_impl_port *port, const struct spa_port_info *info);