diff --git a/src/examples/media-session.c b/src/examples/media-session.c
index 09fcef761..bd4601f02 100644
--- a/src/examples/media-session.c
+++ b/src/examples/media-session.c
@@ -178,8 +178,6 @@ static void add_object(struct impl *impl, struct object *obj)
 static void remove_object(struct impl *impl, struct object *obj)
 {
         pw_map_insert_at(&impl->globals, obj->id, NULL);
-	if (obj->proxy)
-		pw_proxy_destroy(obj->proxy);
 }
 
 static void *find_object(struct impl *impl, uint32_t id)
@@ -238,13 +236,11 @@ static void add_idle_timeout(struct session *sess)
 
 static int unlink_session_dsp(struct impl *impl, struct session *session)
 {
-	if (session->link_proxy == NULL)
-		return 0;
-
-	if (impl->core_proxy)
-		pw_core_proxy_destroy(impl->core_proxy, session->link_proxy);
-
-	session->link_proxy = NULL;
+	if (session->link_proxy != NULL) {
+		pw_log_debug(NAME " %p: destroy session dsp link %p", impl, session->link_proxy);
+		pw_proxy_destroy(session->link_proxy);
+		session->link_proxy = NULL;
+	}
 	return 0;
 }
 
@@ -423,8 +419,7 @@ static void remove_session(struct impl *impl, struct session *sess)
 	}
 	if (sess->dsp_proxy) {
 		pw_log_debug(NAME " %p: destroy dsp %p", impl, sess->dsp_proxy);
-		if (impl->core_proxy)
-			pw_core_proxy_destroy(impl->core_proxy, sess->dsp_proxy);
+		pw_proxy_destroy(sess->dsp_proxy);
 		sess->dsp_proxy = NULL;
 	}
 	if (sess->link_proxy) {
diff --git a/src/pipewire/private.h b/src/pipewire/private.h
index 2ec00b8ed..162ca4b02 100644
--- a/src/pipewire/private.h
+++ b/src/pipewire/private.h
@@ -584,6 +584,7 @@ struct pw_proxy {
 	struct spa_list link;		/**< link in the remote */
 
 	uint32_t id;			/**< client side id */
+	int removed:1;			/**< proxy was removed from server */
 
 	struct spa_hook_list listener_list;
 	struct spa_hook_list proxy_listener_list;
diff --git a/src/pipewire/proxy.c b/src/pipewire/proxy.c
index c3664d3c2..a76c60e3b 100644
--- a/src/pipewire/proxy.c
+++ b/src/pipewire/proxy.c
@@ -136,13 +136,22 @@ SPA_EXPORT
 void pw_proxy_destroy(struct pw_proxy *proxy)
 {
 	struct proxy *impl = SPA_CONTAINER_OF(proxy, struct proxy, this);
+	struct pw_remote *remote = proxy->remote;
 
 	pw_log_debug("proxy %p: destroy %u", proxy, proxy->id);
 	pw_proxy_emit_destroy(proxy);
 
-	pw_map_insert_at(&proxy->remote->objects, proxy->id, NULL);
 	spa_list_remove(&proxy->link);
 
+	if (!proxy->removed) {
+		/* if the server did not remove this proxy, remove ourselves
+		 * from the proxy objects and schedule a destroy, if
+		 * we don't have a remote, we must keep the proxy id
+		 * locked because the server might be using it still. */
+		pw_map_insert_at(&remote->objects, proxy->id, NULL);
+		if (remote->core_proxy)
+			pw_core_proxy_destroy(remote->core_proxy, proxy);
+	}
 	free(impl);
 }
 
diff --git a/src/pipewire/remote.c b/src/pipewire/remote.c
index 05ee53437..cac37a6e6 100644
--- a/src/pipewire/remote.c
+++ b/src/pipewire/remote.c
@@ -136,10 +136,10 @@ static void core_event_remove_id(void *data, uint32_t id)
 	struct pw_proxy *proxy;
 
 	pw_log_debug("remote %p: object remove %u", this, id);
-	proxy = pw_map_lookup(&this->objects, id);
-	if (proxy)
+	if ((proxy = pw_map_lookup(&this->objects, id))) {
+		proxy->removed = true;
 		pw_proxy_destroy(proxy);
-
+	}
 	pw_map_remove(&this->objects, id);
 }
 
@@ -335,6 +335,7 @@ static int do_connect(struct pw_remote *remote)
 	return 0;
 
       clean_core_proxy:
+	((struct pw_proxy*)remote->core_proxy)->removed = true;
 	pw_proxy_destroy((struct pw_proxy*)remote->core_proxy);
       no_proxy:
 	pw_protocol_client_disconnect(remote->conn);
diff --git a/src/pipewire/stream.c b/src/pipewire/stream.c
index 29353df8f..72293a24c 100644
--- a/src/pipewire/stream.c
+++ b/src/pipewire/stream.c
@@ -1228,15 +1228,15 @@ int pw_stream_disconnect(struct pw_stream *stream)
 	pw_log_debug("stream %p: disconnect", stream);
 	impl->disconnecting = true;
 
-	if (impl->node) {
-		pw_node_destroy(impl->node);
-		impl->node = NULL;
-	}
 	if (stream->proxy) {
 		stream->proxy = NULL;
 		spa_hook_remove(&stream->proxy_listener);
 		stream->node_id = SPA_ID_INVALID;
 	}
+	if (impl->node) {
+		pw_node_destroy(impl->node);
+		impl->node = NULL;
+	}
 	stream_set_state(stream, PW_STREAM_STATE_UNCONNECTED, NULL);
 	return 0;
 }