diff --git a/src/pipewire/proxy.c b/src/pipewire/proxy.c
index d7c91e2b9..4a87b07be 100644
--- a/src/pipewire/proxy.c
+++ b/src/pipewire/proxy.c
@@ -121,7 +121,7 @@ void pw_proxy_destroy(struct pw_proxy *proxy)
 	pw_log_debug("proxy %p: destroy %u", proxy, proxy->id);
 	spa_hook_list_call(&proxy->listener_list, struct pw_proxy_events, destroy);
 
-	pw_map_remove(&proxy->remote->objects, proxy->id);
+	pw_map_insert_at(&proxy->remote->objects, proxy->id, NULL);
 	spa_list_remove(&proxy->link);
 
 	free(impl);
diff --git a/src/pipewire/remote.c b/src/pipewire/remote.c
index dbaeaaf5b..6d162817d 100644
--- a/src/pipewire/remote.c
+++ b/src/pipewire/remote.c
@@ -188,6 +188,7 @@ static void core_event_remove_id(void *data, uint32_t id)
 		pw_log_debug("remote %p: object remove %u", this, id);
 		pw_proxy_destroy(proxy);
 	}
+	pw_map_remove(&this->objects, id);
 }
 
 static void