diff --git a/src/pipewire/private.h b/src/pipewire/private.h
index 53a99fb1f..27c184a94 100644
--- a/src/pipewire/private.h
+++ b/src/pipewire/private.h
@@ -740,6 +740,7 @@ struct pw_proxy {
 	unsigned int zombie:1;		/**< proxy is removed locally and waiting to
 					  *  be removed from server */
 	unsigned int removed:1;		/**< proxy was removed from server */
+	unsigned int destroyed:1;	/**< proxy was destroyed by client */
 
 	struct spa_hook_list listener_list;
 	struct spa_hook_list object_listener_list;
diff --git a/src/pipewire/proxy.c b/src/pipewire/proxy.c
index 5d4e81517..d6299212c 100644
--- a/src/pipewire/proxy.c
+++ b/src/pipewire/proxy.c
@@ -22,6 +22,8 @@
  * DEALINGS IN THE SOFTWARE.
  */
 
+#include <assert.h>
+
 #include <pipewire/log.h>
 #include <pipewire/proxy.h>
 #include <pipewire/core.h>
@@ -214,15 +216,20 @@ void pw_proxy_add_object_listener(struct pw_proxy *proxy,
 SPA_EXPORT
 void pw_proxy_destroy(struct pw_proxy *proxy)
 {
+	assert(!proxy->destroyed);
+	proxy->destroyed = true;
+
+	pw_log_debug(NAME" %p: destroy id:%u removed:%u zombie:%u", proxy,
+			proxy->id, proxy->removed, proxy->zombie);
+
 	if (!proxy->zombie) {
-		pw_log_debug(NAME" %p: destroy %u", proxy, proxy->id);
+		proxy->zombie = true;
 		pw_proxy_emit_destroy(proxy);
 	}
 	if (!proxy->removed) {
 		/* if the server did not remove this proxy, remove ourselves
 		 * from the proxy objects and schedule a destroy. */
 		if (proxy->core && !proxy->core->destroyed) {
-			proxy->zombie = true;
 			pw_core_destroy(proxy->core, proxy);
 		} else {
 			proxy->removed = true;
@@ -239,12 +246,14 @@ void pw_proxy_destroy(struct pw_proxy *proxy)
 void pw_proxy_remove(struct pw_proxy *proxy)
 {
 	proxy->removed = true;
+	proxy->destroyed = false;
 	pw_proxy_destroy(proxy);
 }
 
 SPA_EXPORT
 void pw_proxy_unref(struct pw_proxy *proxy)
 {
+	assert(proxy->refcount > 0);
 	if (--proxy->refcount > 0)
 		return;