diff --git a/src/pipewire/core.c b/src/pipewire/core.c
index 9e7cef651..637a84a51 100644
--- a/src/pipewire/core.c
+++ b/src/pipewire/core.c
@@ -115,7 +115,7 @@ static void core_event_remove_mem(void *data, uint32_t id)
 {
 	struct pw_core *this = data;
 	pw_log_debug(NAME" %p: remove mem %u", this, id);
-	pw_mempool_unref_id(this->pool, id);
+	pw_mempool_remove_id(this->pool, id);
 }
 
 static const struct pw_core_events core_events = {
diff --git a/src/pipewire/mem.c b/src/pipewire/mem.c
index 30bb91589..50ee6b406 100644
--- a/src/pipewire/mem.c
+++ b/src/pipewire/mem.c
@@ -623,8 +623,7 @@ struct pw_memmap * pw_mempool_import_map(struct pw_mempool *pool,
 	return map;
 }
 
-
-int pw_mempool_unref_id(struct pw_mempool *pool, uint32_t id)
+int pw_mempool_remove_id(struct pw_mempool *pool, uint32_t id)
 {
 	struct mempool *impl = SPA_CONTAINER_OF(pool, struct mempool, this);
 	struct memblock *b;
@@ -636,7 +635,10 @@ int pw_mempool_unref_id(struct pw_mempool *pool, uint32_t id)
 	pw_log_debug(NAME" %p: block:%p id:%d fd:%d ref:%d",
 			pool, b, id, b->this.fd, b->this.ref);
 
+	b->this.id = SPA_ID_INVALID;
+	pw_map_remove(&impl->map, id);
 	pw_memblock_unref(&b->this);
+
 	return 0;
 }
 
@@ -661,7 +663,8 @@ void pw_memblock_free(struct pw_memblock *block)
 	if (block->map)
 		block->ref++;
 
-	pw_map_remove(&impl->map, block->id);
+	if (block->id != SPA_ID_INVALID)
+		pw_map_remove(&impl->map, block->id);
 	spa_list_remove(&b->link);
 
 	pw_mempool_emit_removed(impl, block);
diff --git a/src/pipewire/mem.h b/src/pipewire/mem.h
index 433b2c7bc..06d92b3a1 100644
--- a/src/pipewire/mem.h
+++ b/src/pipewire/mem.h
@@ -134,8 +134,8 @@ static inline void pw_memblock_unref(struct pw_memblock *mem)
 		pw_memblock_free(mem);
 }
 
-/** Unref a memblock for given \a id */
-int pw_mempool_unref_id(struct pw_mempool *pool, uint32_t id);
+/** Remove a memblock for given \a id */
+int pw_mempool_remove_id(struct pw_mempool *pool, uint32_t id);
 
 /** Find memblock for given \a ptr */
 struct pw_memblock * pw_mempool_find_ptr(struct pw_mempool *pool, const void *ptr);