diff --git a/src/pipewire/link.c b/src/pipewire/link.c
index cf54d2e77..718bd9f52 100644
--- a/src/pipewire/link.c
+++ b/src/pipewire/link.c
@@ -489,6 +489,7 @@ static int do_allocation(struct pw_link *this, uint32_t in_state, uint32_t out_s
 	char *error = NULL;
 	struct pw_port *input, *output;
 	struct pw_type *t = &this->core->type;
+	struct allocation allocation;
 
 	if (in_state != PW_PORT_STATE_READY && out_state != PW_PORT_STATE_READY)
 		return 0;
@@ -557,21 +558,23 @@ static int do_allocation(struct pw_link *this, uint32_t in_state, uint32_t out_s
 		spa_debug_port_info(oinfo);
 		spa_debug_port_info(iinfo);
 	}
-	if (this->allocation.buffers == NULL && output->allocation.n_buffers) {
+	if (output->allocation.n_buffers) {
 		out_flags = 0;
 		in_flags = SPA_PORT_INFO_FLAG_CAN_USE_BUFFERS;
-		this->allocation = output->allocation;
-		this->allocation_owner = output;
+
+		allocation = output->allocation;
+
 		pw_log_debug("link %p: reusing %d output buffers %p", this,
-				this->allocation.n_buffers, this->allocation.buffers);
-	} else if (this->allocation.buffers == NULL && input->allocation.n_buffers && input->mix == NULL) {
+				allocation.n_buffers, allocation.buffers);
+	} else if (input->allocation.n_buffers && input->mix == NULL) {
 		out_flags = SPA_PORT_INFO_FLAG_CAN_USE_BUFFERS;
 		in_flags = 0;
-		this->allocation = input->allocation;
-		this->allocation_owner = input;
+
+		allocation = input->allocation;
+
 		pw_log_debug("link %p: reusing %d input buffers %p", this,
-				this->allocation.n_buffers, this->allocation.buffers);
-	} else if (this->allocation.buffers == NULL) {
+				allocation.n_buffers, allocation.buffers);
+	} else {
 		struct spa_pod **params, *param;
 		uint8_t buffer[4096];
 		struct spa_pod_builder b = SPA_POD_BUILDER_INIT(buffer, sizeof(buffer));
@@ -634,88 +637,82 @@ static int do_allocation(struct pw_link *this, uint32_t in_state, uint32_t out_s
 					 params,
 					 1,
 					 data_sizes, data_strides,
-					 &this->allocation)) < 0) {
+					 &allocation)) < 0) {
 			asprintf(&error, "error alloc buffers: %d", res);
 			goto error;
 		}
-		this->allocation_owner = this;
 
 		pw_log_debug("link %p: allocating %d buffers %p %zd %zd", this,
-			     this->allocation.n_buffers, this->allocation.buffers, minsize, stride);
+			     allocation.n_buffers, allocation.buffers, minsize, stride);
 
 		if (out_flags & SPA_PORT_INFO_FLAG_CAN_ALLOC_BUFFERS) {
 			if ((res = pw_port_alloc_buffers(output,
 							 params, n_params,
-							 this->allocation.buffers,
-							 &this->allocation.n_buffers)) < 0) {
+							 allocation.buffers,
+							 &allocation.n_buffers)) < 0) {
 				asprintf(&error, "error alloc output buffers: %d", res);
 				goto error;
 			}
 			if (SPA_RESULT_IS_ASYNC(res))
 				pw_work_queue_add(impl->work, output->node, res, complete_paused, output);
 
-			output->allocation = this->allocation;
-			this->allocation_owner = output;
+			move_allocation(&allocation, &output->allocation);
 
 			pw_log_debug("link %p: allocated %d buffers %p from output port", this,
-				     this->allocation.n_buffers, this->allocation.buffers);
+				     allocation.n_buffers, allocation.buffers);
 		} else if (in_flags & SPA_PORT_INFO_FLAG_CAN_ALLOC_BUFFERS) {
 			if ((res = pw_port_alloc_buffers(input,
 							 params, n_params,
-							 this->allocation.buffers,
-							 &this->allocation.n_buffers)) < 0) {
+							 allocation.buffers,
+							 &allocation.n_buffers)) < 0) {
 				asprintf(&error, "error alloc input buffers: %d", res);
 				goto error;
 			}
 			if (SPA_RESULT_IS_ASYNC(res))
 				pw_work_queue_add(impl->work, input->node, res, complete_paused, input);
 
-			input->allocation = this->allocation;
-			this->allocation_owner = input;
-
 			pw_log_debug("link %p: allocated %d buffers %p from input port", this,
-				     this->allocation.n_buffers, this->allocation.buffers);
+				     allocation.n_buffers, allocation.buffers);
 		}
 	}
 
-	if (in_flags & SPA_PORT_INFO_FLAG_CAN_USE_BUFFERS) {
-		pw_log_debug("link %p: using %d buffers %p on input port", this,
-			     this->allocation.n_buffers, this->allocation.buffers);
-		if ((res = pw_port_use_buffers(input,
-					       this->allocation.buffers,
-					       this->allocation.n_buffers)) < 0) {
-			asprintf(&error, "error use input buffers: %d", res);
-			goto error;
-		}
-		if (SPA_RESULT_IS_ASYNC(res))
-			pw_work_queue_add(impl->work, input->node, res, complete_paused, input);
-	} else if (out_flags & SPA_PORT_INFO_FLAG_CAN_USE_BUFFERS) {
+	if (out_flags & SPA_PORT_INFO_FLAG_CAN_USE_BUFFERS) {
 		pw_log_debug("link %p: using %d buffers %p on output port", this,
-			     this->allocation.n_buffers, this->allocation.buffers);
+			     allocation.n_buffers, allocation.buffers);
 		if ((res = pw_port_use_buffers(output,
-					       this->allocation.buffers,
-					       this->allocation.n_buffers)) < 0) {
+					       allocation.buffers,
+					       allocation.n_buffers)) < 0) {
 			asprintf(&error, "error use output buffers: %d", res);
 			goto error;
 		}
 		if (SPA_RESULT_IS_ASYNC(res))
 			pw_work_queue_add(impl->work, output->node, res, complete_paused, output);
 
-		output->allocation = this->allocation;
-		output->allocated = false;
-		this->allocation_owner = output;
+		move_allocation(&allocation, &output->allocation);
+
+	} else if (in_flags & SPA_PORT_INFO_FLAG_CAN_USE_BUFFERS) {
+		pw_log_debug("link %p: using %d buffers %p on input port", this,
+			     allocation.n_buffers, allocation.buffers);
+		if ((res = pw_port_use_buffers(input,
+					       allocation.buffers,
+					       allocation.n_buffers)) < 0) {
+			asprintf(&error, "error use input buffers: %d", res);
+			goto error;
+		}
+		if (SPA_RESULT_IS_ASYNC(res))
+			pw_work_queue_add(impl->work, input->node, res, complete_paused, input);
+
 	} else {
 		asprintf(&error, "no common buffer alloc found");
 		goto error;
 	}
 
+
 	return 0;
 
       error:
-	drop_allocation(&output->allocation);
-	output->allocated = false;
-	drop_allocation(&input->allocation);
-	input->allocated = false;
+	free_allocation(&output->allocation);
+	free_allocation(&input->allocation);
 	pw_link_update_state(this, PW_LINK_STATE_ERROR, error);
 	return res;
 }
@@ -853,7 +850,7 @@ output_node_async_complete(void *data, uint32_t seq, int res)
 
 static void clear_port_buffers(struct pw_link *link, struct pw_port *port)
 {
-	if (spa_list_is_empty(&port->links) && link->allocation_owner != port)
+	if (spa_list_is_empty(&port->links) && port->allocation.mem == NULL)
 		pw_port_use_buffers(port, NULL, 0);
 }
 
@@ -1305,8 +1302,7 @@ void pw_link_destroy(struct pw_link *link)
 	if (link->info.format)
 		free(link->info.format);
 
-	if (link->allocation_owner == link)
-		free_allocation(&link->allocation);
+	free_allocation(&link->allocation);
 
 	free(impl);
 }
diff --git a/src/pipewire/port.c b/src/pipewire/port.c
index dadb236fc..0c4f786f4 100644
--- a/src/pipewire/port.c
+++ b/src/pipewire/port.c
@@ -556,8 +556,7 @@ void pw_port_destroy(struct pw_port *port)
 	pw_log_debug("port %p: free", port);
 	spa_hook_list_call(&port->listener_list, struct pw_port_events, free);
 
-	if (port->allocated)
-		free_allocation(&port->allocation);
+	free_allocation(&port->allocation);
 
 	if (port->properties)
 		pw_properties_free(port->properties);
@@ -666,11 +665,8 @@ int pw_port_set_param(struct pw_port *port, uint32_t id, uint32_t flags,
 
 	if (id == t->param.idFormat) {
 		if (param == NULL || res < 0) {
-			if (port->allocated) {
-				free_allocation(&port->allocation);
-				port->allocated = false;
-			}
-			drop_allocation(&port->allocation);
+			free_allocation(&port->allocation);
+			port->allocated = false;
 			port_update_state (port, PW_PORT_STATE_CONFIGURE);
 		}
 		else if (!SPA_RESULT_IS_ASYNC(res)) {
@@ -694,18 +690,16 @@ int pw_port_use_buffers(struct pw_port *port, struct spa_buffer **buffers, uint3
 	res = spa_node_port_use_buffers(node->node, port->direction, port->port_id, buffers, n_buffers);
 	pw_log_debug("port %p: use %d buffers: %d (%s)", port, n_buffers, res, spa_strerror(res));
 
-	if (port->allocated) {
-		free_allocation(&port->allocation);
-		port->allocated = false;
-	}
+	port->allocated = false;
+
+	free_allocation(&port->allocation);
+
 	if (res < 0) {
-		drop_allocation(&port->allocation);
-	} else {
-		port->allocation.buffers = buffers;
-		port->allocation.n_buffers = n_buffers;
+		n_buffers = 0;
+		buffers = NULL;
 	}
 
-	if (port->allocation.n_buffers == 0)
+	if (n_buffers == 0)
 		port_update_state (port, PW_PORT_STATE_READY);
 	else if (!SPA_RESULT_IS_ASYNC(res))
 		port_update_state (port, PW_PORT_STATE_PAUSED);
@@ -728,20 +722,18 @@ int pw_port_alloc_buffers(struct pw_port *port,
 					  buffers, n_buffers);
 	pw_log_debug("port %p: alloc %d buffers: %d (%s)", port, *n_buffers, res, spa_strerror(res));
 
-	if (port->allocated) {
-		free_allocation(&port->allocation);
-	}
+	free_allocation(&port->allocation);
+
 	if (res < 0) {
-		drop_allocation(&port->allocation);
+		n_buffers = 0;
+		buffers = NULL;
 		port->allocated = false;
 	}
 	else {
-		port->allocation.buffers = buffers;
-		port->allocation.n_buffers = *n_buffers;
 		port->allocated = true;
 	}
 
-	if (port->allocation.n_buffers == 0)
+	if (n_buffers == 0)
 		port_update_state (port, PW_PORT_STATE_READY);
 	else if (!SPA_RESULT_IS_ASYNC(res))
 		port_update_state (port, PW_PORT_STATE_PAUSED);
diff --git a/src/pipewire/private.h b/src/pipewire/private.h
index a8567beee..3aad48fd8 100644
--- a/src/pipewire/private.h
+++ b/src/pipewire/private.h
@@ -195,16 +195,21 @@ struct allocation {
 	uint32_t n_buffers;		/**< number of port buffers */
 };
 
-static inline void drop_allocation(struct allocation *alloc)
+static inline void move_allocation(struct allocation *alloc, struct allocation *dest)
 {
-	alloc->buffers = NULL;
-	alloc->n_buffers = 0;
+	*dest = *alloc;
+	alloc->mem = NULL;
 }
 
 static inline void free_allocation(struct allocation *alloc)
 {
-	pw_memblock_free(alloc->mem);
-	free(alloc->buffers);
+	if (alloc->mem) {
+		pw_memblock_free(alloc->mem);
+		free(alloc->buffers);
+	}
+	alloc->mem = NULL;
+	alloc->buffers = NULL;
+	alloc->n_buffers = 0;
 }
 
 struct pw_link {
@@ -231,7 +236,6 @@ struct pw_link {
 
 	struct spa_hook_list listener_list;
 
-	void *allocation_owner;
 	struct allocation allocation;
 
 	struct {