diff --git a/spa/plugins/audioconvert/audioadapter.c b/spa/plugins/audioconvert/audioadapter.c
index 249995984..e5108ea4f 100644
--- a/spa/plugins/audioconvert/audioadapter.c
+++ b/spa/plugins/audioconvert/audioadapter.c
@@ -486,6 +486,13 @@ static int negotiate_buffers(struct impl *this)
 	return 0;
 }
 
+static void clear_buffers(struct impl *this)
+{
+	free(this->buffers);
+	this->buffers = NULL;
+	this->n_buffers = 0;
+}
+
 static int configure_format(struct impl *this, uint32_t flags, const struct spa_pod *format)
 {
 	uint8_t buffer[4096];
@@ -529,11 +536,11 @@ static int configure_format(struct impl *this, uint32_t flags, const struct spa_
 	}
 
 	this->have_format = format != NULL;
-	if (format == NULL) {
-		this->n_buffers = 0;
-	} else if (this->target != this->follower) {
+	clear_buffers(this);
+
+	if (format != NULL && this->target != this->follower)
 		res = negotiate_buffers(this);
-	}
+
 	return res;
 }
 
@@ -1842,10 +1849,7 @@ static int impl_clear(struct spa_handle *handle)
 
 	spa_handle_clear(this->hnd_convert);
 
-	if (this->buffers)
-		free(this->buffers);
-	this->buffers = NULL;
-
+	clear_buffers(this);
 	return 0;
 }