diff --git a/spa/plugins/audioconvert/audioconvert.c b/spa/plugins/audioconvert/audioconvert.c
index e4d352202..fcf2b1b64 100644
--- a/spa/plugins/audioconvert/audioconvert.c
+++ b/spa/plugins/audioconvert/audioconvert.c
@@ -1112,11 +1112,12 @@ impl_node_port_set_param(void *object,
 
 	switch (id) {
 	case SPA_PARAM_Latency:
-		target = this->fmt[SPA_DIRECTION_REVERSE(direction)];
-		port_id = 0;
-		if ((res = spa_node_port_set_param(target,
-					direction, port_id, id, flags, param)) < 0)
-			return res;
+		if (port_id == 0) {
+			target = this->fmt[SPA_DIRECTION_REVERSE(direction)];
+			if ((res = spa_node_port_set_param(target,
+						direction, port_id, id, flags, param)) < 0)
+				return res;
+		}
 		break;
 	}
 
diff --git a/spa/plugins/audioconvert/merger.c b/spa/plugins/audioconvert/merger.c
index aeb0ea558..bf67f0c9b 100644
--- a/spa/plugins/audioconvert/merger.c
+++ b/spa/plugins/audioconvert/merger.c
@@ -1031,7 +1031,11 @@ static int port_set_latency(void *object,
 	enum spa_direction other = SPA_DIRECTION_REVERSE(direction);
 	uint32_t i;
 
-	spa_log_debug(this->log, "%p: set latency direction:%d", this, direction);
+	spa_log_debug(this->log, "%p: set latency direction:%d id:%d",
+			this, direction, port_id);
+
+	if (direction == SPA_DIRECTION_OUTPUT && port_id != 0)
+		return 0;
 
 	if (latency == NULL) {
 		this->latency[other] = SPA_LATENCY_INFO(other);