diff --git a/spa/plugins/audioconvert/channelmix-ops.c b/spa/plugins/audioconvert/channelmix-ops.c
index 885fa9bcc..025b2a8eb 100644
--- a/spa/plugins/audioconvert/channelmix-ops.c
+++ b/spa/plugins/audioconvert/channelmix-ops.c
@@ -540,7 +540,7 @@ static void impl_channelmix_set_volume(struct channelmix *mix, float volume, boo
 				mix->matrix[i][j] = mix->matrix_orig[i][j] * volumes[i];
 			}
 		}
-	} else {
+	} else if (n_channel_volumes == 0) {
 		for (i = 0; i < dst_chan; i++) {
 			for (j = 0; j < src_chan; j++) {
 				mix->matrix[i][j] = mix->matrix_orig[i][j] * vol;