diff --git a/src/modules/module-protocol-pulse/pulse-server.c b/src/modules/module-protocol-pulse/pulse-server.c
index b403aba21..32733f369 100644
--- a/src/modules/module-protocol-pulse/pulse-server.c
+++ b/src/modules/module-protocol-pulse/pulse-server.c
@@ -685,9 +685,10 @@ static uint64_t fix_record_buffer_attr(struct stream *s, struct buffer_attr *att
 	if (attr->maxlength < attr->fragsize * 4)
 		attr->maxlength = attr->fragsize * 4;
 
-	latency = attr->fragsize / frame_size;
+	/* pulseaudio configures half the fragsize as latency in the source. */
+	latency = attr->fragsize / 2;
 
-	lat->num = latency;
+	lat->num = latency / frame_size;
 	lat->denom = rate;
 	clamp_latency(s, lat);