diff --git a/src/modules/alsa/alsa-sink.c b/src/modules/alsa/alsa-sink.c
index 401b003f4..6ed1c3adb 100644
--- a/src/modules/alsa/alsa-sink.c
+++ b/src/modules/alsa/alsa-sink.c
@@ -483,7 +483,13 @@ static int mmap_write(struct userdata *u, pa_usec_t *sleep_usec, pa_bool_t polle
         }
     }
 
-    *sleep_usec = pa_bytes_to_usec(left_to_play, &u->sink->sample_spec) - process_usec;
+    *sleep_usec = pa_bytes_to_usec(left_to_play, &u->sink->sample_spec);
+
+    if (*sleep_usec > process_usec)
+        *sleep_usec -= process_usec;
+    else
+        *sleep_usec = 0;
+
     return work_done ? 1 : 0;
 }
 
@@ -605,7 +611,13 @@ static int unix_write(struct userdata *u, pa_usec_t *sleep_usec, pa_bool_t polle
         }
     }
 
-    *sleep_usec = pa_bytes_to_usec(left_to_play, &u->sink->sample_spec) - process_usec;
+    *sleep_usec = pa_bytes_to_usec(left_to_play, &u->sink->sample_spec);
+
+    if (*sleep_usec > process_usec)
+        *sleep_usec -= process_usec;
+    else
+        *sleep_usec = 0;
+
     return work_done ? 1 : 0;
 }
 
diff --git a/src/modules/alsa/alsa-source.c b/src/modules/alsa/alsa-source.c
index 99e825c4e..d87ab8097 100644
--- a/src/modules/alsa/alsa-source.c
+++ b/src/modules/alsa/alsa-source.c
@@ -465,7 +465,13 @@ static int mmap_read(struct userdata *u, pa_usec_t *sleep_usec, pa_bool_t polled
         }
     }
 
-    *sleep_usec = pa_bytes_to_usec(left_to_record, &u->source->sample_spec) - process_usec;
+    *sleep_usec = pa_bytes_to_usec(left_to_record, &u->source->sample_spec);
+
+    if (*sleep_usec > process_usec)
+        *sleep_usec -= process_usec;
+    else
+        *sleep_usec = 0;
+
     return work_done ? 1 : 0;
 }
 
@@ -575,7 +581,13 @@ static int unix_read(struct userdata *u, pa_usec_t *sleep_usec, pa_bool_t polled
         }
     }
 
-    *sleep_usec = pa_bytes_to_usec(left_to_record, &u->source->sample_spec) - process_usec;
+    *sleep_usec = pa_bytes_to_usec(left_to_record, &u->source->sample_spec);
+
+    if (*sleep_usec > process_usec)
+        *sleep_usec -= process_usec;
+    else
+        *sleep_usec = 0;
+
     return work_done ? 1 : 0;
 }