diff --git a/src/modules/alsa/alsa-sink.c b/src/modules/alsa/alsa-sink.c
index c8684531b..3a952517a 100644
--- a/src/modules/alsa/alsa-sink.c
+++ b/src/modules/alsa/alsa-sink.c
@@ -1751,8 +1751,12 @@ static void thread_func(void *userdata) {
         if (u->sink->flags & PA_SINK_DEFERRED_VOLUME) {
             pa_usec_t volume_sleep;
             pa_sink_volume_change_apply(u->sink, &volume_sleep);
-            if (volume_sleep > 0)
-                rtpoll_sleep = PA_MIN(volume_sleep, rtpoll_sleep);
+            if (volume_sleep > 0) {
+                if (rtpoll_sleep > 0)
+                    rtpoll_sleep = PA_MIN(volume_sleep, rtpoll_sleep);
+                else
+                    rtpoll_sleep = volume_sleep;
+            }
         }
 
         if (rtpoll_sleep > 0)
diff --git a/src/modules/alsa/alsa-source.c b/src/modules/alsa/alsa-source.c
index d54482808..46b89a7bd 100644
--- a/src/modules/alsa/alsa-source.c
+++ b/src/modules/alsa/alsa-source.c
@@ -1452,8 +1452,12 @@ static void thread_func(void *userdata) {
         if (u->source->flags & PA_SOURCE_DEFERRED_VOLUME) {
             pa_usec_t volume_sleep;
             pa_source_volume_change_apply(u->source, &volume_sleep);
-            if (volume_sleep > 0)
-                rtpoll_sleep = PA_MIN(volume_sleep, rtpoll_sleep);
+            if (volume_sleep > 0) {
+                if (rtpoll_sleep > 0)
+                    rtpoll_sleep = PA_MIN(volume_sleep, rtpoll_sleep);
+                else
+                    rtpoll_sleep = volume_sleep;
+            }
         }
 
         if (rtpoll_sleep > 0)