diff --git a/src/modules/module-switch-on-port-available.c b/src/modules/module-switch-on-port-available.c
index b9a0f3b3a..4020987fd 100644
--- a/src/modules/module-switch-on-port-available.c
+++ b/src/modules/module-switch-on-port-available.c
@@ -281,11 +281,6 @@ static pa_hook_result_t port_available_hook_callback(pa_core *c, pa_device_port
         return PA_HOOK_OK;
     }
 
-    if (pa_idxset_size(port->card->sinks) == 0 && pa_idxset_size(port->card->sources) == 0)
-        /* This card is not initialized yet. We'll handle it in
-           sink_new / source_new callbacks later. */
-        return PA_HOOK_OK;
-
     switch (port->available) {
     case PA_AVAILABLE_YES:
         switch_to_port(port);
diff --git a/src/pulsecore/device-port.c b/src/pulsecore/device-port.c
index 76a7e80a1..5cf4ac639 100644
--- a/src/pulsecore/device-port.c
+++ b/src/pulsecore/device-port.c
@@ -92,7 +92,7 @@ void pa_device_port_set_available(pa_device_port *p, pa_available_t status) {
      * before the card object has been created. The card object should probably
      * be created before port objects, and then p->card could be non-NULL for
      * the whole lifecycle of pa_device_port. */
-    if (p->card) {
+    if (p->card && p->card->linked) {
         /* A sink or source whose active port is unavailable can't be the
          * default sink/source, so port availability changes may affect the
          * default sink/source choice. */