diff --git a/src/daemon/main.c b/src/daemon/main.c
index 55726fdbe..1d0a65e38 100644
--- a/src/daemon/main.c
+++ b/src/daemon/main.c
@@ -1156,6 +1156,11 @@ finish:
 #endif
 
     if (c) {
+        /* Ensure all the modules/samples are unloaded when the core is still ref'ed,
+         * as unlink callback hooks in modules may need the core to be ref'ed */
+        pa_module_unload_all(c);
+        pa_scache_free_all(c);
+
         pa_core_unref(c);
         pa_log_info(_("Daemon terminated."));
     }
diff --git a/src/pulsecore/core.c b/src/pulsecore/core.c
index 2a68fdf47..0e53b9d98 100644
--- a/src/pulsecore/core.c
+++ b/src/pulsecore/core.c
@@ -166,8 +166,8 @@ static void core_free(pa_object *o) {
 
     c->state = PA_CORE_SHUTDOWN;
 
-    pa_module_unload_all(c);
-    pa_scache_free_all(c);
+    /* Note: All modules and samples in the cache should be unloaded before
+     * we get here */
 
     pa_assert(pa_idxset_isempty(c->scache));
     pa_idxset_free(c->scache, NULL, NULL);