diff --git a/src/modules/module-metadata.c b/src/modules/module-metadata.c
index 7a9f125e4..b8619a458 100644
--- a/src/modules/module-metadata.c
+++ b/src/modules/module-metadata.c
@@ -58,7 +58,8 @@ struct pw_proxy *pw_core_metadata_export(struct pw_core *core,
 int pw_protocol_native_ext_metadata_init(struct pw_context *context);
 
 struct factory_data {
-	struct pw_impl_factory *this;
+	struct pw_impl_factory *factory;
+	struct spa_hook factory_listener;
 
 	struct pw_impl_module *module;
 	struct spa_hook module_listener;
@@ -86,7 +87,7 @@ static void *create_object(void *_data,
 		return NULL;
 
 	pw_properties_setf(properties, PW_KEY_FACTORY_ID, "%d",
-			pw_impl_factory_get_info(data->this)->id);
+			pw_impl_factory_get_info(data->factory)->id);
 	pw_properties_setf(properties, PW_KEY_MODULE_ID, "%d",
 			pw_impl_module_get_info(data->module)->id);
 
@@ -143,22 +144,35 @@ static const struct pw_impl_factory_implementation impl_factory = {
 	.create_object = create_object,
 };
 
+static void factory_destroy(void *data)
+{
+	struct factory_data *d = data;
+	spa_hook_remove(&d->factory_listener);
+	d->factory = NULL;
+	if (d->module)
+		pw_impl_module_destroy(d->module);
+}
+
+static const struct pw_impl_factory_events factory_events = {
+	PW_VERSION_IMPL_FACTORY_EVENTS,
+	.destroy = factory_destroy,
+};
+
 static void module_destroy(void *data)
 {
 	struct factory_data *d = data;
-
 	spa_hook_remove(&d->module_listener);
-
 	spa_list_remove(&d->export_metadata.link);
-
-	pw_impl_factory_destroy(d->this);
+	d->module = NULL;
+	if (d->factory)
+		pw_impl_factory_destroy(d->factory);
 }
 
 static void module_registered(void *data)
 {
 	struct factory_data *d = data;
 	struct pw_impl_module *module = d->module;
-	struct pw_impl_factory *factory = d->this;
+	struct pw_impl_factory *factory = d->factory;
 	struct spa_dict_item items[1];
 	char id[16];
 	int res;
@@ -201,7 +215,7 @@ int pipewire__module_init(struct pw_impl_module *module, const char *args)
 		return -errno;
 
 	data = pw_impl_factory_get_user_data(factory);
-	data->this = factory;
+	data->factory = factory;
 	data->module = module;
 
 	pw_log_debug("module %p: new", module);
@@ -215,12 +229,13 @@ int pipewire__module_init(struct pw_impl_module *module, const char *args)
 	if ((res = pw_context_register_export_type(context, &data->export_metadata)) < 0)
 		goto error;
 
+	pw_impl_factory_add_listener(factory, &data->factory_listener, &factory_events, data);
 	pw_impl_module_add_listener(module, &data->module_listener, &module_events, data);
 
 	pw_impl_module_update_properties(module, &SPA_DICT_INIT_ARRAY(module_props));
 
 	return 0;
 error:
-	pw_impl_factory_destroy(data->this);
+	pw_impl_factory_destroy(data->factory);
 	return res;
 }