diff --git a/src/modules/module-rtp/opus.c b/src/modules/module-rtp/opus.c
index 8a01d36fb..fa646f1da 100644
--- a/src/modules/module-rtp/opus.c
+++ b/src/modules/module-rtp/opus.c
@@ -320,6 +320,16 @@ static void rtp_opus_process_capture(void *data)
 	rtp_opus_flush_packets(impl);
 }
 
+static void rtp_opus_deinit(struct impl *impl, enum spa_direction direction)
+{
+	if (impl->stream_data) {
+		if (direction == SPA_DIRECTION_INPUT)
+			opus_multistream_encoder_destroy(impl->stream_data);
+		else
+			opus_multistream_decoder_destroy(impl->stream_data);
+	}
+}
+
 static int rtp_opus_init(struct impl *impl, enum spa_direction direction)
 {
 	int err;
@@ -342,6 +352,7 @@ static int rtp_opus_init(struct impl *impl, enum spa_direction direction)
 	for (i = 0; i < impl->info.info.opus.channels; i++)
 		mapping[i] = i;
 
+	impl->deinit = rtp_opus_deinit;
 	impl->receive_rtp = rtp_opus_receive;
 	if (direction == SPA_DIRECTION_INPUT) {
 		impl->stream_events.process = rtp_opus_process_capture;
diff --git a/src/modules/module-rtp/stream.c b/src/modules/module-rtp/stream.c
index 9d3713dd5..e44289a7e 100644
--- a/src/modules/module-rtp/stream.c
+++ b/src/modules/module-rtp/stream.c
@@ -57,6 +57,7 @@ struct impl {
 
 	const struct format_info *format_info;
 
+	enum spa_direction direction;
 	void *stream_data;
 
 	uint32_t rate;
@@ -103,6 +104,7 @@ struct impl {
 
 	int (*receive_rtp)(struct impl *impl, uint8_t *buffer, ssize_t len);
 	void (*flush_timeout)(struct impl *impl, uint64_t expirations);
+	void (*deinit)(struct impl *impl, enum spa_direction direction);
 
 	/*
 	 * pw_filter where the filter would be driven at the PTP clock
@@ -304,7 +306,7 @@ static void on_flush_timeout(void *d, uint64_t expirations)
 }
 
 struct rtp_stream *rtp_stream_new(struct pw_core *core,
-		enum pw_direction direction, struct pw_properties *props,
+		enum spa_direction direction, struct pw_properties *props,
 		const struct rtp_stream_events *events, void *data)
 {
 	struct impl *impl;
@@ -326,6 +328,7 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
 	}
 	impl->first = true;
 	spa_hook_list_init(&impl->listener_list);
+	impl->direction = direction;
 	impl->stream_events = stream_events;
 	impl->context = pw_core_get_context(core);
 	impl->main_loop = pw_context_get_main_loop(impl->context);
@@ -631,6 +634,12 @@ void rtp_stream_destroy(struct rtp_stream *s)
 
 	rtp_stream_emit_destroy(impl);
 
+	if (impl->deinit)
+		impl->deinit(impl, impl->direction);
+
+	if (impl->ptp_sender)
+		pw_filter_destroy(impl->ptp_sender);
+
 	if (impl->stream)
 		pw_stream_destroy(impl->stream);