mirrors/pipewire
1
0
Fork 0
mirror of https://gitlab.freedesktop.org/pipewire/pipewire.git synced 2026-07-05 00:06:16 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

module-rtp: Add audio codec support to audio.c and replace opus.c with it

Browse source 
Opus was integrated as a completely separate code path to the PCM audio
processing found in audio.c. This is actually not ideal, since the only
part that actually is Opus specific is the part that en- and decodes from
and to PCM. The rest is 1:1 the same PCM handling.

For this reason, it is much better to instead add audio codec support to
audio.c, meaning that the code in there can now encode PCM audio right
before sending it out as RTP, and decode incoming packets to PCM right
before actually processing the decoded audio data.

This significantly modifies how stream.c initializes the PCM audio path,
since the audio codec feature is new. It now treats the Opus subtype
as an audio codec selector instead of a selector for an entirely
alternate code path (like how MIDI integration remains entirely separate).

Since audio codecs usually require their frames to be decoded in order,
this also integrates the RTP jitter buffer in the RTP module.

Opus is now integrated as such a codec in audio.c. When it is selected,
incoming packets in rtp_audio_receive() are first inserted into the
jitter buffer. That buffer then outputs packets in order, and then, these
packets are decoded to PCM. The rest of the processing chain goes as usual.
A similar route is used for when the jitter buffer signals packet loss
to be able to apply PLC.

For encoding, it is similar (except that no jitter buffer is involved);
in rtp_audio_flush_packets(), when Opus is active, the PCM data is
rerouted to be fed to Opus for encoding, and the Opus output is then
placed into the iovec array instead of the original PCM.

This also improves overall Opus support; it supports S16 PCM data in
addition to F32 data, correctly checks the ptime, sample rate etc. for
Opus compatibility, computes an ideal bitrate, allows for manual bitrate
selection and encoding complexity adjustment (via the new stream properties
"opus.encoder.bitrate" and "opus.encoder.complexity"), sets several other
Opus CTLs to fixed values, supports the Opus restricted-lowdelay mode
(sacrifices Speech code paths for lower latency, enabled by setting the
"opus.encoder.restricted-lowdelay" stream property to true), and also uses
Opus' PLC in case of packet loss.

The audio codec interface is designed such that adding other codecs in
the future is easily doable. New integrations need to implement the
function pointers found in the rtp_audio_codec structure, and expose
an instance of such a custom rtp_audio_codec structure instance (see
the get_rtp_opus_codec() implementation for an example).
This commit is contained in:
Carlos Rafael Giani 2026-06-24 18:48:36 +02:00
parent 31bb82e116
commit 6b524fd596
12 changed files with 1461 additions and 471 deletions
Download patch file Download diff file Expand all files Collapse all files

224
src/modules/module-rtp/stream.c
View file

@ -12,6 +12,7 @@
#include <spa/utils/json.h>
#include <spa/utils/ringbuffer.h>
#include <spa/utils/dll.h>
#include <spa/utils/defs.h>
#include <spa/param/audio/format-utils.h>
#include <spa/param/audio/raw-json.h>
#include <spa/param/latency-utils.h>
@ -27,6 +28,9 @@
#include <module-rtp/rtp.h>
#include <module-rtp/stream.h>
#include <module-rtp/apple-midi.h>
#include <module-rtp/jitter-buffer.h>
#include <module-rtp/audio-codec.h>
#include <module-rtp/opus-codec.h>
PW_LOG_TOPIC_EXTERN(mod_topic);
#define PW_LOG_TOPIC_DEFAULT mod_topic
@ -105,7 +109,6 @@ struct impl {
const struct rtp_format_info *rtp_format_info;
enum spa_direction direction;
void *stream_data;
uint32_t rate;
uint32_t stride;
@ -113,6 +116,15 @@ struct impl {
uint8_t payload;
uint32_t ssrc;
uint16_t seq;
/* This is used for pre-checking for received packets that arrive
* out of order. Depending on the audio type, a jitter buffer
* may be used intermediately, and then, the seq field above
* will be set _after_ the jitter buffer reorders packets,
* so it cannot be used for the pre-checking purpose. Thus,
* this separate seq field is required.
* It is int32_t to be able to store -1 as an indicator that
* no expected seqnum is set yet. */
int32_t next_expected_incoming_seq;
unsigned fixed_ssrc:1;
unsigned have_ssrc:1;
unsigned ignore_ssrc:1;
@ -187,6 +199,19 @@ struct impl {
void (*deinit)(struct impl *impl, enum spa_direction direction);
int (*resend_packets)(struct impl *impl, uint16_t seq, uint16_t num);
struct rtp_jitter_buffer jitter_buffer;
const struct rtp_audio_codec *audio_codec;
struct rtp_audio_codec_context audio_codec_context;
/* This buffer is needed in case the data that is to be encoded is wrapped
* around the ring buffer border. In such a case, the two halves have to
* be copied and merged into this buffer, since audio codecs expect one
* contiguous input memory block as the data to encode. */
uint8_t *audio_encoder_staging_buffer;
/* Delay of the audio codec. Depending on how the audio codec is configured,
* this is either the decoder delay or the encoder delay. In samples. */
size_t codec_delay;
/*
* pw_filter where the filter would be driven at the PTP clock
* rate with RTP sink being driven at the sink driver clock rate
@ -311,7 +336,6 @@ static int do_finish_stopping_state(struct spa_loop *loop, bool async, uint32_t
#include "module-rtp/audio.c"
#include "module-rtp/midi.c"
#include "module-rtp/opus.c"
struct rtp_format_info {
uint32_t media_subtype;
@ -407,6 +431,11 @@ static int stream_start(struct impl *impl)
pw_log_error("error while closing leftover connection: %s", spa_strerror(res));
}
impl->next_expected_incoming_seq = -1;
if (impl->audio_codec != NULL)
reset_rtp_audio_codec(impl, "starting new stream");
impl->reset_ringbuffer(impl);
res = 0;
@ -519,23 +548,18 @@ static void on_stream_state_changed(void *d, enum pw_stream_state old,
}
}
static void update_latency_params(struct impl *impl)
static void fill_latency_params(struct impl *impl, struct spa_pod_builder *b,
const struct spa_pod **params, uint32_t *n_params)
{
uint32_t n_params = 0;
const struct spa_pod *params[2];
uint8_t buffer[1024];
struct spa_pod_builder b;
struct spa_latency_info main_latency;
spa_pod_builder_init(&b, buffer, sizeof(buffer));
/* main_latency is the latency in the direction indicated by impl->direction.
* In RTP streams, this consists solely of the process latency. (In theory,
* PipeWire SPA nodes could have additional latencies on top of the process
* latency, but this is not the case here.) The other direction is already
* handled by pw_stream.
* In RTP streams, this consists of the process latency. In the INPUT direction
* (which is what sinks use), the encoder delay is also part of main_latency.
* The full latency params also include latency in the other direction -
* this is already handled by pw_stream.
*
* The main_latncy is passed as updated SPA_PARAM_Latency params to the stream.
* The main_latency is passed as updated SPA_PARAM_Latency params to the stream.
* That way, the stream always gets information of latency for _both_ directions;
* the direction indicated by impl->direction is covered by main_latency, and
* the opposite direction is already taken care of by the default pw_stream
@ -547,10 +571,27 @@ static void update_latency_params(struct impl *impl)
main_latency = SPA_LATENCY_INFO(impl->direction);
spa_process_latency_info_add(&impl->process_latency, &main_latency);
params[n_params++] = spa_latency_build(&b, SPA_PARAM_Latency, &main_latency);
params[n_params++] = spa_process_latency_build(&b, SPA_PARAM_ProcessLatency,
&impl->process_latency);
if (impl->direction == PW_DIRECTION_INPUT) {
int64_t codec_delay_ns = (int64_t)(impl->codec_delay) * SPA_NSEC_PER_SEC / impl->rate;
main_latency.min_ns += codec_delay_ns;
main_latency.max_ns += codec_delay_ns;
}
params[(*n_params)++] = spa_latency_build(b, SPA_PARAM_Latency, &main_latency);
params[(*n_params)++] = spa_process_latency_build(b, SPA_PARAM_ProcessLatency,
&impl->process_latency);
}
static void update_latency_params(struct impl *impl)
{
const struct spa_pod *params[2];
uint32_t n_params;
uint8_t buffer[1024];
struct spa_pod_builder b;
n_params = 0;
spa_pod_builder_init(&b, buffer, sizeof(buffer));
fill_latency_params(impl, &b, params, &n_params);
pw_stream_update_params(impl->stream, params, n_params);
}
@ -609,11 +650,12 @@ static const struct rtp_format_info *find_rtp_pcm_audio_format_info(const struct
return NULL;
}
static int parse_audio_info(const struct pw_properties *props, struct spa_audio_info_raw *info)
static int parse_audio_info(const struct pw_properties *props, struct spa_audio_info_raw *info,
const char *default_format)
{
return spa_audio_info_raw_init_dict_keys(info,
&SPA_DICT_ITEMS(
SPA_DICT_ITEM(SPA_KEY_AUDIO_FORMAT, DEFAULT_RAW_AUDIO_FORMAT),
SPA_DICT_ITEM(SPA_KEY_AUDIO_FORMAT, default_format),
SPA_DICT_ITEM(SPA_KEY_AUDIO_RATE, SPA_STRINGIFY(DEFAULT_RATE)),
SPA_DICT_ITEM(SPA_KEY_AUDIO_POSITION, DEFAULT_POSITION)),
&props->dict,
@ -660,6 +702,8 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
float latency_msec;
int res;
bool process_latency_from_sess;
uint32_t audio_codec_type = 0;
const char *default_audio_format = NULL;
impl = calloc(1, sizeof(*impl));
if (impl == NULL) {
@ -690,11 +734,15 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
impl->info.media_type = SPA_MEDIA_TYPE_audio;
impl->info.media_subtype = SPA_MEDIA_SUBTYPE_raw;
impl->payload = 127;
audio_codec_type = SPA_MEDIA_SUBTYPE_raw;
default_audio_format = DEFAULT_RAW_AUDIO_FORMAT;
}
else if (spa_streq(str, "raop")) {
impl->info.media_type = SPA_MEDIA_TYPE_audio;
impl->info.media_subtype = SPA_MEDIA_SUBTYPE_raw;
impl->payload = 0x60;
audio_codec_type = SPA_MEDIA_SUBTYPE_raw;
default_audio_format = DEFAULT_RAOP_AUDIO_FORMAT;
}
else if (spa_streq(str, "midi")) {
impl->info.media_type = SPA_MEDIA_TYPE_application;
@ -702,10 +750,14 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
impl->payload = 0x61;
}
#ifdef HAVE_OPUS
/* The "opus" sess.media type is actually raw audio that
* is encoded with Opus before sending it out as RTP. */
else if (spa_streq(str, "opus")) {
impl->info.media_type = SPA_MEDIA_TYPE_audio;
impl->info.media_subtype = SPA_MEDIA_SUBTYPE_opus;
impl->info.media_subtype = SPA_MEDIA_SUBTYPE_raw;
impl->payload = 127;
audio_codec_type = SPA_MEDIA_SUBTYPE_opus;
default_audio_format = DEFAULT_OPUS_AUDIO_FORMAT;
}
#endif
else {
@ -716,22 +768,61 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
switch (impl->info.media_subtype) {
case SPA_MEDIA_SUBTYPE_raw:
if ((res = parse_audio_info(props, &impl->info.info.raw)) < 0) {
if ((res = parse_audio_info(props, &impl->info.info.raw, default_audio_format)) < 0) {
pw_log_error("can't parse format: %s", spa_strerror(res));
goto out;
}
impl->stream_info = impl->info;
impl->rtp_format_info = find_rtp_pcm_audio_format_info(&impl->info);
if (impl->rtp_format_info == NULL) {
pw_log_error("unsupported audio format:%d (%s) channels:%d",
impl->rate = impl->stream_info.info.raw.rate;
/* Pick the RTP information and stride values suitable for
* the specified codec type. If the codec type is set to
* SPA_MEDIA_SUBTYPE_raw, then no special encoding is done,
* and PCM samples are transmitted directly over RTP. */
switch (audio_codec_type) {
case SPA_MEDIA_SUBTYPE_raw:
impl->rtp_format_info = find_rtp_pcm_audio_format_info(&impl->info);
if (impl->rtp_format_info == NULL) {
pw_log_error("unsupported audio format:%d (%s) channels:%d",
impl->stream_info.info.raw.format,
spa_type_audio_format_to_short_name(impl->stream_info.info.raw.format),
impl->stream_info.info.raw.channels);
res = -EINVAL;
goto out;
}
impl->stride = impl->rtp_format_info->size * impl->stream_info.info.raw.channels;
pw_log_info("configured raw PCM RTP payload: MIME: %s format: %s rate: %"
PRIu32 " stride: %" PRIu32, impl->rtp_format_info->mime,
spa_type_audio_format_to_short_name(impl->rtp_format_info->format),
impl->rate, impl->stride);
break;
case SPA_MEDIA_SUBTYPE_opus:
impl->rtp_format_info = &rtp_opus_format_info;
switch (impl->stream_info.info.raw.format) {
case SPA_AUDIO_FORMAT_S16:
impl->stride = 2 * impl->stream_info.info.raw.channels;
break;
case SPA_AUDIO_FORMAT_F32:
impl->stride = 4 * impl->stream_info.info.raw.channels;
break;
default:
pw_log_error("unsupported raw audio format for encoding to Opus:%d (%s)",
impl->stream_info.info.raw.format,
spa_type_audio_format_to_short_name(impl->stream_info.info.raw.format));
res = -EINVAL;
goto out;
}
pw_log_info("configured Opus RTP payload: format: %s rate: %" PRIu32 " stride: %"
PRIu32, spa_type_audio_format_to_short_name(impl->stream_info.info.raw.format),
impl->rate, impl->stride);
break;
default:
pw_log_error("unsupported audio encoding:%d (%s)", audio_codec_type,
spa_type_to_short_name(audio_codec_type,
spa_type_media_subtype, "<unknown>"));
res = -EINVAL;
goto out;
}
impl->stride = impl->rtp_format_info->size * impl->stream_info.info.raw.channels;
impl->rate = impl->stream_info.info.raw.rate;
break;
case SPA_MEDIA_SUBTYPE_control:
impl->stream_info = impl->info;
@ -741,21 +832,8 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
impl->rate = pw_properties_get_uint32(props, "midi.rate", 10000);
if (impl->rate == 0)
impl->rate = 10000;
break;
case SPA_MEDIA_SUBTYPE_opus:
impl->stream_info.media_type = SPA_MEDIA_TYPE_audio;
impl->stream_info.media_subtype = SPA_MEDIA_SUBTYPE_raw;
if ((res = parse_audio_info(props, &impl->stream_info.info.raw)) < 0) {
pw_log_error("can't parse format: %s", spa_strerror(res));
goto out;
}
impl->stream_info.info.raw.format = SPA_AUDIO_FORMAT_F32;
impl->info.info.opus.rate = impl->stream_info.info.raw.rate;
impl->info.info.opus.channels = impl->stream_info.info.raw.channels;
impl->rtp_format_info = &rtp_opus_format_info;
impl->stride = impl->rtp_format_info->size * impl->stream_info.info.raw.channels;
impl->rate = impl->stream_info.info.raw.rate;
pw_log_info("configured MIDI RTP payload: rate: %" PRIu32 " stride: %" PRIu32,
impl->rate, impl->stride);
break;
default:
spa_assert_not_reached();
@ -845,6 +923,7 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
impl->payload_size = impl->mtu - impl->header_size;
impl->seq = pw_rand32();
impl->next_expected_incoming_seq = -1;
str = pw_properties_get(props, "sess.min-ptime");
if (!spa_atof(str, &min_ptime))
@ -954,6 +1033,26 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
spa_dll_set_bw(&impl->dll, SPA_DLL_BW_MIN, 128, impl->rate);
impl->corr = 1.0;
switch (impl->info.media_subtype) {
case SPA_MEDIA_SUBTYPE_raw:
switch (audio_codec_type) {
case SPA_MEDIA_SUBTYPE_opus:
res = setup_rtp_audio_codec(impl, get_rtp_opus_codec(), props);
break;
default:
res = 0;
break;
}
if (SPA_UNLIKELY(res < 0))
goto out;
break;
default:
break;
}
impl->stream = pw_stream_new(core, "rtp-session", spa_steal_ptr(props));
if (impl->stream == NULL) {
res = -errno;
@ -980,12 +1079,6 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_control));
rtp_midi_init(impl, direction);
break;
case SPA_MEDIA_SUBTYPE_opus:
params[n_params++] = spa_format_audio_build(&b,
SPA_PARAM_EnumFormat, &impl->stream_info);
flags |= PW_STREAM_FLAG_AUTOCONNECT;
rtp_opus_init(impl, direction);
break;
default:
res = -EINVAL;
goto out;
@ -998,24 +1091,20 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
* quantity in turn is subjected to constraint checks (see above), it is
* possible that the _actual_ session latency no longer equals the value
* of sess.latency.msec by the time this location is reached. To take into
* account these constraint adjustments, convert back the impl->target_buffer
* to nanoseconds, and use that as the process latency.
* account these constraint adjustments, fill_latency_params() converts
* back the impl->target_buffer to nanoseconds, and uses that as the
* process latency.
*
* Then, just like how update_latency_params() does it, construct the
* SPA_PARAM_Latency and SPA_PARAM_ProcessLatency params to let the new
* Then, just like in update_latency_params(), the SPA_PARAM_Latency
* and SPA_PARAM_ProcessLatency params are constructed to let the new
* pw_stream know of these latency figures right from the start. */
struct spa_latency_info latency;
impl->process_latency.ns = (int64_t)(impl->target_buffer * 1e9 / impl->rate);
pw_log_debug("set process latency to %" PRId64 " based on sess.latency.msec "
"value %f", impl->process_latency.ns, latency_msec);
pw_log_debug("set process latency to %" PRId64 " ns based on sess.latency.msec "
"value %f ms (= %" PRIu32 " samples)", impl->process_latency.ns,
latency_msec, impl->target_buffer);
latency = SPA_LATENCY_INFO(impl->direction);
spa_process_latency_info_add(&(impl->process_latency), &latency);
params[n_params++] = spa_latency_build(&b, SPA_PARAM_Latency, &latency);
params[n_params++] = spa_process_latency_build(&b, SPA_PARAM_ProcessLatency,
&(impl->process_latency));
fill_latency_params(impl, &b, params, &n_params);
}
pw_stream_add_listener(impl->stream,
@ -1040,6 +1129,7 @@ struct rtp_stream *rtp_stream_new(struct pw_core *core,
return (struct rtp_stream*)impl;
out:
teardown_rtp_audio_codec(impl);
pw_properties_free(props);
if (impl) {
if (impl->stream)
@ -1058,6 +1148,8 @@ void rtp_stream_destroy(struct rtp_stream *s)
rtp_stream_emit_destroy(impl);
teardown_rtp_audio_codec(impl);
if (impl->deinit)
impl->deinit(impl, impl->direction);
@ -1142,7 +1234,17 @@ unexpected_ssrc:
int rtp_stream_resend_packets(struct rtp_stream *s, uint16_t seq, uint16_t num)
{
struct impl *impl = (struct impl*)s;
if (impl->resend_packets)
/* Resending only works with raw data, since codecs usually
* have internal state that is updated after encoding a
* packet. This then means that resend attempts would not
* yield the exact same packet, and this can corrupt the
* state of the decoder in a receiver.
* To support resending with codecs, the last N packets
* have to be cached somehow. Given the fact that, thus
* far, only the RAOP sink resends packets, and RAOP only
* supports raw PCM, it is currently easier to just
* disable retransmissions when a codec is in use. */
if (impl->resend_packets && (impl->audio_codec == NULL))
return impl->resend_packets(impl, seq, num);
else
return -ENOTSUP;