From 2d30ab94c2f72d3814cf11a599e7392784eb0437 Mon Sep 17 00:00:00 2001 From: Pauli Virtanen Date: Fri, 23 Feb 2024 21:04:47 +0200 Subject: [PATCH] bluez5: account for codec internal delay in latency values Encoders and some decoders have additional internal latency that needs to be accounted for. This mostly matters for AAC (~40ms), as the other BT codecs have much lower delays (~5ms). --- spa/plugins/bluez5/a2dp-codec-aac.c | 24 ++++++++++++++++++++++ spa/plugins/bluez5/a2dp-codec-aptx.c | 13 +++++++++++- spa/plugins/bluez5/a2dp-codec-faststream.c | 11 +++++++++- spa/plugins/bluez5/a2dp-codec-ldac.c | 20 ++++++++++++++++++ spa/plugins/bluez5/a2dp-codec-opus-g.c | 18 ++++++++++++++++ spa/plugins/bluez5/a2dp-codec-opus.c | 20 +++++++++++++++++- spa/plugins/bluez5/a2dp-codec-sbc.c | 16 +++++++++++++++ spa/plugins/bluez5/media-codecs.h | 13 +++++++++++- spa/plugins/bluez5/media-sink.c | 20 ++++++++++++------ spa/plugins/bluez5/media-source.c | 6 ++++++ 10 files changed, 151 insertions(+), 10 deletions(-) diff --git a/spa/plugins/bluez5/a2dp-codec-aac.c b/spa/plugins/bluez5/a2dp-codec-aac.c index 630c90192..68d891e40 100644 --- a/spa/plugins/bluez5/a2dp-codec-aac.c +++ b/spa/plugins/bluez5/a2dp-codec-aac.c @@ -40,6 +40,9 @@ struct impl { uint32_t rate; uint32_t channels; int samplesize; + + uint32_t enc_delay; + uint32_t dec_delay; }; static bool eld_supported(void) @@ -443,6 +446,8 @@ static void *codec_init(const struct media_codec *codec, uint32_t flags, if (res != AACENC_OK) goto error; + this->enc_delay = enc_info.nDelay; + this->codesize = enc_info.frameLength * this->channels * this->samplesize; this->aacdec = aacDecoder_Open(TT_MP4_LATM_MCP1, 1); @@ -471,6 +476,8 @@ static void *codec_init(const struct media_codec *codec, uint32_t flags, } #endif + this->dec_delay = 0; + return this; error: @@ -650,6 +657,21 @@ static int codec_increase_bitpool(void *data) return codec_change_bitrate(this, (this->cur_bitrate * 4) / 3); } +static void codec_get_delay(void *data, uint32_t *encoder, uint32_t *decoder) +{ + struct impl *this = data; + + if (encoder) + *encoder = this->enc_delay; + + if (decoder) { + CStreamInfo *info = aacDecoder_GetStreamInfo(this->aacdec); + if (info) + this->dec_delay = info->outputDelay; + *decoder = this->dec_delay; + } +} + static void codec_set_log(struct spa_log *global_log) { log = global_log; @@ -678,6 +700,7 @@ const struct media_codec a2dp_codec_aac = { .reduce_bitpool = codec_reduce_bitpool, .increase_bitpool = codec_increase_bitpool, .set_log = codec_set_log, + .get_delay = codec_get_delay, }; const struct media_codec a2dp_codec_aac_eld = { @@ -703,6 +726,7 @@ const struct media_codec a2dp_codec_aac_eld = { .reduce_bitpool = codec_reduce_bitpool, .increase_bitpool = codec_increase_bitpool, .set_log = codec_set_log, + .get_delay = codec_get_delay, }; MEDIA_CODEC_EXPORT_DEF( diff --git a/spa/plugins/bluez5/a2dp-codec-aptx.c b/spa/plugins/bluez5/a2dp-codec-aptx.c index 2682d9d1c..2aa364740 100644 --- a/spa/plugins/bluez5/a2dp-codec-aptx.c +++ b/spa/plugins/bluez5/a2dp-codec-aptx.c @@ -449,6 +449,14 @@ static int codec_decode(void *data, return res; } +static void codec_get_delay(void *data, uint32_t *encoder, uint32_t *decoder) +{ + if (encoder) + *encoder = 90; + if (decoder) + *decoder = 0; +} + /* * mSBC duplex codec * @@ -627,6 +635,7 @@ const struct media_codec a2dp_codec_aptx = { .decode = codec_decode, .reduce_bitpool = codec_reduce_bitpool, .increase_bitpool = codec_increase_bitpool, + .get_delay = codec_get_delay, }; @@ -650,6 +659,7 @@ const struct media_codec a2dp_codec_aptx_hd = { .decode = codec_decode, .reduce_bitpool = codec_reduce_bitpool, .increase_bitpool = codec_increase_bitpool, + .get_delay = codec_get_delay, }; #define APTX_LL_COMMON_DEFS \ @@ -665,7 +675,8 @@ const struct media_codec a2dp_codec_aptx_hd = { .start_encode = codec_start_encode, \ .encode = codec_encode, \ .reduce_bitpool = codec_reduce_bitpool, \ - .increase_bitpool = codec_increase_bitpool + .increase_bitpool = codec_increase_bitpool, \ + .get_delay = codec_get_delay const struct media_codec a2dp_codec_aptx_ll_0 = { diff --git a/spa/plugins/bluez5/a2dp-codec-faststream.c b/spa/plugins/bluez5/a2dp-codec-faststream.c index 9e09d5560..9bcc0310d 100644 --- a/spa/plugins/bluez5/a2dp-codec-faststream.c +++ b/spa/plugins/bluez5/a2dp-codec-faststream.c @@ -554,6 +554,14 @@ static int duplex_decode(void *data, return res; } +static void codec_get_delay(void *data, uint32_t *encoder, uint32_t *decoder) +{ + if (encoder) + *encoder = 73; + if (decoder) + *decoder = 0; +} + /* Voice channel SBC, not a real A2DP codec */ static const struct media_codec duplex_codec = { .codec_id = A2DP_CODEC_VENDOR, @@ -590,7 +598,8 @@ static const struct media_codec duplex_codec = { .start_encode = codec_start_encode, \ .encode = codec_encode, \ .reduce_bitpool = codec_reduce_bitpool, \ - .increase_bitpool = codec_increase_bitpool + .increase_bitpool = codec_increase_bitpool, \ + .get_delay = codec_get_delay const struct media_codec a2dp_codec_faststream = { FASTSTREAM_COMMON_DEFS, diff --git a/spa/plugins/bluez5/a2dp-codec-ldac.c b/spa/plugins/bluez5/a2dp-codec-ldac.c index 3ec20b720..bebdf55fe 100644 --- a/spa/plugins/bluez5/a2dp-codec-ldac.c +++ b/spa/plugins/bluez5/a2dp-codec-ldac.c @@ -551,6 +551,25 @@ static int codec_encode(void *data, return src_used; } +static void codec_get_delay(void *data, uint32_t *encoder, uint32_t *decoder) +{ + struct impl *this = data; + + if (encoder) { + switch (this->frequency) { + case 96000: + case 88200: + *encoder = 256; + break; + default: + *encoder = 128; + break; + } + } + if (decoder) + *decoder = 0; +} + const struct media_codec a2dp_codec_ldac = { .id = SPA_BLUETOOTH_AUDIO_CODEC_LDAC, .codec_id = A2DP_CODEC_VENDOR, @@ -577,6 +596,7 @@ const struct media_codec a2dp_codec_ldac = { .encode = codec_encode, .reduce_bitpool = codec_reduce_bitpool, .increase_bitpool = codec_increase_bitpool, + .get_delay = codec_get_delay, }; MEDIA_CODEC_EXPORT_DEF( diff --git a/spa/plugins/bluez5/a2dp-codec-opus-g.c b/spa/plugins/bluez5/a2dp-codec-opus-g.c index f3b3802b6..313c0a089 100644 --- a/spa/plugins/bluez5/a2dp-codec-opus-g.c +++ b/spa/plugins/bluez5/a2dp-codec-opus-g.c @@ -26,6 +26,7 @@ static struct spa_log *log; struct dec_data { + int32_t delay; }; struct enc_data { @@ -37,6 +38,8 @@ struct enc_data { int frame_dms; int bitrate; int packet_size; + + int32_t delay; }; struct impl { @@ -334,6 +337,8 @@ static void *codec_init(const struct media_codec *codec, uint32_t flags, opus_encoder_ctl(this->enc, OPUS_SET_BITRATE(this->e.bitrate)); + opus_encoder_ctl(this->enc, OPUS_GET_LOOKAHEAD(&this->e.delay)); + /* * Setup decoder */ @@ -343,6 +348,8 @@ static void *codec_init(const struct media_codec *codec, uint32_t flags, goto error; } + opus_decoder_ctl(this->dec, OPUS_GET_LOOKAHEAD(&this->d.delay)); + return this; error_errno: @@ -487,6 +494,16 @@ static int codec_increase_bitpool(void *data) return 0; } +static void codec_get_delay(void *data, uint32_t *encoder, uint32_t *decoder) +{ + struct impl *this = data; + + if (encoder) + *encoder = this->e.delay; + if (decoder) + *decoder = this->d.delay; +} + static void codec_set_log(struct spa_log *global_log) { log = global_log; @@ -516,6 +533,7 @@ const struct media_codec a2dp_codec_opus_g = { .name = "opus_g", .description = "Opus", .fill_caps = codec_fill_caps, + .get_delay = codec_get_delay, }; MEDIA_CODEC_EXPORT_DEF( diff --git a/spa/plugins/bluez5/a2dp-codec-opus.c b/spa/plugins/bluez5/a2dp-codec-opus.c index 92a1bf664..1dcfe2708 100644 --- a/spa/plugins/bluez5/a2dp-codec-opus.c +++ b/spa/plugins/bluez5/a2dp-codec-opus.c @@ -80,6 +80,8 @@ struct dec_data { int fragment_size; int fragment_count; uint8_t fragment[OPUS_05_MAX_BYTES]; + + int32_t delay; }; struct abr { @@ -119,6 +121,8 @@ struct enc_data { int frame_dms; int application; + + int32_t delay; }; struct impl { @@ -1005,6 +1009,7 @@ static void *codec_init(const struct media_codec *codec, uint32_t flags, this->e.samples = this->e.frame_dms * this->samplerate / 10000; this->e.codesize = this->e.samples * (int)this->channels * sizeof(float); + opus_multistream_encoder_ctl(this->enc, OPUS_GET_LOOKAHEAD(&this->e.delay)); /* * Setup decoder @@ -1020,6 +1025,8 @@ static void *codec_init(const struct media_codec *codec, uint32_t flags, goto error; } + opus_multistream_decoder_ctl(this->dec, OPUS_GET_LOOKAHEAD(&this->d.delay)); + return this; error_errno: @@ -1325,6 +1332,16 @@ static int codec_increase_bitpool(void *data) return 0; } +static void codec_get_delay(void *data, uint32_t *encoder, uint32_t *decoder) +{ + struct impl *this = data; + + if (encoder) + *encoder = this->e.delay; + if (decoder) + *decoder = this->d.delay; +} + static void codec_set_log(struct spa_log *global_log) { log = global_log; @@ -1347,7 +1364,8 @@ static void codec_set_log(struct spa_log *global_log) .encode = codec_encode, \ .reduce_bitpool = codec_reduce_bitpool, \ .increase_bitpool = codec_increase_bitpool, \ - .set_log = codec_set_log + .set_log = codec_set_log, \ + .get_delay = codec_get_delay #define OPUS_05_COMMON_FULL_DEFS \ OPUS_05_COMMON_DEFS, \ diff --git a/spa/plugins/bluez5/a2dp-codec-sbc.c b/spa/plugins/bluez5/a2dp-codec-sbc.c index 3397a8f5a..1a9627ad0 100644 --- a/spa/plugins/bluez5/a2dp-codec-sbc.c +++ b/spa/plugins/bluez5/a2dp-codec-sbc.c @@ -29,6 +29,8 @@ struct impl { int min_bitpool; int max_bitpool; + + uint32_t enc_delay; }; static int codec_fill_caps(const struct media_codec *codec, uint32_t flags, @@ -497,9 +499,11 @@ static void *codec_init(const struct media_codec *codec, uint32_t flags, switch (conf->subbands) { case SBC_SUBBANDS_4: this->sbc.subbands = SBC_SB_4; + this->enc_delay = 37; break; case SBC_SUBBANDS_8: this->sbc.subbands = SBC_SB_8; + this->enc_delay = 73; break; default: res = -EINVAL; @@ -618,6 +622,16 @@ static int codec_decode(void *data, return res; } +static void codec_get_delay(void *data, uint32_t *encoder, uint32_t *decoder) +{ + struct impl *this = data; + + if (encoder) + *encoder = this->enc_delay; + if (decoder) + *decoder = 0; +} + const struct media_codec a2dp_codec_sbc = { .id = SPA_BLUETOOTH_AUDIO_CODEC_SBC, .codec_id = A2DP_CODEC_SBC, @@ -638,6 +652,7 @@ const struct media_codec a2dp_codec_sbc = { .decode = codec_decode, .reduce_bitpool = codec_reduce_bitpool, .increase_bitpool = codec_increase_bitpool, + .get_delay = codec_get_delay, }; const struct media_codec a2dp_codec_sbc_xq = { @@ -661,6 +676,7 @@ const struct media_codec a2dp_codec_sbc_xq = { .decode = codec_decode, .reduce_bitpool = codec_reduce_bitpool, .increase_bitpool = codec_increase_bitpool, + .get_delay = codec_get_delay, }; MEDIA_CODEC_EXPORT_DEF( diff --git a/spa/plugins/bluez5/media-codecs.h b/spa/plugins/bluez5/media-codecs.h index a1b0a9f2a..87e2bf88f 100644 --- a/spa/plugins/bluez5/media-codecs.h +++ b/spa/plugins/bluez5/media-codecs.h @@ -26,7 +26,7 @@ #define SPA_TYPE_INTERFACE_Bluez5CodecMedia SPA_TYPE_INFO_INTERFACE_BASE "Bluez5:Codec:Media:Private" -#define SPA_VERSION_BLUEZ5_CODEC_MEDIA 9 +#define SPA_VERSION_BLUEZ5_CODEC_MEDIA 10 struct spa_bluez5_codec_a2dp { struct spa_interface iface; @@ -199,6 +199,17 @@ struct media_codec { int (*increase_bitpool) (void *data); void (*set_log) (struct spa_log *global_log); + + /** + * Get codec internal delays, in samples at input/output rates. + * + * The delay does not include the duration of the PCM input/output + * audio data, but is that internal to the codec. + * + * \param[out] encoder Encoder delay in samples, or NULL + * \param[out] decoder Decoder delay in samples, or NULL + */ + void (*get_delay) (void *data, uint32_t *encoder, uint32_t *decoder); }; struct media_codec_config { diff --git a/spa/plugins/bluez5/media-sink.c b/spa/plugins/bluez5/media-sink.c index 1e822f776..007513e5e 100644 --- a/spa/plugins/bluez5/media-sink.c +++ b/spa/plugins/bluez5/media-sink.c @@ -165,6 +165,8 @@ struct impl { uint64_t packet_delay_ns; struct spa_source *update_delay_event; + uint32_t encoder_delay; + const struct media_codec *codec; bool codec_props_changed; void *codec_props; @@ -380,7 +382,7 @@ static void set_latency(struct impl *this, bool emit_latency) /* in main loop */ - if (this->transport == NULL) + if (this->transport == NULL || !port->have_format) return; /* @@ -388,12 +390,13 @@ static void set_latency(struct impl *this, bool emit_latency) * * (packet delay) + (codec internal delay) + (transport delay) + (latency offset) * - * and doesn't depend on the quantum. The codec internal delay is neglected. - * Kernel knows the latency due to socket/controller queue, but doesn't - * tell us, so not included but hopefully in < 20 ms range. + * and doesn't depend on the quantum. Kernel knows the latency due to + * socket/controller queue, but doesn't tell us, so not included but + * hopefully in < 10 ms range. */ delay = __atomic_load_n(&this->packet_delay_ns, __ATOMIC_RELAXED); + delay += (int64_t)this->encoder_delay * SPA_NSEC_PER_SEC / port->current_format.info.raw.rate; delay += spa_bt_transport_get_delay_nsec(this->transport); delay += SPA_CLAMP(this->props.latency_offset, -delay, INT64_MAX / 2); delay = SPA_MAX(delay, 0); @@ -1250,9 +1253,14 @@ static int transport_start(struct impl *this) this->codec_props_changed = true; } - spa_log_info(this->log, "%p: using %s codec %s, delay:%"PRIi64" ms", this, + this->encoder_delay = 0; + if (this->codec->get_delay) + this->codec->get_delay(this->codec_data, &this->encoder_delay, NULL); + + spa_log_info(this->log, "%p: using %s codec %s, delay:%.2f ms, codec-delay:%.2f ms", this, this->codec->bap ? "BAP" : "A2DP", this->codec->description, - (int64_t)(spa_bt_transport_get_delay_nsec(this->transport) / SPA_NSEC_PER_MSEC)); + (double)spa_bt_transport_get_delay_nsec(this->transport) / SPA_NSEC_PER_MSEC, + (double)this->encoder_delay * SPA_MSEC_PER_SEC / port->current_format.info.raw.rate); this->seqnum = UINT16_MAX; diff --git a/spa/plugins/bluez5/media-source.c b/spa/plugins/bluez5/media-source.c index 5f3fb522e..d35e57d6e 100644 --- a/spa/plugins/bluez5/media-source.c +++ b/spa/plugins/bluez5/media-source.c @@ -1560,6 +1560,12 @@ static void process_buffering(struct impl *this) if (this->update_delay_event) { int32_t target = spa_bt_decode_buffer_get_target_latency(&port->buffer); + uint32_t decoder_delay = 0; + + if (this->codec->get_delay) + this->codec->get_delay(this->codec_data, NULL, &decoder_delay); + + target += decoder_delay; if (target != this->delay.buffer || duration != this->delay.duration) { struct delay_info info = { .buffer = target, .duration = duration };