audioconvert: fix dither scale

Rectangular dither should be [-0.5, 0.5]
Triangular dither should be [-1.0, 1.0]
Noise should add extra bits.
This commit is contained in:
Wim Taymans 2022-07-11 17:19:28 +02:00
parent e313149f7f
commit 68f883ff77
4 changed files with 23 additions and 3 deletions

View file

@ -394,7 +394,10 @@ int convert_init(struct convert *conv)
const struct conv_info *info;
uint32_t i, dither_flags;
conv->scale = 1.0f / (float)(INT32_MAX >> conv->noise);
conv->scale = 1.0f / (float)(INT32_MAX);
if (conv->noise > 0)
conv->scale *= (1 << (conv->noise + 1));
/* disable dither if not needed */
if (!need_dither(conv->dst_fmt))