resample: use fixed point for resample phase and input rate

If phase is float, calculations in impl_native_in_len/out_len can
produce wrong results due to rounding error.

It's probably better to not be in the business of predicting
floating-point rounding, so replace this by fixed-point arithmetic.

Also make sure `offset+1` cannot overflow data->filter array in
do_resample_inter* due to float multiplication possibly rounding up.
This commit is contained in:
Pauli Virtanen 2025-07-26 17:23:44 +03:00 committed by Wim Taymans
parent 3cade43cf3
commit 244d5a1cc1
3 changed files with 61 additions and 32 deletions

View file

@ -126,20 +126,20 @@ static void pull_blocks_out(struct resample *r, uint32_t first, uint32_t size, u
}
}
static void check_inout_len(struct resample *r, uint32_t first, uint32_t size, double rate, double phase)
static void check_inout_len(struct resample *r, uint32_t first, uint32_t size, double rate, float phase)
{
struct native_data *data = r->data;
resample_reset(r);
resample_update_rate(r, rate);
if (phase != 0.0)
data->phase = (float)phase;
data->phase = FLOAT_TO_FIXP(phase);
pull_blocks(r, first, size, 500);
resample_reset(r);
resample_update_rate(r, rate);
if (phase != 0.0)
data->phase = (float)phase;
data->phase = FLOAT_TO_FIXP(phase);
pull_blocks_out(r, first, size, 500);
}
@ -237,7 +237,7 @@ static void test_inout_len(void)
r.options = RESAMPLE_OPTION_PREFILL;
resample_native_init(&r);
check_inout_len(&r, 64, 64, 1.0 + 1e-10, 7999.99);
check_inout_len(&r, 64, 64, 1.0 + 1e-10, 7999.99f);
resample_free(&r);
/* Test value of phase that overflows filter buffer due to floating point rounding