diff --git a/spa/plugins/audioconvert/fmt-ops-avx2.c b/spa/plugins/audioconvert/fmt-ops-avx2.c new file mode 100644 index 000000000..014687144 --- /dev/null +++ b/spa/plugins/audioconvert/fmt-ops-avx2.c @@ -0,0 +1,772 @@ +/* Spa + * + * Copyright © 2018 Wim Taymans + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "fmt-ops.h" + +#include + +static void +conv_s16_to_f32d_1s_avx2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, + uint32_t n_channels, uint32_t n_samples) +{ + const int16_t *s = src; + float *d0 = dst[0]; + uint32_t n, unrolled; + __m128i in; + __m128 out, factor = _mm_set1_ps(1.0f / S16_SCALE); + + if (SPA_LIKELY(SPA_IS_ALIGNED(d0, 16))) + unrolled = n_samples & ~3; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in = _mm_insert_epi16(in, s[0*n_channels], 1); + in = _mm_insert_epi16(in, s[1*n_channels], 3); + in = _mm_insert_epi16(in, s[2*n_channels], 5); + in = _mm_insert_epi16(in, s[3*n_channels], 7); + in = _mm_srai_epi32(in, 16); + out = _mm_cvtepi32_ps(in); + out = _mm_mul_ps(out, factor); + _mm_store_ps(&d0[n], out); + s += 4*n_channels; + } + for(; n < n_samples; n++) { + out = _mm_cvtsi32_ss(out, s[0]); + out = _mm_mul_ss(out, factor); + _mm_store_ss(&d0[n], out); + s += n_channels; + } +} + +void +conv_s16_to_f32d_avx2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], + uint32_t n_samples) +{ + const int16_t *s = src[0]; + uint32_t i = 0, n_channels = conv->n_channels; + + for(; i < n_channels; i++) + conv_s16_to_f32d_1s_avx2(conv, &dst[i], &s[i], n_channels, n_samples); +} + +void +conv_s16_to_f32d_2_avx2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], + uint32_t n_samples) +{ + const int16_t *s = src[0]; + float *d0 = dst[0], *d1 = dst[1]; + uint32_t n, unrolled; + __m128i in[2], t[4]; + __m128 out[4], factor = _mm_set1_ps(1.0f / S16_SCALE); + + if (SPA_IS_ALIGNED(s, 16) && + SPA_IS_ALIGNED(d0, 16) && + SPA_IS_ALIGNED(d1, 16)) + unrolled = n_samples & ~7; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 8) { + in[0] = _mm_load_si128((__m128i*)(s + 0)); + in[1] = _mm_load_si128((__m128i*)(s + 8)); + + t[0] = _mm_slli_epi32(in[0], 16); + t[0] = _mm_srai_epi32(t[0], 16); + out[0] = _mm_cvtepi32_ps(t[0]); + out[0] = _mm_mul_ps(out[0], factor); + + t[1] = _mm_srai_epi32(in[0], 16); + out[1] = _mm_cvtepi32_ps(t[1]); + out[1] = _mm_mul_ps(out[1], factor); + + t[2] = _mm_slli_epi32(in[1], 16); + t[2] = _mm_srai_epi32(t[2], 16); + out[2] = _mm_cvtepi32_ps(t[2]); + out[2] = _mm_mul_ps(out[2], factor); + + t[3] = _mm_srai_epi32(in[1], 16); + out[3] = _mm_cvtepi32_ps(t[3]); + out[3] = _mm_mul_ps(out[3], factor); + + _mm_store_ps(&d0[n + 0], out[0]); + _mm_store_ps(&d1[n + 0], out[1]); + _mm_store_ps(&d0[n + 4], out[2]); + _mm_store_ps(&d1[n + 4], out[3]); + + s += 16; + } + for(; n < n_samples; n++) { + out[0] = _mm_cvtsi32_ss(out[0], s[0]); + out[0] = _mm_mul_ss(out[0], factor); + out[1] = _mm_cvtsi32_ss(out[1], s[1]); + out[1] = _mm_mul_ss(out[1], factor); + _mm_store_ss(&d0[n], out[0]); + _mm_store_ss(&d1[n], out[1]); + s += 2; + } +} + +void +conv_s24_to_f32d_1s_avx2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, + uint32_t n_channels, uint32_t n_samples) +{ + const uint8_t *s = src; + float *d0 = dst[0]; + uint32_t n, unrolled; + __m128i in; + __m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE); + + if (SPA_IS_ALIGNED(d0, 16) && n_samples > 0) { + unrolled = n_samples & ~3; + if ((n_samples & 3) == 0) + unrolled -= 4; + } + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in = _mm_setr_epi32( + *((uint32_t*)&s[0 * n_channels]), + *((uint32_t*)&s[3 * n_channels]), + *((uint32_t*)&s[6 * n_channels]), + *((uint32_t*)&s[9 * n_channels])); + in = _mm_slli_epi32(in, 8); + in = _mm_srai_epi32(in, 8); + out = _mm_cvtepi32_ps(in); + out = _mm_mul_ps(out, factor); + _mm_store_ps(&d0[n], out); + s += 12 * n_channels; + } + for(; n < n_samples; n++) { + out = _mm_cvtsi32_ss(out, read_s24(s)); + out = _mm_mul_ss(out, factor); + _mm_store_ss(&d0[n], out); + s += 3 * n_channels; + } +} + +static void +conv_s24_to_f32d_2s_avx2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, + uint32_t n_channels, uint32_t n_samples) +{ + const uint8_t *s = src; + float *d0 = dst[0], *d1 = dst[1]; + uint32_t n, unrolled; + __m128i in[2]; + __m128 out[2], factor = _mm_set1_ps(1.0f / S24_SCALE); + + if (SPA_IS_ALIGNED(d0, 16) && + SPA_IS_ALIGNED(d1, 16) && + n_samples > 0) { + unrolled = n_samples & ~3; + if ((n_samples & 3) == 0) + unrolled -= 4; + } + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in[0] = _mm_setr_epi32( + *((uint32_t*)&s[0 + 0*n_channels]), + *((uint32_t*)&s[0 + 3*n_channels]), + *((uint32_t*)&s[0 + 6*n_channels]), + *((uint32_t*)&s[0 + 9*n_channels])); + in[1] = _mm_setr_epi32( + *((uint32_t*)&s[3 + 0*n_channels]), + *((uint32_t*)&s[3 + 3*n_channels]), + *((uint32_t*)&s[3 + 6*n_channels]), + *((uint32_t*)&s[3 + 9*n_channels])); + + in[0] = _mm_slli_epi32(in[0], 8); + in[1] = _mm_slli_epi32(in[1], 8); + + in[0] = _mm_srai_epi32(in[0], 8); + in[1] = _mm_srai_epi32(in[1], 8); + + out[0] = _mm_cvtepi32_ps(in[0]); + out[1] = _mm_cvtepi32_ps(in[1]); + + out[0] = _mm_mul_ps(out[0], factor); + out[1] = _mm_mul_ps(out[1], factor); + + _mm_store_ps(&d0[n], out[0]); + _mm_store_ps(&d1[n], out[1]); + + s += 12 * n_channels; + } + for(; n < n_samples; n++) { + out[0] = _mm_cvtsi32_ss(out[0], read_s24(s)); + out[1] = _mm_cvtsi32_ss(out[1], read_s24(s+3)); + out[0] = _mm_mul_ss(out[0], factor); + out[1] = _mm_mul_ss(out[1], factor); + _mm_store_ss(&d0[n], out[0]); + _mm_store_ss(&d1[n], out[1]); + s += 3 * n_channels; + } +} +static void +conv_s24_to_f32d_4s_avx2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, + uint32_t n_channels, uint32_t n_samples) +{ + const uint8_t *s = src; + float *d0 = dst[0], *d1 = dst[1], *d2 = dst[2], *d3 = dst[3]; + uint32_t n, unrolled; + __m128i in[4]; + __m128 out[4], factor = _mm_set1_ps(1.0f / S24_SCALE); + + if (SPA_IS_ALIGNED(d0, 16) && + SPA_IS_ALIGNED(d1, 16) && + SPA_IS_ALIGNED(d2, 16) && + SPA_IS_ALIGNED(d3, 16) && + n_samples > 0) { + unrolled = n_samples & ~3; + if ((n_samples & 3) == 0) + unrolled -= 4; + } + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in[0] = _mm_setr_epi32( + *((uint32_t*)&s[0 + 0*n_channels]), + *((uint32_t*)&s[0 + 3*n_channels]), + *((uint32_t*)&s[0 + 6*n_channels]), + *((uint32_t*)&s[0 + 9*n_channels])); + in[1] = _mm_setr_epi32( + *((uint32_t*)&s[3 + 0*n_channels]), + *((uint32_t*)&s[3 + 3*n_channels]), + *((uint32_t*)&s[3 + 6*n_channels]), + *((uint32_t*)&s[3 + 9*n_channels])); + in[2] = _mm_setr_epi32( + *((uint32_t*)&s[6 + 0*n_channels]), + *((uint32_t*)&s[6 + 3*n_channels]), + *((uint32_t*)&s[6 + 6*n_channels]), + *((uint32_t*)&s[6 + 9*n_channels])); + in[3] = _mm_setr_epi32( + *((uint32_t*)&s[9 + 0*n_channels]), + *((uint32_t*)&s[9 + 3*n_channels]), + *((uint32_t*)&s[9 + 6*n_channels]), + *((uint32_t*)&s[9 + 9*n_channels])); + + in[0] = _mm_slli_epi32(in[0], 8); + in[1] = _mm_slli_epi32(in[1], 8); + in[2] = _mm_slli_epi32(in[2], 8); + in[3] = _mm_slli_epi32(in[3], 8); + + in[0] = _mm_srai_epi32(in[0], 8); + in[1] = _mm_srai_epi32(in[1], 8); + in[2] = _mm_srai_epi32(in[2], 8); + in[3] = _mm_srai_epi32(in[3], 8); + + out[0] = _mm_cvtepi32_ps(in[0]); + out[1] = _mm_cvtepi32_ps(in[1]); + out[2] = _mm_cvtepi32_ps(in[2]); + out[3] = _mm_cvtepi32_ps(in[3]); + + out[0] = _mm_mul_ps(out[0], factor); + out[1] = _mm_mul_ps(out[1], factor); + out[2] = _mm_mul_ps(out[2], factor); + out[3] = _mm_mul_ps(out[3], factor); + + _mm_store_ps(&d0[n], out[0]); + _mm_store_ps(&d1[n], out[1]); + _mm_store_ps(&d2[n], out[2]); + _mm_store_ps(&d3[n], out[3]); + + s += 12 * n_channels; + } + for(; n < n_samples; n++) { + out[0] = _mm_cvtsi32_ss(out[0], read_s24(s)); + out[1] = _mm_cvtsi32_ss(out[1], read_s24(s+3)); + out[2] = _mm_cvtsi32_ss(out[2], read_s24(s+6)); + out[3] = _mm_cvtsi32_ss(out[3], read_s24(s+9)); + out[0] = _mm_mul_ss(out[0], factor); + out[1] = _mm_mul_ss(out[1], factor); + out[2] = _mm_mul_ss(out[2], factor); + out[3] = _mm_mul_ss(out[3], factor); + _mm_store_ss(&d0[n], out[0]); + _mm_store_ss(&d1[n], out[1]); + _mm_store_ss(&d2[n], out[2]); + _mm_store_ss(&d3[n], out[3]); + s += 3 * n_channels; + } +} + +void +conv_s24_to_f32d_avx2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], + uint32_t n_samples) +{ + const int8_t *s = src[0]; + uint32_t i = 0, n_channels = conv->n_channels; + + for(; i + 3 < n_channels; i += 4) + conv_s24_to_f32d_4s_avx2(conv, &dst[i], &s[3*i], n_channels, n_samples); + for(; i + 1 < n_channels; i += 2) + conv_s24_to_f32d_2s_avx2(conv, &dst[i], &s[3*i], n_channels, n_samples); + for(; i < n_channels; i++) + conv_s24_to_f32d_1s_avx2(conv, &dst[i], &s[3*i], n_channels, n_samples); +} + + +void +conv_s32_to_f32d_1s_avx2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, + uint32_t n_channels, uint32_t n_samples) +{ + const int32_t *s = src; + float *d0 = dst[0]; + uint32_t n, unrolled; + __m128i in; + __m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE); + + if (SPA_IS_ALIGNED(d0, 16)) + unrolled = n_samples & ~3; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in = _mm_setr_epi32(s[0*n_channels], + s[1*n_channels], + s[2*n_channels], + s[3*n_channels]); + in = _mm_srai_epi32(in, 8); + out = _mm_cvtepi32_ps(in); + out = _mm_mul_ps(out, factor); + _mm_store_ps(&d0[n], out); + s += 4*n_channels; + } + for(; n < n_samples; n++) { + out = _mm_cvtsi32_ss(out, s[0]>>8); + out = _mm_mul_ss(out, factor); + _mm_store_ss(&d0[n], out); + s += n_channels; + } +} + +void +conv_s32_to_f32d_avx2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], + uint32_t n_samples) +{ + const int32_t *s = src[0]; + uint32_t i = 0, n_channels = conv->n_channels; + + for(; i < n_channels; i++) + conv_s32_to_f32d_1s_avx2(conv, &dst[i], &s[i], n_channels, n_samples); +} + +static void +conv_f32d_to_s32_1s_avx2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], + uint32_t n_channels, uint32_t n_samples) +{ + const float *s0 = src[0]; + int32_t *d = dst; + uint32_t n, unrolled; + __m128 in[1]; + __m128i out[4]; + __m128 scale = _mm_set1_ps(S32_SCALE); + __m128 int_min = _mm_set1_ps(S32_MIN); + + if (SPA_IS_ALIGNED(s0, 16)) + unrolled = n_samples & ~3; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), scale); + in[0] = _mm_min_ps(in[0], int_min); + out[0] = _mm_cvtps_epi32(in[0]); + out[1] = _mm_shuffle_epi32(out[0], _MM_SHUFFLE(0, 3, 2, 1)); + out[2] = _mm_shuffle_epi32(out[0], _MM_SHUFFLE(1, 0, 3, 2)); + out[3] = _mm_shuffle_epi32(out[0], _MM_SHUFFLE(2, 1, 0, 3)); + + d[0*n_channels] = _mm_cvtsi128_si32(out[0]); + d[1*n_channels] = _mm_cvtsi128_si32(out[1]); + d[2*n_channels] = _mm_cvtsi128_si32(out[2]); + d[3*n_channels] = _mm_cvtsi128_si32(out[3]); + d += 4*n_channels; + } + for(; n < n_samples; n++) { + in[0] = _mm_load_ss(&s0[n]); + in[0] = _mm_mul_ss(in[0], scale); + in[0] = _mm_min_ss(in[0], int_min); + *d = _mm_cvtss_si32(in[0]); + d += n_channels; + } +} + +static void +conv_f32d_to_s32_2s_avx2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], + uint32_t n_channels, uint32_t n_samples) +{ + const float *s0 = src[0], *s1 = src[1]; + int32_t *d = dst; + uint32_t n, unrolled; + __m128 in[2]; + __m128i out[2], t[2]; + __m128 scale = _mm_set1_ps(S32_SCALE); + __m128 int_min = _mm_set1_ps(S32_MIN); + + if (SPA_IS_ALIGNED(s0, 16) && + SPA_IS_ALIGNED(s1, 16)) + unrolled = n_samples & ~3; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), scale); + in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), scale); + + in[0] = _mm_min_ps(in[0], int_min); + in[1] = _mm_min_ps(in[1], int_min); + + out[0] = _mm_cvtps_epi32(in[0]); + out[1] = _mm_cvtps_epi32(in[1]); + + t[0] = _mm_unpacklo_epi32(out[0], out[1]); + t[1] = _mm_unpackhi_epi32(out[0], out[1]); + + _mm_storel_pd((double*)(d + 0*n_channels), (__m128d)t[0]); + _mm_storeh_pd((double*)(d + 1*n_channels), (__m128d)t[0]); + _mm_storel_pd((double*)(d + 2*n_channels), (__m128d)t[1]); + _mm_storeh_pd((double*)(d + 3*n_channels), (__m128d)t[1]); + d += 4*n_channels; + } + for(; n < n_samples; n++) { + in[0] = _mm_load_ss(&s0[n]); + in[1] = _mm_load_ss(&s1[n]); + + in[0] = _mm_unpacklo_ps(in[0], in[1]); + + in[0] = _mm_mul_ps(in[0], scale); + in[0] = _mm_min_ps(in[0], int_min); + out[0] = _mm_cvtps_epi32(in[0]); + _mm_storel_epi64((__m128i*)d, out[0]); + d += n_channels; + } +} + +static void +conv_f32d_to_s32_4s_avx2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], + uint32_t n_channels, uint32_t n_samples) +{ + const float *s0 = src[0], *s1 = src[1], *s2 = src[2], *s3 = src[3]; + int32_t *d = dst; + uint32_t n, unrolled; + __m128 in[4]; + __m128i out[4]; + __m128 scale = _mm_set1_ps(S32_SCALE); + __m128 int_min = _mm_set1_ps(S32_MIN); + + if (SPA_IS_ALIGNED(s0, 16) && + SPA_IS_ALIGNED(s1, 16) && + SPA_IS_ALIGNED(s2, 16) && + SPA_IS_ALIGNED(s3, 16)) + unrolled = n_samples & ~3; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), scale); + in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), scale); + in[2] = _mm_mul_ps(_mm_load_ps(&s2[n]), scale); + in[3] = _mm_mul_ps(_mm_load_ps(&s3[n]), scale); + + in[0] = _mm_min_ps(in[0], int_min); + in[1] = _mm_min_ps(in[1], int_min); + in[2] = _mm_min_ps(in[2], int_min); + in[3] = _mm_min_ps(in[3], int_min); + + _MM_TRANSPOSE4_PS(in[0], in[1], in[2], in[3]); + + out[0] = _mm_cvtps_epi32(in[0]); + out[1] = _mm_cvtps_epi32(in[1]); + out[2] = _mm_cvtps_epi32(in[2]); + out[3] = _mm_cvtps_epi32(in[3]); + + _mm_storeu_si128((__m128i*)(d + 0*n_channels), out[0]); + _mm_storeu_si128((__m128i*)(d + 1*n_channels), out[1]); + _mm_storeu_si128((__m128i*)(d + 2*n_channels), out[2]); + _mm_storeu_si128((__m128i*)(d + 3*n_channels), out[3]); + d += 4*n_channels; + } + for(; n < n_samples; n++) { + in[0] = _mm_load_ss(&s0[n]); + in[1] = _mm_load_ss(&s1[n]); + in[2] = _mm_load_ss(&s2[n]); + in[3] = _mm_load_ss(&s3[n]); + + in[0] = _mm_unpacklo_ps(in[0], in[2]); + in[1] = _mm_unpacklo_ps(in[1], in[3]); + in[0] = _mm_unpacklo_ps(in[0], in[1]); + + in[0] = _mm_mul_ps(in[0], scale); + in[0] = _mm_min_ps(in[0], int_min); + out[0] = _mm_cvtps_epi32(in[0]); + _mm_storeu_si128((__m128i*)d, out[0]); + d += n_channels; + } +} + +void +conv_f32d_to_s32_avx2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], + uint32_t n_samples) +{ + int32_t *d = dst[0]; + uint32_t i = 0, n_channels = conv->n_channels; + + for(; i + 3 < n_channels; i += 4) + conv_f32d_to_s32_4s_avx2(conv, &d[i], &src[i], n_channels, n_samples); + for(; i + 1 < n_channels; i += 2) + conv_f32d_to_s32_2s_avx2(conv, &d[i], &src[i], n_channels, n_samples); + for(; i < n_channels; i++) + conv_f32d_to_s32_1s_avx2(conv, &d[i], &src[i], n_channels, n_samples); +} + +static void +conv_f32d_to_s16_1s_avx2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], + uint32_t n_channels, uint32_t n_samples) +{ + const float *s0 = src[0]; + int16_t *d = dst; + uint32_t n, unrolled; + __m128 in[2]; + __m128i out[2]; + __m128 int_max = _mm_set1_ps(S16_MAX_F); + __m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max); + + if (SPA_IS_ALIGNED(s0, 16)) + unrolled = n_samples & ~7; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 8) { + in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), int_max); + in[1] = _mm_mul_ps(_mm_load_ps(&s0[n+4]), int_max); + out[0] = _mm_cvtps_epi32(in[0]); + out[1] = _mm_cvtps_epi32(in[1]); + out[0] = _mm_packs_epi32(out[0], out[1]); + + d[0*n_channels] = _mm_extract_epi16(out[0], 0); + d[1*n_channels] = _mm_extract_epi16(out[0], 1); + d[2*n_channels] = _mm_extract_epi16(out[0], 2); + d[3*n_channels] = _mm_extract_epi16(out[0], 3); + d[4*n_channels] = _mm_extract_epi16(out[0], 4); + d[5*n_channels] = _mm_extract_epi16(out[0], 5); + d[6*n_channels] = _mm_extract_epi16(out[0], 6); + d[7*n_channels] = _mm_extract_epi16(out[0], 7); + d += 8*n_channels; + } + for(; n < n_samples; n++) { + in[0] = _mm_mul_ss(_mm_load_ss(&s0[n]), int_max); + in[0] = _mm_min_ss(int_max, _mm_max_ss(in[0], int_min)); + *d = _mm_cvtss_si32(in[0]); + d += n_channels; + } +} + +static void +conv_f32d_to_s16_2s_avx2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], + uint32_t n_channels, uint32_t n_samples) +{ + const float *s0 = src[0], *s1 = src[1]; + int16_t *d = dst; + uint32_t n, unrolled; + __m128 in[2]; + __m128i out[4], t[2]; + __m128 int_max = _mm_set1_ps(S16_MAX_F); + __m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max); + + if (SPA_IS_ALIGNED(s0, 16) && + SPA_IS_ALIGNED(s1, 16)) + unrolled = n_samples & ~3; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 4) { + in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), int_max); + in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), int_max); + + t[0] = _mm_cvtps_epi32(in[0]); + t[1] = _mm_cvtps_epi32(in[1]); + + t[0] = _mm_packs_epi32(t[0], t[0]); + t[1] = _mm_packs_epi32(t[1], t[1]); + + out[0] = _mm_unpacklo_epi16(t[0], t[1]); + out[1] = _mm_shuffle_epi32(out[0], _MM_SHUFFLE(0, 3, 2, 1)); + out[2] = _mm_shuffle_epi32(out[0], _MM_SHUFFLE(1, 0, 3, 2)); + out[3] = _mm_shuffle_epi32(out[0], _MM_SHUFFLE(2, 1, 0, 3)); + + *((int32_t*)(d + 0*n_channels)) = _mm_cvtsi128_si32(out[0]); + *((int32_t*)(d + 1*n_channels)) = _mm_cvtsi128_si32(out[1]); + *((int32_t*)(d + 2*n_channels)) = _mm_cvtsi128_si32(out[2]); + *((int32_t*)(d + 3*n_channels)) = _mm_cvtsi128_si32(out[3]); + d += 4*n_channels; + } + for(; n < n_samples; n++) { + in[0] = _mm_mul_ss(_mm_load_ss(&s0[n]), int_max); + in[1] = _mm_mul_ss(_mm_load_ss(&s1[n]), int_max); + in[0] = _mm_min_ss(int_max, _mm_max_ss(in[0], int_min)); + in[1] = _mm_min_ss(int_max, _mm_max_ss(in[1], int_min)); + d[0] = _mm_cvtss_si32(in[0]); + d[1] = _mm_cvtss_si32(in[1]); + d += n_channels; + } +} + +static void +conv_f32d_to_s16_4s_avx2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], + uint32_t n_channels, uint32_t n_samples) +{ + const float *s0 = src[0], *s1 = src[1], *s2 = src[2], *s3 = src[3]; + int16_t *d = dst; + uint32_t n, unrolled; + __m256 in[4]; + __m256i out[4], t[4]; + __m256 int_max = _mm256_set1_ps(S16_MAX_F); + + if (SPA_IS_ALIGNED(s0, 16) && + SPA_IS_ALIGNED(s1, 16) && + SPA_IS_ALIGNED(s2, 16) && + SPA_IS_ALIGNED(s3, 16)) + unrolled = n_samples & ~7; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 8) { + in[0] = _mm256_mul_ps(_mm256_load_ps(&s0[n]), int_max); + in[1] = _mm256_mul_ps(_mm256_load_ps(&s1[n]), int_max); + in[2] = _mm256_mul_ps(_mm256_load_ps(&s2[n]), int_max); + in[3] = _mm256_mul_ps(_mm256_load_ps(&s3[n]), int_max); + + t[0] = _mm256_cvtps_epi32(in[0]); + t[1] = _mm256_cvtps_epi32(in[1]); + t[2] = _mm256_cvtps_epi32(in[2]); + t[3] = _mm256_cvtps_epi32(in[3]); + + t[0] = _mm256_packs_epi32(t[0], t[2]); + t[1] = _mm256_packs_epi32(t[1], t[3]); + + out[0] = _mm256_unpacklo_epi16(t[0], t[1]); + out[1] = _mm256_unpackhi_epi16(t[0], t[1]); + out[2] = _mm256_unpacklo_epi32(out[0], out[1]); + out[3] = _mm256_unpackhi_epi32(out[0], out[1]); + + *(int64_t*)(d + 0*n_channels) = _mm256_extract_epi64(out[2], 0); + *(int64_t*)(d + 1*n_channels) = _mm256_extract_epi64(out[2], 1); + *(int64_t*)(d + 2*n_channels) = _mm256_extract_epi64(out[3], 0); + *(int64_t*)(d + 3*n_channels) = _mm256_extract_epi64(out[3], 1); + *(int64_t*)(d + 4*n_channels) = _mm256_extract_epi64(out[2], 2); + *(int64_t*)(d + 5*n_channels) = _mm256_extract_epi64(out[2], 3); + *(int64_t*)(d + 6*n_channels) = _mm256_extract_epi64(out[3], 2); + *(int64_t*)(d + 7*n_channels) = _mm256_extract_epi64(out[3], 3); + + d += 8*n_channels; + } + for(; n < n_samples; n++) { + __m128 in[4]; + __m128 int_max = _mm_set1_ps(S16_MAX_F); + __m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max); + + in[0] = _mm_mul_ss(_mm_load_ss(&s0[n]), int_max); + in[1] = _mm_mul_ss(_mm_load_ss(&s1[n]), int_max); + in[2] = _mm_mul_ss(_mm_load_ss(&s2[n]), int_max); + in[3] = _mm_mul_ss(_mm_load_ss(&s3[n]), int_max); + in[0] = _mm_min_ss(int_max, _mm_max_ss(in[0], int_min)); + in[1] = _mm_min_ss(int_max, _mm_max_ss(in[1], int_min)); + in[2] = _mm_min_ss(int_max, _mm_max_ss(in[2], int_min)); + in[3] = _mm_min_ss(int_max, _mm_max_ss(in[3], int_min)); + d[0] = _mm_cvtss_si32(in[0]); + d[1] = _mm_cvtss_si32(in[1]); + d[2] = _mm_cvtss_si32(in[2]); + d[3] = _mm_cvtss_si32(in[3]); + d += n_channels; + } +} + +void +conv_f32d_to_s16_avx2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], + uint32_t n_samples) +{ + int16_t *d = dst[0]; + uint32_t i = 0, n_channels = conv->n_channels; + + for(; i + 3 < n_channels; i += 4) + conv_f32d_to_s16_4s_avx2(conv, &d[i], &src[i], n_channels, n_samples); + for(; i + 1 < n_channels; i += 2) + conv_f32d_to_s16_2s_avx2(conv, &d[i], &src[i], n_channels, n_samples); + for(; i < n_channels; i++) + conv_f32d_to_s16_1s_avx2(conv, &d[i], &src[i], n_channels, n_samples); +} + +void +conv_f32d_to_s16_2_avx2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], + uint32_t n_samples) +{ + const float *s0 = src[0], *s1 = src[1]; + int16_t *d = dst[0]; + uint32_t n, unrolled; + __m128 in[4]; + __m128i out[4]; + __m128 int_max = _mm_set1_ps(S16_MAX_F); + __m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max); + + if (SPA_IS_ALIGNED(s0, 16) && + SPA_IS_ALIGNED(s1, 16)) + unrolled = n_samples & ~7; + else + unrolled = 0; + + for(n = 0; n < unrolled; n += 8) { + in[0] = _mm_mul_ps(_mm_load_ps(&s0[n+0]), int_max); + in[1] = _mm_mul_ps(_mm_load_ps(&s1[n+0]), int_max); + in[2] = _mm_mul_ps(_mm_load_ps(&s0[n+4]), int_max); + in[3] = _mm_mul_ps(_mm_load_ps(&s1[n+4]), int_max); + + out[0] = _mm_cvtps_epi32(in[0]); + out[1] = _mm_cvtps_epi32(in[1]); + out[2] = _mm_cvtps_epi32(in[2]); + out[3] = _mm_cvtps_epi32(in[3]); + + out[0] = _mm_packs_epi32(out[0], out[2]); + out[1] = _mm_packs_epi32(out[1], out[3]); + + out[2] = _mm_unpacklo_epi16(out[0], out[1]); + out[3] = _mm_unpackhi_epi16(out[0], out[1]); + + _mm_storeu_si128((__m128i*)(d+0), out[2]); + _mm_storeu_si128((__m128i*)(d+8), out[3]); + + d += 16; + } + for(; n < n_samples; n++) { + in[0] = _mm_mul_ss(_mm_load_ss(&s0[n]), int_max); + in[1] = _mm_mul_ss(_mm_load_ss(&s1[n]), int_max); + in[0] = _mm_min_ss(int_max, _mm_max_ss(in[0], int_min)); + in[1] = _mm_min_ss(int_max, _mm_max_ss(in[1], int_min)); + d[0] = _mm_cvtss_si32(in[0]); + d[1] = _mm_cvtss_si32(in[1]); + d += 2; + } +} diff --git a/spa/plugins/audioconvert/fmt-ops-sse2.c b/spa/plugins/audioconvert/fmt-ops-sse2.c index 104bf6827..93ac0fa5d 100644 --- a/spa/plugins/audioconvert/fmt-ops-sse2.c +++ b/spa/plugins/audioconvert/fmt-ops-sse2.c @@ -31,8 +31,7 @@ conv_s16_to_f32d_1s_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA uint32_t n_channels, uint32_t n_samples) { const int16_t *s = src; - float **d = (float **) dst; - float *d0 = d[0]; + float *d0 = dst[0]; uint32_t n, unrolled; __m128i in; __m128 out, factor = _mm_set1_ps(1.0f / S16_SCALE); @@ -77,8 +76,7 @@ conv_s16_to_f32d_2_sse2(struct convert *conv, void * SPA_RESTRICT dst[], const v uint32_t n_samples) { const int16_t *s = src[0]; - float **d = (float **) dst; - float *d0 = d[0], *d1 = d[1]; + float *d0 = dst[0], *d1 = dst[1]; uint32_t n, unrolled; __m128i in[2], t[4]; __m128 out[4], factor = _mm_set1_ps(1.0f / S16_SCALE); @@ -135,8 +133,7 @@ conv_s24_to_f32d_1s_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA uint32_t n_channels, uint32_t n_samples) { const uint8_t *s = src; - float **d = (float **) dst; - float *d0 = d[0]; + float *d0 = dst[0]; uint32_t n, unrolled; __m128i in; __m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE); @@ -175,8 +172,7 @@ conv_s24_to_f32d_2s_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA uint32_t n_channels, uint32_t n_samples) { const uint8_t *s = src; - float **d = (float **) dst; - float *d0 = d[0], *d1 = d[1]; + float *d0 = dst[0], *d1 = dst[1]; uint32_t n, unrolled; __m128i in[2]; __m128 out[2], factor = _mm_set1_ps(1.0f / S24_SCALE); @@ -235,8 +231,7 @@ conv_s24_to_f32d_4s_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA uint32_t n_channels, uint32_t n_samples) { const uint8_t *s = src; - float **d = (float **) dst; - float *d0 = d[0], *d1 = d[1], *d2 = d[2], *d3 = d[3]; + float *d0 = dst[0], *d1 = dst[1], *d2 = dst[2], *d3 = dst[3]; uint32_t n, unrolled; __m128i in[4]; __m128 out[4], factor = _mm_set1_ps(1.0f / S24_SCALE); @@ -340,8 +335,7 @@ conv_s32_to_f32d_1s_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA uint32_t n_channels, uint32_t n_samples) { const int32_t *s = src; - float **d = (float **) dst; - float *d0 = d[0]; + float *d0 = dst[0]; uint32_t n, unrolled; __m128i in; __m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE); @@ -385,8 +379,7 @@ static void conv_f32d_to_s32_1s_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples) { - const float **s = (const float **) src; - const float *s0 = s[0]; + const float *s0 = src[0]; int32_t *d = dst; uint32_t n, unrolled; __m128 in[1]; @@ -426,8 +419,7 @@ static void conv_f32d_to_s32_2s_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples) { - const float **s = (const float **) src; - const float *s0 = s[0], *s1 = s[1]; + const float *s0 = src[0], *s1 = src[1]; int32_t *d = dst; uint32_t n, unrolled; __m128 in[2]; @@ -478,8 +470,7 @@ static void conv_f32d_to_s32_4s_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples) { - const float **s = (const float **) src; - const float *s0 = s[0], *s1 = s[1], *s2 = s[2], *s3 = s[3]; + const float *s0 = src[0], *s1 = src[1], *s2 = src[2], *s3 = src[3]; int32_t *d = dst; uint32_t n, unrolled; __m128 in[4]; @@ -556,8 +547,7 @@ static void conv_f32d_to_s16_1s_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples) { - const float **s = (const float **) src; - const float *s0 = s[0]; + const float *s0 = src[0]; int16_t *d = dst; uint32_t n, unrolled; __m128 in[2]; @@ -599,8 +589,7 @@ static void conv_f32d_to_s16_2s_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples) { - const float **s = (const float **) src; - const float *s0 = s[0], *s1 = s[1]; + const float *s0 = src[0], *s1 = src[1]; int16_t *d = dst; uint32_t n, unrolled; __m128 in[2]; @@ -724,7 +713,6 @@ conv_f32d_to_s16_sse2(struct convert *conv, void * SPA_RESTRICT dst[], const voi conv_f32d_to_s16_1s_sse2(conv, &d[i], &src[i], n_channels, n_samples); } - void conv_f32d_to_s16_2_sse2(struct convert *conv, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], uint32_t n_samples) diff --git a/spa/plugins/audioconvert/fmt-ops-sse41.c b/spa/plugins/audioconvert/fmt-ops-sse41.c index eef05df21..0478555e8 100644 --- a/spa/plugins/audioconvert/fmt-ops-sse41.c +++ b/spa/plugins/audioconvert/fmt-ops-sse41.c @@ -31,8 +31,7 @@ conv_s24_to_f32d_1s_sse41(void *data, void * SPA_RESTRICT dst[], const void * SP uint32_t n_channels, uint32_t n_samples) { const uint8_t *s = src; - float **d = (float **) dst; - float *d0 = d[0]; + float *d0 = dst[0]; uint32_t n, unrolled; __m128i in; __m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE); diff --git a/spa/plugins/audioconvert/fmt-ops-ssse3.c b/spa/plugins/audioconvert/fmt-ops-ssse3.c index 6383147fc..6a7fc2e05 100644 --- a/spa/plugins/audioconvert/fmt-ops-ssse3.c +++ b/spa/plugins/audioconvert/fmt-ops-ssse3.c @@ -31,8 +31,7 @@ conv_s24_to_f32d_4s_ssse3(void *data, void * SPA_RESTRICT dst[], const void * SP uint32_t n_channels, uint32_t n_samples) { const uint8_t *s = src; - float **d = (float **) dst; - float *d0 = d[0], *d1 = d[1], *d2 = d[2], *d3 = d[3]; + float *d0 = dst[0], *d1 = dst[1], *d2 = dst[2], *d3 = dst[3]; uint32_t n, unrolled; __m128i in[4]; __m128 out[4], factor = _mm_set1_ps(1.0f / S24_SCALE); diff --git a/spa/plugins/audioconvert/fmt-ops.c b/spa/plugins/audioconvert/fmt-ops.c index 0cadb8f32..6fd247e64 100644 --- a/spa/plugins/audioconvert/fmt-ops.c +++ b/spa/plugins/audioconvert/fmt-ops.c @@ -55,6 +55,10 @@ static struct conv_info conv_table[] = { SPA_AUDIO_FORMAT_S16, SPA_AUDIO_FORMAT_F32, 0, 0, conv_s16_to_f32_c }, { SPA_AUDIO_FORMAT_S16P, SPA_AUDIO_FORMAT_F32P, 0, 0, conv_s16d_to_f32d_c }, +#if defined (HAVE_AVX2) + { SPA_AUDIO_FORMAT_S16, SPA_AUDIO_FORMAT_F32P, 2, SPA_CPU_FLAG_SSE2, conv_s16_to_f32d_2_avx2 }, + { SPA_AUDIO_FORMAT_S16, SPA_AUDIO_FORMAT_F32P, 0, SPA_CPU_FLAG_SSE2, conv_s16_to_f32d_avx2 }, +#endif #if defined (HAVE_SSE2) { SPA_AUDIO_FORMAT_S16, SPA_AUDIO_FORMAT_F32P, 2, SPA_CPU_FLAG_SSE2, conv_s16_to_f32d_2_sse2 }, { SPA_AUDIO_FORMAT_S16, SPA_AUDIO_FORMAT_F32P, 0, SPA_CPU_FLAG_SSE2, conv_s16_to_f32d_sse2 }, @@ -67,6 +71,9 @@ static struct conv_info conv_table[] = { SPA_AUDIO_FORMAT_F32, SPA_AUDIO_FORMAT_F32P, 0, 0, conv_deinterleave_32_c }, { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_F32, 0, 0, conv_interleave_32_c }, +#if defined (HAVE_AVX2) + { SPA_AUDIO_FORMAT_S32, SPA_AUDIO_FORMAT_F32P, 0, SPA_CPU_FLAG_SSE2, conv_s32_to_f32d_avx2 }, +#endif #if defined (HAVE_SSE2) { SPA_AUDIO_FORMAT_S32, SPA_AUDIO_FORMAT_F32P, 0, SPA_CPU_FLAG_SSE2, conv_s32_to_f32d_sse2 }, #endif @@ -77,6 +84,9 @@ static struct conv_info conv_table[] = { SPA_AUDIO_FORMAT_S24, SPA_AUDIO_FORMAT_F32, 0, 0, conv_s24_to_f32_c }, { SPA_AUDIO_FORMAT_S24P, SPA_AUDIO_FORMAT_F32P, 0, 0, conv_s24d_to_f32d_c }, +#if defined (HAVE_AVX2) + { SPA_AUDIO_FORMAT_S24, SPA_AUDIO_FORMAT_F32P, 0, SPA_CPU_FLAG_SSE2, conv_s24_to_f32d_avx2 }, +#endif #if defined (HAVE_SSSE3) // { SPA_AUDIO_FORMAT_S24, SPA_AUDIO_FORMAT_F32P, 0, SPA_CPU_FLAG_SSSE3, conv_s24_to_f32d_ssse3 }, #endif @@ -105,6 +115,10 @@ static struct conv_info conv_table[] = { SPA_AUDIO_FORMAT_F32, SPA_AUDIO_FORMAT_S16, 0, 0, conv_f32_to_s16_c }, { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S16P, 0, 0, conv_f32d_to_s16d_c }, { SPA_AUDIO_FORMAT_F32, SPA_AUDIO_FORMAT_S16P, 0, 0, conv_f32_to_s16d_c }, +#if defined (HAVE_AVX2) + { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S16, 2, SPA_CPU_FLAG_SSE2, conv_f32d_to_s16_2_avx2 }, + { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S16, 0, SPA_CPU_FLAG_SSE2, conv_f32d_to_s16_avx2 }, +#endif #if defined (HAVE_SSE2) { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S16, 2, SPA_CPU_FLAG_SSE2, conv_f32d_to_s16_2_sse2 }, { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S16, 0, SPA_CPU_FLAG_SSE2, conv_f32d_to_s16_sse2 }, @@ -114,6 +128,9 @@ static struct conv_info conv_table[] = { SPA_AUDIO_FORMAT_F32, SPA_AUDIO_FORMAT_S32, 0, 0, conv_f32_to_s32_c }, { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S32P, 0, 0, conv_f32d_to_s32d_c }, { SPA_AUDIO_FORMAT_F32, SPA_AUDIO_FORMAT_S32P, 0, 0, conv_f32_to_s32d_c }, +#if defined (HAVE_AVX2) + { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S32, 0, SPA_CPU_FLAG_SSE2, conv_f32d_to_s32_avx2 }, +#endif #if defined (HAVE_SSE2) { SPA_AUDIO_FORMAT_F32P, SPA_AUDIO_FORMAT_S32, 0, SPA_CPU_FLAG_SSE2, conv_f32d_to_s32_sse2 }, #endif diff --git a/spa/plugins/audioconvert/fmt-ops.h b/spa/plugins/audioconvert/fmt-ops.h index d8bb509c7..e296ea508 100644 --- a/spa/plugins/audioconvert/fmt-ops.h +++ b/spa/plugins/audioconvert/fmt-ops.h @@ -200,5 +200,13 @@ DEFINE_FUNCTION(s24_to_f32d, ssse3); #endif #if defined(HAVE_SSE41) DEFINE_FUNCTION(s24_to_f32d, sse41); - +#endif +#if defined(HAVE_AVX2) +DEFINE_FUNCTION(s16_to_f32d_2, avx2); +DEFINE_FUNCTION(s16_to_f32d, avx2); +DEFINE_FUNCTION(s24_to_f32d, avx2); +DEFINE_FUNCTION(s32_to_f32d, avx2); +DEFINE_FUNCTION(f32d_to_s32, avx2); +DEFINE_FUNCTION(f32d_to_s16_2, avx2); +DEFINE_FUNCTION(f32d_to_s16, avx2); #endif diff --git a/spa/plugins/audioconvert/meson.build b/spa/plugins/audioconvert/meson.build index 35515bd9a..ccb06abd0 100644 --- a/spa/plugins/audioconvert/meson.build +++ b/spa/plugins/audioconvert/meson.build @@ -74,6 +74,16 @@ if have_avx and have_fma simd_cargs += ['-DHAVE_AVX', '-DHAVE_FMA'] simd_dependencies += audioconvert_avx endif +if have_avx2 + audioconvert_avx2 = static_library('audioconvert_avx2', + ['fmt-ops-avx2.c'], + c_args : [avx2_args, '-O3', '-DHAVE_AVX2'], + include_directories : [spa_inc], + install : false + ) + simd_cargs += ['-DHAVE_AVX2'] + simd_dependencies += audioconvert_avx2 +endif audioconvertlib = shared_library('spa-audioconvert', audioconvert_sources,