From e7ef13e3109799fb7ba62855d3d384bb9f70426e Mon Sep 17 00:00:00 2001 From: Wim Taymans Date: Thu, 28 Mar 2019 16:45:57 +0100 Subject: [PATCH] audioconvert: add avx optimizations --- meson.build | 6 ++ spa/plugins/audioconvert/channelmix-ops-sse.c | 28 +++--- spa/plugins/audioconvert/fmt-ops-sse2.c | 50 +++++------ spa/plugins/audioconvert/fmt-ops-sse41.c | 4 +- spa/plugins/audioconvert/fmt-ops-ssse3.c | 4 +- spa/plugins/audioconvert/meson.build | 10 +++ .../audioconvert/resample-native-avx.c | 88 +++++++++++++++++++ .../audioconvert/resample-native-impl.h | 4 + .../audioconvert/resample-native-sse.c | 25 +++++- spa/plugins/audioconvert/resample-native.h | 4 + src/modules/module-audio-dsp/floatmix.c | 4 +- 11 files changed, 180 insertions(+), 47 deletions(-) create mode 100644 spa/plugins/audioconvert/resample-native-avx.c diff --git a/meson.build b/meson.build index 8a6c3b07d..81303d272 100644 --- a/meson.build +++ b/meson.build @@ -54,11 +54,17 @@ sse_args = '-msse' sse2_args = '-msse2' ssse3_args = '-mssse3' sse41_args = '-msse4.1' +fma_args = '-mfma' +avx_args = '-mavx' +avx2_args = '-mavx2' have_sse = cc.has_argument(sse_args) have_sse2 = cc.has_argument(sse2_args) have_ssse3 = cc.has_argument(ssse3_args) have_sse41 = cc.has_argument(sse41_args) +have_fma = cc.has_argument(fma_args) +have_avx = cc.has_argument(avx_args) +have_avx2 = cc.has_argument(avx2_args) cdata = configuration_data() cdata.set('PIPEWIRE_VERSION_MAJOR', pipewire_version_major) diff --git a/spa/plugins/audioconvert/channelmix-ops-sse.c b/spa/plugins/audioconvert/channelmix-ops-sse.c index 6cd325411..d9e302397 100644 --- a/spa/plugins/audioconvert/channelmix-ops-sse.c +++ b/spa/plugins/audioconvert/channelmix-ops-sse.c @@ -50,11 +50,11 @@ channelmix_copy_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], if (SPA_IS_ALIGNED(di, 16) && SPA_IS_ALIGNED(si, 16)) - unrolled = n_samples / 16; + unrolled = n_samples & ~15; else unrolled = 0; - for(n = 0; unrolled--; n += 16) { + for(n = 0; n < unrolled; n += 16) { t[0] = _mm_load_ps(&si[n]); t[1] = _mm_load_ps(&si[n+4]); t[2] = _mm_load_ps(&si[n+8]); @@ -89,7 +89,7 @@ channelmix_f32_2_4_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], SPA_IS_ALIGNED(dFR, 16) && SPA_IS_ALIGNED(dRL, 16) && SPA_IS_ALIGNED(dRR, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; @@ -98,7 +98,7 @@ channelmix_f32_2_4_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], memset(d[i], 0, n_samples * sizeof(float)); } else if (v == VOLUME_NORM) { - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in = _mm_load_ps(&sFL[n]); _mm_store_ps(&dFL[n], in); _mm_store_ps(&dRL[n], in); @@ -116,7 +116,7 @@ channelmix_f32_2_4_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], } } else { - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in = _mm_mul_ps(_mm_load_ps(&sFL[n]), vol); _mm_store_ps(&dFL[n], in); _mm_store_ps(&dRL[n], in); @@ -161,7 +161,7 @@ channelmix_f32_5p1_2_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], SPA_IS_ALIGNED(sSR, 16) && SPA_IS_ALIGNED(dFL, 16) && SPA_IS_ALIGNED(dFR, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; @@ -170,7 +170,7 @@ channelmix_f32_5p1_2_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], memset(dFR, 0, n_samples * sizeof(float)); } else if (v == VOLUME_NORM) { - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { ctr = _mm_mul_ps(_mm_load_ps(&sFC[n]), clev); ctr = _mm_add_ps(ctr, _mm_mul_ps(_mm_load_ps(&sLFE[n]), llev)); in = _mm_mul_ps(_mm_load_ps(&sSL[n]), slev); @@ -196,7 +196,7 @@ channelmix_f32_5p1_2_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], } } else { - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { ctr = _mm_mul_ps(_mm_load_ps(&sFC[n]), clev); ctr = _mm_add_ps(ctr, _mm_mul_ps(_mm_load_ps(&sLFE[n]), llev)); in = _mm_mul_ps(_mm_load_ps(&sSL[n]), slev); @@ -252,7 +252,7 @@ channelmix_f32_5p1_3p1_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst] SPA_IS_ALIGNED(dFR, 16) && SPA_IS_ALIGNED(dFC, 16) && SPA_IS_ALIGNED(dLFE, 16)) - unrolled = n_samples / 8; + unrolled = n_samples & ~7; else unrolled = 0; @@ -261,7 +261,7 @@ channelmix_f32_5p1_3p1_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst] memset(d[i], 0, n_samples * sizeof(float)); } else if (v == VOLUME_NORM) { - for(n = 0; unrolled--; n += 8) { + for(n = 0; n < unrolled; n += 8) { avg[0] = _mm_add_ps(_mm_load_ps(&sFL[n]), _mm_load_ps(&sSL[n])); avg[1] = _mm_add_ps(_mm_load_ps(&sFL[n+4]), _mm_load_ps(&sSL[n+4])); _mm_store_ps(&dFL[n], _mm_mul_ps(avg[0], mix)); @@ -285,7 +285,7 @@ channelmix_f32_5p1_3p1_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst] } } else { - for(n = 0; unrolled--; n += 8) { + for(n = 0; n < unrolled; n += 8) { avg[0] = _mm_add_ps(_mm_load_ps(&sFL[n]), _mm_load_ps(&sSL[n])); avg[1] = _mm_add_ps(_mm_load_ps(&sFL[n+4]), _mm_load_ps(&sSL[n+4])); _mm_store_ps(&dFL[n], _mm_mul_ps(avg[0], mix)); @@ -337,7 +337,7 @@ channelmix_f32_5p1_4_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], SPA_IS_ALIGNED(dFR, 16) && SPA_IS_ALIGNED(dRL, 16) && SPA_IS_ALIGNED(dRR, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; @@ -346,7 +346,7 @@ channelmix_f32_5p1_4_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], memset(d[i], 0, n_samples * sizeof(float)); } else if (v == VOLUME_NORM) { - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { ctr = _mm_mul_ps(_mm_load_ps(&sFC[n]), clev); ctr = _mm_add_ps(ctr, _mm_mul_ps(_mm_load_ps(&sLFE[n]), llev)); _mm_store_ps(&dFL[n], _mm_add_ps(_mm_load_ps(&sFL[n]), ctr)); @@ -364,7 +364,7 @@ channelmix_f32_5p1_4_sse(void *data, int n_dst, void * SPA_RESTRICT dst[n_dst], } } else { - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { ctr = _mm_mul_ps(_mm_load_ps(&sFC[n]), clev); ctr = _mm_add_ps(ctr, _mm_mul_ps(_mm_load_ps(&sLFE[n]), llev)); _mm_store_ps(&dFL[n], _mm_mul_ps(_mm_add_ps(_mm_load_ps(&sFL[n]), ctr), vol)); diff --git a/spa/plugins/audioconvert/fmt-ops-sse2.c b/spa/plugins/audioconvert/fmt-ops-sse2.c index 3e2e58000..2dd265787 100644 --- a/spa/plugins/audioconvert/fmt-ops-sse2.c +++ b/spa/plugins/audioconvert/fmt-ops-sse2.c @@ -37,11 +37,11 @@ conv_s16_to_f32d_1_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_ __m128 out, factor = _mm_set1_ps(1.0f / S16_SCALE); if (SPA_IS_ALIGNED(d0, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in = _mm_insert_epi16(in, s[0*n_channels], 1); in = _mm_insert_epi16(in, s[1*n_channels], 3); in = _mm_insert_epi16(in, s[2*n_channels], 5); @@ -74,11 +74,11 @@ conv_s16_to_f32d_2_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_ SPA_IS_ALIGNED(s, 16) && SPA_IS_ALIGNED(d0, 16) && SPA_IS_ALIGNED(d1, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in = _mm_load_si128((__m128i*)s); t[0] = _mm_slli_epi32(in, 16); @@ -129,14 +129,14 @@ conv_s24_to_f32d_1_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_ __m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE); if (SPA_IS_ALIGNED(d0, 16) && n_samples > 0) { - unrolled = n_samples / 4; + unrolled = n_samples & ~3; if ((n_samples & 3) == 0) - unrolled--; + unrolled -= 4; } else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in = _mm_setr_epi32( *((uint32_t*)&s[0 * n_channels]), *((uint32_t*)&s[3 * n_channels]), @@ -170,14 +170,14 @@ conv_s24_to_f32d_2_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_ if (SPA_IS_ALIGNED(d0, 16) && SPA_IS_ALIGNED(d1, 16) && n_samples > 0) { - unrolled = n_samples / 4; + unrolled = n_samples & ~3; if ((n_samples & 3) == 0) - unrolled--; + unrolled -= 4; } else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_setr_epi32( *((uint32_t*)&s[0 + 0*n_channels]), *((uint32_t*)&s[0 + 3*n_channels]), @@ -231,14 +231,14 @@ conv_s24_to_f32d_4_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_ SPA_IS_ALIGNED(d2, 16) && SPA_IS_ALIGNED(d3, 16) && n_samples > 0) { - unrolled = n_samples / 4; + unrolled = n_samples & ~3; if ((n_samples & 3) == 0) - unrolled--; + unrolled -= 4; } else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_setr_epi32( *((uint32_t*)&s[0 + 0*n_channels]), *((uint32_t*)&s[0 + 3*n_channels]), @@ -331,11 +331,11 @@ conv_f32d_to_s32_1_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RE __m128 int_min = _mm_set1_ps(S32_MIN); if (SPA_IS_ALIGNED(s0, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), scale); in[0] = _mm_min_ps(in[0], int_min); out[0] = _mm_cvtps_epi32(in[0]); @@ -372,11 +372,11 @@ conv_f32d_to_s32_2_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RE if (SPA_IS_ALIGNED(s0, 16) && SPA_IS_ALIGNED(s1, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), scale); in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), scale); @@ -425,11 +425,11 @@ conv_f32d_to_s32_4_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RE SPA_IS_ALIGNED(s1, 16) && SPA_IS_ALIGNED(s2, 16) && SPA_IS_ALIGNED(s3, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), scale); in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), scale); in[2] = _mm_mul_ps(_mm_load_ps(&s2[n]), scale); @@ -498,11 +498,11 @@ conv_f32d_to_s16_1_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RE __m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max); if (SPA_IS_ALIGNED(s0, 16)) - unrolled = n_samples / 8; + unrolled = n_samples & ~7; else unrolled = 0; - for(n = 0; unrolled--; n += 8) { + for(n = 0; n < unrolled; n += 8) { in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), int_max); in[1] = _mm_mul_ps(_mm_load_ps(&s0[n+4]), int_max); out[0] = _mm_cvtps_epi32(in[0]); @@ -541,11 +541,11 @@ conv_f32d_to_s16_2_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RE if (SPA_IS_ALIGNED(s0, 16) && SPA_IS_ALIGNED(s1, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), int_max); in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), int_max); @@ -593,11 +593,11 @@ conv_f32d_to_s16_4_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RE SPA_IS_ALIGNED(s1, 16) && SPA_IS_ALIGNED(s2, 16) && SPA_IS_ALIGNED(s3, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), int_max); in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), int_max); in[2] = _mm_mul_ps(_mm_load_ps(&s2[n]), int_max); diff --git a/spa/plugins/audioconvert/fmt-ops-sse41.c b/spa/plugins/audioconvert/fmt-ops-sse41.c index 2cfe83c43..3db80caf8 100644 --- a/spa/plugins/audioconvert/fmt-ops-sse41.c +++ b/spa/plugins/audioconvert/fmt-ops-sse41.c @@ -37,11 +37,11 @@ conv_s24_to_f32d_1_sse41(void *data, void * SPA_RESTRICT dst[], const void * SPA __m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE); if (SPA_IS_ALIGNED(d0, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in = _mm_insert_epi32(in, *((uint32_t*)&s[0 * n_channels]), 0); in = _mm_insert_epi32(in, *((uint32_t*)&s[3 * n_channels]), 1); in = _mm_insert_epi32(in, *((uint32_t*)&s[6 * n_channels]), 2); diff --git a/spa/plugins/audioconvert/fmt-ops-ssse3.c b/spa/plugins/audioconvert/fmt-ops-ssse3.c index 772354ba0..aa7f51d9e 100644 --- a/spa/plugins/audioconvert/fmt-ops-ssse3.c +++ b/spa/plugins/audioconvert/fmt-ops-ssse3.c @@ -42,11 +42,11 @@ conv_s24_to_f32d_4_ssse3(void *data, void * SPA_RESTRICT dst[], const void * SPA SPA_IS_ALIGNED(d1, 16) && SPA_IS_ALIGNED(d2, 16) && SPA_IS_ALIGNED(d3, 16)) - unrolled = n_samples / 4; + unrolled = n_samples & ~3; else unrolled = 0; - for(n = 0; unrolled--; n += 4) { + for(n = 0; n < unrolled; n += 4) { in[0] = _mm_loadu_si128((__m128i*)(s + 0*n_channels)); in[1] = _mm_loadu_si128((__m128i*)(s + 3*n_channels)); in[2] = _mm_loadu_si128((__m128i*)(s + 6*n_channels)); diff --git a/spa/plugins/audioconvert/meson.build b/spa/plugins/audioconvert/meson.build index c916cee61..2da4d5bdd 100644 --- a/spa/plugins/audioconvert/meson.build +++ b/spa/plugins/audioconvert/meson.build @@ -50,6 +50,16 @@ if have_sse41 simd_cargs += ['-DHAVE_SSE41'] simd_dependencies += audioconvert_sse41 endif +if have_avx and have_fma + audioconvert_avx = static_library('audioconvert_avx', + ['resample-native-avx.c'], + c_args : [avx2_args, fma_args], + include_directories : [spa_inc], + install : false + ) + simd_cargs += ['-DHAVE_AVX', '-DHAVE_FMA'] + simd_dependencies += audioconvert_avx +endif audioconvertlib = shared_library('spa-audioconvert', audioconvert_sources, diff --git a/spa/plugins/audioconvert/resample-native-avx.c b/spa/plugins/audioconvert/resample-native-avx.c new file mode 100644 index 000000000..a4f6d6403 --- /dev/null +++ b/spa/plugins/audioconvert/resample-native-avx.c @@ -0,0 +1,88 @@ +/* Spa + * + * Copyright © 2019 Wim Taymans + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "resample-native-impl.h" + +#include + +static void inner_product_avx(float *d, const float * SPA_RESTRICT s, + const float * SPA_RESTRICT taps, uint32_t n_taps) +{ + __m256 sy[2] = { _mm256_setzero_ps(), _mm256_setzero_ps() }; + __m128 sx[2]; + uint32_t i, n_taps4 = n_taps & ~0xf; + + for (i = 0; i < n_taps4; i += 16) { + sy[0] = _mm256_fmadd_ps(_mm256_loadu_ps(s + i + 0), _mm256_load_ps(taps + i + 0), sy[0]); + sy[1] = _mm256_fmadd_ps(_mm256_loadu_ps(s + i + 8), _mm256_load_ps(taps + i + 8), sy[1]); + } + sy[0] = _mm256_add_ps(sy[0], sy[1]); + sx[0] = _mm256_extractf128_ps(sy[0], 0); + sx[1] = _mm256_extractf128_ps(sy[0], 1); + for (; i < n_taps; i += 8) { + sx[0] = _mm_fmadd_ps(_mm_loadu_ps(s + i + 0), _mm_load_ps(taps + i + 0), sx[0]); + sx[1] = _mm_fmadd_ps(_mm_loadu_ps(s + i + 4), _mm_load_ps(taps + i + 4), sx[1]); + } + sx[0] = _mm_add_ps(sx[0], sx[1]); + sx[0] = _mm_hadd_ps(sx[0], sx[0]); + sx[0] = _mm_hadd_ps(sx[0], sx[0]); + _mm_store_ss(d, sx[0]); +} + +static void inner_product_ip_avx(float *d, const float * SPA_RESTRICT s, + const float * SPA_RESTRICT t0, const float * SPA_RESTRICT t1, float x, + uint32_t n_taps) +{ + __m256 sy[2] = { _mm256_setzero_ps(), _mm256_setzero_ps() }, ty; + __m128 sx[2], tx; + uint32_t i, n_taps4 = n_taps & ~0xf; + + for (i = 0; i < n_taps4; i += 16) { + ty = _mm256_loadu_ps(s + i + 0); + sy[0] = _mm256_fmadd_ps(ty, _mm256_load_ps(t0 + i + 0), sy[0]); + sy[1] = _mm256_fmadd_ps(ty, _mm256_load_ps(t1 + i + 0), sy[1]); + ty = _mm256_loadu_ps(s + i + 8); + sy[0] = _mm256_fmadd_ps(ty, _mm256_load_ps(t0 + i + 8), sy[0]); + sy[1] = _mm256_fmadd_ps(ty, _mm256_load_ps(t1 + i + 8), sy[1]); + } + sy[0] = _mm256_add_ps(sy[0], sy[1]); + sx[0] = _mm256_extractf128_ps(sy[0], 0); + sx[1] = _mm256_extractf128_ps(sy[0], 1); + for (; i < n_taps; i += 8) { + tx = _mm_loadu_ps(s + i + 0); + sx[0] = _mm_fmadd_ps(tx, _mm_load_ps(t0 + i + 0), sx[0]); + sx[1] = _mm_fmadd_ps(tx, _mm_load_ps(t1 + i + 0), sx[1]); + tx = _mm_loadu_ps(s + i + 4); + sx[0] = _mm_fmadd_ps(tx, _mm_load_ps(t0 + i + 4), sx[0]); + sx[1] = _mm_fmadd_ps(tx, _mm_load_ps(t1 + i + 4), sx[1]); + } + sx[1] = _mm_mul_ps(_mm_sub_ps(sx[1], sx[0]), _mm_load1_ps(&x)); + sx[0] = _mm_add_ps(sx[0], sx[1]); + sx[0] = _mm_hadd_ps(sx[0], sx[0]); + sx[0] = _mm_hadd_ps(sx[0], sx[0]); + _mm_store_ss(d, sx[0]); +} + +MAKE_RESAMPLER_FULL(avx); +MAKE_RESAMPLER_INTER(avx); diff --git a/spa/plugins/audioconvert/resample-native-impl.h b/spa/plugins/audioconvert/resample-native-impl.h index 36b8cec3d..be85bf353 100644 --- a/spa/plugins/audioconvert/resample-native-impl.h +++ b/spa/plugins/audioconvert/resample-native-impl.h @@ -188,3 +188,7 @@ DEFINE_RESAMPLER_INTER(sse); DEFINE_RESAMPLER_FULL(ssse3); DEFINE_RESAMPLER_INTER(ssse3); #endif +#if defined (HAVE_AVX) && defined(HAVE_FMA) +DEFINE_RESAMPLER_FULL(avx); +DEFINE_RESAMPLER_INTER(avx); +#endif diff --git a/spa/plugins/audioconvert/resample-native-sse.c b/spa/plugins/audioconvert/resample-native-sse.c index 479bacdc8..d0ebe39ec 100644 --- a/spa/plugins/audioconvert/resample-native-sse.c +++ b/spa/plugins/audioconvert/resample-native-sse.c @@ -30,9 +30,30 @@ static void inner_product_sse(float *d, const float * SPA_RESTRICT s, const float * SPA_RESTRICT taps, uint32_t n_taps) { __m128 sum = _mm_setzero_ps(); - uint32_t i; + uint32_t i = 0; +#if 0 + uint32_t unrolled = n_taps & ~15; - for (i = 0; i < n_taps; i += 8) { + for (i = 0; i < unrolled; i += 16) { + sum = _mm_add_ps(sum, + _mm_mul_ps( + _mm_loadu_ps(s + i + 0), + _mm_load_ps(taps + i + 0))); + sum = _mm_add_ps(sum, + _mm_mul_ps( + _mm_loadu_ps(s + i + 4), + _mm_load_ps(taps + i + 4))); + sum = _mm_add_ps(sum, + _mm_mul_ps( + _mm_loadu_ps(s + i + 8), + _mm_load_ps(taps + i + 8))); + sum = _mm_add_ps(sum, + _mm_mul_ps( + _mm_loadu_ps(s + i + 12), + _mm_load_ps(taps + i + 12))); + } +#endif + for (; i < n_taps; i += 8) { sum = _mm_add_ps(sum, _mm_mul_ps( _mm_loadu_ps(s + i + 0), diff --git a/spa/plugins/audioconvert/resample-native.h b/spa/plugins/audioconvert/resample-native.h index dd2fac94e..491a2f441 100644 --- a/spa/plugins/audioconvert/resample-native.h +++ b/spa/plugins/audioconvert/resample-native.h @@ -125,6 +125,10 @@ static void impl_native_update_rate(struct resample *r, double rate) #if defined (HAVE_SSSE3) if (SPA_FLAG_CHECK(r->cpu_flags, SPA_CPU_FLAG_SSSE3 | SPA_CPU_FLAG_SLOW_UNALIGNED)) data->func = is_full ? do_resample_full_ssse3 : do_resample_inter_ssse3; +#endif +#if defined(HAVE_AVX) && defined(HAVE_FMA) + if (SPA_FLAG_CHECK(r->cpu_flags, SPA_CPU_FLAG_AVX | SPA_CPU_FLAG_FMA3)) + data->func = is_full ? do_resample_full_avx : do_resample_inter_avx; #endif } } diff --git a/src/modules/module-audio-dsp/floatmix.c b/src/modules/module-audio-dsp/floatmix.c index f7d2aff1d..15f58e31b 100644 --- a/src/modules/module-audio-dsp/floatmix.c +++ b/src/modules/module-audio-dsp/floatmix.c @@ -693,11 +693,11 @@ static void mix_2(float * dst, const float * SPA_RESTRICT src1, if (SPA_IS_ALIGNED(src1, 16) && SPA_IS_ALIGNED(src2, 16) && SPA_IS_ALIGNED(dst, 16)) - unrolled = n_samples / 16; + unrolled = n_samples & ~15; else unrolled = 0; - for (n = 0; unrolled--; n += 16) { + for (n = 0; n < unrolled; n += 16) { in1[0] = _mm_load_ps(&src1[n+ 0]); in1[1] = _mm_load_ps(&src1[n+ 4]); in1[2] = _mm_load_ps(&src1[n+ 8]);