From 67a5dd35be529c567d3b54460b213c061119bb32 Mon Sep 17 00:00:00 2001 From: Wim Taymans Date: Fri, 27 Mar 2020 08:27:11 -0400 Subject: [PATCH] resample: add neon optimizations --- meson.build | 15 ++ spa/plugins/audioconvert/meson.build | 13 +- .../audioconvert/resample-native-impl.h | 4 + .../audioconvert/resample-native-neon.c | 222 ++++++++++++++++++ spa/plugins/audioconvert/resample-native.h | 4 + 5 files changed, 257 insertions(+), 1 deletion(-) create mode 100644 spa/plugins/audioconvert/resample-native-neon.c diff --git a/meson.build b/meson.build index 558ab4f29..2acf85e28 100644 --- a/meson.build +++ b/meson.build @@ -80,6 +80,21 @@ have_fma = cc.has_argument(fma_args) have_avx = cc.has_argument(avx_args) have_avx2 = cc.has_argument(avx2_args) +have_neon = false +if host_machine.cpu_family() == 'arm' or host_machine.cpu_family() == 'aarch64' + if cc.compiles( +''' +#include +int32x4_t testfunc(int16_t *a, int16_t *b) { +return vmull_s16(vld1_s16(a), vld1_s16(b)); +} +''', + name : 'NEON support') + have_neon = true + endif +endif + + cdata = configuration_data() cdata.set('PIPEWIRE_VERSION_MAJOR', pipewire_version_major) cdata.set('PIPEWIRE_VERSION_MINOR', pipewire_version_minor) diff --git a/spa/plugins/audioconvert/meson.build b/spa/plugins/audioconvert/meson.build index ccb06abd0..9a5fe9a89 100644 --- a/spa/plugins/audioconvert/meson.build +++ b/spa/plugins/audioconvert/meson.build @@ -85,6 +85,17 @@ if have_avx2 simd_dependencies += audioconvert_avx2 endif +if have_neon + audioconvert_neon = static_library('audioconvert_neon', + ['resample-native-neon.c'], + c_args : ['-O3', '-DHAVE_NEON'], + include_directories : [spa_inc], + install : false + ) + simd_cargs += ['-DHAVE_NEON'] + simd_dependencies += audioconvert_neon +endif + audioconvertlib = shared_library('spa-audioconvert', audioconvert_sources, c_args : simd_cargs, @@ -146,7 +157,7 @@ if sndfile_dep.found() ] executable('spa-resample', sparesample_sources, - c_args : [ '-D_GNU_SOURCE' ], + c_args : [ simd_cargs, '-D_GNU_SOURCE' ], install: false, include_directories : [spa_inc ], link_with : [ simd_dependencies, test_lib, audioconvertlib ], diff --git a/spa/plugins/audioconvert/resample-native-impl.h b/spa/plugins/audioconvert/resample-native-impl.h index f8f184f17..48913355c 100644 --- a/spa/plugins/audioconvert/resample-native-impl.h +++ b/spa/plugins/audioconvert/resample-native-impl.h @@ -171,6 +171,10 @@ DEFINE_RESAMPLER(copy,c); DEFINE_RESAMPLER(full,c); DEFINE_RESAMPLER(inter,c); +#if defined (HAVE_NEON) +DEFINE_RESAMPLER(full,neon); +DEFINE_RESAMPLER(inter,neon); +#endif #if defined (HAVE_SSE) DEFINE_RESAMPLER(full,sse); DEFINE_RESAMPLER(inter,sse); diff --git a/spa/plugins/audioconvert/resample-native-neon.c b/spa/plugins/audioconvert/resample-native-neon.c new file mode 100644 index 000000000..8602c3228 --- /dev/null +++ b/spa/plugins/audioconvert/resample-native-neon.c @@ -0,0 +1,222 @@ +/* Spa + * + * Copyright © 2019 Wim Taymans + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#include "resample-native-impl.h" + +#include + +static void inner_product_neon(float *d, const float * SPA_RESTRICT s, + const float * SPA_RESTRICT taps, uint32_t n_taps) +{ + unsigned int remainder = n_taps % 16; + n_taps = n_taps - remainder; + +#ifdef __aarch64__ + asm volatile( + " cmp %[n_taps], #0\n" + " bne 1f\n" + " ld1 {v4.4s}, [%[taps]], #16\n" + " ld1 {v8.4s}, [%[s]], #16\n" + " subs %[remainder], %[remainder], #4\n" + " fmul v0.4s, v4.4s, v8.4s\n" + " bne 4f\n" + " b 5f\n" + "1:" + " ld1 {v4.4s, v5.4s}, [%[taps]], #32\n" + " ld1 {v8.4s, v9.4s}, [%[s]], #32\n" + " ld1 {v6.4s, v7.4s}, [%[taps]], #32\n" + " ld1 {v10.4s, v11.4s}, [%[s]], #32\n" + " subs %[n_taps], %[n_taps], #16\n" + " fmul v0.4s, v4.4s, v8.4s\n" + " fmul v1.4s, v5.4s, v9.4s\n" + " fmul v2.4s, v6.4s, v10.4s\n" + " fmul v3.4s, v7.4s, v11.4s\n" + " beq 3f\n" + "2:" + " ld1 { v4.4s, v5.4s}, [%[taps]], #32\n" + " ld1 { v8.4s, v9.4s}, [%[s]], #32\n" + " ld1 { v6.4s, v7.4s}, [%[taps]], #32\n" + " ld1 { v10.4s, v11.4s}, [%[s]], #32\n" + " subs %[n_taps], %[n_taps], #16\n" + " fmla v0.4s, v4.4s, v8.4s\n" + " fmla v1.4s, v5.4s, v9.4s\n" + " fmla v2.4s, v6.4s, v10.4s\n" + " fmla v3.4s, v7.4s, v11.4s\n" + " bne 2b\n" + "3:" + " fadd v4.4s, v0.4s, v1.4s\n" + " fadd v5.4s, v2.4s, v3.4s\n" + " cmp %[remainder], #0\n" + " fadd v0.4s, v4.4s, v5.4s\n" + " beq 5f\n" + "4:" + " ld1 { v6.4s }, [%[taps]], #16\n" + " ld1 { v10.4s }, [%[s]], #16\n" + " subs %[remainder], %[remainder], #4\n" + " fmla v0.4s, v6.4s, v10.4s\n" + " bne 4b\n" + "5:" + " faddp v0.4s, v0.4s, v0.4s\n" + " faddp v0.2s, v0.2s, v0.2s\n" + " str s0, [%[d]]\n" + : [d] "+X" (d), [s] "+X" (s), [taps] "+X" (taps), + [n_taps] "+r" (n_taps), [remainder] "+r" (remainder) + : + : "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11"); +#else + asm volatile ( + " cmp %[n_taps], #0\n" + " bne 1f\n" + " vld1.32 {q4}, [%[taps]]!\n" + " vld1.32 {q8}, [%[s]]!\n" + " subs %[remainder], %[remainder], #4\n" + " vmul.f32 q0, q4, q8\n" + " bne 4f\n" + " b 5f\n" + "1:" + " vld1.32 {q4, q5}, [%[taps]]!\n" + " vld1.32 {q8, q9}, [%[s]]!\n" + " vld1.32 {q6, q7}, [%[taps]]!\n" + " vld1.32 {q10, q11}, [%[s]]!\n" + " subs %[n_taps], %[n_taps], #16\n" + " vmul.f32 q0, q4, q8\n" + " vmul.f32 q1, q5, q9\n" + " vmul.f32 q2, q6, q10\n" + " vmul.f32 q3, q7, q11\n" + " beq 3f\n" + "2:" + " vld1.32 {q4, q5}, [%[taps]]!\n" + " vld1.32 {q8, q9}, [%[s]]!\n" + " vld1.32 {q6, q7}, [%[taps]]!\n" + " vld1.32 {q10, q11}, [%[s]]!\n" + " subs %[n_taps], %[n_taps], #16\n" + " vmla.f32 q0, q4, q8\n" + " vmla.f32 q1, q5, q9\n" + " vmla.f32 q2, q6, q10\n" + " vmla.f32 q3, q7, q11\n" + " bne 2b\n" + "3:" + " vadd.f32 q4, q0, q1\n" + " vadd.f32 q5, q2, q3\n" + " cmp %[remainder], #0\n" + " vadd.f32 q0, q4, q5\n" + " beq 5f\n" + "4:" + " vld1.32 {q6}, [%[taps]]!\n" + " vld1.32 {q10}, [%[s]]!\n" + " subs %[remainder], %[remainder], #4\n" + " vmla.f32 q0, q6, q10\n" + " bne 4b\n" + "5:" + " vadd.f32 d0, d0, d1\n" + " vpadd.f32 d0, d0, d0\n" + " str d0, [%[d]]\n" + : [d] "+r" (d), [s] "+r" (s), [taps] "+r" (taps), + [n_taps] "+l" (n_taps), [remainder] "+l" (remainder) + : + : "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10", "q11"); +#endif +} + +static void inner_product_ip_neon(float *d, const float * SPA_RESTRICT s, + const float * SPA_RESTRICT t0, const float * SPA_RESTRICT t1, float x, + uint32_t n_taps) +{ +#ifdef __aarch64__ + asm volatile( + " dup v10.4s, %w[x]\n" + " ld1 {v4.4s, v5.4s}, [%[t0]], #32\n" + " ld1 {v8.4s, v9.4s}, [%[s]], #32\n" + " ld1 {v6.4s, v7.4s}, [%[t1]], #32\n" + " subs %[n_taps], %[n_taps], #8\n" + " fmul v0.4s, v4.4s, v8.4s\n" + " fmul v1.4s, v5.4s, v9.4s\n" + " fmul v2.4s, v6.4s, v8.4s\n" + " fmul v3.4s, v7.4s, v9.4s\n" + " beq 3f\n" + "2:" + " ld1 { v4.4s, v5.4s}, [%[t0]], #32\n" + " ld1 { v8.4s, v9.4s}, [%[s]], #32\n" + " ld1 { v6.4s, v7.4s}, [%[t1]], #32\n" + " subs %[n_taps], %[n_taps], #8\n" + " fmla v0.4s, v4.4s, v8.4s\n" + " fmla v1.4s, v5.4s, v9.4s\n" + " fmla v2.4s, v6.4s, v8.4s\n" + " fmla v3.4s, v7.4s, v9.4s\n" + " bne 2b\n" + "3:" + " fadd v0.4s, v0.4s, v1.4s\n" /* sum[0] */ + " fadd v2.4s, v2.4s, v3.4s\n" /* sum[1] */ + " fsub v2.4s, v2.4s, v0.4s\n" /* sum[1] -= sum[0] */ + " fmla v0.4s, v1.4s, v10.4s\n" /* sum[0] += sum[1] * x */ + " faddp v0.4s, v0.4s, v0.4s\n" + " faddp v0.2s, v0.2s, v0.2s\n" + " str s0, [%[d]]\n" + : [d] "+X" (d), [s] "+X" (s), [t0] "+X" (t0), [t1] "+X" (t1), + [n_taps] "+r" (n_taps), [x] "+r" (x) + : + : "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10"); +#else + asm volatile( + " vdup.32 q10, %[x]\n" + " vld1.32 {q4, q5}, [%[taps]]!\n" + " vld1.32 {q8, q9}, [%[s]]!\n" + " vld1.32 {q6, q7}, [%[taps]]!\n" + " subs %[n_taps], %[n_taps], #8\n" + " vmul.f32 q0, q4, q8\n" + " vmul.f32 q1, q5, q9\n" + " vmul.f32 q2, q6, q8\n" + " vmul.f32 q3, q7, q9\n" + " beq 3f\n" + "2:" + " vld1.32 {q4, q5}, [%[taps]]!\n" + " vld1.32 {q8, q9}, [%[s]]!\n" + " vld1.32 {q6, q7}, [%[taps]]!\n" + " subs %[n_taps], %[n_taps], #8\n" + " vmla.f32 q0, q4, q8\n" + " vmla.f32 q1, q5, q9\n" + " vmla.f32 q2, q6, q8\n" + " vmla.f32 q3, q7, q9\n" + " bne 2b\n" + "3:" + " vadd.f32 q0, q0, q1\n" /* sum[0] */ + " vadd.f32 q2, q2, q3\n" /* sum[1] */ + " vsub.f32 q2, q2, q0\n" /* sum[1] - sum[0] */ + " vmla.f32 q0, q1, q10\n" /* sum[0] += sum[1] * x */ + " vadd.f32 d0, d0, d1\n" + " vpadd.f32 d0, d0, d0\n" + " str d0, [%[d]]\n" + : [d] "+r" (d), [s] "+r" (s), [t0] "+r" (t0), [t1] "+r" (t1), + [n_taps] "+l" (n_taps), [x] "+l" (x) + : + : "cc", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", "q8", + "q9", "q10"); +#endif +} + +MAKE_RESAMPLER_FULL(neon); +MAKE_RESAMPLER_INTER(neon); diff --git a/spa/plugins/audioconvert/resample-native.h b/spa/plugins/audioconvert/resample-native.h index 43d354b16..52885cec9 100644 --- a/spa/plugins/audioconvert/resample-native.h +++ b/spa/plugins/audioconvert/resample-native.h @@ -127,6 +127,10 @@ static void impl_native_update_rate(struct resample *r, double rate) bool is_full = rate == 1.0; data->func = is_full ? do_resample_full_c : do_resample_inter_c; +#if defined (HAVE_NEON) + if (SPA_FLAG_IS_SET(r->cpu_flags, SPA_CPU_FLAG_NEON)) + data->func = is_full ? do_resample_full_neon : do_resample_inter_neon; +#endif #if defined (HAVE_SSE) if (SPA_FLAG_IS_SET(r->cpu_flags, SPA_CPU_FLAG_SSE)) data->func = is_full ? do_resample_full_sse : do_resample_inter_sse;