audioconvert: some more optimizations

This commit is contained in:
Wim Taymans 2019-03-20 13:04:44 +01:00
parent 67f26c9caf
commit fa3bcabbca
6 changed files with 580 additions and 150 deletions

View file

@ -30,12 +30,12 @@
#include <emmintrin.h>
static void
conv_s16_to_f32d_1_sse2(void *data, void *dst[], const void *src, int n_channels, int n_samples)
conv_s16_to_f32d_1_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, uint32_t n_channels, uint32_t n_samples)
{
const int16_t *s = src;
float **d = (float **) dst;
float *d0 = d[0];
int n, unrolled;
uint32_t n, unrolled;
__m128i in;
__m128 out, factor = _mm_set1_ps(1.0f / S16_SCALE);
@ -64,12 +64,12 @@ conv_s16_to_f32d_1_sse2(void *data, void *dst[], const void *src, int n_channels
}
static void
conv_s16_to_f32d_2_sse2(void *data, void *dst[], const void *src, int n_channels, int n_samples)
conv_s16_to_f32d_2_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, uint32_t n_channels, uint32_t n_samples)
{
const int16_t *s = src;
float **d = (float **) dst;
float *d0 = d[0], *d1 = d[1];
int n, unrolled;
uint32_t n, unrolled;
__m128i in, t[2];
__m128 out[2], factor = _mm_set1_ps(1.0f / S16_SCALE);
@ -110,10 +110,10 @@ conv_s16_to_f32d_2_sse2(void *data, void *dst[], const void *src, int n_channels
}
static void
conv_s16_to_f32d_sse2(void *data, void *dst[], const void *src[], int n_channels, int n_samples)
conv_s16_to_f32d_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const int16_t *s = src[0];
int i = 0;
uint32_t i = 0;
for(; i + 1 < n_channels; i += 2)
conv_s16_to_f32d_2_sse2(data, &dst[i], &s[i], n_channels, n_samples);
@ -122,16 +122,16 @@ conv_s16_to_f32d_sse2(void *data, void *dst[], const void *src[], int n_channels
}
static void
conv_s24_to_f32d_1_sse2(void *data, void *dst[], const void *src, int n_channels, int n_samples)
conv_s24_to_f32d_1_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, uint32_t n_channels, uint32_t n_samples)
{
const uint8_t *s = src;
float **d = (float **) dst;
float *d0 = d[0];
int n, unrolled;
uint32_t n, unrolled;
__m128i in;
__m128 out, factor = _mm_set1_ps(1.0f / S24_SCALE);
if (SPA_IS_ALIGNED(d0, 16) && n_samples > 4) {
if (SPA_IS_ALIGNED(d0, 16)) {
unrolled = n_samples / 4;
if ((n_samples & 3) == 0)
unrolled--;
@ -161,22 +161,167 @@ conv_s24_to_f32d_1_sse2(void *data, void *dst[], const void *src, int n_channels
}
static void
conv_s24_to_f32d_sse2(void *data, void *dst[], const void *src[], int n_channels, int n_samples)
conv_s24_to_f32d_2_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, uint32_t n_channels, uint32_t n_samples)
{
const uint8_t *s = src;
float **d = (float **) dst;
float *d0 = d[0], *d1 = d[1];
uint32_t n, unrolled;
__m128i in[2];
__m128 out[2], factor = _mm_set1_ps(1.0f / S24_SCALE);
if (SPA_IS_ALIGNED(d0, 16)) {
unrolled = n_samples / 4;
if ((n_samples & 3) == 0)
unrolled--;
}
else
unrolled = 0;
for(n = 0; unrolled--; n += 4) {
in[0] = _mm_setr_epi32(
*((uint32_t*)&s[0 + 0*n_channels]),
*((uint32_t*)&s[0 + 3*n_channels]),
*((uint32_t*)&s[0 + 6*n_channels]),
*((uint32_t*)&s[0 + 9*n_channels]));
in[1] = _mm_setr_epi32(
*((uint32_t*)&s[3 + 0*n_channels]),
*((uint32_t*)&s[3 + 3*n_channels]),
*((uint32_t*)&s[3 + 6*n_channels]),
*((uint32_t*)&s[3 + 9*n_channels]));
in[0] = _mm_slli_epi32(in[0], 8);
in[1] = _mm_slli_epi32(in[1], 8);
in[0] = _mm_srai_epi32(in[0], 8);
in[1] = _mm_srai_epi32(in[1], 8);
out[0] = _mm_cvtepi32_ps(in[0]);
out[1] = _mm_cvtepi32_ps(in[1]);
out[0] = _mm_mul_ps(out[0], factor);
out[1] = _mm_mul_ps(out[1], factor);
_mm_store_ps(&d0[n], out[0]);
_mm_store_ps(&d1[n], out[1]);
s += 12 * n_channels;
}
for(; n < n_samples; n++) {
out[0] = _mm_cvtsi32_ss(out[0], read_s24(s));
out[1] = _mm_cvtsi32_ss(out[1], read_s24(s+3));
out[0] = _mm_mul_ss(out[0], factor);
out[1] = _mm_mul_ss(out[1], factor);
_mm_store_ss(&d0[n], out[0]);
_mm_store_ss(&d1[n], out[1]);
s += 3 * n_channels;
}
}
static void
conv_s24_to_f32d_4_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src, uint32_t n_channels, uint32_t n_samples)
{
const uint8_t *s = src;
float **d = (float **) dst;
float *d0 = d[0], *d1 = d[1], *d2 = d[2], *d3 = d[3];
uint32_t n, unrolled;
__m128i in[4];
__m128 out[4], factor = _mm_set1_ps(1.0f / S24_SCALE);
if (SPA_IS_ALIGNED(d0, 16)) {
unrolled = n_samples / 4;
if ((n_samples & 3) == 0)
unrolled--;
}
else
unrolled = 0;
for(n = 0; unrolled--; n += 4) {
in[0] = _mm_setr_epi32(
*((uint32_t*)&s[0 + 0*n_channels]),
*((uint32_t*)&s[0 + 3*n_channels]),
*((uint32_t*)&s[0 + 6*n_channels]),
*((uint32_t*)&s[0 + 9*n_channels]));
in[1] = _mm_setr_epi32(
*((uint32_t*)&s[3 + 0*n_channels]),
*((uint32_t*)&s[3 + 3*n_channels]),
*((uint32_t*)&s[3 + 6*n_channels]),
*((uint32_t*)&s[3 + 9*n_channels]));
in[2] = _mm_setr_epi32(
*((uint32_t*)&s[6 + 0*n_channels]),
*((uint32_t*)&s[6 + 3*n_channels]),
*((uint32_t*)&s[6 + 6*n_channels]),
*((uint32_t*)&s[6 + 9*n_channels]));
in[3] = _mm_setr_epi32(
*((uint32_t*)&s[9 + 0*n_channels]),
*((uint32_t*)&s[9 + 3*n_channels]),
*((uint32_t*)&s[9 + 6*n_channels]),
*((uint32_t*)&s[9 + 9*n_channels]));
in[0] = _mm_slli_epi32(in[0], 8);
in[1] = _mm_slli_epi32(in[1], 8);
in[2] = _mm_slli_epi32(in[2], 8);
in[3] = _mm_slli_epi32(in[3], 8);
in[0] = _mm_srai_epi32(in[0], 8);
in[1] = _mm_srai_epi32(in[1], 8);
in[2] = _mm_srai_epi32(in[2], 8);
in[3] = _mm_srai_epi32(in[3], 8);
out[0] = _mm_cvtepi32_ps(in[0]);
out[1] = _mm_cvtepi32_ps(in[1]);
out[2] = _mm_cvtepi32_ps(in[2]);
out[3] = _mm_cvtepi32_ps(in[3]);
out[0] = _mm_mul_ps(out[0], factor);
out[1] = _mm_mul_ps(out[1], factor);
out[2] = _mm_mul_ps(out[2], factor);
out[3] = _mm_mul_ps(out[3], factor);
_mm_store_ps(&d0[n], out[0]);
_mm_store_ps(&d1[n], out[1]);
_mm_store_ps(&d2[n], out[2]);
_mm_store_ps(&d3[n], out[3]);
s += 12 * n_channels;
}
for(; n < n_samples; n++) {
out[0] = _mm_cvtsi32_ss(out[0], read_s24(s));
out[1] = _mm_cvtsi32_ss(out[1], read_s24(s+3));
out[2] = _mm_cvtsi32_ss(out[2], read_s24(s+6));
out[3] = _mm_cvtsi32_ss(out[3], read_s24(s+9));
out[0] = _mm_mul_ss(out[0], factor);
out[1] = _mm_mul_ss(out[1], factor);
out[2] = _mm_mul_ss(out[2], factor);
out[3] = _mm_mul_ss(out[3], factor);
_mm_store_ss(&d0[n], out[0]);
_mm_store_ss(&d1[n], out[1]);
_mm_store_ss(&d2[n], out[2]);
_mm_store_ss(&d3[n], out[3]);
s += 3 * n_channels;
}
}
static void
conv_s24_to_f32d_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const int8_t *s = src[0];
int i = 0;
uint32_t i = 0;
for(; i + 3 < n_channels; i += 4)
conv_s24_to_f32d_4_sse2(data, &dst[i], &s[3*i], n_channels, n_samples);
for(; i + 1 < n_channels; i += 2)
conv_s24_to_f32d_2_sse2(data, &dst[i], &s[3*i], n_channels, n_samples);
for(; i < n_channels; i++)
conv_s24_to_f32d_1_sse2(data, &dst[i], &s[3*i], n_channels, n_samples);
}
static void
conv_f32d_to_s32_1_sse2(void *data, void *dst, const void *src[], int n_channels, int n_samples)
conv_f32d_to_s32_1_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const float **s = (const float **) src;
const float *s0 = s[0];
int32_t *d = dst;
int n, unrolled;
uint32_t n, unrolled;
__m128 in[1];
__m128i out[4];
__m128 int_max = _mm_set1_ps(S24_MAX_F);
@ -212,12 +357,12 @@ conv_f32d_to_s32_1_sse2(void *data, void *dst, const void *src[], int n_channels
}
static void
conv_f32d_to_s32_2_sse2(void *data, void *dst, const void *src[], int n_channels, int n_samples)
conv_f32d_to_s32_2_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const float **s = (const float **) src;
const float *s0 = s[0], *s1 = s[1];
int32_t *d = dst;
int n, unrolled;
uint32_t n, unrolled;
__m128 in[2];
__m128i out[2], t[2];
__m128 int_max = _mm_set1_ps(S24_MAX_F);
@ -265,12 +410,12 @@ conv_f32d_to_s32_2_sse2(void *data, void *dst, const void *src[], int n_channels
}
static void
conv_f32d_to_s32_4_sse2(void *data, void *dst, const void *src[], int n_channels, int n_samples)
conv_f32d_to_s32_4_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const float **s = (const float **) src;
const float *s0 = s[0], *s1 = s[1], *s2 = s[2], *s3 = s[3];
int32_t *d = dst;
int n, unrolled;
uint32_t n, unrolled;
__m128 in[4];
__m128i out[4], t[4];
__m128 int_max = _mm_set1_ps(S24_MAX_F);
@ -279,7 +424,8 @@ conv_f32d_to_s32_4_sse2(void *data, void *dst, const void *src[], int n_channels
if (SPA_IS_ALIGNED(s0, 16) &&
SPA_IS_ALIGNED(s1, 16) &&
SPA_IS_ALIGNED(s2, 16) &&
SPA_IS_ALIGNED(s3, 16))
SPA_IS_ALIGNED(s3, 16) &&
SPA_IS_ALIGNED(d, 16))
unrolled = n_samples / 4;
else
unrolled = 0;
@ -310,10 +456,10 @@ conv_f32d_to_s32_4_sse2(void *data, void *dst, const void *src[], int n_channels
out[2] = _mm_unpacklo_epi64(t[2], t[3]);
out[3] = _mm_unpackhi_epi64(t[2], t[3]);
_mm_storeu_si128((__m128i*)(d + 0*n_channels), out[0]);
_mm_storeu_si128((__m128i*)(d + 1*n_channels), out[1]);
_mm_storeu_si128((__m128i*)(d + 2*n_channels), out[2]);
_mm_storeu_si128((__m128i*)(d + 3*n_channels), out[3]);
_mm_store_si128((__m128i*)(d + 0*n_channels), out[0]);
_mm_store_si128((__m128i*)(d + 1*n_channels), out[1]);
_mm_store_si128((__m128i*)(d + 2*n_channels), out[2]);
_mm_store_si128((__m128i*)(d + 3*n_channels), out[3]);
d += 4*n_channels;
}
for(; n < n_samples; n++) {
@ -335,10 +481,10 @@ conv_f32d_to_s32_4_sse2(void *data, void *dst, const void *src[], int n_channels
}
static void
conv_f32d_to_s32_sse2(void *data, void *dst[], const void *src[], int n_channels, int n_samples)
conv_f32d_to_s32_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
int32_t *d = dst[0];
int i = 0;
uint32_t i = 0;
for(; i + 3 < n_channels; i += 4)
conv_f32d_to_s32_4_sse2(data, &d[i], &src[i], n_channels, n_samples);
@ -349,12 +495,12 @@ conv_f32d_to_s32_sse2(void *data, void *dst[], const void *src[], int n_channels
}
static void
conv_f32d_to_s16_1_sse2(void *data, void *dst, const void *src[], int n_channels, int n_samples)
conv_f32d_to_s16_1_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const float **s = (const float **) src;
const float *s0 = s[0];
int16_t *d = dst;
int n, unrolled;
uint32_t n, unrolled;
__m128 in[2];
__m128i out[2];
__m128 int_max = _mm_set1_ps(S16_MAX_F);
@ -391,12 +537,12 @@ conv_f32d_to_s16_1_sse2(void *data, void *dst, const void *src[], int n_channels
}
static void
conv_f32d_to_s16_2_sse2(void *data, void *dst, const void *src[], int n_channels, int n_samples)
conv_f32d_to_s16_2_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const float **s = (const float **) src;
const float *s0 = s[0], *s1 = s[1];
int16_t *d = dst;
int n, unrolled;
uint32_t n, unrolled;
__m128 in[2];
__m128i out[4], t[2];
__m128 int_max = _mm_set1_ps(S16_MAX_F);
@ -441,11 +587,76 @@ conv_f32d_to_s16_2_sse2(void *data, void *dst, const void *src[], int n_channels
}
static void
conv_f32d_to_s16_sse2(void *data, void *dst[], const void *src[], int n_channels, int n_samples)
conv_f32d_to_s16_4_sse2(void *data, void * SPA_RESTRICT dst, const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
const float **s = (const float **) src;
const float *s0 = s[0], *s1 = s[1], *s2 = s[2], *s3 = s[3];
int16_t *d = dst;
uint32_t n, unrolled;
__m128 in[4];
__m128i out[4], t[4];
__m128 int_max = _mm_set1_ps(S16_MAX_F);
__m128 int_min = _mm_sub_ps(_mm_setzero_ps(), int_max);
if (SPA_IS_ALIGNED(s0, 16) &&
SPA_IS_ALIGNED(s1, 16) &&
SPA_IS_ALIGNED(s2, 16) &&
SPA_IS_ALIGNED(s3, 16))
unrolled = n_samples / 4;
else
unrolled = 0;
for(n = 0; unrolled--; n += 4) {
in[0] = _mm_mul_ps(_mm_load_ps(&s0[n]), int_max);
in[1] = _mm_mul_ps(_mm_load_ps(&s1[n]), int_max);
in[2] = _mm_mul_ps(_mm_load_ps(&s2[n]), int_max);
in[3] = _mm_mul_ps(_mm_load_ps(&s3[n]), int_max);
t[0] = _mm_cvtps_epi32(in[0]);
t[1] = _mm_cvtps_epi32(in[1]);
t[2] = _mm_cvtps_epi32(in[2]);
t[3] = _mm_cvtps_epi32(in[3]);
t[0] = _mm_packs_epi32(t[0], t[2]);
t[1] = _mm_packs_epi32(t[1], t[3]);
out[0] = _mm_unpacklo_epi16(t[0], t[1]);
out[1] = _mm_unpackhi_epi16(t[0], t[1]);
out[2] = _mm_unpacklo_epi32(out[0], out[1]);
out[3] = _mm_unpackhi_epi32(out[0], out[1]);
_mm_storel_pi((__m64*)(d + 0*n_channels), (__m128)out[2]);
_mm_storeh_pi((__m64*)(d + 1*n_channels), (__m128)out[2]);
_mm_storel_pi((__m64*)(d + 2*n_channels), (__m128)out[3]);
_mm_storeh_pi((__m64*)(d + 3*n_channels), (__m128)out[3]);
d += 4*n_channels;
}
for(; n < n_samples; n++) {
in[0] = _mm_mul_ss(_mm_load_ss(&s0[n]), int_max);
in[1] = _mm_mul_ss(_mm_load_ss(&s1[n]), int_max);
in[2] = _mm_mul_ss(_mm_load_ss(&s2[n]), int_max);
in[3] = _mm_mul_ss(_mm_load_ss(&s3[n]), int_max);
in[0] = _mm_min_ss(int_max, _mm_max_ss(in[0], int_min));
in[1] = _mm_min_ss(int_max, _mm_max_ss(in[1], int_min));
in[2] = _mm_min_ss(int_max, _mm_max_ss(in[2], int_min));
in[3] = _mm_min_ss(int_max, _mm_max_ss(in[3], int_min));
d[0] = _mm_cvtss_si32(in[0]);
d[1] = _mm_cvtss_si32(in[1]);
d[2] = _mm_cvtss_si32(in[2]);
d[3] = _mm_cvtss_si32(in[3]);
d += n_channels;
}
}
static void
conv_f32d_to_s16_sse2(void *data, void * SPA_RESTRICT dst[], const void * SPA_RESTRICT src[], uint32_t n_channels, uint32_t n_samples)
{
int16_t *d = dst[0];
int i = 0;
uint32_t i = 0;
for(; i + 3 < n_channels; i += 4)
conv_f32d_to_s16_4_sse2(data, &d[i], &src[i], n_channels, n_samples);
for(; i + 1 < n_channels; i += 2)
conv_f32d_to_s16_2_sse2(data, &d[i], &src[i], n_channels, n_samples);
for(; i < n_channels; i++)