media/libaudioprocessing/AudioResamplerFirProcessNeon.h

/*
 * Copyright (C) 2013 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H
#define ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H

namespace android {

// depends on AudioResamplerFirOps.h, AudioResamplerFirProcess.h

#if USE_NEON

// use intrinsics if inline arm32 assembly is not possible
#if !USE_INLINE_ASSEMBLY
#define USE_INTRINSIC
#endif

// following intrinsics available only on ARM 64 bit ACLE
#ifndef __aarch64__
#undef vld1q_f32_x2
#undef vld1q_s32_x2
#endif

#define TO_STRING2(x) #x
#define TO_STRING(x) TO_STRING2(x)
// uncomment to print GCC version, may be relevant for intrinsic optimizations
/* #pragma message ("GCC version: " TO_STRING(__GNUC__) \
        "." TO_STRING(__GNUC_MINOR__) \
        "." TO_STRING(__GNUC_PATCHLEVEL__)) */

//
// NEON specializations are enabled for Process() and ProcessL() in AudioResamplerFirProcess.h
//
// Two variants are presented here:
// ARM NEON inline assembly which appears up to 10-15% faster than intrinsics (gcc 4.9) for arm32.
// ARM NEON intrinsics which can also be used by arm64 and x86/64 with NEON header.
//

// Macros to save a mono/stereo accumulator sample in q0 (and q4) as stereo out.
// These are only used for inline assembly.
#define ASSEMBLY_ACCUMULATE_MONO \
        "vld1.s32       {d2}, [%[vLR]:64]        \n"/* (1) load volumes */\
        "vld1.s32       {d3}, %[out]             \n"/* (2) unaligned load the output */\
        "vpadd.s32      d0, d0, d1               \n"/* (1) add all 4 partial sums */\
        "vpadd.s32      d0, d0, d0               \n"/* (1+4d) and replicate L/R */\
        "vqrdmulh.s32   d0, d0, d2               \n"/* (2+3d) apply volume */\
        "vqadd.s32      d3, d3, d0               \n"/* (1+4d) accumulate result (saturating) */\
        "vst1.s32       {d3}, %[out]             \n"/* (2+2d) store result */

#define ASSEMBLY_ACCUMULATE_STEREO \
        "vld1.s32       {d2}, [%[vLR]:64]        \n"/* (1) load volumes*/\
        "vld1.s32       {d3}, %[out]             \n"/* (2) unaligned load the output*/\
        "vpadd.s32      d0, d0, d1               \n"/* (1) add all 4 partial sums from q0*/\
        "vpadd.s32      d8, d8, d9               \n"/* (1) add all 4 partial sums from q4*/\
        "vpadd.s32      d0, d0, d8               \n"/* (1+4d) combine into L/R*/\
        "vqrdmulh.s32   d0, d0, d2               \n"/* (2+3d) apply volume*/\
        "vqadd.s32      d3, d3, d0               \n"/* (1+4d) accumulate result (saturating)*/\
        "vst1.s32       {d3}, %[out]             \n"/* (2+2d)store result*/

template <int CHANNELS, int STRIDE, bool FIXED>
static inline void ProcessNeonIntrinsic(int32_t* out,
        int count,
        const int16_t* coefsP,
        const int16_t* coefsN,
        const int16_t* sP,
        const int16_t* sN,
        const int32_t* volumeLR,
        uint32_t lerpP,
        const int16_t* coefsP1,
        const int16_t* coefsN1)
{
    ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8
    static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2");

    sP -= CHANNELS*((STRIDE>>1)-1);
    coefsP = (const int16_t*)__builtin_assume_aligned(coefsP, 16);
    coefsN = (const int16_t*)__builtin_assume_aligned(coefsN, 16);

    int16x4_t interp;
    if (!FIXED) {
        interp = vdup_n_s16(lerpP);
        //interp = (int16x4_t)vset_lane_s32 ((int32x2_t)lerpP, interp, 0);
        coefsP1 = (const int16_t*)__builtin_assume_aligned(coefsP1, 16);
        coefsN1 = (const int16_t*)__builtin_assume_aligned(coefsN1, 16);
    }
    int32x4_t accum, accum2;
    // warning uninitialized if we use veorq_s32
    // (alternative to below) accum = veorq_s32(accum, accum);
    accum = vdupq_n_s32(0);
    if (CHANNELS == 2) {
        // (alternative to below) accum2 = veorq_s32(accum2, accum2);
        accum2 = vdupq_n_s32(0);
    }
    do {
        int16x8_t posCoef = vld1q_s16(coefsP);
        coefsP += 8;
        int16x8_t negCoef = vld1q_s16(coefsN);
        coefsN += 8;
        if (!FIXED) { // interpolate
            int16x8_t posCoef1 = vld1q_s16(coefsP1);
            coefsP1 += 8;
            int16x8_t negCoef1 = vld1q_s16(coefsN1);
            coefsN1 += 8;

            posCoef1 = vsubq_s16(posCoef1, posCoef);
            negCoef = vsubq_s16(negCoef, negCoef1);

            posCoef1 = vqrdmulhq_lane_s16(posCoef1, interp, 0);
            negCoef = vqrdmulhq_lane_s16(negCoef, interp, 0);

            posCoef = vaddq_s16(posCoef, posCoef1);
            negCoef = vaddq_s16(negCoef, negCoef1);
        }
        switch (CHANNELS) {
        case 1: {
            int16x8_t posSamp = vld1q_s16(sP);
            int16x8_t negSamp = vld1q_s16(sN);
            sN += 8;
            posSamp = vrev64q_s16(posSamp);

            // dot product
            accum = vmlal_s16(accum, vget_low_s16(posSamp), vget_high_s16(posCoef)); // reversed
            accum = vmlal_s16(accum, vget_high_s16(posSamp), vget_low_s16(posCoef)); // reversed
            accum = vmlal_s16(accum, vget_low_s16(negSamp), vget_low_s16(negCoef));
            accum = vmlal_s16(accum, vget_high_s16(negSamp), vget_high_s16(negCoef));
            sP -= 8;
        } break;
        case 2: {
            int16x8x2_t posSamp = vld2q_s16(sP);
            int16x8x2_t negSamp = vld2q_s16(sN);
            sN += 16;
            posSamp.val[0] = vrev64q_s16(posSamp.val[0]);
            posSamp.val[1] = vrev64q_s16(posSamp.val[1]);

            // dot product
            accum = vmlal_s16(accum, vget_low_s16(posSamp.val[0]), vget_high_s16(posCoef)); // r
            accum = vmlal_s16(accum, vget_high_s16(posSamp.val[0]), vget_low_s16(posCoef)); // r
            accum2 = vmlal_s16(accum2, vget_low_s16(posSamp.val[1]), vget_high_s16(posCoef)); // r
            accum2 = vmlal_s16(accum2, vget_high_s16(posSamp.val[1]), vget_low_s16(posCoef)); // r
            accum = vmlal_s16(accum, vget_low_s16(negSamp.val[0]), vget_low_s16(negCoef));
            accum = vmlal_s16(accum, vget_high_s16(negSamp.val[0]), vget_high_s16(negCoef));
            accum2 = vmlal_s16(accum2, vget_low_s16(negSamp.val[1]), vget_low_s16(negCoef));
            accum2 = vmlal_s16(accum2, vget_high_s16(negSamp.val[1]), vget_high_s16(negCoef));
            sP -= 16;
        } break;
        }
    } while (count -= 8);

    // multiply by volume and save
    volumeLR = (const int32_t*)__builtin_assume_aligned(volumeLR, 8);
    int32x2_t vLR = vld1_s32(volumeLR);
    int32x2_t outSamp = vld1_s32(out);
    // combine and funnel down accumulator
    int32x2_t outAccum = vpadd_s32(vget_low_s32(accum), vget_high_s32(accum));
    if (CHANNELS == 1) {
        // duplicate accum to both L and R
        outAccum = vpadd_s32(outAccum, outAccum);
    } else if (CHANNELS == 2) {
        // accum2 contains R, fold in
        int32x2_t outAccum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
        outAccum = vpadd_s32(outAccum, outAccum2);
    }
    outAccum = vqrdmulh_s32(outAccum, vLR);
    outSamp = vqadd_s32(outSamp, outAccum);
    vst1_s32(out, outSamp);
}

template <int CHANNELS, int STRIDE, bool FIXED>
static inline void ProcessNeonIntrinsic(int32_t* out,
        int count,
        const int32_t* coefsP,
        const int32_t* coefsN,
        const int16_t* sP,
        const int16_t* sN,
        const int32_t* volumeLR,
        uint32_t lerpP,
        const int32_t* coefsP1,
        const int32_t* coefsN1)
{
    ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8
    static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2");

    sP -= CHANNELS*((STRIDE>>1)-1);
    coefsP = (const int32_t*)__builtin_assume_aligned(coefsP, 16);
    coefsN = (const int32_t*)__builtin_assume_aligned(coefsN, 16);

    int32x2_t interp;
    if (!FIXED) {
        interp = vdup_n_s32(lerpP);
        coefsP1 = (const int32_t*)__builtin_assume_aligned(coefsP1, 16);
        coefsN1 = (const int32_t*)__builtin_assume_aligned(coefsN1, 16);
    }
    int32x4_t accum, accum2;
    // warning uninitialized if we use veorq_s32
    // (alternative to below) accum = veorq_s32(accum, accum);
    accum = vdupq_n_s32(0);
    if (CHANNELS == 2) {
        // (alternative to below) accum2 = veorq_s32(accum2, accum2);
        accum2 = vdupq_n_s32(0);
    }
    do {
#ifdef vld1q_s32_x2
        int32x4x2_t posCoef = vld1q_s32_x2(coefsP);
        coefsP += 8;
        int32x4x2_t negCoef = vld1q_s32_x2(coefsN);
        coefsN += 8;
#else
        int32x4x2_t posCoef;
        posCoef.val[0] = vld1q_s32(coefsP);
        coefsP += 4;
        posCoef.val[1] = vld1q_s32(coefsP);
        coefsP += 4;
        int32x4x2_t negCoef;
        negCoef.val[0] = vld1q_s32(coefsN);
        coefsN += 4;
        negCoef.val[1] = vld1q_s32(coefsN);
        coefsN += 4;
#endif
        if (!FIXED) { // interpolate
#ifdef vld1q_s32_x2
            int32x4x2_t posCoef1 = vld1q_s32_x2(coefsP1);
            coefsP1 += 8;
            int32x4x2_t negCoef1 = vld1q_s32_x2(coefsN1);
            coefsN1 += 8;
#else
            int32x4x2_t posCoef1;
            posCoef1.val[0] = vld1q_s32(coefsP1);
            coefsP1 += 4;
            posCoef1.val[1] = vld1q_s32(coefsP1);
            coefsP1 += 4;
            int32x4x2_t negCoef1;
            negCoef1.val[0] = vld1q_s32(coefsN1);
            coefsN1 += 4;
            negCoef1.val[1] = vld1q_s32(coefsN1);
            coefsN1 += 4;
#endif

            posCoef1.val[0] = vsubq_s32(posCoef1.val[0], posCoef.val[0]);
            posCoef1.val[1] = vsubq_s32(posCoef1.val[1], posCoef.val[1]);
            negCoef.val[0] = vsubq_s32(negCoef.val[0], negCoef1.val[0]);
            negCoef.val[1] = vsubq_s32(negCoef.val[1], negCoef1.val[1]);

            posCoef1.val[0] = vqrdmulhq_lane_s32(posCoef1.val[0], interp, 0);
            posCoef1.val[1] = vqrdmulhq_lane_s32(posCoef1.val[1], interp, 0);
            negCoef.val[0] = vqrdmulhq_lane_s32(negCoef.val[0], interp, 0);
            negCoef.val[1] = vqrdmulhq_lane_s32(negCoef.val[1], interp, 0);

            posCoef.val[0] = vaddq_s32(posCoef.val[0], posCoef1.val[0]);
            posCoef.val[1] = vaddq_s32(posCoef.val[1], posCoef1.val[1]);
            negCoef.val[0] = vaddq_s32(negCoef.val[0], negCoef1.val[0]);
            negCoef.val[1] = vaddq_s32(negCoef.val[1], negCoef1.val[1]);
        }
        switch (CHANNELS) {
        case 1: {
            int16x8_t posSamp = vld1q_s16(sP);
            int16x8_t negSamp = vld1q_s16(sN);
            sN += 8;
            posSamp = vrev64q_s16(posSamp);

            int32x4_t posSamp0 = vshll_n_s16(vget_low_s16(posSamp), 15);
            int32x4_t posSamp1 = vshll_n_s16(vget_high_s16(posSamp), 15);
            int32x4_t negSamp0 = vshll_n_s16(vget_low_s16(negSamp), 15);
            int32x4_t negSamp1 = vshll_n_s16(vget_high_s16(negSamp), 15);

            // dot product
            posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed
            posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed
            negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]);
            negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]);

            accum = vaddq_s32(accum, posSamp0);
            negSamp0 = vaddq_s32(negSamp0, negSamp1);
            accum = vaddq_s32(accum, posSamp1);
            accum = vaddq_s32(accum, negSamp0);

            sP -= 8;
        } break;
        case 2: {
            int16x8x2_t posSamp = vld2q_s16(sP);
            int16x8x2_t negSamp = vld2q_s16(sN);
            sN += 16;
            posSamp.val[0] = vrev64q_s16(posSamp.val[0]);
            posSamp.val[1] = vrev64q_s16(posSamp.val[1]);

            // left
            int32x4_t posSamp0 = vshll_n_s16(vget_low_s16(posSamp.val[0]), 15);
            int32x4_t posSamp1 = vshll_n_s16(vget_high_s16(posSamp.val[0]), 15);
            int32x4_t negSamp0 = vshll_n_s16(vget_low_s16(negSamp.val[0]), 15);
            int32x4_t negSamp1 = vshll_n_s16(vget_high_s16(negSamp.val[0]), 15);

            // dot product
            posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed
            posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed
            negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]);
            negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]);

            accum = vaddq_s32(accum, posSamp0);
            negSamp0 = vaddq_s32(negSamp0, negSamp1);
            accum = vaddq_s32(accum, posSamp1);
            accum = vaddq_s32(accum, negSamp0);

            // right
            posSamp0 = vshll_n_s16(vget_low_s16(posSamp.val[1]), 15);
            posSamp1 = vshll_n_s16(vget_high_s16(posSamp.val[1]), 15);
            negSamp0 = vshll_n_s16(vget_low_s16(negSamp.val[1]), 15);
            negSamp1 = vshll_n_s16(vget_high_s16(negSamp.val[1]), 15);

            // dot product
            posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed
            posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed
            negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]);
            negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]);

            accum2 = vaddq_s32(accum2, posSamp0);
            negSamp0 = vaddq_s32(negSamp0, negSamp1);
            accum2 = vaddq_s32(accum2, posSamp1);
            accum2 = vaddq_s32(accum2, negSamp0);

            sP -= 16;
        } break;
        }
    } while (count -= 8);

    // multiply by volume and save
    volumeLR = (const int32_t*)__builtin_assume_aligned(volumeLR, 8);
    int32x2_t vLR = vld1_s32(volumeLR);
    int32x2_t outSamp = vld1_s32(out);
    // combine and funnel down accumulator
    int32x2_t outAccum = vpadd_s32(vget_low_s32(accum), vget_high_s32(accum));
    if (CHANNELS == 1) {
        // duplicate accum to both L and R
        outAccum = vpadd_s32(outAccum, outAccum);
    } else if (CHANNELS == 2) {
        // accum2 contains R, fold in
        int32x2_t outAccum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2));
        outAccum = vpadd_s32(outAccum, outAccum2);
    }
    outAccum = vqrdmulh_s32(outAccum, vLR);
    outSamp = vqadd_s32(outSamp, outAccum);
    vst1_s32(out, outSamp);
}

template <int CHANNELS, int STRIDE, bool FIXED>
static inline void ProcessNeonIntrinsic(float* out,
        int count,
        const float* coefsP,
        const float* coefsN,
        const float* sP,
        const float* sN,
        const float* volumeLR,
        float lerpP,
        const float* coefsP1,
        const float* coefsN1)
{
    ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8
    static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2");

    sP -= CHANNELS*((STRIDE>>1)-1);
    coefsP = (const float*)__builtin_assume_aligned(coefsP, 16);
    coefsN = (const float*)__builtin_assume_aligned(coefsN, 16);

    float32x2_t interp;
    if (!FIXED) {
        interp = vdup_n_f32(lerpP);
        coefsP1 = (const float*)__builtin_assume_aligned(coefsP1, 16);
        coefsN1 = (const float*)__builtin_assume_aligned(coefsN1, 16);
    }
    float32x4_t accum, accum2;
    // warning uninitialized if we use veorq_s32
    // (alternative to below) accum = veorq_s32(accum, accum);
    accum = vdupq_n_f32(0);
    if (CHANNELS == 2) {
        // (alternative to below) accum2 = veorq_s32(accum2, accum2);
        accum2 = vdupq_n_f32(0);
    }
    do {
#ifdef vld1q_f32_x2
        float32x4x2_t posCoef = vld1q_f32_x2(coefsP);
        coefsP += 8;
        float32x4x2_t negCoef = vld1q_f32_x2(coefsN);
        coefsN += 8;
#else
        float32x4x2_t posCoef;
        posCoef.val[0] = vld1q_f32(coefsP);
        coefsP += 4;
        posCoef.val[1] = vld1q_f32(coefsP);
        coefsP += 4;
        float32x4x2_t negCoef;
        negCoef.val[0] = vld1q_f32(coefsN);
        coefsN += 4;
        negCoef.val[1] = vld1q_f32(coefsN);
        coefsN += 4;
#endif
        if (!FIXED) { // interpolate
#ifdef vld1q_f32_x2
            float32x4x2_t posCoef1 = vld1q_f32_x2(coefsP1);
            coefsP1 += 8;
            float32x4x2_t negCoef1 = vld1q_f32_x2(coefsN1);
            coefsN1 += 8;
#else
            float32x4x2_t posCoef1;
            posCoef1.val[0] = vld1q_f32(coefsP1);
            coefsP1 += 4;
            posCoef1.val[1] = vld1q_f32(coefsP1);
            coefsP1 += 4;
            float32x4x2_t negCoef1;
            negCoef1.val[0] = vld1q_f32(coefsN1);
            coefsN1 += 4;
            negCoef1.val[1] = vld1q_f32(coefsN1);
            coefsN1 += 4;
#endif
            posCoef1.val[0] = vsubq_f32(posCoef1.val[0], posCoef.val[0]);
            posCoef1.val[1] = vsubq_f32(posCoef1.val[1], posCoef.val[1]);
            negCoef.val[0] = vsubq_f32(negCoef.val[0], negCoef1.val[0]);
            negCoef.val[1] = vsubq_f32(negCoef.val[1], negCoef1.val[1]);

            posCoef.val[0] = vmlaq_lane_f32(posCoef.val[0], posCoef1.val[0], interp, 0);
            posCoef.val[1] = vmlaq_lane_f32(posCoef.val[1], posCoef1.val[1], interp, 0);
            negCoef.val[0] = vmlaq_lane_f32(negCoef1.val[0], negCoef.val[0], interp, 0); // rev
            negCoef.val[1] = vmlaq_lane_f32(negCoef1.val[1], negCoef.val[1], interp, 0); // rev
        }
        switch (CHANNELS) {
        case 1: {
#ifdef vld1q_f32_x2
            float32x4x2_t posSamp = vld1q_f32_x2(sP);
            float32x4x2_t negSamp = vld1q_f32_x2(sN);
            sN += 8;
            sP -= 8;
#else
            float32x4x2_t posSamp;
            posSamp.val[0] = vld1q_f32(sP);
            sP += 4;
            posSamp.val[1] = vld1q_f32(sP);
            sP -= 12;
            float32x4x2_t negSamp;
            negSamp.val[0] = vld1q_f32(sN);
            sN += 4;
            negSamp.val[1] = vld1q_f32(sN);
            sN += 4;
#endif
            // effectively we want a vrev128q_f32()
            posSamp.val[0] = vrev64q_f32(posSamp.val[0]);
            posSamp.val[1] = vrev64q_f32(posSamp.val[1]);
            posSamp.val[0] = vcombine_f32(
                    vget_high_f32(posSamp.val[0]), vget_low_f32(posSamp.val[0]));
            posSamp.val[1] = vcombine_f32(
                    vget_high_f32(posSamp.val[1]), vget_low_f32(posSamp.val[1]));

            accum = vmlaq_f32(accum, posSamp.val[0], posCoef.val[1]);
            accum = vmlaq_f32(accum, posSamp.val[1], posCoef.val[0]);
            accum = vmlaq_f32(accum, negSamp.val[0], negCoef.val[0]);
            accum = vmlaq_f32(accum, negSamp.val[1], negCoef.val[1]);
        } break;
        case 2: {
            float32x4x2_t posSamp0 = vld2q_f32(sP);
            sP += 8;
            float32x4x2_t negSamp0 = vld2q_f32(sN);
            sN += 8;
            posSamp0.val[0] = vrev64q_f32(posSamp0.val[0]);
            posSamp0.val[1] = vrev64q_f32(posSamp0.val[1]);
            posSamp0.val[0] = vcombine_f32(
                    vget_high_f32(posSamp0.val[0]), vget_low_f32(posSamp0.val[0]));
            posSamp0.val[1] = vcombine_f32(
                    vget_high_f32(posSamp0.val[1]), vget_low_f32(posSamp0.val[1]));

            float32x4x2_t posSamp1 = vld2q_f32(sP);
            sP -= 24;
            float32x4x2_t negSamp1 = vld2q_f32(sN);
            sN += 8;
            posSamp1.val[0] = vrev64q_f32(posSamp1.val[0]);
            posSamp1.val[1] = vrev64q_f32(posSamp1.val[1]);
            posSamp1.val[0] = vcombine_f32(
                    vget_high_f32(posSamp1.val[0]), vget_low_f32(posSamp1.val[0]));
            posSamp1.val[1] = vcombine_f32(
                    vget_high_f32(posSamp1.val[1]), vget_low_f32(posSamp1.val[1]));

            // Note: speed is affected by accumulation order.
            // Also, speed appears slower using vmul/vadd instead of vmla for
            // stereo case, comparable for mono.

            accum = vmlaq_f32(accum, negSamp0.val[0], negCoef.val[0]);
            accum = vmlaq_f32(accum, negSamp1.val[0], negCoef.val[1]);
            accum2 = vmlaq_f32(accum2, negSamp0.val[1], negCoef.val[0]);
            accum2 = vmlaq_f32(accum2, negSamp1.val[1], negCoef.val[1]);

            accum = vmlaq_f32(accum, posSamp0.val[0], posCoef.val[1]); // reversed
            accum = vmlaq_f32(accum, posSamp1.val[0], posCoef.val[0]); // reversed
            accum2 = vmlaq_f32(accum2, posSamp0.val[1], posCoef.val[1]); // reversed
            accum2 = vmlaq_f32(accum2, posSamp1.val[1], posCoef.val[0]); // reversed
        } break;
        }
    } while (count -= 8);

    // multiply by volume and save
    volumeLR = (const float*)__builtin_assume_aligned(volumeLR, 8);
    float32x2_t vLR = vld1_f32(volumeLR);
    float32x2_t outSamp = vld1_f32(out);
    // combine and funnel down accumulator
    float32x2_t outAccum = vpadd_f32(vget_low_f32(accum), vget_high_f32(accum));
    if (CHANNELS == 1) {
        // duplicate accum to both L and R
        outAccum = vpadd_f32(outAccum, outAccum);
    } else if (CHANNELS == 2) {
        // accum2 contains R, fold in
        float32x2_t outAccum2 = vpadd_f32(vget_low_f32(accum2), vget_high_f32(accum2));
        outAccum = vpadd_f32(outAccum, outAccum2);
    }
    outSamp = vmla_f32(outSamp, outAccum, vLR);
    vst1_f32(out, outSamp);
}

template <>
inline void ProcessL<1, 16>(int32_t* const out,
        int count,
        const int16_t* coefsP,
        const int16_t* coefsN,
        const int16_t* sP,
        const int16_t* sN,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
#else
    const int CHANNELS = 1; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "veor           q0, q0, q0               \n"// (0 - combines+) accumulator = 0

        "1:                                      \n"

        "vld1.16        {q2}, [%[sP]]            \n"// (2+0d) load 8 16-bits mono samples
        "vld1.16        {q3}, [%[sN]]!           \n"// (2) load 8 16-bits mono samples
        "vld1.16        {q8}, [%[coefsP0]:128]!  \n"// (1) load 8 16-bits coefs
        "vld1.16        {q10}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs

        "vrev64.16      q2, q2                   \n"// (1) reverse s3, s2, s1, s0, s7, s6, s5, s4

        // reordering the vmal to do d6, d7 before d4, d5 is slower(?)
        "vmlal.s16      q0, d4, d17              \n"// (1+0d) multiply (reversed)samples by coef
        "vmlal.s16      q0, d5, d16              \n"// (1) multiply (reversed)samples by coef
        "vmlal.s16      q0, d6, d20              \n"// (1) multiply neg samples
        "vmlal.s16      q0, d7, d21              \n"// (1) multiply neg samples

        // moving these ARM instructions before neon above seems to be slower
        "subs           %[count], %[count], #8   \n"// (1) update loop counter
        "sub            %[sP], %[sP], #16        \n"// (0) move pointer to next set of samples

        // sP used after branch (warning)
        "bne            1b                       \n"// loop

         ASSEMBLY_ACCUMULATE_MONO

        : [out]     "=Uv" (out[0]),
          [count]   "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [sP]      "+r" (sP),
          [sN]      "+r" (sN)
        : [vLR]     "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q8", "q10"
    );
#endif
}

template <>
inline void ProcessL<2, 16>(int32_t* const out,
        int count,
        const int16_t* coefsP,
        const int16_t* coefsN,
        const int16_t* sP,
        const int16_t* sN,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
#else
    const int CHANNELS = 2; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "veor           q0, q0, q0               \n"// (1) acc_L = 0
        "veor           q4, q4, q4               \n"// (0 combines+) acc_R = 0

        "1:                                      \n"

        "vld2.16        {q2, q3}, [%[sP]]        \n"// (3+0d) load 8 16-bits stereo frames
        "vld2.16        {q5, q6}, [%[sN]]!       \n"// (3) load 8 16-bits stereo frames
        "vld1.16        {q8}, [%[coefsP0]:128]!  \n"// (1) load 8 16-bits coefs
        "vld1.16        {q10}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs

        "vrev64.16      q2, q2                   \n"// (1) reverse 8 samples of positive left
        "vrev64.16      q3, q3                   \n"// (0 combines+) reverse positive right

        "vmlal.s16      q0, d4, d17              \n"// (1) multiply (reversed) samples left
        "vmlal.s16      q0, d5, d16              \n"// (1) multiply (reversed) samples left
        "vmlal.s16      q4, d6, d17              \n"// (1) multiply (reversed) samples right
        "vmlal.s16      q4, d7, d16              \n"// (1) multiply (reversed) samples right
        "vmlal.s16      q0, d10, d20             \n"// (1) multiply samples left
        "vmlal.s16      q0, d11, d21             \n"// (1) multiply samples left
        "vmlal.s16      q4, d12, d20             \n"// (1) multiply samples right
        "vmlal.s16      q4, d13, d21             \n"// (1) multiply samples right

        // moving these ARM before neon seems to be slower
        "subs           %[count], %[count], #8   \n"// (1) update loop counter
        "sub            %[sP], %[sP], #32        \n"// (0) move pointer to next set of samples

        // sP used after branch (warning)
        "bne            1b                       \n"// loop

        ASSEMBLY_ACCUMULATE_STEREO

        : [out] "=Uv" (out[0]),
          [count] "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [sP] "+r" (sP),
          [sN] "+r" (sN)
        : [vLR] "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q4", "q5", "q6",
          "q8", "q10"
     );
#endif
}

template <>
inline void Process<1, 16>(int32_t* const out,
        int count,
        const int16_t* coefsP,
        const int16_t* coefsN,
        const int16_t* coefsP1,
        const int16_t* coefsN1,
        const int16_t* sP,
        const int16_t* sN,
        uint32_t lerpP,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            lerpP, coefsP1, coefsN1);
#else

    const int CHANNELS = 1; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "vmov.32        d2[0], %[lerpP]          \n"// load the positive phase S32 Q15
        "veor           q0, q0, q0               \n"// (0 - combines+) accumulator = 0

        "1:                                      \n"

        "vld1.16        {q2}, [%[sP]]            \n"// (2+0d) load 8 16-bits mono samples
        "vld1.16        {q3}, [%[sN]]!           \n"// (2) load 8 16-bits mono samples
        "vld1.16        {q8}, [%[coefsP0]:128]!  \n"// (1) load 8 16-bits coefs
        "vld1.16        {q9}, [%[coefsP1]:128]!  \n"// (1) load 8 16-bits coefs for interpolation
        "vld1.16        {q10}, [%[coefsN1]:128]! \n"// (1) load 8 16-bits coefs
        "vld1.16        {q11}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs for interpolation

        "vsub.s16       q9, q9, q8               \n"// (1) interpolate (step1) 1st set of coefs
        "vsub.s16       q11, q11, q10            \n"// (1) interpolate (step1) 2nd set of coets

        "vqrdmulh.s16   q9, q9, d2[0]            \n"// (2) interpolate (step2) 1st set of coefs
        "vqrdmulh.s16   q11, q11, d2[0]          \n"// (2) interpolate (step2) 2nd set of coefs

        "vrev64.16      q2, q2                   \n"// (1) reverse s3, s2, s1, s0, s7, s6, s5, s4

        "vadd.s16       q8, q8, q9               \n"// (1+2d) interpolate (step3) 1st set
        "vadd.s16       q10, q10, q11            \n"// (1+1d) interpolate (step3) 2nd set

        // reordering the vmal to do d6, d7 before d4, d5 is slower(?)
        "vmlal.s16      q0, d4, d17              \n"// (1+0d) multiply reversed samples by coef
        "vmlal.s16      q0, d5, d16              \n"// (1) multiply reversed samples by coef
        "vmlal.s16      q0, d6, d20              \n"// (1) multiply neg samples
        "vmlal.s16      q0, d7, d21              \n"// (1) multiply neg samples

        // moving these ARM instructions before neon above seems to be slower
        "subs           %[count], %[count], #8   \n"// (1) update loop counter
        "sub            %[sP], %[sP], #16        \n"// (0) move pointer to next set of samples

        // sP used after branch (warning)
        "bne            1b                       \n"// loop

        ASSEMBLY_ACCUMULATE_MONO

        : [out]     "=Uv" (out[0]),
          [count]   "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [coefsP1] "+r" (coefsP1),
          [coefsN1] "+r" (coefsN1),
          [sP]      "+r" (sP),
          [sN]      "+r" (sN)
        : [lerpP]   "r" (lerpP),
          [vLR]     "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q8", "q9", "q10", "q11"
    );
#endif
}

template <>
inline void Process<2, 16>(int32_t* const out,
        int count,
        const int16_t* coefsP,
        const int16_t* coefsN,
        const int16_t* coefsP1,
        const int16_t* coefsN1,
        const int16_t* sP,
        const int16_t* sN,
        uint32_t lerpP,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            lerpP, coefsP1, coefsN1);
#else
    const int CHANNELS = 2; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "vmov.32        d2[0], %[lerpP]          \n"// load the positive phase
        "veor           q0, q0, q0               \n"// (1) acc_L = 0
        "veor           q4, q4, q4               \n"// (0 combines+) acc_R = 0

        "1:                                      \n"

        "vld2.16        {q2, q3}, [%[sP]]        \n"// (3+0d) load 8 16-bits stereo frames
        "vld2.16        {q5, q6}, [%[sN]]!       \n"// (3) load 8 16-bits stereo frames
        "vld1.16        {q8}, [%[coefsP0]:128]!  \n"// (1) load 8 16-bits coefs
        "vld1.16        {q9}, [%[coefsP1]:128]!  \n"// (1) load 8 16-bits coefs for interpolation
        "vld1.16        {q10}, [%[coefsN1]:128]! \n"// (1) load 8 16-bits coefs
        "vld1.16        {q11}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs for interpolation

        "vsub.s16       q9, q9, q8               \n"// (1) interpolate (step1) 1st set of coefs
        "vsub.s16       q11, q11, q10            \n"// (1) interpolate (step1) 2nd set of coets

        "vqrdmulh.s16   q9, q9, d2[0]            \n"// (2) interpolate (step2) 1st set of coefs
        "vqrdmulh.s16   q11, q11, d2[0]          \n"// (2) interpolate (step2) 2nd set of coefs

        "vrev64.16      q2, q2                   \n"// (1) reverse 8 samples of positive left
        "vrev64.16      q3, q3                   \n"// (1) reverse 8 samples of positive right

        "vadd.s16       q8, q8, q9               \n"// (1+1d) interpolate (step3) 1st set
        "vadd.s16       q10, q10, q11            \n"// (1+1d) interpolate (step3) 2nd set

        "vmlal.s16      q0, d4, d17              \n"// (1) multiply reversed samples left
        "vmlal.s16      q0, d5, d16              \n"// (1) multiply reversed samples left
        "vmlal.s16      q4, d6, d17              \n"// (1) multiply reversed samples right
        "vmlal.s16      q4, d7, d16              \n"// (1) multiply reversed samples right
        "vmlal.s16      q0, d10, d20             \n"// (1) multiply samples left
        "vmlal.s16      q0, d11, d21             \n"// (1) multiply samples left
        "vmlal.s16      q4, d12, d20             \n"// (1) multiply samples right
        "vmlal.s16      q4, d13, d21             \n"// (1) multiply samples right

        // moving these ARM before neon seems to be slower
        "subs           %[count], %[count], #8   \n"// (1) update loop counter
        "sub            %[sP], %[sP], #32        \n"// (0) move pointer to next set of samples

        // sP used after branch (warning)
        "bne            1b                       \n"// loop

        ASSEMBLY_ACCUMULATE_STEREO

        : [out] "=Uv" (out[0]),
          [count] "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [coefsP1] "+r" (coefsP1),
          [coefsN1] "+r" (coefsN1),
          [sP] "+r" (sP),
          [sN] "+r" (sN)
        : [lerpP]   "r" (lerpP),
          [vLR] "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q4", "q5", "q6",
          "q8", "q9", "q10", "q11"
    );
#endif
}

template <>
inline void ProcessL<1, 16>(int32_t* const out,
        int count,
        const int32_t* coefsP,
        const int32_t* coefsN,
        const int16_t* sP,
        const int16_t* sN,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
#else
    const int CHANNELS = 1; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "veor           q0, q0, q0                    \n"// result, initialize to 0

        "1:                                           \n"

        "vld1.16        {q2}, [%[sP]]                 \n"// load 8 16-bits mono samples
        "vld1.16        {q3}, [%[sN]]!                \n"// load 8 16-bits mono samples
        "vld1.32        {q8, q9}, [%[coefsP0]:128]!   \n"// load 8 32-bits coefs
        "vld1.32        {q10, q11}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs

        "vrev64.16      q2, q2                        \n"// reverse 8 samples of the positive side

        "vshll.s16      q12, d4, #15                  \n"// extend samples to 31 bits
        "vshll.s16      q13, d5, #15                  \n"// extend samples to 31 bits

        "vshll.s16      q14, d6, #15                  \n"// extend samples to 31 bits
        "vshll.s16      q15, d7, #15                  \n"// extend samples to 31 bits

        "vqrdmulh.s32   q12, q12, q9                  \n"// multiply samples
        "vqrdmulh.s32   q13, q13, q8                  \n"// multiply samples
        "vqrdmulh.s32   q14, q14, q10                 \n"// multiply samples
        "vqrdmulh.s32   q15, q15, q11                 \n"// multiply samples

        "vadd.s32       q0, q0, q12                   \n"// accumulate result
        "vadd.s32       q13, q13, q14                 \n"// accumulate result
        "vadd.s32       q0, q0, q15                   \n"// accumulate result
        "vadd.s32       q0, q0, q13                   \n"// accumulate result

        "sub            %[sP], %[sP], #16             \n"// move pointer to next set of samples
        "subs           %[count], %[count], #8        \n"// update loop counter

        "bne            1b                            \n"// loop

        ASSEMBLY_ACCUMULATE_MONO

        : [out]     "=Uv" (out[0]),
          [count]   "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [sP]      "+r" (sP),
          [sN]      "+r" (sN)
        : [vLR]     "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q8", "q9", "q10", "q11",
          "q12", "q13", "q14", "q15"
    );
#endif
}

template <>
inline void ProcessL<2, 16>(int32_t* const out,
        int count,
        const int32_t* coefsP,
        const int32_t* coefsN,
        const int16_t* sP,
        const int16_t* sN,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
#else
    const int CHANNELS = 2; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "veor           q0, q0, q0                    \n"// result, initialize to 0
        "veor           q4, q4, q4                    \n"// result, initialize to 0

        "1:                                           \n"

        "vld2.16        {q2, q3}, [%[sP]]             \n"// load 8 16-bits stereo frames
        "vld2.16        {q5, q6}, [%[sN]]!            \n"// load 8 16-bits stereo frames
        "vld1.32        {q8, q9}, [%[coefsP0]:128]!   \n"// load 8 32-bits coefs
        "vld1.32        {q10, q11}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs

        "vrev64.16      q2, q2                        \n"// reverse 8 samples of positive left
        "vrev64.16      q3, q3                        \n"// reverse 8 samples of positive right

        "vshll.s16      q12,  d4, #15                 \n"// extend samples to 31 bits
        "vshll.s16      q13,  d5, #15                 \n"// extend samples to 31 bits

        "vshll.s16      q14,  d10, #15                \n"// extend samples to 31 bits
        "vshll.s16      q15,  d11, #15                \n"// extend samples to 31 bits

        "vqrdmulh.s32   q12, q12, q9                  \n"// multiply samples by coef
        "vqrdmulh.s32   q13, q13, q8                  \n"// multiply samples by coef
        "vqrdmulh.s32   q14, q14, q10                 \n"// multiply samples by coef
        "vqrdmulh.s32   q15, q15, q11                 \n"// multiply samples by coef

        "vadd.s32       q0, q0, q12                   \n"// accumulate result
        "vadd.s32       q13, q13, q14                 \n"// accumulate result
        "vadd.s32       q0, q0, q15                   \n"// accumulate result
        "vadd.s32       q0, q0, q13                   \n"// accumulate result

        "vshll.s16      q12,  d6, #15                 \n"// extend samples to 31 bits
        "vshll.s16      q13,  d7, #15                 \n"// extend samples to 31 bits

        "vshll.s16      q14,  d12, #15                \n"// extend samples to 31 bits
        "vshll.s16      q15,  d13, #15                \n"// extend samples to 31 bits

        "vqrdmulh.s32   q12, q12, q9                  \n"// multiply samples by coef
        "vqrdmulh.s32   q13, q13, q8                  \n"// multiply samples by coef
        "vqrdmulh.s32   q14, q14, q10                 \n"// multiply samples by coef
        "vqrdmulh.s32   q15, q15, q11                 \n"// multiply samples by coef

        "vadd.s32       q4, q4, q12                   \n"// accumulate result
        "vadd.s32       q13, q13, q14                 \n"// accumulate result
        "vadd.s32       q4, q4, q15                   \n"// accumulate result
        "vadd.s32       q4, q4, q13                   \n"// accumulate result

        "subs           %[count], %[count], #8        \n"// update loop counter
        "sub            %[sP], %[sP], #32             \n"// move pointer to next set of samples

        "bne            1b                            \n"// loop

        ASSEMBLY_ACCUMULATE_STEREO

        : [out]     "=Uv" (out[0]),
          [count]   "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [sP]      "+r" (sP),
          [sN]      "+r" (sN)
        : [vLR]     "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q4", "q5", "q6",
          "q8", "q9", "q10", "q11",
          "q12", "q13", "q14", "q15"
    );
#endif
}

template <>
inline void Process<1, 16>(int32_t* const out,
        int count,
        const int32_t* coefsP,
        const int32_t* coefsN,
        const int32_t* coefsP1,
        const int32_t* coefsN1,
        const int16_t* sP,
        const int16_t* sN,
        uint32_t lerpP,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            lerpP, coefsP1, coefsN1);
#else
    const int CHANNELS = 1; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "vmov.32        d2[0], %[lerpP]               \n"// load the positive phase
        "veor           q0, q0, q0                    \n"// result, initialize to 0

        "1:                                           \n"

        "vld1.16        {q2}, [%[sP]]                 \n"// load 8 16-bits mono samples
        "vld1.16        {q3}, [%[sN]]!                \n"// load 8 16-bits mono samples
        "vld1.32        {q8, q9}, [%[coefsP0]:128]!   \n"// load 8 32-bits coefs
        "vld1.32        {q12, q13}, [%[coefsP1]:128]! \n"// load 8 32-bits coefs
        "vld1.32        {q10, q11}, [%[coefsN1]:128]! \n"// load 8 32-bits coefs
        "vld1.32        {q14, q15}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs

        "vsub.s32       q12, q12, q8                  \n"// interpolate (step1)
        "vsub.s32       q13, q13, q9                  \n"// interpolate (step1)
        "vsub.s32       q14, q14, q10                 \n"// interpolate (step1)
        "vsub.s32       q15, q15, q11                 \n"// interpolate (step1)

        "vqrdmulh.s32   q12, q12, d2[0]               \n"// interpolate (step2)
        "vqrdmulh.s32   q13, q13, d2[0]               \n"// interpolate (step2)
        "vqrdmulh.s32   q14, q14, d2[0]               \n"// interpolate (step2)
        "vqrdmulh.s32   q15, q15, d2[0]               \n"// interpolate (step2)

        "vadd.s32       q8, q8, q12                   \n"// interpolate (step3)
        "vadd.s32       q9, q9, q13                   \n"// interpolate (step3)
        "vadd.s32       q10, q10, q14                 \n"// interpolate (step3)
        "vadd.s32       q11, q11, q15                 \n"// interpolate (step3)

        "vrev64.16      q2, q2                        \n"// reverse 8 samples of the positive side

        "vshll.s16      q12,  d4, #15                 \n"// extend samples to 31 bits
        "vshll.s16      q13,  d5, #15                 \n"// extend samples to 31 bits

        "vshll.s16      q14,  d6, #15                 \n"// extend samples to 31 bits
        "vshll.s16      q15,  d7, #15                 \n"// extend samples to 31 bits

        "vqrdmulh.s32   q12, q12, q9                  \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q13, q13, q8                  \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q14, q14, q10                 \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q15, q15, q11                 \n"// multiply samples by interpolated coef

        "vadd.s32       q0, q0, q12                   \n"// accumulate result
        "vadd.s32       q13, q13, q14                 \n"// accumulate result
        "vadd.s32       q0, q0, q15                   \n"// accumulate result
        "vadd.s32       q0, q0, q13                   \n"// accumulate result

        "sub            %[sP], %[sP], #16             \n"// move pointer to next set of samples
        "subs           %[count], %[count], #8        \n"// update loop counter

        "bne            1b                            \n"// loop

        ASSEMBLY_ACCUMULATE_MONO

        : [out]     "=Uv" (out[0]),
          [count]   "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [coefsP1] "+r" (coefsP1),
          [coefsN1] "+r" (coefsN1),
          [sP]      "+r" (sP),
          [sN]      "+r" (sN)
        : [lerpP]   "r" (lerpP),
          [vLR]     "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q8", "q9", "q10", "q11",
          "q12", "q13", "q14", "q15"
    );
#endif
}

template <>
inline void Process<2, 16>(int32_t* const out,
        int count,
        const int32_t* coefsP,
        const int32_t* coefsN,
        const int32_t* coefsP1,
        const int32_t* coefsN1,
        const int16_t* sP,
        const int16_t* sN,
        uint32_t lerpP,
        const int32_t* const volumeLR)
{
#ifdef USE_INTRINSIC
    ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            lerpP, coefsP1, coefsN1);
#else
    const int CHANNELS = 2; // template specialization does not preserve params
    const int STRIDE = 16;
    sP -= CHANNELS*((STRIDE>>1)-1);
    asm (
        "vmov.32        d2[0], %[lerpP]               \n"// load the positive phase
        "veor           q0, q0, q0                    \n"// result, initialize to 0
        "veor           q4, q4, q4                    \n"// result, initialize to 0

        "1:                                           \n"

        "vld2.16        {q2, q3}, [%[sP]]             \n"// load 8 16-bits stereo frames
        "vld2.16        {q5, q6}, [%[sN]]!            \n"// load 8 16-bits stereo frames
        "vld1.32        {q8, q9}, [%[coefsP0]:128]!   \n"// load 8 32-bits coefs
        "vld1.32        {q12, q13}, [%[coefsP1]:128]! \n"// load 8 32-bits coefs
        "vld1.32        {q10, q11}, [%[coefsN1]:128]! \n"// load 8 32-bits coefs
        "vld1.32        {q14, q15}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs

        "vsub.s32       q12, q12, q8                  \n"// interpolate (step1)
        "vsub.s32       q13, q13, q9                  \n"// interpolate (step1)
        "vsub.s32       q14, q14, q10                 \n"// interpolate (step1)
        "vsub.s32       q15, q15, q11                 \n"// interpolate (step1)

        "vqrdmulh.s32   q12, q12, d2[0]               \n"// interpolate (step2)
        "vqrdmulh.s32   q13, q13, d2[0]               \n"// interpolate (step2)
        "vqrdmulh.s32   q14, q14, d2[0]               \n"// interpolate (step2)
        "vqrdmulh.s32   q15, q15, d2[0]               \n"// interpolate (step2)

        "vadd.s32       q8, q8, q12                   \n"// interpolate (step3)
        "vadd.s32       q9, q9, q13                   \n"// interpolate (step3)
        "vadd.s32       q10, q10, q14                 \n"// interpolate (step3)
        "vadd.s32       q11, q11, q15                 \n"// interpolate (step3)

        "vrev64.16      q2, q2                        \n"// reverse 8 samples of positive left
        "vrev64.16      q3, q3                        \n"// reverse 8 samples of positive right

        "vshll.s16      q12,  d4, #15                 \n"// extend samples to 31 bits
        "vshll.s16      q13,  d5, #15                 \n"// extend samples to 31 bits

        "vshll.s16      q14,  d10, #15                \n"// extend samples to 31 bits
        "vshll.s16      q15,  d11, #15                \n"// extend samples to 31 bits

        "vqrdmulh.s32   q12, q12, q9                  \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q13, q13, q8                  \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q14, q14, q10                 \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q15, q15, q11                 \n"// multiply samples by interpolated coef

        "vadd.s32       q0, q0, q12                   \n"// accumulate result
        "vadd.s32       q13, q13, q14                 \n"// accumulate result
        "vadd.s32       q0, q0, q15                   \n"// accumulate result
        "vadd.s32       q0, q0, q13                   \n"// accumulate result

        "vshll.s16      q12,  d6, #15                 \n"// extend samples to 31 bits
        "vshll.s16      q13,  d7, #15                 \n"// extend samples to 31 bits

        "vshll.s16      q14,  d12, #15                \n"// extend samples to 31 bits
        "vshll.s16      q15,  d13, #15                \n"// extend samples to 31 bits

        "vqrdmulh.s32   q12, q12, q9                  \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q13, q13, q8                  \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q14, q14, q10                 \n"// multiply samples by interpolated coef
        "vqrdmulh.s32   q15, q15, q11                 \n"// multiply samples by interpolated coef

        "vadd.s32       q4, q4, q12                   \n"// accumulate result
        "vadd.s32       q13, q13, q14                 \n"// accumulate result
        "vadd.s32       q4, q4, q15                   \n"// accumulate result
        "vadd.s32       q4, q4, q13                   \n"// accumulate result

        "subs           %[count], %[count], #8        \n"// update loop counter
        "sub            %[sP], %[sP], #32             \n"// move pointer to next set of samples

        "bne            1b                            \n"// loop

        ASSEMBLY_ACCUMULATE_STEREO

        : [out]     "=Uv" (out[0]),
          [count]   "+r" (count),
          [coefsP0] "+r" (coefsP),
          [coefsN0] "+r" (coefsN),
          [coefsP1] "+r" (coefsP1),
          [coefsN1] "+r" (coefsN1),
          [sP]      "+r" (sP),
          [sN]      "+r" (sN)
        : [lerpP]   "r" (lerpP),
          [vLR]     "r" (volumeLR)
        : "cc", "memory",
          "q0", "q1", "q2", "q3",
          "q4", "q5", "q6",
          "q8", "q9", "q10", "q11",
          "q12", "q13", "q14", "q15"
    );
#endif
}

template<>
inline void ProcessL<1, 16>(float* const out,
        int count,
        const float* coefsP,
        const float* coefsN,
        const float* sP,
        const float* sN,
        const float* const volumeLR)
{
    ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
}

template<>
inline void ProcessL<2, 16>(float* const out,
        int count,
        const float* coefsP,
        const float* coefsN,
        const float* sP,
        const float* sN,
        const float* const volumeLR)
{
    ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/);
}

template<>
inline void Process<1, 16>(float* const out,
        int count,
        const float* coefsP,
        const float* coefsN,
        const float* coefsP1,
        const float* coefsN1,
        const float* sP,
        const float* sN,
        float lerpP,
        const float* const volumeLR)
{
    ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            lerpP, coefsP1, coefsN1);
}

template<>
inline void Process<2, 16>(float* const out,
        int count,
        const float* coefsP,
        const float* coefsN,
        const float* coefsP1,
        const float* coefsN1,
        const float* sP,
        const float* sN,
        float lerpP,
        const float* const volumeLR)
{
    ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR,
            lerpP, coefsP1, coefsN1);
}

#endif //USE_NEON

} // namespace android

#endif /*ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H*/