path: root/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst/generic.cpp

/*
 * Copyright (c) 2021-2023 Arm Limited.
 *
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#if defined(__aarch64__)

#include "arm_gemm.hpp"
#include <cstdint>

namespace arm_conv {
namespace depthwise {

void a64_s8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const int8_t *const *const inptrs, const int8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, int8_t *const *const outptrs)
{
  __asm__ __volatile__(
    "mov x20, #0x1\n"
    "orr x20, x20, #0x100\n"
    "ldp x15, x14, [%x[inptrs], #0x0]\n"
    "ldp x13, x12, [%x[inptrs], #0x10]\n"
    "orr x20, x20, #0x10000\n"
    "lsr x11, %x[n_channels], #0x4\n"
    "dup v12.4s, w20\n"
    "ldp x10, x9, [%x[inptrs], #0x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_minval]\n"
    "ld1r { v13.4s }, [x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n"
    "ld1r { v11.4s }, [x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n"
    "ld1r { v16.4s }, [x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n"
    "ld1r { v14.4s }, [x20]\n"
    "mov x28, #0x0\n"
    "mov x27, #0x0\n"
    "ldp x26, x21, [%x[inptrs], #0x30]\n"
    "ldp x25, x24, [%x[outptrs], #0x0]\n"
    "ldp x23, x22, [%x[outptrs], #0x10]\n"
    "cbz x11, 3f\n"
    "ldr q15, [x15, x28]\n"
    "ldr q28, [x14, x28]\n"
    "subs x11, x11, #0x1\n"
    "ldr q30, [x13, x28]\n"
    "ldr q8, [x12, x28]\n"
    "zip2 v19.16b, v15.16b, v30.16b\n"
    "zip1 v15.16b, v15.16b, v30.16b\n"
    "ldr q26, [x10, x28]\n"
    "ldr q0, [x9, x28]\n"
    "zip1 v7.16b, v28.16b, v8.16b\n"
    "zip2 v8.16b, v28.16b, v8.16b\n"
    "ldr q29, [x26, x28]\n"
    "ldr q10, [x21, x28]\n"
    "zip2 v25.16b, v15.16b, v7.16b\n"
    "zip1 v15.16b, v15.16b, v7.16b\n"
    "ldr q1, [%x[params], #0x10]\n"
    "ldr q6, [%x[params], #0x20]\n"
    "zip1 v7.16b, v19.16b, v8.16b\n"
    "zip2 v8.16b, v19.16b, v8.16b\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q20, [%x[params], #0x30]\n"
    "zip2 v21.16b, v26.16b, v29.16b\n"
    "zip1 v26.16b, v26.16b, v29.16b\n"
    "ldp x21, x20, [%x[inptrs], #0x40]\n"
    "ldr q22, [x21, x28]\n"
    "zip1 v27.16b, v0.16b, v10.16b\n"
    "zip2 v10.16b, v0.16b, v10.16b\n"
    "ldr q17, [x20, x28]\n"
    "ldp x21, x20, [%x[inptrs], #0x50]\n"
    "zip2 v23.16b, v26.16b, v27.16b\n"
    "zip1 v26.16b, v26.16b, v27.16b\n"
    "ldr q9, [x21, x28]\n"
    "ldr q5, [x20, x28]\n"
    "zip2 v28.16b, v22.16b, v9.16b\n"
    "zip1 v22.16b, v22.16b, v9.16b\n"
    "ldp x21, x20, [%x[inptrs], #0x60]\n"
    "ldr q27, [x21, x28]\n"
    "zip1 v24.16b, v17.16b, v5.16b\n"
    "zip2 v5.16b, v17.16b, v5.16b\n"
    "ldr q18, [x20, x28]\n"
    "ldp x21, x20, [%x[inptrs], #0x70]\n"
    "zip1 v3.16b, v21.16b, v10.16b\n"
    "zip2 v10.16b, v21.16b, v10.16b\n"
    "ldr q4, [x21, x28]\n"
    "ldr q9, [x20, x28]\n"
    "zip2 v17.16b, v27.16b, v4.16b\n"
    "zip1 v27.16b, v27.16b, v4.16b\n"
    "zip1 v4.16b, v18.16b, v9.16b\n"
    "zip2 v9.16b, v18.16b, v9.16b\n"
    "ldp x15, x14, [%x[inptrs], #0x0]\n"
    "ldp x13, x12, [%x[inptrs], #0x10]\n"
    "ldp x10, x9, [%x[inptrs], #0x20]\n"
    "ldp x26, x21, [%x[inptrs], #0x30]\n"
    "zip2 v19.16b, v22.16b, v24.16b\n"
    "zip1 v22.16b, v22.16b, v24.16b\n"
    "zip1 v0.16b, v28.16b, v5.16b\n"
    "zip2 v5.16b, v28.16b, v5.16b\n"
    "add %x[params], %x[params], #0x40\n"
    "zip2 v24.16b, v27.16b, v4.16b\n"
    "zip1 v27.16b, v27.16b, v4.16b\n"
    "zip1 v2.16b, v17.16b, v9.16b\n"
    "zip2 v9.16b, v17.16b, v9.16b\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "mov v28.16b, v31.16b\n"
    "beq 2f\n"
    "1:"  // Loop
    "movi v21.4s, #0x0\n"
    ".inst 0x4e9a9595  // sdot v21.4s, v12.16b, v26.16b\n"
    ".inst 0x4e8f943f  // sdot v31.4s, v1.16b, v15.16b\n"
    "add x28, x28, #0x10\n"
    ".inst 0x4e969595  // sdot v21.4s, v12.16b, v22.16b\n"
    ".inst 0x4e9a943d  // sdot v29.4s, v1.16b, v26.16b\n"
    "movi v18.4s, #0x0\n"
    "subs x11, x11, #0x1\n"
    ".inst 0x4e9a94df  // sdot v31.4s, v6.16b, v26.16b\n"
    "ext v26.16b, v26.16b, v26.16b, #0x1\n"
    "mov v17.16b, v21.16b\n .inst 0x4e9b9591  // sdot v17.4s, v12.16b, v27.16b\n"
    ".inst 0x4e8f9595  // sdot v21.4s, v12.16b, v15.16b\n"
    "ext v15.16b, v15.16b, v15.16b, #0x1\n"
    ".inst 0x4e9a9592  // sdot v18.4s, v12.16b, v26.16b\n"
    ".inst 0x4e9694dd  // sdot v29.4s, v6.16b, v22.16b\n"
    ".inst 0x4e96969f  // sdot v31.4s, v20.16b, v22.16b\n"
    "ext v22.16b, v22.16b, v22.16b, #0x1\n"
    ".inst 0x4e8f943e  // sdot v30.4s, v1.16b, v15.16b\n"
    ".inst 0x4e9a943c  // sdot v28.4s, v1.16b, v26.16b\n"
    "mls v31.4s, v21.4s, v16.4s\n"
    ".inst 0x4e969592  // sdot v18.4s, v12.16b, v22.16b\n"
    ".inst 0x4e9b969d  // sdot v29.4s, v20.16b, v27.16b\n"
    "ext v27.16b, v27.16b, v27.16b, #0x1\n"
    ".inst 0x4e9a94de  // sdot v30.4s, v6.16b, v26.16b\n"
    "ldr q26, [%x[params], #0x10]\n"
    ".inst 0x4e9694dc  // sdot v28.4s, v6.16b, v22.16b\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mov v21.16b, v18.16b\n .inst 0x4e9b9595  // sdot v21.4s, v12.16b, v27.16b\n"
    ".inst 0x4e8f9592  // sdot v18.4s, v12.16b, v15.16b\n"
    "ldr q17, [%x[params], #0x0]\n"
    "sqrdmulh v31.4s, v31.4s, v17.4s\n"
    ".inst 0x4e96969e  // sdot v30.4s, v20.16b, v22.16b\n"
    ".inst 0x4e9b969c  // sdot v28.4s, v20.16b, v27.16b\n"
    "mls v30.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v21.4s, v16.4s\n"
    "and v15.16b, v31.16b, v26.16b\n"
    "sshr v15.4s, v15.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v17.4s\n"
    "sqrdmulh v29.4s, v29.4s, v17.4s\n"
    "sqrdmulh v28.4s, v28.4s, v17.4s\n"
    "ldr q1, [%x[params], #0x60]\n"
    "sqadd v31.4s, v31.4s, v15.4s\n"
    "and v18.16b, v30.16b, v26.16b\n"
    "and v21.16b, v29.16b, v26.16b\n"
    "and v17.16b, v28.16b, v26.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v21.4s, v21.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v26.4s\n"
    "sqadd v30.4s, v30.4s, v18.4s\n"
    "ldr q18, [%x[params], #0x40]\n"
    "sqadd v29.4s, v29.4s, v21.4s\n"
    "ldr q27, [%x[params], #0x50]\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "ldr q15, [%x[params], #0x30]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v30.4s, v30.4s, v26.4s\n"
    "srshl v29.4s, v29.4s, v26.4s\n"
    "srshl v28.4s, v28.4s, v26.4s\n"
    "ldr q20, [%x[params], #0x70]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v22.4s, #0x0\n"
    ".inst 0x4e979596  // sdot v22.4s, v12.16b, v23.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q26, [%x[params], #0x20]\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    ".inst 0x4e939596  // sdot v22.4s, v12.16b, v19.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s30, [x24, x27]\n"
    "mov v6.16b, v22.16b\n .inst 0x4e989586  // sdot v6.4s, v12.16b, v24.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "str s29, [x23, x27]\n"
    "mov v30.16b, v26.16b\n"
    ".inst 0x4e999596  // sdot v22.4s, v12.16b, v25.16b\n"
    "str s28, [x22, x27]\n"
    "mov v29.16b, v26.16b\n"
    "mov v21.16b, v26.16b\n"
    ".inst 0x4e9995fa  // sdot v26.4s, v15.16b, v25.16b\n"
    ".inst 0x4e9795fd  // sdot v29.4s, v15.16b, v23.16b\n"
    ".inst 0x4e97965a  // sdot v26.4s, v18.16b, v23.16b\n"
    "ext v25.16b, v25.16b, v25.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v23.16b, v23.16b, v23.16b, #0x1\n"
    "movi v28.4s, #0x0\n"
    ".inst 0x4e9995fe  // sdot v30.4s, v15.16b, v25.16b\n"
    ".inst 0x4e9795f5  // sdot v21.4s, v15.16b, v23.16b\n"
    ".inst 0x4e97959c  // sdot v28.4s, v12.16b, v23.16b\n"
    ".inst 0x4e93965d  // sdot v29.4s, v18.16b, v19.16b\n"
    ".inst 0x4e93977a  // sdot v26.4s, v27.16b, v19.16b\n"
    "ext v19.16b, v19.16b, v19.16b, #0x1\n"
    ".inst 0x4e97965e  // sdot v30.4s, v18.16b, v23.16b\n"
    "ldr q4, [x9, x28]\n"
    ".inst 0x4e939655  // sdot v21.4s, v18.16b, v19.16b\n"
    "mls v26.4s, v22.4s, v16.4s\n"
    ".inst 0x4e93959c  // sdot v28.4s, v12.16b, v19.16b\n"
    ".inst 0x4e98977d  // sdot v29.4s, v27.16b, v24.16b\n"
    "ext v24.16b, v24.16b, v24.16b, #0x1\n"
    ".inst 0x4e93977e  // sdot v30.4s, v27.16b, v19.16b\n"
    ".inst 0x4e989775  // sdot v21.4s, v27.16b, v24.16b\n"
    "sqrdmulh v26.4s, v26.4s, v1.4s\n"
    "mov v17.16b, v28.16b\n .inst 0x4e989591  // sdot v17.4s, v12.16b, v24.16b\n"
    ".inst 0x4e99959c  // sdot v28.4s, v12.16b, v25.16b\n"
    "ldr q31, [x14, x28]\n"
    "mls v30.4s, v28.4s, v16.4s\n"
    "mls v29.4s, v6.4s, v16.4s\n"
    "mls v21.4s, v17.4s, v16.4s\n"
    "and v17.16b, v26.16b, v20.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v1.4s\n"
    "sqrdmulh v29.4s, v29.4s, v1.4s\n"
    "sqrdmulh v21.4s, v21.4s, v1.4s\n"
    "ldr q27, [%x[params], #0xc0]\n"
    "sqadd v26.4s, v26.4s, v17.4s\n"
    "and v18.16b, v30.16b, v20.16b\n"
    "and v6.16b, v29.16b, v20.16b\n"
    "and v17.16b, v21.16b, v20.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v6.4s, v6.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v26.4s, v26.4s, v20.4s\n"
    "sqadd v30.4s, v30.4s, v18.4s\n"
    "ldr q28, [%x[params], #0xa0]\n"
    "sqadd v29.4s, v29.4s, v6.4s\n"
    "ldr q24, [%x[params], #0xb0]\n"
    "sqadd v21.4s, v21.4s, v17.4s\n"
    "ldr q15, [%x[params], #0x90]\n"
    "add v26.4s, v26.4s, v14.4s\n"
    "srshl v30.4s, v30.4s, v20.4s\n"
    "srshl v29.4s, v29.4s, v20.4s\n"
    "srshl v21.4s, v21.4s, v20.4s\n"
    "ldr q1, [%x[params], #0xd0]\n"
    "smax v26.4s, v26.4s, v13.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "smin v26.4s, v26.4s, v11.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "movi v22.4s, #0x0\n"
    ".inst 0x4e839596  // sdot v22.4s, v12.16b, v3.16b\n"
    ".inst 0x4e809596  // sdot v22.4s, v12.16b, v0.16b\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "str s26, [x25, x27]\n"
    "ldr q26, [%x[params], #0x80]\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "mov v18.16b, v22.16b\n .inst 0x4e829592  // sdot v18.4s, v12.16b, v2.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s30, [x24, x27]\n"
    ".inst 0x4e879596  // sdot v22.4s, v12.16b, v7.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s29, [x23, x27]\n"
    "mov v6.16b, v26.16b\n"
    "str s21, [x22, x27]\n"
    "mov v25.16b, v26.16b\n"
    "mov v20.16b, v26.16b\n"
    ".inst 0x4e8795fa  // sdot v26.4s, v15.16b, v7.16b\n"
    ".inst 0x4e8395f9  // sdot v25.4s, v15.16b, v3.16b\n"
    ".inst 0x4e83979a  // sdot v26.4s, v28.16b, v3.16b\n"
    "ext v7.16b, v7.16b, v7.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v3.16b, v3.16b, v3.16b, #0x1\n"
    "movi v23.4s, #0x0\n"
    ".inst 0x4e8795e6  // sdot v6.4s, v15.16b, v7.16b\n"
    ".inst 0x4e8395f4  // sdot v20.4s, v15.16b, v3.16b\n"
    ".inst 0x4e839597  // sdot v23.4s, v12.16b, v3.16b\n"
    ".inst 0x4e809799  // sdot v25.4s, v28.16b, v0.16b\n"
    ".inst 0x4e80971a  // sdot v26.4s, v24.16b, v0.16b\n"
    "ext v0.16b, v0.16b, v0.16b, #0x1\n"
    ".inst 0x4e839786  // sdot v6.4s, v28.16b, v3.16b\n"
    "ldr q19, [x26, x28]\n"
    ".inst 0x4e809794  // sdot v20.4s, v28.16b, v0.16b\n"
    "mls v26.4s, v22.4s, v16.4s\n"
    ".inst 0x4e809597  // sdot v23.4s, v12.16b, v0.16b\n"
    ".inst 0x4e829719  // sdot v25.4s, v24.16b, v2.16b\n"
    "ext v2.16b, v2.16b, v2.16b, #0x1\n"
    ".inst 0x4e809706  // sdot v6.4s, v24.16b, v0.16b\n"
    ".inst 0x4e829714  // sdot v20.4s, v24.16b, v2.16b\n"
    "sqrdmulh v26.4s, v26.4s, v27.4s\n"
    "mov v17.16b, v23.16b\n .inst 0x4e829591  // sdot v17.4s, v12.16b, v2.16b\n"
    ".inst 0x4e879597  // sdot v23.4s, v12.16b, v7.16b\n"
    "ldr q21, [x13, x28]\n"
    "mls v6.4s, v23.4s, v16.4s\n"
    "mls v25.4s, v18.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v26.16b, v1.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v6.4s, v6.4s, v27.4s\n"
    "sqrdmulh v25.4s, v25.4s, v27.4s\n"
    "sqrdmulh v20.4s, v20.4s, v27.4s\n"
    "ldr q15, [%x[params], #0x120]\n"
    "sqadd v26.4s, v26.4s, v17.4s\n"
    "and v18.16b, v6.16b, v1.16b\n"
    "and v22.16b, v25.16b, v1.16b\n"
    "and v17.16b, v20.16b, v1.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v22.4s, v22.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v26.4s, v26.4s, v1.4s\n"
    "sqadd v6.4s, v6.4s, v18.4s\n"
    "ldr q30, [%x[params], #0x100]\n"
    "sqadd v25.4s, v25.4s, v22.4s\n"
    "ldr q27, [%x[params], #0x110]\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "ldr q24, [%x[params], #0xf0]\n"
    "add v26.4s, v26.4s, v14.4s\n"
    "srshl v6.4s, v6.4s, v1.4s\n"
    "srshl v25.4s, v25.4s, v1.4s\n"
    "srshl v20.4s, v20.4s, v1.4s\n"
    "ldr q23, [%x[params], #0x130]\n"
    "smax v26.4s, v26.4s, v13.4s\n"
    "add v6.4s, v6.4s, v14.4s\n"
    "add v25.4s, v25.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smin v26.4s, v26.4s, v11.4s\n"
    "smax v6.4s, v6.4s, v13.4s\n"
    "smax v25.4s, v25.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v6.4s, v6.4s, v11.4s\n"
    "smin v25.4s, v25.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "movi v0.4s, #0x0\n"
    ".inst 0x4e8a9580  // sdot v0.4s, v12.16b, v10.16b\n"
    ".inst 0x4e859580  // sdot v0.4s, v12.16b, v5.16b\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "uzp1 v6.16b, v6.16b, v6.16b\n"
    "str s26, [x25, x27]\n"
    "ldr q28, [%x[params], #0xe0]\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "mov v22.16b, v0.16b\n .inst 0x4e899596  // sdot v22.4s, v12.16b, v9.16b\n"
    "uzp1 v6.16b, v6.16b, v6.16b\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "str s6, [x24, x27]\n"
    ".inst 0x4e889580  // sdot v0.4s, v12.16b, v8.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s25, [x23, x27]\n"
    "mov v29.16b, v28.16b\n"
    "str s20, [x22, x27]\n"
    "mov v25.16b, v28.16b\n"
    "mov v7.16b, v28.16b\n"
    ".inst 0x4e88971c  // sdot v28.4s, v24.16b, v8.16b\n"
    ".inst 0x4e8a9719  // sdot v25.4s, v24.16b, v10.16b\n"
    ".inst 0x4e8a97dc  // sdot v28.4s, v30.16b, v10.16b\n"
    "ext v8.16b, v8.16b, v8.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v10.16b, v10.16b, v10.16b, #0x1\n"
    "movi v17.4s, #0x0\n"
    ".inst 0x4e88971d  // sdot v29.4s, v24.16b, v8.16b\n"
    ".inst 0x4e8a9707  // sdot v7.4s, v24.16b, v10.16b\n"
    ".inst 0x4e8a9591  // sdot v17.4s, v12.16b, v10.16b\n"
    ".inst 0x4e8597d9  // sdot v25.4s, v30.16b, v5.16b\n"
    ".inst 0x4e85977c  // sdot v28.4s, v27.16b, v5.16b\n"
    "ext v5.16b, v5.16b, v5.16b, #0x1\n"
    ".inst 0x4e8a97dd  // sdot v29.4s, v30.16b, v10.16b\n"
    "ldr q10, [x21, x28]\n"
    ".inst 0x4e8597c7  // sdot v7.4s, v30.16b, v5.16b\n"
    "mls v28.4s, v0.4s, v16.4s\n"
    ".inst 0x4e859591  // sdot v17.4s, v12.16b, v5.16b\n"
    ".inst 0x4e899779  // sdot v25.4s, v27.16b, v9.16b\n"
    "ext v9.16b, v9.16b, v9.16b, #0x1\n"
    ".inst 0x4e85977d  // sdot v29.4s, v27.16b, v5.16b\n"
    ".inst 0x4e899767  // sdot v7.4s, v27.16b, v9.16b\n"
    "sqrdmulh v28.4s, v28.4s, v15.4s\n"
    "mov v18.16b, v17.16b\n .inst 0x4e899592  // sdot v18.4s, v12.16b, v9.16b\n"
    ".inst 0x4e889591  // sdot v17.4s, v12.16b, v8.16b\n"
    "ldr q8, [x12, x28]\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mls v25.4s, v22.4s, v16.4s\n"
    "mls v7.4s, v18.4s, v16.4s\n"
    "and v17.16b, v28.16b, v23.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v29.4s, v29.4s, v15.4s\n"
    "sqrdmulh v25.4s, v25.4s, v15.4s\n"
    "sqrdmulh v7.4s, v7.4s, v15.4s\n"
    "ldr q15, [x15, x28]\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "ldp x21, x20, [%x[inptrs], #0x40]\n"
    "ldr q22, [x21, x28]\n"
    "ldr q3, [x20, x28]\n"
    "and v24.16b, v29.16b, v23.16b\n"
    "and v20.16b, v25.16b, v23.16b\n"
    "and v17.16b, v7.16b, v23.16b\n"
    "sshr v24.4s, v24.4s, #0x1f\n"
    "ldp x21, x20, [%x[inptrs], #0x50]\n"
    "ldr q2, [x21, x28]\n"
    "ldr q5, [x20, x28]\n"
    "sshr v20.4s, v20.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v28.4s, v28.4s, v23.4s\n"
    "sqadd v29.4s, v29.4s, v24.4s\n"
    "ldr q6, [%x[params], #0x160]\n"
    "sqadd v25.4s, v25.4s, v20.4s\n"
    "ldr q20, [%x[params], #0x170]\n"
    "sqadd v7.4s, v7.4s, v17.4s\n"
    "ldr q1, [%x[params], #0x150]\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "srshl v29.4s, v29.4s, v23.4s\n"
    "srshl v25.4s, v25.4s, v23.4s\n"
    "srshl v7.4s, v7.4s, v23.4s\n"
    "ldr q26, [x10, x28]\n"
    "ldp x21, x20, [%x[inptrs], #0x60]\n"
    "ldr q27, [x21, x28]\n"
    "ldr q30, [x20, x28]\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v25.4s, v25.4s, v14.4s\n"
    "add v7.4s, v7.4s, v14.4s\n"
    "ldp x21, x20, [%x[inptrs], #0x70]\n"
    "ldr q23, [x21, x28]\n"
    "ldr q9, [x20, x28]\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "ldp x15, x14, [%x[inptrs], #0x0]\n"
    "smax v25.4s, v25.4s, v13.4s\n"
    "smax v7.4s, v7.4s, v13.4s\n"
    "ldp x13, x12, [%x[inptrs], #0x10]\n"
    "ldp x10, x9, [%x[inptrs], #0x20]\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v25.4s, v25.4s, v11.4s\n"
    "ldp x26, x21, [%x[inptrs], #0x30]\n"
    "smin v7.4s, v7.4s, v11.4s\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s28, [x25, x27]\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "uzp1 v7.16b, v7.16b, v7.16b\n"
    "zip2 v17.16b, v15.16b, v21.16b\n"
    "zip1 v15.16b, v15.16b, v21.16b\n"
    "zip1 v18.16b, v31.16b, v8.16b\n"
    "zip2 v8.16b, v31.16b, v8.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "str s29, [x24, x27]\n"
    "uzp1 v7.16b, v7.16b, v7.16b\n"
    "str s25, [x23, x27]\n"
    "zip2 v25.16b, v15.16b, v18.16b\n"
    "str s7, [x22, x27]\n"
    "zip1 v15.16b, v15.16b, v18.16b\n"
    "zip1 v7.16b, v17.16b, v8.16b\n"
    "add x27, x27, #0x4\n"
    "zip2 v8.16b, v17.16b, v8.16b\n"
    "ldr q31, [%x[params], #0x140]\n"
    "zip2 v29.16b, v26.16b, v19.16b\n"
    "add %x[params], %x[params], #0x180\n"
    "zip1 v26.16b, v26.16b, v19.16b\n"
    "zip1 v28.16b, v4.16b, v10.16b\n"
    "zip2 v10.16b, v4.16b, v10.16b\n"
    "zip2 v24.16b, v22.16b, v2.16b\n"
    "zip1 v22.16b, v22.16b, v2.16b\n"
    "zip1 v21.16b, v3.16b, v5.16b\n"
    "zip2 v5.16b, v3.16b, v5.16b\n"
    "zip2 v18.16b, v27.16b, v23.16b\n"
    "zip1 v27.16b, v27.16b, v23.16b\n"
    "zip1 v17.16b, v30.16b, v9.16b\n"
    "zip2 v9.16b, v30.16b, v9.16b\n"
    "zip2 v23.16b, v26.16b, v28.16b\n"
    "zip1 v26.16b, v26.16b, v28.16b\n"
    "zip1 v3.16b, v29.16b, v10.16b\n"
    "zip2 v10.16b, v29.16b, v10.16b\n"
    "zip2 v19.16b, v22.16b, v21.16b\n"
    "zip1 v22.16b, v22.16b, v21.16b\n"
    "zip1 v0.16b, v24.16b, v5.16b\n"
    "zip2 v5.16b, v24.16b, v5.16b\n"
    "zip2 v24.16b, v27.16b, v17.16b\n"
    "zip1 v27.16b, v27.16b, v17.16b\n"
    "zip1 v2.16b, v18.16b, v9.16b\n"
    "zip2 v9.16b, v18.16b, v9.16b\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "mov v28.16b, v31.16b\n"
    "bgt 1b\n"
    "2:"  // Detached iteration
    "movi v21.4s, #0x0\n"
    ".inst 0x4e9a9595  // sdot v21.4s, v12.16b, v26.16b\n"
    ".inst 0x4e8f943f  // sdot v31.4s, v1.16b, v15.16b\n"
    "tst %x[n_channels], #0xf\n"
    ".inst 0x4e969595  // sdot v21.4s, v12.16b, v22.16b\n"
    ".inst 0x4e9a943d  // sdot v29.4s, v1.16b, v26.16b\n"
    "movi v18.4s, #0x0\n"
    "add x28, x28, #0x10\n"
    ".inst 0x4e9a94df  // sdot v31.4s, v6.16b, v26.16b\n"
    "ext v26.16b, v26.16b, v26.16b, #0x1\n"
    "mov v17.16b, v21.16b\n .inst 0x4e9b9591  // sdot v17.4s, v12.16b, v27.16b\n"
    ".inst 0x4e8f9595  // sdot v21.4s, v12.16b, v15.16b\n"
    "ext v15.16b, v15.16b, v15.16b, #0x1\n"
    ".inst 0x4e9a9592  // sdot v18.4s, v12.16b, v26.16b\n"
    ".inst 0x4e9694dd  // sdot v29.4s, v6.16b, v22.16b\n"
    ".inst 0x4e96969f  // sdot v31.4s, v20.16b, v22.16b\n"
    "ext v22.16b, v22.16b, v22.16b, #0x1\n"
    ".inst 0x4e8f943e  // sdot v30.4s, v1.16b, v15.16b\n"
    ".inst 0x4e9a943c  // sdot v28.4s, v1.16b, v26.16b\n"
    "mls v31.4s, v21.4s, v16.4s\n"
    ".inst 0x4e969592  // sdot v18.4s, v12.16b, v22.16b\n"
    ".inst 0x4e9b969d  // sdot v29.4s, v20.16b, v27.16b\n"
    "ext v27.16b, v27.16b, v27.16b, #0x1\n"
    ".inst 0x4e9a94de  // sdot v30.4s, v6.16b, v26.16b\n"
    "ldr q4, [%x[params], #0x10]\n"
    ".inst 0x4e9694dc  // sdot v28.4s, v6.16b, v22.16b\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mov v21.16b, v18.16b\n .inst 0x4e9b9595  // sdot v21.4s, v12.16b, v27.16b\n"
    ".inst 0x4e8f9592  // sdot v18.4s, v12.16b, v15.16b\n"
    "ldr q17, [%x[params], #0x0]\n"
    "sqrdmulh v31.4s, v31.4s, v17.4s\n"
    ".inst 0x4e96969e  // sdot v30.4s, v20.16b, v22.16b\n"
    ".inst 0x4e9b969c  // sdot v28.4s, v20.16b, v27.16b\n"
    "mls v30.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v21.4s, v16.4s\n"
    "and v27.16b, v31.16b, v4.16b\n"
    "sshr v27.4s, v27.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v17.4s\n"
    "sqrdmulh v29.4s, v29.4s, v17.4s\n"
    "sqrdmulh v28.4s, v28.4s, v17.4s\n"
    "ldr q15, [%x[params], #0x60]\n"
    "sqadd v31.4s, v31.4s, v27.4s\n"
    "and v20.16b, v30.16b, v4.16b\n"
    "and v18.16b, v29.16b, v4.16b\n"
    "and v17.16b, v28.16b, v4.16b\n"
    "sshr v20.4s, v20.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v4.4s\n"
    "sqadd v30.4s, v30.4s, v20.4s\n"
    "ldr q27, [%x[params], #0x40]\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "ldr q26, [%x[params], #0x50]\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "ldr q6, [%x[params], #0x30]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v30.4s, v30.4s, v4.4s\n"
    "srshl v29.4s, v29.4s, v4.4s\n"
    "srshl v28.4s, v28.4s, v4.4s\n"
    "ldr q4, [%x[params], #0x70]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v1.4s, #0x0\n"
    ".inst 0x4e979581  // sdot v1.4s, v12.16b, v23.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q31, [%x[params], #0x20]\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    ".inst 0x4e939581  // sdot v1.4s, v12.16b, v19.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s30, [x24, x27]\n"
    "mov v22.16b, v1.16b\n .inst 0x4e989596  // sdot v22.4s, v12.16b, v24.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "str s29, [x23, x27]\n"
    "mov v29.16b, v31.16b\n"
    ".inst 0x4e999581  // sdot v1.4s, v12.16b, v25.16b\n"
    "str s28, [x22, x27]\n"
    "mov v21.16b, v31.16b\n"
    "mov v20.16b, v31.16b\n"
    ".inst 0x4e9994df  // sdot v31.4s, v6.16b, v25.16b\n"
    ".inst 0x4e9794d5  // sdot v21.4s, v6.16b, v23.16b\n"
    ".inst 0x4e97977f  // sdot v31.4s, v27.16b, v23.16b\n"
    "ext v25.16b, v25.16b, v25.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v23.16b, v23.16b, v23.16b, #0x1\n"
    "movi v18.4s, #0x0\n"
    ".inst 0x4e9994dd  // sdot v29.4s, v6.16b, v25.16b\n"
    ".inst 0x4e9794d4  // sdot v20.4s, v6.16b, v23.16b\n"
    ".inst 0x4e979592  // sdot v18.4s, v12.16b, v23.16b\n"
    ".inst 0x4e939775  // sdot v21.4s, v27.16b, v19.16b\n"
    ".inst 0x4e93975f  // sdot v31.4s, v26.16b, v19.16b\n"
    "ext v19.16b, v19.16b, v19.16b, #0x1\n"
    ".inst 0x4e97977d  // sdot v29.4s, v27.16b, v23.16b\n"
    ".inst 0x4e939774  // sdot v20.4s, v27.16b, v19.16b\n"
    "mls v31.4s, v1.4s, v16.4s\n"
    ".inst 0x4e939592  // sdot v18.4s, v12.16b, v19.16b\n"
    ".inst 0x4e989755  // sdot v21.4s, v26.16b, v24.16b\n"
    "ext v24.16b, v24.16b, v24.16b, #0x1\n"
    ".inst 0x4e93975d  // sdot v29.4s, v26.16b, v19.16b\n"
    ".inst 0x4e989754  // sdot v20.4s, v26.16b, v24.16b\n"
    "sqrdmulh v31.4s, v31.4s, v15.4s\n"
    "mov v17.16b, v18.16b\n .inst 0x4e989591  // sdot v17.4s, v12.16b, v24.16b\n"
    ".inst 0x4e999592  // sdot v18.4s, v12.16b, v25.16b\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mls v21.4s, v22.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v4.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v29.4s, v29.4s, v15.4s\n"
    "sqrdmulh v21.4s, v21.4s, v15.4s\n"
    "sqrdmulh v20.4s, v20.4s, v15.4s\n"
    "ldr q27, [%x[params], #0xc0]\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v29.16b, v4.16b\n"
    "and v18.16b, v21.16b, v4.16b\n"
    "and v17.16b, v20.16b, v4.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v4.4s\n"
    "sqadd v29.4s, v29.4s, v19.4s\n"
    "ldr q26, [%x[params], #0xa0]\n"
    "sqadd v21.4s, v21.4s, v18.4s\n"
    "ldr q25, [%x[params], #0xb0]\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "ldr q24, [%x[params], #0x90]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v29.4s, v29.4s, v4.4s\n"
    "srshl v21.4s, v21.4s, v4.4s\n"
    "srshl v20.4s, v20.4s, v4.4s\n"
    "ldr q1, [%x[params], #0xd0]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v23.4s, #0x0\n"
    ".inst 0x4e839597  // sdot v23.4s, v12.16b, v3.16b\n"
    ".inst 0x4e809597  // sdot v23.4s, v12.16b, v0.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q31, [%x[params], #0x80]\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "mov v22.16b, v23.16b\n .inst 0x4e829596  // sdot v22.4s, v12.16b, v2.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s29, [x24, x27]\n"
    ".inst 0x4e879597  // sdot v23.4s, v12.16b, v7.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s21, [x23, x27]\n"
    "mov v21.16b, v31.16b\n"
    "str s20, [x22, x27]\n"
    "mov v4.16b, v31.16b\n"
    "mov v20.16b, v31.16b\n"
    ".inst 0x4e87971f  // sdot v31.4s, v24.16b, v7.16b\n"
    ".inst 0x4e839704  // sdot v4.4s, v24.16b, v3.16b\n"
    ".inst 0x4e83975f  // sdot v31.4s, v26.16b, v3.16b\n"
    "ext v7.16b, v7.16b, v7.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v3.16b, v3.16b, v3.16b, #0x1\n"
    "movi v18.4s, #0x0\n"
    ".inst 0x4e879715  // sdot v21.4s, v24.16b, v7.16b\n"
    ".inst 0x4e839714  // sdot v20.4s, v24.16b, v3.16b\n"
    ".inst 0x4e839592  // sdot v18.4s, v12.16b, v3.16b\n"
    ".inst 0x4e809744  // sdot v4.4s, v26.16b, v0.16b\n"
    ".inst 0x4e80973f  // sdot v31.4s, v25.16b, v0.16b\n"
    "ext v0.16b, v0.16b, v0.16b, #0x1\n"
    ".inst 0x4e839755  // sdot v21.4s, v26.16b, v3.16b\n"
    ".inst 0x4e809754  // sdot v20.4s, v26.16b, v0.16b\n"
    "mls v31.4s, v23.4s, v16.4s\n"
    ".inst 0x4e809592  // sdot v18.4s, v12.16b, v0.16b\n"
    ".inst 0x4e829724  // sdot v4.4s, v25.16b, v2.16b\n"
    "ext v2.16b, v2.16b, v2.16b, #0x1\n"
    ".inst 0x4e809735  // sdot v21.4s, v25.16b, v0.16b\n"
    ".inst 0x4e829734  // sdot v20.4s, v25.16b, v2.16b\n"
    "sqrdmulh v31.4s, v31.4s, v27.4s\n"
    "mov v17.16b, v18.16b\n .inst 0x4e829591  // sdot v17.4s, v12.16b, v2.16b\n"
    ".inst 0x4e879592  // sdot v18.4s, v12.16b, v7.16b\n"
    "mls v21.4s, v18.4s, v16.4s\n"
    "mls v4.4s, v22.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v1.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v21.4s, v21.4s, v27.4s\n"
    "sqrdmulh v4.4s, v4.4s, v27.4s\n"
    "sqrdmulh v20.4s, v20.4s, v27.4s\n"
    "ldr q30, [%x[params], #0x120]\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v21.16b, v1.16b\n"
    "and v18.16b, v4.16b, v1.16b\n"
    "and v17.16b, v20.16b, v1.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v1.4s\n"
    "sqadd v21.4s, v21.4s, v19.4s\n"
    "ldr q29, [%x[params], #0x100]\n"
    "sqadd v4.4s, v4.4s, v18.4s\n"
    "ldr q28, [%x[params], #0x110]\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "ldr q27, [%x[params], #0xf0]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v21.4s, v21.4s, v1.4s\n"
    "srshl v4.4s, v4.4s, v1.4s\n"
    "srshl v20.4s, v20.4s, v1.4s\n"
    "ldr q26, [%x[params], #0x130]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "add v4.4s, v4.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smax v4.4s, v4.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "smin v4.4s, v4.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v25.4s, #0x0\n"
    ".inst 0x4e8a9599  // sdot v25.4s, v12.16b, v10.16b\n"
    ".inst 0x4e859599  // sdot v25.4s, v12.16b, v5.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q24, [%x[params], #0xe0]\n"
    "uzp1 v4.16b, v4.16b, v4.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "mov v23.16b, v25.16b\n .inst 0x4e899597  // sdot v23.4s, v12.16b, v9.16b\n"
    "add %x[params], %x[params], #0x140\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "uzp1 v4.16b, v4.16b, v4.16b\n"
    "str s21, [x24, x27]\n"
    ".inst 0x4e889599  // sdot v25.4s, v12.16b, v8.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s4, [x23, x27]\n"
    "mov v22.16b, v24.16b\n"
    "str s20, [x22, x27]\n"
    "mov v21.16b, v24.16b\n"
    "mov v20.16b, v24.16b\n"
    ".inst 0x4e889778  // sdot v24.4s, v27.16b, v8.16b\n"
    ".inst 0x4e8a9775  // sdot v21.4s, v27.16b, v10.16b\n"
    ".inst 0x4e8a97b8  // sdot v24.4s, v29.16b, v10.16b\n"
    "ext v8.16b, v8.16b, v8.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v10.16b, v10.16b, v10.16b, #0x1\n"
    "movi v18.4s, #0x0\n"
    ".inst 0x4e889776  // sdot v22.4s, v27.16b, v8.16b\n"
    ".inst 0x4e8a9774  // sdot v20.4s, v27.16b, v10.16b\n"
    ".inst 0x4e8a9592  // sdot v18.4s, v12.16b, v10.16b\n"
    ".inst 0x4e8597b5  // sdot v21.4s, v29.16b, v5.16b\n"
    ".inst 0x4e859798  // sdot v24.4s, v28.16b, v5.16b\n"
    "ext v5.16b, v5.16b, v5.16b, #0x1\n"
    ".inst 0x4e8a97b6  // sdot v22.4s, v29.16b, v10.16b\n"
    ".inst 0x4e8597b4  // sdot v20.4s, v29.16b, v5.16b\n"
    "mls v24.4s, v25.4s, v16.4s\n"
    ".inst 0x4e859592  // sdot v18.4s, v12.16b, v5.16b\n"
    ".inst 0x4e899795  // sdot v21.4s, v28.16b, v9.16b\n"
    "ext v9.16b, v9.16b, v9.16b, #0x1\n"
    ".inst 0x4e859796  // sdot v22.4s, v28.16b, v5.16b\n"
    ".inst 0x4e899794  // sdot v20.4s, v28.16b, v9.16b\n"
    "sqrdmulh v24.4s, v24.4s, v30.4s\n"
    "mov v17.16b, v18.16b\n .inst 0x4e899591  // sdot v17.4s, v12.16b, v9.16b\n"
    ".inst 0x4e889592  // sdot v18.4s, v12.16b, v8.16b\n"
    "mls v22.4s, v18.4s, v16.4s\n"
    "mls v21.4s, v23.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v24.16b, v26.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v22.4s, v22.4s, v30.4s\n"
    "sqrdmulh v21.4s, v21.4s, v30.4s\n"
    "sqrdmulh v20.4s, v20.4s, v30.4s\n"
    "sqadd v24.4s, v24.4s, v17.4s\n"
    "and v19.16b, v22.16b, v26.16b\n"
    "and v18.16b, v21.16b, v26.16b\n"
    "and v17.16b, v20.16b, v26.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqadd v22.4s, v22.4s, v19.4s\n"
    "sqadd v21.4s, v21.4s, v18.4s\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "srshl v24.4s, v24.4s, v26.4s\n"
    "srshl v22.4s, v22.4s, v26.4s\n"
    "srshl v21.4s, v21.4s, v26.4s\n"
    "srshl v20.4s, v20.4s, v26.4s\n"
    "add v24.4s, v24.4s, v14.4s\n"
    "add v22.4s, v22.4s, v14.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smax v24.4s, v24.4s, v13.4s\n"
    "smax v22.4s, v22.4s, v13.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v24.4s, v24.4s, v11.4s\n"
    "smin v22.4s, v22.4s, v11.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v24.16b, v24.16b, v24.16b\n"
    "uzp1 v22.16b, v22.16b, v22.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "uzp1 v24.16b, v24.16b, v24.16b\n"
    "str s24, [x25, x27]\n"
    "uzp1 v22.16b, v22.16b, v22.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s22, [x24, x27]\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s21, [x23, x27]\n"
    "str s20, [x22, x27]\n"
    "add x27, x27, #0x4\n"
    "beq 35f\n"
    "3:"  // Oddments
    "and x20, %x[n_channels], #0xf\n"
    "add x15, x15, x28\n"
    "add x14, x14, x28\n"
    "add x13, x13, x28\n"
    "add x12, x12, x28\n"
    "add x10, x10, x28\n"
    "add x9, x9, x28\n"
    "add x26, x26, x28\n"
    "add x21, x21, x28\n"
    "tbz %x[n_channels], #3, 7f\n"
    "ldr d15, [x15], #0x8\n"
    "ldr d25, [x14], #0x8\n"
    "ldr d7, [x13], #0x8\n"
    "ldr d8, [x12], #0x8\n"
    "ldr d26, [x10], #0x8\n"
    "ldr d23, [x9], #0x8\n"
    "ldr d3, [x26], #0x8\n"
    "ldr d10, [x21], #0x8\n"
    "tbz %x[n_channels], #2, 5f\n"
    "ld1 { v15.s }[2], [x15], #0x4\n"
    "ld1 { v25.s }[2], [x14], #0x4\n"
    "ld1 { v7.s }[2], [x13], #0x4\n"
    "ld1 { v8.s }[2], [x12], #0x4\n"
    "ld1 { v26.s }[2], [x10], #0x4\n"
    "ld1 { v23.s }[2], [x9], #0x4\n"
    "ld1 { v3.s }[2], [x26], #0x4\n"
    "ld1 { v10.s }[2], [x21], #0x4\n"
    "tbz %x[n_channels], #1, 4f\n"
    "ld1 { v15.h }[6], [x15], #0x2\n"
    "ld1 { v25.h }[6], [x14], #0x2\n"
    "ld1 { v7.h }[6], [x13], #0x2\n"
    "ld1 { v8.h }[6], [x12], #0x2\n"
    "ld1 { v26.h }[6], [x10], #0x2\n"
    "ld1 { v23.h }[6], [x9], #0x2\n"
    "ld1 { v3.h }[6], [x26], #0x2\n"
    "ld1 { v10.h }[6], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[14], [x15], #0x1\n"
    "ld1 { v25.b }[14], [x14], #0x1\n"
    "ld1 { v7.b }[14], [x13], #0x1\n"
    "ld1 { v8.b }[14], [x12], #0x1\n"
    "ld1 { v26.b }[14], [x10], #0x1\n"
    "ld1 { v23.b }[14], [x9], #0x1\n"
    "ld1 { v3.b }[14], [x26], #0x1\n"
    "ld1 { v10.b }[14], [x21], #0x1\n"
    "b 11f\n"
    "4:"  // Oddments: Load (A): Bit 3: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[12], [x15], #0x1\n"
    "ld1 { v25.b }[12], [x14], #0x1\n"
    "ld1 { v7.b }[12], [x13], #0x1\n"
    "ld1 { v8.b }[12], [x12], #0x1\n"
    "ld1 { v26.b }[12], [x10], #0x1\n"
    "ld1 { v23.b }[12], [x9], #0x1\n"
    "ld1 { v3.b }[12], [x26], #0x1\n"
    "ld1 { v10.b }[12], [x21], #0x1\n"
    "b 11f\n"
    "5:"  // Oddments: Load (A): Bit 3: Bit 2: Unset
    "tbz %x[n_channels], #1, 6f\n"
    "ld1 { v15.h }[4], [x15], #0x2\n"
    "ld1 { v25.h }[4], [x14], #0x2\n"
    "ld1 { v7.h }[4], [x13], #0x2\n"
    "ld1 { v8.h }[4], [x12], #0x2\n"
    "ld1 { v26.h }[4], [x10], #0x2\n"
    "ld1 { v23.h }[4], [x9], #0x2\n"
    "ld1 { v3.h }[4], [x26], #0x2\n"
    "ld1 { v10.h }[4], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[10], [x15], #0x1\n"
    "ld1 { v25.b }[10], [x14], #0x1\n"
    "ld1 { v7.b }[10], [x13], #0x1\n"
    "ld1 { v8.b }[10], [x12], #0x1\n"
    "ld1 { v26.b }[10], [x10], #0x1\n"
    "ld1 { v23.b }[10], [x9], #0x1\n"
    "ld1 { v3.b }[10], [x26], #0x1\n"
    "ld1 { v10.b }[10], [x21], #0x1\n"
    "b 11f\n"
    "6:"  // Oddments: Load (A): Bit 3: Bit 2: Unset: Bit 1: Unset
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[8], [x15], #0x1\n"
    "ld1 { v25.b }[8], [x14], #0x1\n"
    "ld1 { v7.b }[8], [x13], #0x1\n"
    "ld1 { v8.b }[8], [x12], #0x1\n"
    "ld1 { v26.b }[8], [x10], #0x1\n"
    "ld1 { v23.b }[8], [x9], #0x1\n"
    "ld1 { v3.b }[8], [x26], #0x1\n"
    "ld1 { v10.b }[8], [x21], #0x1\n"
    "b 11f\n"
    "7:"  // Oddments: Load (A): Bit 3: Unset
    "tbz %x[n_channels], #2, 9f\n"
    "ldr s15, [x15], #0x4\n"
    "ldr s25, [x14], #0x4\n"
    "ldr s7, [x13], #0x4\n"
    "ldr s8, [x12], #0x4\n"
    "ldr s26, [x10], #0x4\n"
    "ldr s23, [x9], #0x4\n"
    "ldr s3, [x26], #0x4\n"
    "ldr s10, [x21], #0x4\n"
    "tbz %x[n_channels], #1, 8f\n"
    "ld1 { v15.h }[2], [x15], #0x2\n"
    "ld1 { v25.h }[2], [x14], #0x2\n"
    "ld1 { v7.h }[2], [x13], #0x2\n"
    "ld1 { v8.h }[2], [x12], #0x2\n"
    "ld1 { v26.h }[2], [x10], #0x2\n"
    "ld1 { v23.h }[2], [x9], #0x2\n"
    "ld1 { v3.h }[2], [x26], #0x2\n"
    "ld1 { v10.h }[2], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[6], [x15], #0x1\n"
    "ld1 { v25.b }[6], [x14], #0x1\n"
    "ld1 { v7.b }[6], [x13], #0x1\n"
    "ld1 { v8.b }[6], [x12], #0x1\n"
    "ld1 { v26.b }[6], [x10], #0x1\n"
    "ld1 { v23.b }[6], [x9], #0x1\n"
    "ld1 { v3.b }[6], [x26], #0x1\n"
    "ld1 { v10.b }[6], [x21], #0x1\n"
    "b 11f\n"
    "8:"  // Oddments: Load (A): Bit 3: Unset: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[4], [x15], #0x1\n"
    "ld1 { v25.b }[4], [x14], #0x1\n"
    "ld1 { v7.b }[4], [x13], #0x1\n"
    "ld1 { v8.b }[4], [x12], #0x1\n"
    "ld1 { v26.b }[4], [x10], #0x1\n"
    "ld1 { v23.b }[4], [x9], #0x1\n"
    "ld1 { v3.b }[4], [x26], #0x1\n"
    "ld1 { v10.b }[4], [x21], #0x1\n"
    "b 11f\n"
    "9:"  // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset
    "tbz %x[n_channels], #1, 10f\n"
    "ldr h15, [x15], #0x2\n"
    "ldr h25, [x14], #0x2\n"
    "ldr h7, [x13], #0x2\n"
    "ldr h8, [x12], #0x2\n"
    "ldr h26, [x10], #0x2\n"
    "ldr h23, [x9], #0x2\n"
    "ldr h3, [x26], #0x2\n"
    "ldr h10, [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[2], [x15], #0x1\n"
    "ld1 { v25.b }[2], [x14], #0x1\n"
    "ld1 { v7.b }[2], [x13], #0x1\n"
    "ld1 { v8.b }[2], [x12], #0x1\n"
    "ld1 { v26.b }[2], [x10], #0x1\n"
    "ld1 { v23.b }[2], [x9], #0x1\n"
    "ld1 { v3.b }[2], [x26], #0x1\n"
    "ld1 { v10.b }[2], [x21], #0x1\n"
    "b 11f\n"
    "10:"  // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
    "ldr b15, [x15], #0x1\n"
    "ldr b25, [x14], #0x1\n"
    "ldr b7, [x13], #0x1\n"
    "ldr b8, [x12], #0x1\n"
    "ldr b26, [x10], #0x1\n"
    "ldr b23, [x9], #0x1\n"
    "ldr b3, [x26], #0x1\n"
    "ldr b10, [x21], #0x1\n"
    "11:"  // Oddments: Load (A): Bit 3: End
    "ldp x15, x14, [%x[inptrs], #0x40]\n"
    "ldp x13, x12, [%x[inptrs], #0x50]\n"
    "add x15, x15, x28\n"
    "add x14, x14, x28\n"
    "ldp x10, x9, [%x[inptrs], #0x60]\n"
    "ldp x26, x21, [%x[inptrs], #0x70]\n"
    "add x13, x13, x28\n"
    "add x12, x12, x28\n"
    "add x10, x10, x28\n"
    "add x9, x9, x28\n"
    "add x26, x26, x28\n"
    "add x21, x21, x28\n"
    "tbz %x[n_channels], #3, 15f\n"
    "ldr d22, [x15], #0x8\n"
    "ldr d19, [x14], #0x8\n"
    "ldr d0, [x13], #0x8\n"
    "ldr d5, [x12], #0x8\n"
    "ldr d27, [x10], #0x8\n"
    "ldr d24, [x9], #0x8\n"
    "ldr d2, [x26], #0x8\n"
    "ldr d9, [x21], #0x8\n"
    "tbz %x[n_channels], #2, 13f\n"
    "ld1 { v22.s }[2], [x15], #0x4\n"
    "ld1 { v19.s }[2], [x14], #0x4\n"
    "ld1 { v0.s }[2], [x13], #0x4\n"
    "ld1 { v5.s }[2], [x12], #0x4\n"
    "ld1 { v27.s }[2], [x10], #0x4\n"
    "ld1 { v24.s }[2], [x9], #0x4\n"
    "ld1 { v2.s }[2], [x26], #0x4\n"
    "ld1 { v9.s }[2], [x21], #0x4\n"
    "tbz %x[n_channels], #1, 12f\n"
    "ld1 { v22.h }[6], [x15], #0x2\n"
    "ld1 { v19.h }[6], [x14], #0x2\n"
    "ld1 { v0.h }[6], [x13], #0x2\n"
    "ld1 { v5.h }[6], [x12], #0x2\n"
    "ld1 { v27.h }[6], [x10], #0x2\n"
    "ld1 { v24.h }[6], [x9], #0x2\n"
    "ld1 { v2.h }[6], [x26], #0x2\n"
    "ld1 { v9.h }[6], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[14], [x15], #0x1\n"
    "ld1 { v19.b }[14], [x14], #0x1\n"
    "ld1 { v0.b }[14], [x13], #0x1\n"
    "ld1 { v5.b }[14], [x12], #0x1\n"
    "ld1 { v27.b }[14], [x10], #0x1\n"
    "ld1 { v24.b }[14], [x9], #0x1\n"
    "ld1 { v2.b }[14], [x26], #0x1\n"
    "ld1 { v9.b }[14], [x21], #0x1\n"
    "b 19f\n"
    "12:"  // Oddments: Load (B): Bit 3: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[12], [x15], #0x1\n"
    "ld1 { v19.b }[12], [x14], #0x1\n"
    "ld1 { v0.b }[12], [x13], #0x1\n"
    "ld1 { v5.b }[12], [x12], #0x1\n"
    "ld1 { v27.b }[12], [x10], #0x1\n"
    "ld1 { v24.b }[12], [x9], #0x1\n"
    "ld1 { v2.b }[12], [x26], #0x1\n"
    "ld1 { v9.b }[12], [x21], #0x1\n"
    "b 19f\n"
    "13:"  // Oddments: Load (B): Bit 3: Bit 2: Unset
    "tbz %x[n_channels], #1, 14f\n"
    "ld1 { v22.h }[4], [x15], #0x2\n"
    "ld1 { v19.h }[4], [x14], #0x2\n"
    "ld1 { v0.h }[4], [x13], #0x2\n"
    "ld1 { v5.h }[4], [x12], #0x2\n"
    "ld1 { v27.h }[4], [x10], #0x2\n"
    "ld1 { v24.h }[4], [x9], #0x2\n"
    "ld1 { v2.h }[4], [x26], #0x2\n"
    "ld1 { v9.h }[4], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[10], [x15], #0x1\n"
    "ld1 { v19.b }[10], [x14], #0x1\n"
    "ld1 { v0.b }[10], [x13], #0x1\n"
    "ld1 { v5.b }[10], [x12], #0x1\n"
    "ld1 { v27.b }[10], [x10], #0x1\n"
    "ld1 { v24.b }[10], [x9], #0x1\n"
    "ld1 { v2.b }[10], [x26], #0x1\n"
    "ld1 { v9.b }[10], [x21], #0x1\n"
    "b 19f\n"
    "14:"  // Oddments: Load (B): Bit 3: Bit 2: Unset: Bit 1: Unset
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[8], [x15], #0x1\n"
    "ld1 { v19.b }[8], [x14], #0x1\n"
    "ld1 { v0.b }[8], [x13], #0x1\n"
    "ld1 { v5.b }[8], [x12], #0x1\n"
    "ld1 { v27.b }[8], [x10], #0x1\n"
    "ld1 { v24.b }[8], [x9], #0x1\n"
    "ld1 { v2.b }[8], [x26], #0x1\n"
    "ld1 { v9.b }[8], [x21], #0x1\n"
    "b 19f\n"
    "15:"  // Oddments: Load (B): Bit 3: Unset
    "tbz %x[n_channels], #2, 17f\n"
    "ldr s22, [x15], #0x4\n"
    "ldr s19, [x14], #0x4\n"
    "ldr s0, [x13], #0x4\n"
    "ldr s5, [x12], #0x4\n"
    "ldr s27, [x10], #0x4\n"
    "ldr s24, [x9], #0x4\n"
    "ldr s2, [x26], #0x4\n"
    "ldr s9, [x21], #0x4\n"
    "tbz %x[n_channels], #1, 16f\n"
    "ld1 { v22.h }[2], [x15], #0x2\n"
    "ld1 { v19.h }[2], [x14], #0x2\n"
    "ld1 { v0.h }[2], [x13], #0x2\n"
    "ld1 { v5.h }[2], [x12], #0x2\n"
    "ld1 { v27.h }[2], [x10], #0x2\n"
    "ld1 { v24.h }[2], [x9], #0x2\n"
    "ld1 { v2.h }[2], [x26], #0x2\n"
    "ld1 { v9.h }[2], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[6], [x15], #0x1\n"
    "ld1 { v19.b }[6], [x14], #0x1\n"
    "ld1 { v0.b }[6], [x13], #0x1\n"
    "ld1 { v5.b }[6], [x12], #0x1\n"
    "ld1 { v27.b }[6], [x10], #0x1\n"
    "ld1 { v24.b }[6], [x9], #0x1\n"
    "ld1 { v2.b }[6], [x26], #0x1\n"
    "ld1 { v9.b }[6], [x21], #0x1\n"
    "b 19f\n"
    "16:"  // Oddments: Load (B): Bit 3: Unset: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[4], [x15], #0x1\n"
    "ld1 { v19.b }[4], [x14], #0x1\n"
    "ld1 { v0.b }[4], [x13], #0x1\n"
    "ld1 { v5.b }[4], [x12], #0x1\n"
    "ld1 { v27.b }[4], [x10], #0x1\n"
    "ld1 { v24.b }[4], [x9], #0x1\n"
    "ld1 { v2.b }[4], [x26], #0x1\n"
    "ld1 { v9.b }[4], [x21], #0x1\n"
    "b 19f\n"
    "17:"  // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset
    "tbz %x[n_channels], #1, 18f\n"
    "ldr h22, [x15], #0x2\n"
    "ldr h19, [x14], #0x2\n"
    "ldr h0, [x13], #0x2\n"
    "ldr h5, [x12], #0x2\n"
    "ldr h27, [x10], #0x2\n"
    "ldr h24, [x9], #0x2\n"
    "ldr h2, [x26], #0x2\n"
    "ldr h9, [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[2], [x15], #0x1\n"
    "ld1 { v19.b }[2], [x14], #0x1\n"
    "ld1 { v0.b }[2], [x13], #0x1\n"
    "ld1 { v5.b }[2], [x12], #0x1\n"
    "ld1 { v27.b }[2], [x10], #0x1\n"
    "ld1 { v24.b }[2], [x9], #0x1\n"
    "ld1 { v2.b }[2], [x26], #0x1\n"
    "ld1 { v9.b }[2], [x21], #0x1\n"
    "b 19f\n"
    "18:"  // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
    "ldr b22, [x15], #0x1\n"
    "ldr b19, [x14], #0x1\n"
    "ldr b0, [x13], #0x1\n"
    "ldr b5, [x12], #0x1\n"
    "ldr b27, [x10], #0x1\n"
    "ldr b24, [x9], #0x1\n"
    "ldr b2, [x26], #0x1\n"
    "ldr b9, [x21], #0x1\n"
    "19:"  // Oddments: Load (B): Bit 3: End
    "ldr q20, [%x[params], #0x10]\n"
    "ldr q6, [%x[params], #0x20]\n"
    "zip2 v1.16b, v26.16b, v3.16b\n"
    "zip1 v26.16b, v26.16b, v3.16b\n"
    "ldr q4, [%x[params], #0x30]\n"
    "zip1 v18.16b, v23.16b, v10.16b\n"
    "zip2 v30.16b, v15.16b, v7.16b\n"
    "cmp x20, #0x4\n"
    "zip1 v15.16b, v15.16b, v7.16b\n"
    "zip1 v29.16b, v25.16b, v8.16b\n"
    "zip2 v8.16b, v25.16b, v8.16b\n"
    "zip2 v10.16b, v23.16b, v10.16b\n"
    "zip2 v23.16b, v26.16b, v18.16b\n"
    "zip1 v26.16b, v26.16b, v18.16b\n"
    "zip2 v28.16b, v22.16b, v0.16b\n"
    "zip1 v22.16b, v22.16b, v0.16b\n"
    "zip1 v21.16b, v19.16b, v5.16b\n"
    "movi v17.4s, #0x0\n"
    ".inst 0x4e9a9591  // sdot v17.4s, v12.16b, v26.16b\n"
    "zip2 v25.16b, v15.16b, v29.16b\n"
    "zip1 v15.16b, v15.16b, v29.16b\n"
    "zip1 v7.16b, v30.16b, v8.16b\n"
    "zip2 v8.16b, v30.16b, v8.16b\n"
    "ldr q31, [%x[params], #0x0]\n"
    "zip2 v5.16b, v19.16b, v5.16b\n"
    "zip2 v30.16b, v27.16b, v2.16b\n"
    "zip1 v27.16b, v27.16b, v2.16b\n"
    "zip1 v18.16b, v24.16b, v9.16b\n"
    "zip2 v9.16b, v24.16b, v9.16b\n"
    "zip2 v19.16b, v22.16b, v21.16b\n"
    "zip1 v22.16b, v22.16b, v21.16b\n"
    "zip1 v3.16b, v1.16b, v10.16b\n"
    ".inst 0x4e969591  // sdot v17.4s, v12.16b, v22.16b\n"
    "zip2 v10.16b, v1.16b, v10.16b\n"
    "zip1 v0.16b, v28.16b, v5.16b\n"
    "zip2 v5.16b, v28.16b, v5.16b\n"
    "zip2 v24.16b, v27.16b, v18.16b\n"
    "zip1 v27.16b, v27.16b, v18.16b\n"
    "zip1 v2.16b, v30.16b, v9.16b\n"
    "mov v18.16b, v17.16b\n .inst 0x4e9b9592  // sdot v18.4s, v12.16b, v27.16b\n"
    "zip2 v9.16b, v30.16b, v9.16b\n"
    "mov v30.16b, v31.16b\n"
    ".inst 0x4e8f9591  // sdot v17.4s, v12.16b, v15.16b\n"
    "mov v29.16b, v31.16b\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x4e8f969f  // sdot v31.4s, v20.16b, v15.16b\n"
    ".inst 0x4e9a969d  // sdot v29.4s, v20.16b, v26.16b\n"
    ".inst 0x4e9a94df  // sdot v31.4s, v6.16b, v26.16b\n"
    "ext v26.16b, v26.16b, v26.16b, #0x1\n"
    "movi v1.4s, #0x0\n"
    "ext v15.16b, v15.16b, v15.16b, #0x1\n"
    ".inst 0x4e9a9581  // sdot v1.4s, v12.16b, v26.16b\n"
    ".inst 0x4e9694dd  // sdot v29.4s, v6.16b, v22.16b\n"
    ".inst 0x4e96949f  // sdot v31.4s, v4.16b, v22.16b\n"
    "ext v22.16b, v22.16b, v22.16b, #0x1\n"
    ".inst 0x4e8f969e  // sdot v30.4s, v20.16b, v15.16b\n"
    ".inst 0x4e9a969c  // sdot v28.4s, v20.16b, v26.16b\n"
    "mls v31.4s, v17.4s, v16.4s\n"
    ".inst 0x4e969581  // sdot v1.4s, v12.16b, v22.16b\n"
    ".inst 0x4e9b949d  // sdot v29.4s, v4.16b, v27.16b\n"
    "ext v27.16b, v27.16b, v27.16b, #0x1\n"
    ".inst 0x4e9a94de  // sdot v30.4s, v6.16b, v26.16b\n"
    "ldr q21, [%x[params], #0x50]\n"
    ".inst 0x4e9694dc  // sdot v28.4s, v6.16b, v22.16b\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mov v20.16b, v1.16b\n .inst 0x4e9b9594  // sdot v20.4s, v12.16b, v27.16b\n"
    ".inst 0x4e8f9581  // sdot v1.4s, v12.16b, v15.16b\n"
    "ldr q18, [%x[params], #0x40]\n"
    "sqrdmulh v31.4s, v31.4s, v18.4s\n"
    ".inst 0x4e96949e  // sdot v30.4s, v4.16b, v22.16b\n"
    ".inst 0x4e9b949c  // sdot v28.4s, v4.16b, v27.16b\n"
    "mls v30.4s, v1.4s, v16.4s\n"
    "add %x[params], %x[params], #0x60\n"
    "mls v28.4s, v20.4s, v16.4s\n"
    "and v17.16b, v31.16b, v21.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v18.4s\n"
    "sqrdmulh v29.4s, v29.4s, v18.4s\n"
    "sqrdmulh v28.4s, v28.4s, v18.4s\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v17.16b, v30.16b, v21.16b\n"
    "and v18.16b, v29.16b, v21.16b\n"
    "and v26.16b, v28.16b, v21.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v26.4s, v26.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v17.4s\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "sqadd v28.4s, v28.4s, v26.4s\n"
    "srshl v31.4s, v31.4s, v21.4s\n"
    "srshl v30.4s, v30.4s, v21.4s\n"
    "srshl v29.4s, v29.4s, v21.4s\n"
    "srshl v28.4s, v28.4s, v21.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "blt 20f\n"
    "str s31, [x25, x27]\n"
    "str s30, [x24, x27]\n"
    "str s29, [x23, x27]\n"
    "str s28, [x22, x27]\n"
    "b 23f\n"
    "20:"  // Oddments: Unroll 0: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 21f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 22f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 22f\n"
    "21:"  // Oddments: Unroll 0: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "22:"  // Oddments: Unroll 0: Oddment store: Bit 1: End
    "23:"  // Oddments: Unroll 0: After oddment store
    "subs x20, x20, #0x4\n"
    "add x27, x27, #0x4\n"
    "ble 35f\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q27, [%x[params], #0x10]\n"
    "movi v1.4s, #0x0\n"
    ".inst 0x4e979581  // sdot v1.4s, v12.16b, v23.16b\n"
    "ldr q26, [%x[params], #0x20]\n"
    "ldr q22, [%x[params], #0x30]\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "ldr q4, [%x[params], #0x40]\n"
    "ldr q21, [%x[params], #0x50]\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x4e99977f  // sdot v31.4s, v27.16b, v25.16b\n"
    ".inst 0x4e939581  // sdot v1.4s, v12.16b, v19.16b\n"
    ".inst 0x4e97977d  // sdot v29.4s, v27.16b, v23.16b\n"
    "movi v20.4s, #0x0\n"
    "cmp x20, #0x4\n"
    ".inst 0x4e97975f  // sdot v31.4s, v26.16b, v23.16b\n"
    "mov v18.16b, v1.16b\n .inst 0x4e989592  // sdot v18.4s, v12.16b, v24.16b\n"
    "ext v23.16b, v23.16b, v23.16b, #0x1\n"
    "add %x[params], %x[params], #0x60\n"
    ".inst 0x4e999581  // sdot v1.4s, v12.16b, v25.16b\n"
    "ext v25.16b, v25.16b, v25.16b, #0x1\n"
    ".inst 0x4e99977e  // sdot v30.4s, v27.16b, v25.16b\n"
    ".inst 0x4e97977c  // sdot v28.4s, v27.16b, v23.16b\n"
    ".inst 0x4e979594  // sdot v20.4s, v12.16b, v23.16b\n"
    ".inst 0x4e93975d  // sdot v29.4s, v26.16b, v19.16b\n"
    ".inst 0x4e9396df  // sdot v31.4s, v22.16b, v19.16b\n"
    "ext v19.16b, v19.16b, v19.16b, #0x1\n"
    ".inst 0x4e97975e  // sdot v30.4s, v26.16b, v23.16b\n"
    ".inst 0x4e93975c  // sdot v28.4s, v26.16b, v19.16b\n"
    "mls v31.4s, v1.4s, v16.4s\n"
    ".inst 0x4e939594  // sdot v20.4s, v12.16b, v19.16b\n"
    ".inst 0x4e9896dd  // sdot v29.4s, v22.16b, v24.16b\n"
    "ext v24.16b, v24.16b, v24.16b, #0x1\n"
    ".inst 0x4e9396de  // sdot v30.4s, v22.16b, v19.16b\n"
    ".inst 0x4e9896dc  // sdot v28.4s, v22.16b, v24.16b\n"
    "sqrdmulh v31.4s, v31.4s, v4.4s\n"
    "mov v17.16b, v20.16b\n .inst 0x4e989591  // sdot v17.4s, v12.16b, v24.16b\n"
    ".inst 0x4e999594  // sdot v20.4s, v12.16b, v25.16b\n"
    "mls v30.4s, v20.4s, v16.4s\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v21.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v4.4s\n"
    "sqrdmulh v29.4s, v29.4s, v4.4s\n"
    "sqrdmulh v28.4s, v28.4s, v4.4s\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v30.16b, v21.16b\n"
    "and v18.16b, v29.16b, v21.16b\n"
    "and v17.16b, v28.16b, v21.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v19.4s\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "srshl v31.4s, v31.4s, v21.4s\n"
    "srshl v30.4s, v30.4s, v21.4s\n"
    "srshl v29.4s, v29.4s, v21.4s\n"
    "srshl v28.4s, v28.4s, v21.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "blt 24f\n"
    "str s31, [x25, x27]\n"
    "str s30, [x24, x27]\n"
    "str s29, [x23, x27]\n"
    "str s28, [x22, x27]\n"
    "b 27f\n"
    "24:"  // Oddments: Unroll 1: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 25f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 26f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 26f\n"
    "25:"  // Oddments: Unroll 1: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "26:"  // Oddments: Unroll 1: Oddment store: Bit 1: End
    "27:"  // Oddments: Unroll 1: After oddment store
    "subs x20, x20, #0x4\n"
    "add x27, x27, #0x4\n"
    "ble 35f\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q25, [%x[params], #0x10]\n"
    "movi v24.4s, #0x0\n"
    ".inst 0x4e839598  // sdot v24.4s, v12.16b, v3.16b\n"
    "ldr q23, [%x[params], #0x20]\n"
    "ldr q22, [%x[params], #0x30]\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "ldr q21, [%x[params], #0x40]\n"
    "ldr q20, [%x[params], #0x50]\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x4e87973f  // sdot v31.4s, v25.16b, v7.16b\n"
    ".inst 0x4e809598  // sdot v24.4s, v12.16b, v0.16b\n"
    ".inst 0x4e83973d  // sdot v29.4s, v25.16b, v3.16b\n"
    "movi v19.4s, #0x0\n"
    "cmp x20, #0x4\n"
    ".inst 0x4e8396ff  // sdot v31.4s, v23.16b, v3.16b\n"
    "mov v18.16b, v24.16b\n .inst 0x4e829592  // sdot v18.4s, v12.16b, v2.16b\n"
    "ext v3.16b, v3.16b, v3.16b, #0x1\n"
    "add %x[params], %x[params], #0x60\n"
    ".inst 0x4e879598  // sdot v24.4s, v12.16b, v7.16b\n"
    "ext v7.16b, v7.16b, v7.16b, #0x1\n"
    ".inst 0x4e87973e  // sdot v30.4s, v25.16b, v7.16b\n"
    ".inst 0x4e83973c  // sdot v28.4s, v25.16b, v3.16b\n"
    ".inst 0x4e839593  // sdot v19.4s, v12.16b, v3.16b\n"
    ".inst 0x4e8096fd  // sdot v29.4s, v23.16b, v0.16b\n"
    ".inst 0x4e8096df  // sdot v31.4s, v22.16b, v0.16b\n"
    "ext v0.16b, v0.16b, v0.16b, #0x1\n"
    ".inst 0x4e8396fe  // sdot v30.4s, v23.16b, v3.16b\n"
    ".inst 0x4e8096fc  // sdot v28.4s, v23.16b, v0.16b\n"
    "mls v31.4s, v24.4s, v16.4s\n"
    ".inst 0x4e809593  // sdot v19.4s, v12.16b, v0.16b\n"
    ".inst 0x4e8296dd  // sdot v29.4s, v22.16b, v2.16b\n"
    "ext v2.16b, v2.16b, v2.16b, #0x1\n"
    ".inst 0x4e8096de  // sdot v30.4s, v22.16b, v0.16b\n"
    ".inst 0x4e8296dc  // sdot v28.4s, v22.16b, v2.16b\n"
    "sqrdmulh v31.4s, v31.4s, v21.4s\n"
    "mov v17.16b, v19.16b\n .inst 0x4e829591  // sdot v17.4s, v12.16b, v2.16b\n"
    ".inst 0x4e879593  // sdot v19.4s, v12.16b, v7.16b\n"
    "mls v30.4s, v19.4s, v16.4s\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v20.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v21.4s\n"
    "sqrdmulh v29.4s, v29.4s, v21.4s\n"
    "sqrdmulh v28.4s, v28.4s, v21.4s\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v30.16b, v20.16b\n"
    "and v18.16b, v29.16b, v20.16b\n"
    "and v17.16b, v28.16b, v20.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v19.4s\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "srshl v31.4s, v31.4s, v20.4s\n"
    "srshl v30.4s, v30.4s, v20.4s\n"
    "srshl v29.4s, v29.4s, v20.4s\n"
    "srshl v28.4s, v28.4s, v20.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "blt 28f\n"
    "str s31, [x25, x27]\n"
    "str s30, [x24, x27]\n"
    "str s29, [x23, x27]\n"
    "str s28, [x22, x27]\n"
    "b 31f\n"
    "28:"  // Oddments: Unroll 2: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 29f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 30f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 30f\n"
    "29:"  // Oddments: Unroll 2: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "30:"  // Oddments: Unroll 2: Oddment store: Bit 1: End
    "31:"  // Oddments: Unroll 2: After oddment store
    "subs x20, x20, #0x4\n"
    "add x27, x27, #0x4\n"
    "ble 35f\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q23, [%x[params], #0x10]\n"
    "movi v22.4s, #0x0\n"
    ".inst 0x4e8a9596  // sdot v22.4s, v12.16b, v10.16b\n"
    "ldr q21, [%x[params], #0x20]\n"
    "ldr q19, [%x[params], #0x30]\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "ldr q20, [%x[params], #0x40]\n"
    "ldr q26, [%x[params], #0x50]\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x4e8896ff  // sdot v31.4s, v23.16b, v8.16b\n"
    ".inst 0x4e859596  // sdot v22.4s, v12.16b, v5.16b\n"
    ".inst 0x4e8a96fd  // sdot v29.4s, v23.16b, v10.16b\n"
    "movi v18.4s, #0x0\n"
    "add %x[params], %x[params], #0x60\n"
    ".inst 0x4e8a96bf  // sdot v31.4s, v21.16b, v10.16b\n"
    "mov v17.16b, v22.16b\n .inst 0x4e899591  // sdot v17.4s, v12.16b, v9.16b\n"
    "ext v10.16b, v10.16b, v10.16b, #0x1\n"
    ".inst 0x4e889596  // sdot v22.4s, v12.16b, v8.16b\n"
    "ext v8.16b, v8.16b, v8.16b, #0x1\n"
    ".inst 0x4e8896fe  // sdot v30.4s, v23.16b, v8.16b\n"
    ".inst 0x4e8a96fc  // sdot v28.4s, v23.16b, v10.16b\n"
    ".inst 0x4e8a9592  // sdot v18.4s, v12.16b, v10.16b\n"
    ".inst 0x4e8596bd  // sdot v29.4s, v21.16b, v5.16b\n"
    ".inst 0x4e85967f  // sdot v31.4s, v19.16b, v5.16b\n"
    "ext v5.16b, v5.16b, v5.16b, #0x1\n"
    ".inst 0x4e8a96be  // sdot v30.4s, v21.16b, v10.16b\n"
    ".inst 0x4e8596bc  // sdot v28.4s, v21.16b, v5.16b\n"
    "mls v31.4s, v22.4s, v16.4s\n"
    ".inst 0x4e859592  // sdot v18.4s, v12.16b, v5.16b\n"
    ".inst 0x4e89967d  // sdot v29.4s, v19.16b, v9.16b\n"
    "ext v9.16b, v9.16b, v9.16b, #0x1\n"
    ".inst 0x4e85967e  // sdot v30.4s, v19.16b, v5.16b\n"
    ".inst 0x4e89967c  // sdot v28.4s, v19.16b, v9.16b\n"
    "sqrdmulh v31.4s, v31.4s, v20.4s\n"
    "mov v7.16b, v18.16b\n .inst 0x4e899587  // sdot v7.4s, v12.16b, v9.16b\n"
    ".inst 0x4e889592  // sdot v18.4s, v12.16b, v8.16b\n"
    "mls v30.4s, v18.4s, v16.4s\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mls v28.4s, v7.4s, v16.4s\n"
    "and v16.16b, v31.16b, v26.16b\n"
    "sshr v16.4s, v16.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v20.4s\n"
    "sqrdmulh v29.4s, v29.4s, v20.4s\n"
    "sqrdmulh v28.4s, v28.4s, v20.4s\n"
    "sqadd v31.4s, v31.4s, v16.4s\n"
    "and v18.16b, v30.16b, v26.16b\n"
    "and v17.16b, v29.16b, v26.16b\n"
    "and v16.16b, v28.16b, v26.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sshr v16.4s, v16.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v18.4s\n"
    "sqadd v29.4s, v29.4s, v17.4s\n"
    "sqadd v28.4s, v28.4s, v16.4s\n"
    "srshl v31.4s, v31.4s, v26.4s\n"
    "srshl v30.4s, v30.4s, v26.4s\n"
    "srshl v29.4s, v29.4s, v26.4s\n"
    "srshl v28.4s, v28.4s, v26.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "32:"  // Oddments: Unroll 3: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 33f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 34f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 34f\n"
    "33:"  // Oddments: Unroll 3: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "34:"  // Oddments: Unroll 3: Oddment store: Bit 1: End
    "35:"  // End
    : [params] "+&r" (params)
    : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp)
    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
  );
}

}  // namespace depthwise
}  // namespace arm_conv

#endif  // defined(__aarch64__)