/*
 * Copyright (c) 2021-2023 Arm Limited.
 *
 * SPDX-License-Identifier: MIT
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to
 * deal in the Software without restriction, including without limitation the
 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
 * sell copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in all
 * copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#if defined(__aarch64__)

#include "arm_gemm.hpp"
#include <cstdint>

namespace arm_conv {
namespace depthwise {

void a64_u8q_nhwc_3x3_s1_output2x2_dot_depthfirst_impl(const unsigned int n_channels, const uint8_t *const *const inptrs, const uint8_t *params, const int32_t *, const arm_gemm::Requantize32& qp, const int32_t *, const int32_t *, uint8_t *const *const outptrs)
{
  __asm__ __volatile__(
    "mov x20, #0x1\n"
    "orr x20, x20, #0x100\n"
    "ldp x15, x14, [%x[inptrs], #0x0]\n"
    "ldp x13, x12, [%x[inptrs], #0x10]\n"
    "orr x20, x20, #0x10000\n"
    "lsr x11, %x[n_channels], #0x4\n"
    "dup v12.4s, w20\n"
    "ldp x10, x9, [%x[inptrs], #0x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_minval]\n"
    "ld1r { v13.4s }, [x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_maxval]\n"
    "ld1r { v11.4s }, [x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_b_offset]\n"
    "ld1r { v16.4s }, [x20]\n"
    "add x20, %x[qp], %[offsetof_Requantize32_c_offset]\n"
    "ld1r { v14.4s }, [x20]\n"
    "mov x28, #0x0\n"
    "mov x27, #0x0\n"
    "ldp x26, x21, [%x[inptrs], #0x30]\n"
    "ldp x25, x24, [%x[outptrs], #0x0]\n"
    "ldp x23, x22, [%x[outptrs], #0x10]\n"
    "cbz x11, 3f\n"
    "ldr q15, [x15, x28]\n"
    "ldr q28, [x14, x28]\n"
    "subs x11, x11, #0x1\n"
    "ldr q30, [x13, x28]\n"
    "ldr q8, [x12, x28]\n"
    "zip2 v19.16b, v15.16b, v30.16b\n"
    "zip1 v15.16b, v15.16b, v30.16b\n"
    "ldr q26, [x10, x28]\n"
    "ldr q0, [x9, x28]\n"
    "zip1 v7.16b, v28.16b, v8.16b\n"
    "zip2 v8.16b, v28.16b, v8.16b\n"
    "ldr q29, [x26, x28]\n"
    "ldr q10, [x21, x28]\n"
    "zip2 v25.16b, v15.16b, v7.16b\n"
    "zip1 v15.16b, v15.16b, v7.16b\n"
    "ldr q1, [%x[params], #0x10]\n"
    "ldr q6, [%x[params], #0x20]\n"
    "zip1 v7.16b, v19.16b, v8.16b\n"
    "zip2 v8.16b, v19.16b, v8.16b\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q20, [%x[params], #0x30]\n"
    "zip2 v21.16b, v26.16b, v29.16b\n"
    "zip1 v26.16b, v26.16b, v29.16b\n"
    "ldp x21, x20, [%x[inptrs], #0x40]\n"
    "ldr q22, [x21, x28]\n"
    "zip1 v27.16b, v0.16b, v10.16b\n"
    "zip2 v10.16b, v0.16b, v10.16b\n"
    "ldr q17, [x20, x28]\n"
    "ldp x21, x20, [%x[inptrs], #0x50]\n"
    "zip2 v23.16b, v26.16b, v27.16b\n"
    "zip1 v26.16b, v26.16b, v27.16b\n"
    "ldr q9, [x21, x28]\n"
    "ldr q5, [x20, x28]\n"
    "zip2 v28.16b, v22.16b, v9.16b\n"
    "zip1 v22.16b, v22.16b, v9.16b\n"
    "ldp x21, x20, [%x[inptrs], #0x60]\n"
    "ldr q27, [x21, x28]\n"
    "zip1 v24.16b, v17.16b, v5.16b\n"
    "zip2 v5.16b, v17.16b, v5.16b\n"
    "ldr q18, [x20, x28]\n"
    "ldp x21, x20, [%x[inptrs], #0x70]\n"
    "zip1 v3.16b, v21.16b, v10.16b\n"
    "zip2 v10.16b, v21.16b, v10.16b\n"
    "ldr q4, [x21, x28]\n"
    "ldr q9, [x20, x28]\n"
    "zip2 v17.16b, v27.16b, v4.16b\n"
    "zip1 v27.16b, v27.16b, v4.16b\n"
    "zip1 v4.16b, v18.16b, v9.16b\n"
    "zip2 v9.16b, v18.16b, v9.16b\n"
    "ldp x15, x14, [%x[inptrs], #0x0]\n"
    "ldp x13, x12, [%x[inptrs], #0x10]\n"
    "ldp x10, x9, [%x[inptrs], #0x20]\n"
    "ldp x26, x21, [%x[inptrs], #0x30]\n"
    "zip2 v19.16b, v22.16b, v24.16b\n"
    "zip1 v22.16b, v22.16b, v24.16b\n"
    "zip1 v0.16b, v28.16b, v5.16b\n"
    "zip2 v5.16b, v28.16b, v5.16b\n"
    "add %x[params], %x[params], #0x40\n"
    "zip2 v24.16b, v27.16b, v4.16b\n"
    "zip1 v27.16b, v27.16b, v4.16b\n"
    "zip1 v2.16b, v17.16b, v9.16b\n"
    "zip2 v9.16b, v17.16b, v9.16b\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "mov v28.16b, v31.16b\n"
    "beq 2f\n"
    "1:"  // Loop
    "movi v21.4s, #0x0\n"
    ".inst 0x6e9a9595  // udot v21.4s, v12.16b, v26.16b\n"
    ".inst 0x6e8f943f  // udot v31.4s, v1.16b, v15.16b\n"
    "add x28, x28, #0x10\n"
    ".inst 0x6e969595  // udot v21.4s, v12.16b, v22.16b\n"
    ".inst 0x6e9a943d  // udot v29.4s, v1.16b, v26.16b\n"
    "movi v18.4s, #0x0\n"
    "subs x11, x11, #0x1\n"
    ".inst 0x6e9a94df  // udot v31.4s, v6.16b, v26.16b\n"
    "ext v26.16b, v26.16b, v26.16b, #0x1\n"
    "mov v17.16b, v21.16b\n .inst 0x6e9b9591  // udot v17.4s, v12.16b, v27.16b\n"
    ".inst 0x6e8f9595  // udot v21.4s, v12.16b, v15.16b\n"
    "ext v15.16b, v15.16b, v15.16b, #0x1\n"
    ".inst 0x6e9a9592  // udot v18.4s, v12.16b, v26.16b\n"
    ".inst 0x6e9694dd  // udot v29.4s, v6.16b, v22.16b\n"
    ".inst 0x6e96969f  // udot v31.4s, v20.16b, v22.16b\n"
    "ext v22.16b, v22.16b, v22.16b, #0x1\n"
    ".inst 0x6e8f943e  // udot v30.4s, v1.16b, v15.16b\n"
    ".inst 0x6e9a943c  // udot v28.4s, v1.16b, v26.16b\n"
    "mls v31.4s, v21.4s, v16.4s\n"
    ".inst 0x6e969592  // udot v18.4s, v12.16b, v22.16b\n"
    ".inst 0x6e9b969d  // udot v29.4s, v20.16b, v27.16b\n"
    "ext v27.16b, v27.16b, v27.16b, #0x1\n"
    ".inst 0x6e9a94de  // udot v30.4s, v6.16b, v26.16b\n"
    "ldr q26, [%x[params], #0x10]\n"
    ".inst 0x6e9694dc  // udot v28.4s, v6.16b, v22.16b\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mov v21.16b, v18.16b\n .inst 0x6e9b9595  // udot v21.4s, v12.16b, v27.16b\n"
    ".inst 0x6e8f9592  // udot v18.4s, v12.16b, v15.16b\n"
    "ldr q17, [%x[params], #0x0]\n"
    "sqrdmulh v31.4s, v31.4s, v17.4s\n"
    ".inst 0x6e96969e  // udot v30.4s, v20.16b, v22.16b\n"
    ".inst 0x6e9b969c  // udot v28.4s, v20.16b, v27.16b\n"
    "mls v30.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v21.4s, v16.4s\n"
    "and v15.16b, v31.16b, v26.16b\n"
    "sshr v15.4s, v15.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v17.4s\n"
    "sqrdmulh v29.4s, v29.4s, v17.4s\n"
    "sqrdmulh v28.4s, v28.4s, v17.4s\n"
    "ldr q1, [%x[params], #0x60]\n"
    "sqadd v31.4s, v31.4s, v15.4s\n"
    "and v18.16b, v30.16b, v26.16b\n"
    "and v21.16b, v29.16b, v26.16b\n"
    "and v17.16b, v28.16b, v26.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v21.4s, v21.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v26.4s\n"
    "sqadd v30.4s, v30.4s, v18.4s\n"
    "ldr q18, [%x[params], #0x40]\n"
    "sqadd v29.4s, v29.4s, v21.4s\n"
    "ldr q27, [%x[params], #0x50]\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "ldr q15, [%x[params], #0x30]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v30.4s, v30.4s, v26.4s\n"
    "srshl v29.4s, v29.4s, v26.4s\n"
    "srshl v28.4s, v28.4s, v26.4s\n"
    "ldr q20, [%x[params], #0x70]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v22.4s, #0x0\n"
    ".inst 0x6e979596  // udot v22.4s, v12.16b, v23.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q26, [%x[params], #0x20]\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    ".inst 0x6e939596  // udot v22.4s, v12.16b, v19.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s30, [x24, x27]\n"
    "mov v6.16b, v22.16b\n .inst 0x6e989586  // udot v6.4s, v12.16b, v24.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "str s29, [x23, x27]\n"
    "mov v30.16b, v26.16b\n"
    ".inst 0x6e999596  // udot v22.4s, v12.16b, v25.16b\n"
    "str s28, [x22, x27]\n"
    "mov v29.16b, v26.16b\n"
    "mov v21.16b, v26.16b\n"
    ".inst 0x6e9995fa  // udot v26.4s, v15.16b, v25.16b\n"
    ".inst 0x6e9795fd  // udot v29.4s, v15.16b, v23.16b\n"
    ".inst 0x6e97965a  // udot v26.4s, v18.16b, v23.16b\n"
    "ext v25.16b, v25.16b, v25.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v23.16b, v23.16b, v23.16b, #0x1\n"
    "movi v28.4s, #0x0\n"
    ".inst 0x6e9995fe  // udot v30.4s, v15.16b, v25.16b\n"
    ".inst 0x6e9795f5  // udot v21.4s, v15.16b, v23.16b\n"
    ".inst 0x6e97959c  // udot v28.4s, v12.16b, v23.16b\n"
    ".inst 0x6e93965d  // udot v29.4s, v18.16b, v19.16b\n"
    ".inst 0x6e93977a  // udot v26.4s, v27.16b, v19.16b\n"
    "ext v19.16b, v19.16b, v19.16b, #0x1\n"
    ".inst 0x6e97965e  // udot v30.4s, v18.16b, v23.16b\n"
    "ldr q4, [x9, x28]\n"
    ".inst 0x6e939655  // udot v21.4s, v18.16b, v19.16b\n"
    "mls v26.4s, v22.4s, v16.4s\n"
    ".inst 0x6e93959c  // udot v28.4s, v12.16b, v19.16b\n"
    ".inst 0x6e98977d  // udot v29.4s, v27.16b, v24.16b\n"
    "ext v24.16b, v24.16b, v24.16b, #0x1\n"
    ".inst 0x6e93977e  // udot v30.4s, v27.16b, v19.16b\n"
    ".inst 0x6e989775  // udot v21.4s, v27.16b, v24.16b\n"
    "sqrdmulh v26.4s, v26.4s, v1.4s\n"
    "mov v17.16b, v28.16b\n .inst 0x6e989591  // udot v17.4s, v12.16b, v24.16b\n"
    ".inst 0x6e99959c  // udot v28.4s, v12.16b, v25.16b\n"
    "ldr q31, [x14, x28]\n"
    "mls v30.4s, v28.4s, v16.4s\n"
    "mls v29.4s, v6.4s, v16.4s\n"
    "mls v21.4s, v17.4s, v16.4s\n"
    "and v17.16b, v26.16b, v20.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v1.4s\n"
    "sqrdmulh v29.4s, v29.4s, v1.4s\n"
    "sqrdmulh v21.4s, v21.4s, v1.4s\n"
    "ldr q27, [%x[params], #0xc0]\n"
    "sqadd v26.4s, v26.4s, v17.4s\n"
    "and v18.16b, v30.16b, v20.16b\n"
    "and v6.16b, v29.16b, v20.16b\n"
    "and v17.16b, v21.16b, v20.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v6.4s, v6.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v26.4s, v26.4s, v20.4s\n"
    "sqadd v30.4s, v30.4s, v18.4s\n"
    "ldr q28, [%x[params], #0xa0]\n"
    "sqadd v29.4s, v29.4s, v6.4s\n"
    "ldr q24, [%x[params], #0xb0]\n"
    "sqadd v21.4s, v21.4s, v17.4s\n"
    "ldr q15, [%x[params], #0x90]\n"
    "add v26.4s, v26.4s, v14.4s\n"
    "srshl v30.4s, v30.4s, v20.4s\n"
    "srshl v29.4s, v29.4s, v20.4s\n"
    "srshl v21.4s, v21.4s, v20.4s\n"
    "ldr q1, [%x[params], #0xd0]\n"
    "smax v26.4s, v26.4s, v13.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "smin v26.4s, v26.4s, v11.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "movi v22.4s, #0x0\n"
    ".inst 0x6e839596  // udot v22.4s, v12.16b, v3.16b\n"
    ".inst 0x6e809596  // udot v22.4s, v12.16b, v0.16b\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "str s26, [x25, x27]\n"
    "ldr q26, [%x[params], #0x80]\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "mov v18.16b, v22.16b\n .inst 0x6e829592  // udot v18.4s, v12.16b, v2.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s30, [x24, x27]\n"
    ".inst 0x6e879596  // udot v22.4s, v12.16b, v7.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s29, [x23, x27]\n"
    "mov v6.16b, v26.16b\n"
    "str s21, [x22, x27]\n"
    "mov v25.16b, v26.16b\n"
    "mov v20.16b, v26.16b\n"
    ".inst 0x6e8795fa  // udot v26.4s, v15.16b, v7.16b\n"
    ".inst 0x6e8395f9  // udot v25.4s, v15.16b, v3.16b\n"
    ".inst 0x6e83979a  // udot v26.4s, v28.16b, v3.16b\n"
    "ext v7.16b, v7.16b, v7.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v3.16b, v3.16b, v3.16b, #0x1\n"
    "movi v23.4s, #0x0\n"
    ".inst 0x6e8795e6  // udot v6.4s, v15.16b, v7.16b\n"
    ".inst 0x6e8395f4  // udot v20.4s, v15.16b, v3.16b\n"
    ".inst 0x6e839597  // udot v23.4s, v12.16b, v3.16b\n"
    ".inst 0x6e809799  // udot v25.4s, v28.16b, v0.16b\n"
    ".inst 0x6e80971a  // udot v26.4s, v24.16b, v0.16b\n"
    "ext v0.16b, v0.16b, v0.16b, #0x1\n"
    ".inst 0x6e839786  // udot v6.4s, v28.16b, v3.16b\n"
    "ldr q19, [x26, x28]\n"
    ".inst 0x6e809794  // udot v20.4s, v28.16b, v0.16b\n"
    "mls v26.4s, v22.4s, v16.4s\n"
    ".inst 0x6e809597  // udot v23.4s, v12.16b, v0.16b\n"
    ".inst 0x6e829719  // udot v25.4s, v24.16b, v2.16b\n"
    "ext v2.16b, v2.16b, v2.16b, #0x1\n"
    ".inst 0x6e809706  // udot v6.4s, v24.16b, v0.16b\n"
    ".inst 0x6e829714  // udot v20.4s, v24.16b, v2.16b\n"
    "sqrdmulh v26.4s, v26.4s, v27.4s\n"
    "mov v17.16b, v23.16b\n .inst 0x6e829591  // udot v17.4s, v12.16b, v2.16b\n"
    ".inst 0x6e879597  // udot v23.4s, v12.16b, v7.16b\n"
    "ldr q21, [x13, x28]\n"
    "mls v6.4s, v23.4s, v16.4s\n"
    "mls v25.4s, v18.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v26.16b, v1.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v6.4s, v6.4s, v27.4s\n"
    "sqrdmulh v25.4s, v25.4s, v27.4s\n"
    "sqrdmulh v20.4s, v20.4s, v27.4s\n"
    "ldr q15, [%x[params], #0x120]\n"
    "sqadd v26.4s, v26.4s, v17.4s\n"
    "and v18.16b, v6.16b, v1.16b\n"
    "and v22.16b, v25.16b, v1.16b\n"
    "and v17.16b, v20.16b, v1.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v22.4s, v22.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v26.4s, v26.4s, v1.4s\n"
    "sqadd v6.4s, v6.4s, v18.4s\n"
    "ldr q30, [%x[params], #0x100]\n"
    "sqadd v25.4s, v25.4s, v22.4s\n"
    "ldr q27, [%x[params], #0x110]\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "ldr q24, [%x[params], #0xf0]\n"
    "add v26.4s, v26.4s, v14.4s\n"
    "srshl v6.4s, v6.4s, v1.4s\n"
    "srshl v25.4s, v25.4s, v1.4s\n"
    "srshl v20.4s, v20.4s, v1.4s\n"
    "ldr q23, [%x[params], #0x130]\n"
    "smax v26.4s, v26.4s, v13.4s\n"
    "add v6.4s, v6.4s, v14.4s\n"
    "add v25.4s, v25.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smin v26.4s, v26.4s, v11.4s\n"
    "smax v6.4s, v6.4s, v13.4s\n"
    "smax v25.4s, v25.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v6.4s, v6.4s, v11.4s\n"
    "smin v25.4s, v25.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "movi v0.4s, #0x0\n"
    ".inst 0x6e8a9580  // udot v0.4s, v12.16b, v10.16b\n"
    ".inst 0x6e859580  // udot v0.4s, v12.16b, v5.16b\n"
    "uzp1 v26.16b, v26.16b, v26.16b\n"
    "uzp1 v6.16b, v6.16b, v6.16b\n"
    "str s26, [x25, x27]\n"
    "ldr q28, [%x[params], #0xe0]\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "mov v22.16b, v0.16b\n .inst 0x6e899596  // udot v22.4s, v12.16b, v9.16b\n"
    "uzp1 v6.16b, v6.16b, v6.16b\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "str s6, [x24, x27]\n"
    ".inst 0x6e889580  // udot v0.4s, v12.16b, v8.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s25, [x23, x27]\n"
    "mov v29.16b, v28.16b\n"
    "str s20, [x22, x27]\n"
    "mov v25.16b, v28.16b\n"
    "mov v7.16b, v28.16b\n"
    ".inst 0x6e88971c  // udot v28.4s, v24.16b, v8.16b\n"
    ".inst 0x6e8a9719  // udot v25.4s, v24.16b, v10.16b\n"
    ".inst 0x6e8a97dc  // udot v28.4s, v30.16b, v10.16b\n"
    "ext v8.16b, v8.16b, v8.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v10.16b, v10.16b, v10.16b, #0x1\n"
    "movi v17.4s, #0x0\n"
    ".inst 0x6e88971d  // udot v29.4s, v24.16b, v8.16b\n"
    ".inst 0x6e8a9707  // udot v7.4s, v24.16b, v10.16b\n"
    ".inst 0x6e8a9591  // udot v17.4s, v12.16b, v10.16b\n"
    ".inst 0x6e8597d9  // udot v25.4s, v30.16b, v5.16b\n"
    ".inst 0x6e85977c  // udot v28.4s, v27.16b, v5.16b\n"
    "ext v5.16b, v5.16b, v5.16b, #0x1\n"
    ".inst 0x6e8a97dd  // udot v29.4s, v30.16b, v10.16b\n"
    "ldr q10, [x21, x28]\n"
    ".inst 0x6e8597c7  // udot v7.4s, v30.16b, v5.16b\n"
    "mls v28.4s, v0.4s, v16.4s\n"
    ".inst 0x6e859591  // udot v17.4s, v12.16b, v5.16b\n"
    ".inst 0x6e899779  // udot v25.4s, v27.16b, v9.16b\n"
    "ext v9.16b, v9.16b, v9.16b, #0x1\n"
    ".inst 0x6e85977d  // udot v29.4s, v27.16b, v5.16b\n"
    ".inst 0x6e899767  // udot v7.4s, v27.16b, v9.16b\n"
    "sqrdmulh v28.4s, v28.4s, v15.4s\n"
    "mov v18.16b, v17.16b\n .inst 0x6e899592  // udot v18.4s, v12.16b, v9.16b\n"
    ".inst 0x6e889591  // udot v17.4s, v12.16b, v8.16b\n"
    "ldr q8, [x12, x28]\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mls v25.4s, v22.4s, v16.4s\n"
    "mls v7.4s, v18.4s, v16.4s\n"
    "and v17.16b, v28.16b, v23.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v29.4s, v29.4s, v15.4s\n"
    "sqrdmulh v25.4s, v25.4s, v15.4s\n"
    "sqrdmulh v7.4s, v7.4s, v15.4s\n"
    "ldr q15, [x15, x28]\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "ldp x21, x20, [%x[inptrs], #0x40]\n"
    "ldr q22, [x21, x28]\n"
    "ldr q3, [x20, x28]\n"
    "and v24.16b, v29.16b, v23.16b\n"
    "and v20.16b, v25.16b, v23.16b\n"
    "and v17.16b, v7.16b, v23.16b\n"
    "sshr v24.4s, v24.4s, #0x1f\n"
    "ldp x21, x20, [%x[inptrs], #0x50]\n"
    "ldr q2, [x21, x28]\n"
    "ldr q5, [x20, x28]\n"
    "sshr v20.4s, v20.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v28.4s, v28.4s, v23.4s\n"
    "sqadd v29.4s, v29.4s, v24.4s\n"
    "ldr q6, [%x[params], #0x160]\n"
    "sqadd v25.4s, v25.4s, v20.4s\n"
    "ldr q20, [%x[params], #0x170]\n"
    "sqadd v7.4s, v7.4s, v17.4s\n"
    "ldr q1, [%x[params], #0x150]\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "srshl v29.4s, v29.4s, v23.4s\n"
    "srshl v25.4s, v25.4s, v23.4s\n"
    "srshl v7.4s, v7.4s, v23.4s\n"
    "ldr q26, [x10, x28]\n"
    "ldp x21, x20, [%x[inptrs], #0x60]\n"
    "ldr q27, [x21, x28]\n"
    "ldr q30, [x20, x28]\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v25.4s, v25.4s, v14.4s\n"
    "add v7.4s, v7.4s, v14.4s\n"
    "ldp x21, x20, [%x[inptrs], #0x70]\n"
    "ldr q23, [x21, x28]\n"
    "ldr q9, [x20, x28]\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "ldp x15, x14, [%x[inptrs], #0x0]\n"
    "smax v25.4s, v25.4s, v13.4s\n"
    "smax v7.4s, v7.4s, v13.4s\n"
    "ldp x13, x12, [%x[inptrs], #0x10]\n"
    "ldp x10, x9, [%x[inptrs], #0x20]\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v25.4s, v25.4s, v11.4s\n"
    "ldp x26, x21, [%x[inptrs], #0x30]\n"
    "smin v7.4s, v7.4s, v11.4s\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s28, [x25, x27]\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "uzp1 v7.16b, v7.16b, v7.16b\n"
    "zip2 v17.16b, v15.16b, v21.16b\n"
    "zip1 v15.16b, v15.16b, v21.16b\n"
    "zip1 v18.16b, v31.16b, v8.16b\n"
    "zip2 v8.16b, v31.16b, v8.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v25.16b, v25.16b, v25.16b\n"
    "str s29, [x24, x27]\n"
    "uzp1 v7.16b, v7.16b, v7.16b\n"
    "str s25, [x23, x27]\n"
    "zip2 v25.16b, v15.16b, v18.16b\n"
    "str s7, [x22, x27]\n"
    "zip1 v15.16b, v15.16b, v18.16b\n"
    "zip1 v7.16b, v17.16b, v8.16b\n"
    "add x27, x27, #0x4\n"
    "zip2 v8.16b, v17.16b, v8.16b\n"
    "ldr q31, [%x[params], #0x140]\n"
    "zip2 v29.16b, v26.16b, v19.16b\n"
    "add %x[params], %x[params], #0x180\n"
    "zip1 v26.16b, v26.16b, v19.16b\n"
    "zip1 v28.16b, v4.16b, v10.16b\n"
    "zip2 v10.16b, v4.16b, v10.16b\n"
    "zip2 v24.16b, v22.16b, v2.16b\n"
    "zip1 v22.16b, v22.16b, v2.16b\n"
    "zip1 v21.16b, v3.16b, v5.16b\n"
    "zip2 v5.16b, v3.16b, v5.16b\n"
    "zip2 v18.16b, v27.16b, v23.16b\n"
    "zip1 v27.16b, v27.16b, v23.16b\n"
    "zip1 v17.16b, v30.16b, v9.16b\n"
    "zip2 v9.16b, v30.16b, v9.16b\n"
    "zip2 v23.16b, v26.16b, v28.16b\n"
    "zip1 v26.16b, v26.16b, v28.16b\n"
    "zip1 v3.16b, v29.16b, v10.16b\n"
    "zip2 v10.16b, v29.16b, v10.16b\n"
    "zip2 v19.16b, v22.16b, v21.16b\n"
    "zip1 v22.16b, v22.16b, v21.16b\n"
    "zip1 v0.16b, v24.16b, v5.16b\n"
    "zip2 v5.16b, v24.16b, v5.16b\n"
    "zip2 v24.16b, v27.16b, v17.16b\n"
    "zip1 v27.16b, v27.16b, v17.16b\n"
    "zip1 v2.16b, v18.16b, v9.16b\n"
    "zip2 v9.16b, v18.16b, v9.16b\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "mov v28.16b, v31.16b\n"
    "bgt 1b\n"
    "2:"  // Detached iteration
    "movi v21.4s, #0x0\n"
    ".inst 0x6e9a9595  // udot v21.4s, v12.16b, v26.16b\n"
    ".inst 0x6e8f943f  // udot v31.4s, v1.16b, v15.16b\n"
    "tst %x[n_channels], #0xf\n"
    ".inst 0x6e969595  // udot v21.4s, v12.16b, v22.16b\n"
    ".inst 0x6e9a943d  // udot v29.4s, v1.16b, v26.16b\n"
    "movi v18.4s, #0x0\n"
    "add x28, x28, #0x10\n"
    ".inst 0x6e9a94df  // udot v31.4s, v6.16b, v26.16b\n"
    "ext v26.16b, v26.16b, v26.16b, #0x1\n"
    "mov v17.16b, v21.16b\n .inst 0x6e9b9591  // udot v17.4s, v12.16b, v27.16b\n"
    ".inst 0x6e8f9595  // udot v21.4s, v12.16b, v15.16b\n"
    "ext v15.16b, v15.16b, v15.16b, #0x1\n"
    ".inst 0x6e9a9592  // udot v18.4s, v12.16b, v26.16b\n"
    ".inst 0x6e9694dd  // udot v29.4s, v6.16b, v22.16b\n"
    ".inst 0x6e96969f  // udot v31.4s, v20.16b, v22.16b\n"
    "ext v22.16b, v22.16b, v22.16b, #0x1\n"
    ".inst 0x6e8f943e  // udot v30.4s, v1.16b, v15.16b\n"
    ".inst 0x6e9a943c  // udot v28.4s, v1.16b, v26.16b\n"
    "mls v31.4s, v21.4s, v16.4s\n"
    ".inst 0x6e969592  // udot v18.4s, v12.16b, v22.16b\n"
    ".inst 0x6e9b969d  // udot v29.4s, v20.16b, v27.16b\n"
    "ext v27.16b, v27.16b, v27.16b, #0x1\n"
    ".inst 0x6e9a94de  // udot v30.4s, v6.16b, v26.16b\n"
    "ldr q4, [%x[params], #0x10]\n"
    ".inst 0x6e9694dc  // udot v28.4s, v6.16b, v22.16b\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mov v21.16b, v18.16b\n .inst 0x6e9b9595  // udot v21.4s, v12.16b, v27.16b\n"
    ".inst 0x6e8f9592  // udot v18.4s, v12.16b, v15.16b\n"
    "ldr q17, [%x[params], #0x0]\n"
    "sqrdmulh v31.4s, v31.4s, v17.4s\n"
    ".inst 0x6e96969e  // udot v30.4s, v20.16b, v22.16b\n"
    ".inst 0x6e9b969c  // udot v28.4s, v20.16b, v27.16b\n"
    "mls v30.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v21.4s, v16.4s\n"
    "and v27.16b, v31.16b, v4.16b\n"
    "sshr v27.4s, v27.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v17.4s\n"
    "sqrdmulh v29.4s, v29.4s, v17.4s\n"
    "sqrdmulh v28.4s, v28.4s, v17.4s\n"
    "ldr q15, [%x[params], #0x60]\n"
    "sqadd v31.4s, v31.4s, v27.4s\n"
    "and v20.16b, v30.16b, v4.16b\n"
    "and v18.16b, v29.16b, v4.16b\n"
    "and v17.16b, v28.16b, v4.16b\n"
    "sshr v20.4s, v20.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v4.4s\n"
    "sqadd v30.4s, v30.4s, v20.4s\n"
    "ldr q27, [%x[params], #0x40]\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "ldr q26, [%x[params], #0x50]\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "ldr q6, [%x[params], #0x30]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v30.4s, v30.4s, v4.4s\n"
    "srshl v29.4s, v29.4s, v4.4s\n"
    "srshl v28.4s, v28.4s, v4.4s\n"
    "ldr q4, [%x[params], #0x70]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v1.4s, #0x0\n"
    ".inst 0x6e979581  // udot v1.4s, v12.16b, v23.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q31, [%x[params], #0x20]\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    ".inst 0x6e939581  // udot v1.4s, v12.16b, v19.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s30, [x24, x27]\n"
    "mov v22.16b, v1.16b\n .inst 0x6e989596  // udot v22.4s, v12.16b, v24.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "str s29, [x23, x27]\n"
    "mov v29.16b, v31.16b\n"
    ".inst 0x6e999581  // udot v1.4s, v12.16b, v25.16b\n"
    "str s28, [x22, x27]\n"
    "mov v21.16b, v31.16b\n"
    "mov v20.16b, v31.16b\n"
    ".inst 0x6e9994df  // udot v31.4s, v6.16b, v25.16b\n"
    ".inst 0x6e9794d5  // udot v21.4s, v6.16b, v23.16b\n"
    ".inst 0x6e97977f  // udot v31.4s, v27.16b, v23.16b\n"
    "ext v25.16b, v25.16b, v25.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v23.16b, v23.16b, v23.16b, #0x1\n"
    "movi v18.4s, #0x0\n"
    ".inst 0x6e9994dd  // udot v29.4s, v6.16b, v25.16b\n"
    ".inst 0x6e9794d4  // udot v20.4s, v6.16b, v23.16b\n"
    ".inst 0x6e979592  // udot v18.4s, v12.16b, v23.16b\n"
    ".inst 0x6e939775  // udot v21.4s, v27.16b, v19.16b\n"
    ".inst 0x6e93975f  // udot v31.4s, v26.16b, v19.16b\n"
    "ext v19.16b, v19.16b, v19.16b, #0x1\n"
    ".inst 0x6e97977d  // udot v29.4s, v27.16b, v23.16b\n"
    ".inst 0x6e939774  // udot v20.4s, v27.16b, v19.16b\n"
    "mls v31.4s, v1.4s, v16.4s\n"
    ".inst 0x6e939592  // udot v18.4s, v12.16b, v19.16b\n"
    ".inst 0x6e989755  // udot v21.4s, v26.16b, v24.16b\n"
    "ext v24.16b, v24.16b, v24.16b, #0x1\n"
    ".inst 0x6e93975d  // udot v29.4s, v26.16b, v19.16b\n"
    ".inst 0x6e989754  // udot v20.4s, v26.16b, v24.16b\n"
    "sqrdmulh v31.4s, v31.4s, v15.4s\n"
    "mov v17.16b, v18.16b\n .inst 0x6e989591  // udot v17.4s, v12.16b, v24.16b\n"
    ".inst 0x6e999592  // udot v18.4s, v12.16b, v25.16b\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mls v21.4s, v22.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v4.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v29.4s, v29.4s, v15.4s\n"
    "sqrdmulh v21.4s, v21.4s, v15.4s\n"
    "sqrdmulh v20.4s, v20.4s, v15.4s\n"
    "ldr q27, [%x[params], #0xc0]\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v29.16b, v4.16b\n"
    "and v18.16b, v21.16b, v4.16b\n"
    "and v17.16b, v20.16b, v4.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v4.4s\n"
    "sqadd v29.4s, v29.4s, v19.4s\n"
    "ldr q26, [%x[params], #0xa0]\n"
    "sqadd v21.4s, v21.4s, v18.4s\n"
    "ldr q25, [%x[params], #0xb0]\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "ldr q24, [%x[params], #0x90]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v29.4s, v29.4s, v4.4s\n"
    "srshl v21.4s, v21.4s, v4.4s\n"
    "srshl v20.4s, v20.4s, v4.4s\n"
    "ldr q1, [%x[params], #0xd0]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v23.4s, #0x0\n"
    ".inst 0x6e839597  // udot v23.4s, v12.16b, v3.16b\n"
    ".inst 0x6e809597  // udot v23.4s, v12.16b, v0.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q31, [%x[params], #0x80]\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "mov v22.16b, v23.16b\n .inst 0x6e829596  // udot v22.4s, v12.16b, v2.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s29, [x24, x27]\n"
    ".inst 0x6e879597  // udot v23.4s, v12.16b, v7.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s21, [x23, x27]\n"
    "mov v21.16b, v31.16b\n"
    "str s20, [x22, x27]\n"
    "mov v4.16b, v31.16b\n"
    "mov v20.16b, v31.16b\n"
    ".inst 0x6e87971f  // udot v31.4s, v24.16b, v7.16b\n"
    ".inst 0x6e839704  // udot v4.4s, v24.16b, v3.16b\n"
    ".inst 0x6e83975f  // udot v31.4s, v26.16b, v3.16b\n"
    "ext v7.16b, v7.16b, v7.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v3.16b, v3.16b, v3.16b, #0x1\n"
    "movi v18.4s, #0x0\n"
    ".inst 0x6e879715  // udot v21.4s, v24.16b, v7.16b\n"
    ".inst 0x6e839714  // udot v20.4s, v24.16b, v3.16b\n"
    ".inst 0x6e839592  // udot v18.4s, v12.16b, v3.16b\n"
    ".inst 0x6e809744  // udot v4.4s, v26.16b, v0.16b\n"
    ".inst 0x6e80973f  // udot v31.4s, v25.16b, v0.16b\n"
    "ext v0.16b, v0.16b, v0.16b, #0x1\n"
    ".inst 0x6e839755  // udot v21.4s, v26.16b, v3.16b\n"
    ".inst 0x6e809754  // udot v20.4s, v26.16b, v0.16b\n"
    "mls v31.4s, v23.4s, v16.4s\n"
    ".inst 0x6e809592  // udot v18.4s, v12.16b, v0.16b\n"
    ".inst 0x6e829724  // udot v4.4s, v25.16b, v2.16b\n"
    "ext v2.16b, v2.16b, v2.16b, #0x1\n"
    ".inst 0x6e809735  // udot v21.4s, v25.16b, v0.16b\n"
    ".inst 0x6e829734  // udot v20.4s, v25.16b, v2.16b\n"
    "sqrdmulh v31.4s, v31.4s, v27.4s\n"
    "mov v17.16b, v18.16b\n .inst 0x6e829591  // udot v17.4s, v12.16b, v2.16b\n"
    ".inst 0x6e879592  // udot v18.4s, v12.16b, v7.16b\n"
    "mls v21.4s, v18.4s, v16.4s\n"
    "mls v4.4s, v22.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v1.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v21.4s, v21.4s, v27.4s\n"
    "sqrdmulh v4.4s, v4.4s, v27.4s\n"
    "sqrdmulh v20.4s, v20.4s, v27.4s\n"
    "ldr q30, [%x[params], #0x120]\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v21.16b, v1.16b\n"
    "and v18.16b, v4.16b, v1.16b\n"
    "and v17.16b, v20.16b, v1.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "srshl v31.4s, v31.4s, v1.4s\n"
    "sqadd v21.4s, v21.4s, v19.4s\n"
    "ldr q29, [%x[params], #0x100]\n"
    "sqadd v4.4s, v4.4s, v18.4s\n"
    "ldr q28, [%x[params], #0x110]\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "ldr q27, [%x[params], #0xf0]\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "srshl v21.4s, v21.4s, v1.4s\n"
    "srshl v4.4s, v4.4s, v1.4s\n"
    "srshl v20.4s, v20.4s, v1.4s\n"
    "ldr q26, [%x[params], #0x130]\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "add v4.4s, v4.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smax v4.4s, v4.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "smin v4.4s, v4.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "movi v25.4s, #0x0\n"
    ".inst 0x6e8a9599  // udot v25.4s, v12.16b, v10.16b\n"
    ".inst 0x6e859599  // udot v25.4s, v12.16b, v5.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s31, [x25, x27]\n"
    "ldr q24, [%x[params], #0xe0]\n"
    "uzp1 v4.16b, v4.16b, v4.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "mov v23.16b, v25.16b\n .inst 0x6e899597  // udot v23.4s, v12.16b, v9.16b\n"
    "add %x[params], %x[params], #0x140\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "uzp1 v4.16b, v4.16b, v4.16b\n"
    "str s21, [x24, x27]\n"
    ".inst 0x6e889599  // udot v25.4s, v12.16b, v8.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s4, [x23, x27]\n"
    "mov v22.16b, v24.16b\n"
    "str s20, [x22, x27]\n"
    "mov v21.16b, v24.16b\n"
    "mov v20.16b, v24.16b\n"
    ".inst 0x6e889778  // udot v24.4s, v27.16b, v8.16b\n"
    ".inst 0x6e8a9775  // udot v21.4s, v27.16b, v10.16b\n"
    ".inst 0x6e8a97b8  // udot v24.4s, v29.16b, v10.16b\n"
    "ext v8.16b, v8.16b, v8.16b, #0x1\n"
    "add x27, x27, #0x4\n"
    "ext v10.16b, v10.16b, v10.16b, #0x1\n"
    "movi v18.4s, #0x0\n"
    ".inst 0x6e889776  // udot v22.4s, v27.16b, v8.16b\n"
    ".inst 0x6e8a9774  // udot v20.4s, v27.16b, v10.16b\n"
    ".inst 0x6e8a9592  // udot v18.4s, v12.16b, v10.16b\n"
    ".inst 0x6e8597b5  // udot v21.4s, v29.16b, v5.16b\n"
    ".inst 0x6e859798  // udot v24.4s, v28.16b, v5.16b\n"
    "ext v5.16b, v5.16b, v5.16b, #0x1\n"
    ".inst 0x6e8a97b6  // udot v22.4s, v29.16b, v10.16b\n"
    ".inst 0x6e8597b4  // udot v20.4s, v29.16b, v5.16b\n"
    "mls v24.4s, v25.4s, v16.4s\n"
    ".inst 0x6e859592  // udot v18.4s, v12.16b, v5.16b\n"
    ".inst 0x6e899795  // udot v21.4s, v28.16b, v9.16b\n"
    "ext v9.16b, v9.16b, v9.16b, #0x1\n"
    ".inst 0x6e859796  // udot v22.4s, v28.16b, v5.16b\n"
    ".inst 0x6e899794  // udot v20.4s, v28.16b, v9.16b\n"
    "sqrdmulh v24.4s, v24.4s, v30.4s\n"
    "mov v17.16b, v18.16b\n .inst 0x6e899591  // udot v17.4s, v12.16b, v9.16b\n"
    ".inst 0x6e889592  // udot v18.4s, v12.16b, v8.16b\n"
    "mls v22.4s, v18.4s, v16.4s\n"
    "mls v21.4s, v23.4s, v16.4s\n"
    "mls v20.4s, v17.4s, v16.4s\n"
    "and v17.16b, v24.16b, v26.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v22.4s, v22.4s, v30.4s\n"
    "sqrdmulh v21.4s, v21.4s, v30.4s\n"
    "sqrdmulh v20.4s, v20.4s, v30.4s\n"
    "sqadd v24.4s, v24.4s, v17.4s\n"
    "and v19.16b, v22.16b, v26.16b\n"
    "and v18.16b, v21.16b, v26.16b\n"
    "and v17.16b, v20.16b, v26.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqadd v22.4s, v22.4s, v19.4s\n"
    "sqadd v21.4s, v21.4s, v18.4s\n"
    "sqadd v20.4s, v20.4s, v17.4s\n"
    "srshl v24.4s, v24.4s, v26.4s\n"
    "srshl v22.4s, v22.4s, v26.4s\n"
    "srshl v21.4s, v21.4s, v26.4s\n"
    "srshl v20.4s, v20.4s, v26.4s\n"
    "add v24.4s, v24.4s, v14.4s\n"
    "add v22.4s, v22.4s, v14.4s\n"
    "add v21.4s, v21.4s, v14.4s\n"
    "add v20.4s, v20.4s, v14.4s\n"
    "smax v24.4s, v24.4s, v13.4s\n"
    "smax v22.4s, v22.4s, v13.4s\n"
    "smax v21.4s, v21.4s, v13.4s\n"
    "smax v20.4s, v20.4s, v13.4s\n"
    "smin v24.4s, v24.4s, v11.4s\n"
    "smin v22.4s, v22.4s, v11.4s\n"
    "smin v21.4s, v21.4s, v11.4s\n"
    "smin v20.4s, v20.4s, v11.4s\n"
    "uzp1 v24.16b, v24.16b, v24.16b\n"
    "uzp1 v22.16b, v22.16b, v22.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "uzp1 v24.16b, v24.16b, v24.16b\n"
    "str s24, [x25, x27]\n"
    "uzp1 v22.16b, v22.16b, v22.16b\n"
    "uzp1 v21.16b, v21.16b, v21.16b\n"
    "str s22, [x24, x27]\n"
    "uzp1 v20.16b, v20.16b, v20.16b\n"
    "str s21, [x23, x27]\n"
    "str s20, [x22, x27]\n"
    "add x27, x27, #0x4\n"
    "beq 35f\n"
    "3:"  // Oddments
    "and x20, %x[n_channels], #0xf\n"
    "add x15, x15, x28\n"
    "add x14, x14, x28\n"
    "add x13, x13, x28\n"
    "add x12, x12, x28\n"
    "add x10, x10, x28\n"
    "add x9, x9, x28\n"
    "add x26, x26, x28\n"
    "add x21, x21, x28\n"
    "tbz %x[n_channels], #3, 7f\n"
    "ldr d15, [x15], #0x8\n"
    "ldr d25, [x14], #0x8\n"
    "ldr d7, [x13], #0x8\n"
    "ldr d8, [x12], #0x8\n"
    "ldr d26, [x10], #0x8\n"
    "ldr d23, [x9], #0x8\n"
    "ldr d3, [x26], #0x8\n"
    "ldr d10, [x21], #0x8\n"
    "tbz %x[n_channels], #2, 5f\n"
    "ld1 { v15.s }[2], [x15], #0x4\n"
    "ld1 { v25.s }[2], [x14], #0x4\n"
    "ld1 { v7.s }[2], [x13], #0x4\n"
    "ld1 { v8.s }[2], [x12], #0x4\n"
    "ld1 { v26.s }[2], [x10], #0x4\n"
    "ld1 { v23.s }[2], [x9], #0x4\n"
    "ld1 { v3.s }[2], [x26], #0x4\n"
    "ld1 { v10.s }[2], [x21], #0x4\n"
    "tbz %x[n_channels], #1, 4f\n"
    "ld1 { v15.h }[6], [x15], #0x2\n"
    "ld1 { v25.h }[6], [x14], #0x2\n"
    "ld1 { v7.h }[6], [x13], #0x2\n"
    "ld1 { v8.h }[6], [x12], #0x2\n"
    "ld1 { v26.h }[6], [x10], #0x2\n"
    "ld1 { v23.h }[6], [x9], #0x2\n"
    "ld1 { v3.h }[6], [x26], #0x2\n"
    "ld1 { v10.h }[6], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[14], [x15], #0x1\n"
    "ld1 { v25.b }[14], [x14], #0x1\n"
    "ld1 { v7.b }[14], [x13], #0x1\n"
    "ld1 { v8.b }[14], [x12], #0x1\n"
    "ld1 { v26.b }[14], [x10], #0x1\n"
    "ld1 { v23.b }[14], [x9], #0x1\n"
    "ld1 { v3.b }[14], [x26], #0x1\n"
    "ld1 { v10.b }[14], [x21], #0x1\n"
    "b 11f\n"
    "4:"  // Oddments: Load (A): Bit 3: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[12], [x15], #0x1\n"
    "ld1 { v25.b }[12], [x14], #0x1\n"
    "ld1 { v7.b }[12], [x13], #0x1\n"
    "ld1 { v8.b }[12], [x12], #0x1\n"
    "ld1 { v26.b }[12], [x10], #0x1\n"
    "ld1 { v23.b }[12], [x9], #0x1\n"
    "ld1 { v3.b }[12], [x26], #0x1\n"
    "ld1 { v10.b }[12], [x21], #0x1\n"
    "b 11f\n"
    "5:"  // Oddments: Load (A): Bit 3: Bit 2: Unset
    "tbz %x[n_channels], #1, 6f\n"
    "ld1 { v15.h }[4], [x15], #0x2\n"
    "ld1 { v25.h }[4], [x14], #0x2\n"
    "ld1 { v7.h }[4], [x13], #0x2\n"
    "ld1 { v8.h }[4], [x12], #0x2\n"
    "ld1 { v26.h }[4], [x10], #0x2\n"
    "ld1 { v23.h }[4], [x9], #0x2\n"
    "ld1 { v3.h }[4], [x26], #0x2\n"
    "ld1 { v10.h }[4], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[10], [x15], #0x1\n"
    "ld1 { v25.b }[10], [x14], #0x1\n"
    "ld1 { v7.b }[10], [x13], #0x1\n"
    "ld1 { v8.b }[10], [x12], #0x1\n"
    "ld1 { v26.b }[10], [x10], #0x1\n"
    "ld1 { v23.b }[10], [x9], #0x1\n"
    "ld1 { v3.b }[10], [x26], #0x1\n"
    "ld1 { v10.b }[10], [x21], #0x1\n"
    "b 11f\n"
    "6:"  // Oddments: Load (A): Bit 3: Bit 2: Unset: Bit 1: Unset
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[8], [x15], #0x1\n"
    "ld1 { v25.b }[8], [x14], #0x1\n"
    "ld1 { v7.b }[8], [x13], #0x1\n"
    "ld1 { v8.b }[8], [x12], #0x1\n"
    "ld1 { v26.b }[8], [x10], #0x1\n"
    "ld1 { v23.b }[8], [x9], #0x1\n"
    "ld1 { v3.b }[8], [x26], #0x1\n"
    "ld1 { v10.b }[8], [x21], #0x1\n"
    "b 11f\n"
    "7:"  // Oddments: Load (A): Bit 3: Unset
    "tbz %x[n_channels], #2, 9f\n"
    "ldr s15, [x15], #0x4\n"
    "ldr s25, [x14], #0x4\n"
    "ldr s7, [x13], #0x4\n"
    "ldr s8, [x12], #0x4\n"
    "ldr s26, [x10], #0x4\n"
    "ldr s23, [x9], #0x4\n"
    "ldr s3, [x26], #0x4\n"
    "ldr s10, [x21], #0x4\n"
    "tbz %x[n_channels], #1, 8f\n"
    "ld1 { v15.h }[2], [x15], #0x2\n"
    "ld1 { v25.h }[2], [x14], #0x2\n"
    "ld1 { v7.h }[2], [x13], #0x2\n"
    "ld1 { v8.h }[2], [x12], #0x2\n"
    "ld1 { v26.h }[2], [x10], #0x2\n"
    "ld1 { v23.h }[2], [x9], #0x2\n"
    "ld1 { v3.h }[2], [x26], #0x2\n"
    "ld1 { v10.h }[2], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[6], [x15], #0x1\n"
    "ld1 { v25.b }[6], [x14], #0x1\n"
    "ld1 { v7.b }[6], [x13], #0x1\n"
    "ld1 { v8.b }[6], [x12], #0x1\n"
    "ld1 { v26.b }[6], [x10], #0x1\n"
    "ld1 { v23.b }[6], [x9], #0x1\n"
    "ld1 { v3.b }[6], [x26], #0x1\n"
    "ld1 { v10.b }[6], [x21], #0x1\n"
    "b 11f\n"
    "8:"  // Oddments: Load (A): Bit 3: Unset: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[4], [x15], #0x1\n"
    "ld1 { v25.b }[4], [x14], #0x1\n"
    "ld1 { v7.b }[4], [x13], #0x1\n"
    "ld1 { v8.b }[4], [x12], #0x1\n"
    "ld1 { v26.b }[4], [x10], #0x1\n"
    "ld1 { v23.b }[4], [x9], #0x1\n"
    "ld1 { v3.b }[4], [x26], #0x1\n"
    "ld1 { v10.b }[4], [x21], #0x1\n"
    "b 11f\n"
    "9:"  // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset
    "tbz %x[n_channels], #1, 10f\n"
    "ldr h15, [x15], #0x2\n"
    "ldr h25, [x14], #0x2\n"
    "ldr h7, [x13], #0x2\n"
    "ldr h8, [x12], #0x2\n"
    "ldr h26, [x10], #0x2\n"
    "ldr h23, [x9], #0x2\n"
    "ldr h3, [x26], #0x2\n"
    "ldr h10, [x21], #0x2\n"
    "tbz %x[n_channels], #0, 11f\n"
    "ld1 { v15.b }[2], [x15], #0x1\n"
    "ld1 { v25.b }[2], [x14], #0x1\n"
    "ld1 { v7.b }[2], [x13], #0x1\n"
    "ld1 { v8.b }[2], [x12], #0x1\n"
    "ld1 { v26.b }[2], [x10], #0x1\n"
    "ld1 { v23.b }[2], [x9], #0x1\n"
    "ld1 { v3.b }[2], [x26], #0x1\n"
    "ld1 { v10.b }[2], [x21], #0x1\n"
    "b 11f\n"
    "10:"  // Oddments: Load (A): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
    "ldr b15, [x15], #0x1\n"
    "ldr b25, [x14], #0x1\n"
    "ldr b7, [x13], #0x1\n"
    "ldr b8, [x12], #0x1\n"
    "ldr b26, [x10], #0x1\n"
    "ldr b23, [x9], #0x1\n"
    "ldr b3, [x26], #0x1\n"
    "ldr b10, [x21], #0x1\n"
    "11:"  // Oddments: Load (A): Bit 3: End
    "ldp x15, x14, [%x[inptrs], #0x40]\n"
    "ldp x13, x12, [%x[inptrs], #0x50]\n"
    "add x15, x15, x28\n"
    "add x14, x14, x28\n"
    "ldp x10, x9, [%x[inptrs], #0x60]\n"
    "ldp x26, x21, [%x[inptrs], #0x70]\n"
    "add x13, x13, x28\n"
    "add x12, x12, x28\n"
    "add x10, x10, x28\n"
    "add x9, x9, x28\n"
    "add x26, x26, x28\n"
    "add x21, x21, x28\n"
    "tbz %x[n_channels], #3, 15f\n"
    "ldr d22, [x15], #0x8\n"
    "ldr d19, [x14], #0x8\n"
    "ldr d0, [x13], #0x8\n"
    "ldr d5, [x12], #0x8\n"
    "ldr d27, [x10], #0x8\n"
    "ldr d24, [x9], #0x8\n"
    "ldr d2, [x26], #0x8\n"
    "ldr d9, [x21], #0x8\n"
    "tbz %x[n_channels], #2, 13f\n"
    "ld1 { v22.s }[2], [x15], #0x4\n"
    "ld1 { v19.s }[2], [x14], #0x4\n"
    "ld1 { v0.s }[2], [x13], #0x4\n"
    "ld1 { v5.s }[2], [x12], #0x4\n"
    "ld1 { v27.s }[2], [x10], #0x4\n"
    "ld1 { v24.s }[2], [x9], #0x4\n"
    "ld1 { v2.s }[2], [x26], #0x4\n"
    "ld1 { v9.s }[2], [x21], #0x4\n"
    "tbz %x[n_channels], #1, 12f\n"
    "ld1 { v22.h }[6], [x15], #0x2\n"
    "ld1 { v19.h }[6], [x14], #0x2\n"
    "ld1 { v0.h }[6], [x13], #0x2\n"
    "ld1 { v5.h }[6], [x12], #0x2\n"
    "ld1 { v27.h }[6], [x10], #0x2\n"
    "ld1 { v24.h }[6], [x9], #0x2\n"
    "ld1 { v2.h }[6], [x26], #0x2\n"
    "ld1 { v9.h }[6], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[14], [x15], #0x1\n"
    "ld1 { v19.b }[14], [x14], #0x1\n"
    "ld1 { v0.b }[14], [x13], #0x1\n"
    "ld1 { v5.b }[14], [x12], #0x1\n"
    "ld1 { v27.b }[14], [x10], #0x1\n"
    "ld1 { v24.b }[14], [x9], #0x1\n"
    "ld1 { v2.b }[14], [x26], #0x1\n"
    "ld1 { v9.b }[14], [x21], #0x1\n"
    "b 19f\n"
    "12:"  // Oddments: Load (B): Bit 3: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[12], [x15], #0x1\n"
    "ld1 { v19.b }[12], [x14], #0x1\n"
    "ld1 { v0.b }[12], [x13], #0x1\n"
    "ld1 { v5.b }[12], [x12], #0x1\n"
    "ld1 { v27.b }[12], [x10], #0x1\n"
    "ld1 { v24.b }[12], [x9], #0x1\n"
    "ld1 { v2.b }[12], [x26], #0x1\n"
    "ld1 { v9.b }[12], [x21], #0x1\n"
    "b 19f\n"
    "13:"  // Oddments: Load (B): Bit 3: Bit 2: Unset
    "tbz %x[n_channels], #1, 14f\n"
    "ld1 { v22.h }[4], [x15], #0x2\n"
    "ld1 { v19.h }[4], [x14], #0x2\n"
    "ld1 { v0.h }[4], [x13], #0x2\n"
    "ld1 { v5.h }[4], [x12], #0x2\n"
    "ld1 { v27.h }[4], [x10], #0x2\n"
    "ld1 { v24.h }[4], [x9], #0x2\n"
    "ld1 { v2.h }[4], [x26], #0x2\n"
    "ld1 { v9.h }[4], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[10], [x15], #0x1\n"
    "ld1 { v19.b }[10], [x14], #0x1\n"
    "ld1 { v0.b }[10], [x13], #0x1\n"
    "ld1 { v5.b }[10], [x12], #0x1\n"
    "ld1 { v27.b }[10], [x10], #0x1\n"
    "ld1 { v24.b }[10], [x9], #0x1\n"
    "ld1 { v2.b }[10], [x26], #0x1\n"
    "ld1 { v9.b }[10], [x21], #0x1\n"
    "b 19f\n"
    "14:"  // Oddments: Load (B): Bit 3: Bit 2: Unset: Bit 1: Unset
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[8], [x15], #0x1\n"
    "ld1 { v19.b }[8], [x14], #0x1\n"
    "ld1 { v0.b }[8], [x13], #0x1\n"
    "ld1 { v5.b }[8], [x12], #0x1\n"
    "ld1 { v27.b }[8], [x10], #0x1\n"
    "ld1 { v24.b }[8], [x9], #0x1\n"
    "ld1 { v2.b }[8], [x26], #0x1\n"
    "ld1 { v9.b }[8], [x21], #0x1\n"
    "b 19f\n"
    "15:"  // Oddments: Load (B): Bit 3: Unset
    "tbz %x[n_channels], #2, 17f\n"
    "ldr s22, [x15], #0x4\n"
    "ldr s19, [x14], #0x4\n"
    "ldr s0, [x13], #0x4\n"
    "ldr s5, [x12], #0x4\n"
    "ldr s27, [x10], #0x4\n"
    "ldr s24, [x9], #0x4\n"
    "ldr s2, [x26], #0x4\n"
    "ldr s9, [x21], #0x4\n"
    "tbz %x[n_channels], #1, 16f\n"
    "ld1 { v22.h }[2], [x15], #0x2\n"
    "ld1 { v19.h }[2], [x14], #0x2\n"
    "ld1 { v0.h }[2], [x13], #0x2\n"
    "ld1 { v5.h }[2], [x12], #0x2\n"
    "ld1 { v27.h }[2], [x10], #0x2\n"
    "ld1 { v24.h }[2], [x9], #0x2\n"
    "ld1 { v2.h }[2], [x26], #0x2\n"
    "ld1 { v9.h }[2], [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[6], [x15], #0x1\n"
    "ld1 { v19.b }[6], [x14], #0x1\n"
    "ld1 { v0.b }[6], [x13], #0x1\n"
    "ld1 { v5.b }[6], [x12], #0x1\n"
    "ld1 { v27.b }[6], [x10], #0x1\n"
    "ld1 { v24.b }[6], [x9], #0x1\n"
    "ld1 { v2.b }[6], [x26], #0x1\n"
    "ld1 { v9.b }[6], [x21], #0x1\n"
    "b 19f\n"
    "16:"  // Oddments: Load (B): Bit 3: Unset: Bit 2: Bit 1: Unset
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[4], [x15], #0x1\n"
    "ld1 { v19.b }[4], [x14], #0x1\n"
    "ld1 { v0.b }[4], [x13], #0x1\n"
    "ld1 { v5.b }[4], [x12], #0x1\n"
    "ld1 { v27.b }[4], [x10], #0x1\n"
    "ld1 { v24.b }[4], [x9], #0x1\n"
    "ld1 { v2.b }[4], [x26], #0x1\n"
    "ld1 { v9.b }[4], [x21], #0x1\n"
    "b 19f\n"
    "17:"  // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset
    "tbz %x[n_channels], #1, 18f\n"
    "ldr h22, [x15], #0x2\n"
    "ldr h19, [x14], #0x2\n"
    "ldr h0, [x13], #0x2\n"
    "ldr h5, [x12], #0x2\n"
    "ldr h27, [x10], #0x2\n"
    "ldr h24, [x9], #0x2\n"
    "ldr h2, [x26], #0x2\n"
    "ldr h9, [x21], #0x2\n"
    "tbz %x[n_channels], #0, 19f\n"
    "ld1 { v22.b }[2], [x15], #0x1\n"
    "ld1 { v19.b }[2], [x14], #0x1\n"
    "ld1 { v0.b }[2], [x13], #0x1\n"
    "ld1 { v5.b }[2], [x12], #0x1\n"
    "ld1 { v27.b }[2], [x10], #0x1\n"
    "ld1 { v24.b }[2], [x9], #0x1\n"
    "ld1 { v2.b }[2], [x26], #0x1\n"
    "ld1 { v9.b }[2], [x21], #0x1\n"
    "b 19f\n"
    "18:"  // Oddments: Load (B): Bit 3: Unset: Bit 2: Unset: Bit 1: Unset
    "ldr b22, [x15], #0x1\n"
    "ldr b19, [x14], #0x1\n"
    "ldr b0, [x13], #0x1\n"
    "ldr b5, [x12], #0x1\n"
    "ldr b27, [x10], #0x1\n"
    "ldr b24, [x9], #0x1\n"
    "ldr b2, [x26], #0x1\n"
    "ldr b9, [x21], #0x1\n"
    "19:"  // Oddments: Load (B): Bit 3: End
    "ldr q20, [%x[params], #0x10]\n"
    "ldr q6, [%x[params], #0x20]\n"
    "zip2 v1.16b, v26.16b, v3.16b\n"
    "zip1 v26.16b, v26.16b, v3.16b\n"
    "ldr q4, [%x[params], #0x30]\n"
    "zip1 v18.16b, v23.16b, v10.16b\n"
    "zip2 v30.16b, v15.16b, v7.16b\n"
    "cmp x20, #0x4\n"
    "zip1 v15.16b, v15.16b, v7.16b\n"
    "zip1 v29.16b, v25.16b, v8.16b\n"
    "zip2 v8.16b, v25.16b, v8.16b\n"
    "zip2 v10.16b, v23.16b, v10.16b\n"
    "zip2 v23.16b, v26.16b, v18.16b\n"
    "zip1 v26.16b, v26.16b, v18.16b\n"
    "zip2 v28.16b, v22.16b, v0.16b\n"
    "zip1 v22.16b, v22.16b, v0.16b\n"
    "zip1 v21.16b, v19.16b, v5.16b\n"
    "movi v17.4s, #0x0\n"
    ".inst 0x6e9a9591  // udot v17.4s, v12.16b, v26.16b\n"
    "zip2 v25.16b, v15.16b, v29.16b\n"
    "zip1 v15.16b, v15.16b, v29.16b\n"
    "zip1 v7.16b, v30.16b, v8.16b\n"
    "zip2 v8.16b, v30.16b, v8.16b\n"
    "ldr q31, [%x[params], #0x0]\n"
    "zip2 v5.16b, v19.16b, v5.16b\n"
    "zip2 v30.16b, v27.16b, v2.16b\n"
    "zip1 v27.16b, v27.16b, v2.16b\n"
    "zip1 v18.16b, v24.16b, v9.16b\n"
    "zip2 v9.16b, v24.16b, v9.16b\n"
    "zip2 v19.16b, v22.16b, v21.16b\n"
    "zip1 v22.16b, v22.16b, v21.16b\n"
    "zip1 v3.16b, v1.16b, v10.16b\n"
    ".inst 0x6e969591  // udot v17.4s, v12.16b, v22.16b\n"
    "zip2 v10.16b, v1.16b, v10.16b\n"
    "zip1 v0.16b, v28.16b, v5.16b\n"
    "zip2 v5.16b, v28.16b, v5.16b\n"
    "zip2 v24.16b, v27.16b, v18.16b\n"
    "zip1 v27.16b, v27.16b, v18.16b\n"
    "zip1 v2.16b, v30.16b, v9.16b\n"
    "mov v18.16b, v17.16b\n .inst 0x6e9b9592  // udot v18.4s, v12.16b, v27.16b\n"
    "zip2 v9.16b, v30.16b, v9.16b\n"
    "mov v30.16b, v31.16b\n"
    ".inst 0x6e8f9591  // udot v17.4s, v12.16b, v15.16b\n"
    "mov v29.16b, v31.16b\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x6e8f969f  // udot v31.4s, v20.16b, v15.16b\n"
    ".inst 0x6e9a969d  // udot v29.4s, v20.16b, v26.16b\n"
    ".inst 0x6e9a94df  // udot v31.4s, v6.16b, v26.16b\n"
    "ext v26.16b, v26.16b, v26.16b, #0x1\n"
    "movi v1.4s, #0x0\n"
    "ext v15.16b, v15.16b, v15.16b, #0x1\n"
    ".inst 0x6e9a9581  // udot v1.4s, v12.16b, v26.16b\n"
    ".inst 0x6e9694dd  // udot v29.4s, v6.16b, v22.16b\n"
    ".inst 0x6e96949f  // udot v31.4s, v4.16b, v22.16b\n"
    "ext v22.16b, v22.16b, v22.16b, #0x1\n"
    ".inst 0x6e8f969e  // udot v30.4s, v20.16b, v15.16b\n"
    ".inst 0x6e9a969c  // udot v28.4s, v20.16b, v26.16b\n"
    "mls v31.4s, v17.4s, v16.4s\n"
    ".inst 0x6e969581  // udot v1.4s, v12.16b, v22.16b\n"
    ".inst 0x6e9b949d  // udot v29.4s, v4.16b, v27.16b\n"
    "ext v27.16b, v27.16b, v27.16b, #0x1\n"
    ".inst 0x6e9a94de  // udot v30.4s, v6.16b, v26.16b\n"
    "ldr q21, [%x[params], #0x50]\n"
    ".inst 0x6e9694dc  // udot v28.4s, v6.16b, v22.16b\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mov v20.16b, v1.16b\n .inst 0x6e9b9594  // udot v20.4s, v12.16b, v27.16b\n"
    ".inst 0x6e8f9581  // udot v1.4s, v12.16b, v15.16b\n"
    "ldr q18, [%x[params], #0x40]\n"
    "sqrdmulh v31.4s, v31.4s, v18.4s\n"
    ".inst 0x6e96949e  // udot v30.4s, v4.16b, v22.16b\n"
    ".inst 0x6e9b949c  // udot v28.4s, v4.16b, v27.16b\n"
    "mls v30.4s, v1.4s, v16.4s\n"
    "add %x[params], %x[params], #0x60\n"
    "mls v28.4s, v20.4s, v16.4s\n"
    "and v17.16b, v31.16b, v21.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v18.4s\n"
    "sqrdmulh v29.4s, v29.4s, v18.4s\n"
    "sqrdmulh v28.4s, v28.4s, v18.4s\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v17.16b, v30.16b, v21.16b\n"
    "and v18.16b, v29.16b, v21.16b\n"
    "and v26.16b, v28.16b, v21.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v26.4s, v26.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v17.4s\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "sqadd v28.4s, v28.4s, v26.4s\n"
    "srshl v31.4s, v31.4s, v21.4s\n"
    "srshl v30.4s, v30.4s, v21.4s\n"
    "srshl v29.4s, v29.4s, v21.4s\n"
    "srshl v28.4s, v28.4s, v21.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "blt 20f\n"
    "str s31, [x25, x27]\n"
    "str s30, [x24, x27]\n"
    "str s29, [x23, x27]\n"
    "str s28, [x22, x27]\n"
    "b 23f\n"
    "20:"  // Oddments: Unroll 0: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 21f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 22f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 22f\n"
    "21:"  // Oddments: Unroll 0: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "22:"  // Oddments: Unroll 0: Oddment store: Bit 1: End
    "23:"  // Oddments: Unroll 0: After oddment store
    "subs x20, x20, #0x4\n"
    "add x27, x27, #0x4\n"
    "ble 35f\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q27, [%x[params], #0x10]\n"
    "movi v1.4s, #0x0\n"
    ".inst 0x6e979581  // udot v1.4s, v12.16b, v23.16b\n"
    "ldr q26, [%x[params], #0x20]\n"
    "ldr q22, [%x[params], #0x30]\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "ldr q4, [%x[params], #0x40]\n"
    "ldr q21, [%x[params], #0x50]\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x6e99977f  // udot v31.4s, v27.16b, v25.16b\n"
    ".inst 0x6e939581  // udot v1.4s, v12.16b, v19.16b\n"
    ".inst 0x6e97977d  // udot v29.4s, v27.16b, v23.16b\n"
    "movi v20.4s, #0x0\n"
    "cmp x20, #0x4\n"
    ".inst 0x6e97975f  // udot v31.4s, v26.16b, v23.16b\n"
    "mov v18.16b, v1.16b\n .inst 0x6e989592  // udot v18.4s, v12.16b, v24.16b\n"
    "ext v23.16b, v23.16b, v23.16b, #0x1\n"
    "add %x[params], %x[params], #0x60\n"
    ".inst 0x6e999581  // udot v1.4s, v12.16b, v25.16b\n"
    "ext v25.16b, v25.16b, v25.16b, #0x1\n"
    ".inst 0x6e99977e  // udot v30.4s, v27.16b, v25.16b\n"
    ".inst 0x6e97977c  // udot v28.4s, v27.16b, v23.16b\n"
    ".inst 0x6e979594  // udot v20.4s, v12.16b, v23.16b\n"
    ".inst 0x6e93975d  // udot v29.4s, v26.16b, v19.16b\n"
    ".inst 0x6e9396df  // udot v31.4s, v22.16b, v19.16b\n"
    "ext v19.16b, v19.16b, v19.16b, #0x1\n"
    ".inst 0x6e97975e  // udot v30.4s, v26.16b, v23.16b\n"
    ".inst 0x6e93975c  // udot v28.4s, v26.16b, v19.16b\n"
    "mls v31.4s, v1.4s, v16.4s\n"
    ".inst 0x6e939594  // udot v20.4s, v12.16b, v19.16b\n"
    ".inst 0x6e9896dd  // udot v29.4s, v22.16b, v24.16b\n"
    "ext v24.16b, v24.16b, v24.16b, #0x1\n"
    ".inst 0x6e9396de  // udot v30.4s, v22.16b, v19.16b\n"
    ".inst 0x6e9896dc  // udot v28.4s, v22.16b, v24.16b\n"
    "sqrdmulh v31.4s, v31.4s, v4.4s\n"
    "mov v17.16b, v20.16b\n .inst 0x6e989591  // udot v17.4s, v12.16b, v24.16b\n"
    ".inst 0x6e999594  // udot v20.4s, v12.16b, v25.16b\n"
    "mls v30.4s, v20.4s, v16.4s\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v21.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v4.4s\n"
    "sqrdmulh v29.4s, v29.4s, v4.4s\n"
    "sqrdmulh v28.4s, v28.4s, v4.4s\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v30.16b, v21.16b\n"
    "and v18.16b, v29.16b, v21.16b\n"
    "and v17.16b, v28.16b, v21.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v19.4s\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "srshl v31.4s, v31.4s, v21.4s\n"
    "srshl v30.4s, v30.4s, v21.4s\n"
    "srshl v29.4s, v29.4s, v21.4s\n"
    "srshl v28.4s, v28.4s, v21.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "blt 24f\n"
    "str s31, [x25, x27]\n"
    "str s30, [x24, x27]\n"
    "str s29, [x23, x27]\n"
    "str s28, [x22, x27]\n"
    "b 27f\n"
    "24:"  // Oddments: Unroll 1: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 25f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 26f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 26f\n"
    "25:"  // Oddments: Unroll 1: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "26:"  // Oddments: Unroll 1: Oddment store: Bit 1: End
    "27:"  // Oddments: Unroll 1: After oddment store
    "subs x20, x20, #0x4\n"
    "add x27, x27, #0x4\n"
    "ble 35f\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q25, [%x[params], #0x10]\n"
    "movi v24.4s, #0x0\n"
    ".inst 0x6e839598  // udot v24.4s, v12.16b, v3.16b\n"
    "ldr q23, [%x[params], #0x20]\n"
    "ldr q22, [%x[params], #0x30]\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "ldr q21, [%x[params], #0x40]\n"
    "ldr q20, [%x[params], #0x50]\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x6e87973f  // udot v31.4s, v25.16b, v7.16b\n"
    ".inst 0x6e809598  // udot v24.4s, v12.16b, v0.16b\n"
    ".inst 0x6e83973d  // udot v29.4s, v25.16b, v3.16b\n"
    "movi v19.4s, #0x0\n"
    "cmp x20, #0x4\n"
    ".inst 0x6e8396ff  // udot v31.4s, v23.16b, v3.16b\n"
    "mov v18.16b, v24.16b\n .inst 0x6e829592  // udot v18.4s, v12.16b, v2.16b\n"
    "ext v3.16b, v3.16b, v3.16b, #0x1\n"
    "add %x[params], %x[params], #0x60\n"
    ".inst 0x6e879598  // udot v24.4s, v12.16b, v7.16b\n"
    "ext v7.16b, v7.16b, v7.16b, #0x1\n"
    ".inst 0x6e87973e  // udot v30.4s, v25.16b, v7.16b\n"
    ".inst 0x6e83973c  // udot v28.4s, v25.16b, v3.16b\n"
    ".inst 0x6e839593  // udot v19.4s, v12.16b, v3.16b\n"
    ".inst 0x6e8096fd  // udot v29.4s, v23.16b, v0.16b\n"
    ".inst 0x6e8096df  // udot v31.4s, v22.16b, v0.16b\n"
    "ext v0.16b, v0.16b, v0.16b, #0x1\n"
    ".inst 0x6e8396fe  // udot v30.4s, v23.16b, v3.16b\n"
    ".inst 0x6e8096fc  // udot v28.4s, v23.16b, v0.16b\n"
    "mls v31.4s, v24.4s, v16.4s\n"
    ".inst 0x6e809593  // udot v19.4s, v12.16b, v0.16b\n"
    ".inst 0x6e8296dd  // udot v29.4s, v22.16b, v2.16b\n"
    "ext v2.16b, v2.16b, v2.16b, #0x1\n"
    ".inst 0x6e8096de  // udot v30.4s, v22.16b, v0.16b\n"
    ".inst 0x6e8296dc  // udot v28.4s, v22.16b, v2.16b\n"
    "sqrdmulh v31.4s, v31.4s, v21.4s\n"
    "mov v17.16b, v19.16b\n .inst 0x6e829591  // udot v17.4s, v12.16b, v2.16b\n"
    ".inst 0x6e879593  // udot v19.4s, v12.16b, v7.16b\n"
    "mls v30.4s, v19.4s, v16.4s\n"
    "mls v29.4s, v18.4s, v16.4s\n"
    "mls v28.4s, v17.4s, v16.4s\n"
    "and v17.16b, v31.16b, v20.16b\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v21.4s\n"
    "sqrdmulh v29.4s, v29.4s, v21.4s\n"
    "sqrdmulh v28.4s, v28.4s, v21.4s\n"
    "sqadd v31.4s, v31.4s, v17.4s\n"
    "and v19.16b, v30.16b, v20.16b\n"
    "and v18.16b, v29.16b, v20.16b\n"
    "and v17.16b, v28.16b, v20.16b\n"
    "sshr v19.4s, v19.4s, #0x1f\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v19.4s\n"
    "sqadd v29.4s, v29.4s, v18.4s\n"
    "sqadd v28.4s, v28.4s, v17.4s\n"
    "srshl v31.4s, v31.4s, v20.4s\n"
    "srshl v30.4s, v30.4s, v20.4s\n"
    "srshl v29.4s, v29.4s, v20.4s\n"
    "srshl v28.4s, v28.4s, v20.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "blt 28f\n"
    "str s31, [x25, x27]\n"
    "str s30, [x24, x27]\n"
    "str s29, [x23, x27]\n"
    "str s28, [x22, x27]\n"
    "b 31f\n"
    "28:"  // Oddments: Unroll 2: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 29f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 30f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 30f\n"
    "29:"  // Oddments: Unroll 2: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "30:"  // Oddments: Unroll 2: Oddment store: Bit 1: End
    "31:"  // Oddments: Unroll 2: After oddment store
    "subs x20, x20, #0x4\n"
    "add x27, x27, #0x4\n"
    "ble 35f\n"
    "ldr q31, [%x[params], #0x0]\n"
    "ldr q23, [%x[params], #0x10]\n"
    "movi v22.4s, #0x0\n"
    ".inst 0x6e8a9596  // udot v22.4s, v12.16b, v10.16b\n"
    "ldr q21, [%x[params], #0x20]\n"
    "ldr q19, [%x[params], #0x30]\n"
    "mov v30.16b, v31.16b\n"
    "mov v29.16b, v31.16b\n"
    "ldr q20, [%x[params], #0x40]\n"
    "ldr q26, [%x[params], #0x50]\n"
    "mov v28.16b, v31.16b\n"
    ".inst 0x6e8896ff  // udot v31.4s, v23.16b, v8.16b\n"
    ".inst 0x6e859596  // udot v22.4s, v12.16b, v5.16b\n"
    ".inst 0x6e8a96fd  // udot v29.4s, v23.16b, v10.16b\n"
    "movi v18.4s, #0x0\n"
    "add %x[params], %x[params], #0x60\n"
    ".inst 0x6e8a96bf  // udot v31.4s, v21.16b, v10.16b\n"
    "mov v17.16b, v22.16b\n .inst 0x6e899591  // udot v17.4s, v12.16b, v9.16b\n"
    "ext v10.16b, v10.16b, v10.16b, #0x1\n"
    ".inst 0x6e889596  // udot v22.4s, v12.16b, v8.16b\n"
    "ext v8.16b, v8.16b, v8.16b, #0x1\n"
    ".inst 0x6e8896fe  // udot v30.4s, v23.16b, v8.16b\n"
    ".inst 0x6e8a96fc  // udot v28.4s, v23.16b, v10.16b\n"
    ".inst 0x6e8a9592  // udot v18.4s, v12.16b, v10.16b\n"
    ".inst 0x6e8596bd  // udot v29.4s, v21.16b, v5.16b\n"
    ".inst 0x6e85967f  // udot v31.4s, v19.16b, v5.16b\n"
    "ext v5.16b, v5.16b, v5.16b, #0x1\n"
    ".inst 0x6e8a96be  // udot v30.4s, v21.16b, v10.16b\n"
    ".inst 0x6e8596bc  // udot v28.4s, v21.16b, v5.16b\n"
    "mls v31.4s, v22.4s, v16.4s\n"
    ".inst 0x6e859592  // udot v18.4s, v12.16b, v5.16b\n"
    ".inst 0x6e89967d  // udot v29.4s, v19.16b, v9.16b\n"
    "ext v9.16b, v9.16b, v9.16b, #0x1\n"
    ".inst 0x6e85967e  // udot v30.4s, v19.16b, v5.16b\n"
    ".inst 0x6e89967c  // udot v28.4s, v19.16b, v9.16b\n"
    "sqrdmulh v31.4s, v31.4s, v20.4s\n"
    "mov v7.16b, v18.16b\n .inst 0x6e899587  // udot v7.4s, v12.16b, v9.16b\n"
    ".inst 0x6e889592  // udot v18.4s, v12.16b, v8.16b\n"
    "mls v30.4s, v18.4s, v16.4s\n"
    "mls v29.4s, v17.4s, v16.4s\n"
    "mls v28.4s, v7.4s, v16.4s\n"
    "and v16.16b, v31.16b, v26.16b\n"
    "sshr v16.4s, v16.4s, #0x1f\n"
    "sqrdmulh v30.4s, v30.4s, v20.4s\n"
    "sqrdmulh v29.4s, v29.4s, v20.4s\n"
    "sqrdmulh v28.4s, v28.4s, v20.4s\n"
    "sqadd v31.4s, v31.4s, v16.4s\n"
    "and v18.16b, v30.16b, v26.16b\n"
    "and v17.16b, v29.16b, v26.16b\n"
    "and v16.16b, v28.16b, v26.16b\n"
    "sshr v18.4s, v18.4s, #0x1f\n"
    "sshr v17.4s, v17.4s, #0x1f\n"
    "sshr v16.4s, v16.4s, #0x1f\n"
    "sqadd v30.4s, v30.4s, v18.4s\n"
    "sqadd v29.4s, v29.4s, v17.4s\n"
    "sqadd v28.4s, v28.4s, v16.4s\n"
    "srshl v31.4s, v31.4s, v26.4s\n"
    "srshl v30.4s, v30.4s, v26.4s\n"
    "srshl v29.4s, v29.4s, v26.4s\n"
    "srshl v28.4s, v28.4s, v26.4s\n"
    "add v31.4s, v31.4s, v14.4s\n"
    "add v30.4s, v30.4s, v14.4s\n"
    "add v29.4s, v29.4s, v14.4s\n"
    "add v28.4s, v28.4s, v14.4s\n"
    "smax v31.4s, v31.4s, v13.4s\n"
    "smax v30.4s, v30.4s, v13.4s\n"
    "smax v29.4s, v29.4s, v13.4s\n"
    "smax v28.4s, v28.4s, v13.4s\n"
    "smin v31.4s, v31.4s, v11.4s\n"
    "smin v30.4s, v30.4s, v11.4s\n"
    "smin v29.4s, v29.4s, v11.4s\n"
    "smin v28.4s, v28.4s, v11.4s\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "uzp1 v31.16b, v31.16b, v31.16b\n"
    "uzp1 v30.16b, v30.16b, v30.16b\n"
    "uzp1 v29.16b, v29.16b, v29.16b\n"
    "uzp1 v28.16b, v28.16b, v28.16b\n"
    "32:"  // Oddments: Unroll 3: Oddment store
    "add x25, x25, x27\n"
    "add x24, x24, x27\n"
    "add x23, x23, x27\n"
    "add x22, x22, x27\n"
    "tbz x20, #1, 33f\n"
    "st1 { v31.h }[0], [x25], #0x2\n"
    "st1 { v30.h }[0], [x24], #0x2\n"
    "st1 { v29.h }[0], [x23], #0x2\n"
    "st1 { v28.h }[0], [x22], #0x2\n"
    "tbz x20, #0, 34f\n"
    "st1 { v31.b }[2], [x25], #0x1\n"
    "st1 { v30.b }[2], [x24], #0x1\n"
    "st1 { v29.b }[2], [x23], #0x1\n"
    "st1 { v28.b }[2], [x22], #0x1\n"
    "b 34f\n"
    "33:"  // Oddments: Unroll 3: Oddment store: Bit 1: Unset
    "st1 { v31.b }[0], [x25], #0x1\n"
    "st1 { v30.b }[0], [x24], #0x1\n"
    "st1 { v29.b }[0], [x23], #0x1\n"
    "st1 { v28.b }[0], [x22], #0x1\n"
    "34:"  // Oddments: Unroll 3: Oddment store: Bit 1: End
    "35:"  // End
    : [params] "+&r" (params)
    : [inptrs] "r" (inptrs), [n_channels] "r" (n_channels), [offsetof_Requantize32_b_offset] "I" (offsetof(arm_gemm::Requantize32, b_offset)), [offsetof_Requantize32_c_offset] "I" (offsetof(arm_gemm::Requantize32, c_offset)), [offsetof_Requantize32_maxval] "I" (offsetof(arm_gemm::Requantize32, maxval)), [offsetof_Requantize32_minval] "I" (offsetof(arm_gemm::Requantize32, minval)), [outptrs] "r" (outptrs), [qp] "r" (&qp)
    : "cc", "memory", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28"
  );
}

}  // namespace depthwise
}  // namespace arm_conv

#endif  // defined(__aarch64__)