diff options
Diffstat (limited to 'src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp')
| -rw-r--r-- | src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp | 891 |
1 files changed, 891 insertions, 0 deletions
diff --git a/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp new file mode 100644 index 0000000000..043914d590 --- /dev/null +++ b/src/core/NEON/kernels/convolution/winograd/output_transforms/sme_fp32_mopa_4x4_3x3.cpp @@ -0,0 +1,891 @@ +/* + * Copyright (c) 2022-2024 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#if defined(ARM_COMPUTE_ENABLE_SME) + +#include <cstddef> + +namespace arm_conv { +namespace winograd { +namespace output_transform { + +void sme_fp32_mopa_4x4_3x3( + unsigned int n_channels, + const float* inptr, + size_t matrix_stride, + const float* bptr, + float* const output, + size_t output_row_stride, + size_t output_col_stride, + float output_min, + float output_max +) +{ + // The below assembler uses the Kronecker product and the "vec trick" to + // implement the Winograd output transform (y = AT Y A) using the SME + // array. This code REQUIRES that the vectors are 512b long (or longer, if + // we add some predication). + // + // The "vec trick" uses the identity $vec(AT Y A) = (AT (x) AT) vec(Y)$ to + // convert the chain of matrix multiplications into a matrix-vector + // product. We then stack multiple channels of vec(Y) together to allow us + // to perform multiple channels of the transformation simultaneously. + // + // Since the complete matrix (AT (x) AT) is quite big [16 x 36], we compute + // it on the fly. To do so, we store two representations of the matrix AT. + // The first representation (the outer terms) contains, within each quad, + // four coefficients of the matrix AT. + const float outer_terms[32] = { + 1, 1, 1, 1, + 0, 1, -1, 2, + 0, 1, 1, 4, + 0, 1, -1, 8, + // The following rows are continuations of the first four rows, and each + // contains two columns of padding values which aren't used in the + // computation but are there to ensure that the coefficients end up in + // the right quads of the vector into which they're read. + 1, 0, 0, 0, + -2, 0, 0, 0, + 4, 0, 0, 0, + -8, 1, 0, 0 + }; + // This should be loaded completely into two Z registers. + // + // We can then use by-element FMLA to construct columns of (AT (x) AT) by + // multiplying elements of the outer terms against the following inner + // terms (again split into quads, but expected to be loaded replicated such + // that each of the six required Z registers contains a repeated quad of + // the values). + const float inner_terms[24] = { + 1, 0, 0, 0, + 1, 1, 1, 1, + 1, -1, 1, -1, + 1, 2, 4, 8, + 1, -2, 4, -8, + 0, 0, 0, 1 + }; + + struct Params + { + const float *outer_terms; + const float *inner_terms; + float act_min; + float act_max; + + Params(const float *outer_terms, + const float *inner_terms, + float act_min, + float act_max) + : outer_terms(outer_terms), inner_terms(inner_terms), + act_min(act_min), act_max(act_max) + { + } + }; + + Params params(outer_terms, inner_terms, output_min, output_max); + + __asm__ __volatile__( + "ldr x20, [%x[params], %[offsetof_Params_outer_terms]]\n" + ".inst 0xd503477f // SMSTART ZA\n" + "ptrue p5.b\n" + "ld1rw { z12.s }, p5/Z, [%x[params], %[offsetof_Params_act_min]]\n" + "ld1rw { z10.s }, p5/Z, [%x[params], %[offsetof_Params_act_max]]\n" + "pfalse p8.b\n" + "ldr x8, [%x[params], %[offsetof_Params_inner_terms]]\n" + "ld1w { z6.s }, p5/Z, [x20]\n" + "ld1w { z7.s }, p5/Z, [x20, #1, MUL VL]\n" + "ld1rqw { z9.s }, p5/Z, [x8]\n" + "ld1rqw { z8.s }, p5/Z, [x8, #16]\n" + "ld1rqw { z15.s }, p5/Z, [x8, #32]\n" + "fmul z11.s, z9.s, z6.s[0]\n" + "fmul z5.s, z9.s, z6.s[1]\n" + "ld1rqw { z4.s }, p5/Z, [x8, #48]\n" + "ld1rqw { z3.s }, p5/Z, [x8, #64]\n" + "ld1rqw { z2.s }, p5/Z, [x8, #80]\n" + "cbz %x[bptr], 1f\n" + "ptrue p8.s\n" + "1:" // Set bias predicate: Done + ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" + "fmov z1.s, #1.0\n" + "mov x25, #0x0\n" + "cntw x24\n" + "cntw x23, ALL, MUL #2\n" + "cntw x22, ALL, MUL #3\n" + "whilelt p4.s, x25, %x[n_channels]\n" + "whilelt p3.s, x24, %x[n_channels]\n" + "ld1w { z31.s }, p4/Z, [%x[inptr], x25, LSL #2]\n" + "ld1w { z30.s }, p3/Z, [%x[inptr], x24, LSL #2]\n" + "whilelt p2.s, x23, %x[n_channels]\n" + "whilelt p1.s, x22, %x[n_channels]\n" + "ld1w { z29.s }, p2/Z, [%x[inptr], x23, LSL #2]\n" + "add x21, %x[inptr], %x[matrix_stride], LSL #2\n" + "and p0.b, p5/Z, p8.b, p4.b\n" + "ld1w { z28.s }, p1/Z, [%x[inptr], x22, LSL #2]\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x25, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p3.b\n" + ".inst 0x8080b420 // fmopa za0.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x24, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p2.b\n" + ".inst 0x8080b421 // fmopa za1.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x23, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p1.b\n" + ".inst 0x8080b422 // fmopa za2.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x22, LSL #2]\n" + ".inst 0x8080b423 // fmopa za3.s, p5/M, p5/M, z1.s, z0.s\n" + "2:" // Loop + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "mov x14, #0xc\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + "whilelt p0.s, x25, %x[n_channels]\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z9.s, z6.s[2]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z9.s, z6.s[3]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z9.s, z7.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z9.s, z7.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z8.s, z6.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z8.s, z6.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z8.s, z6.s[2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z8.s, z6.s[3]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z8.s, z7.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z8.s, z7.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z15.s, z6.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z15.s, z6.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z15.s, z6.s[2]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z15.s, z6.s[3]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z15.s, z7.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z15.s, z7.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z4.s, z6.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z4.s, z6.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z4.s, z6.s[2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z4.s, z6.s[3]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z4.s, z7.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z4.s, z7.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z3.s, z6.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z3.s, z6.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z3.s, z6.s[2]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z3.s, z6.s[3]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z3.s, z7.s[0]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z3.s, z7.s[1]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z2.s, z6.s[0]\n" + "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z2.s, z6.s[1]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z2.s, z6.s[2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z2.s, z6.s[3]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" + ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" + ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" + "fmul z11.s, z2.s, z7.s[0]\n" + ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" + ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" + ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" + ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" + "fmul z5.s, z2.s, z7.s[1]\n" + ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" + ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" + ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" + ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" + "fmul z11.s, z9.s, z6.s[0]\n" + ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" + ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" + ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" + ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" + "fmul z5.s, z9.s, z6.s[1]\n" + ".inst 0xc082741f // mova z31.s, p5/M, za0h.s[XZR]\n" + ".inst 0xc082541c // mova z28.s, p5/M, za0h.s[x14]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082743b // mova z27.s, p5/M, za0h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc0825438 // mova z24.s, p5/M, za0h.s[x14, #1]\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + "mov x13, #0x4\n" + "mov x12, #0x8\n" + ".inst 0xc082341e // mova z30.s, p5/M, za0h.s[x13]\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc082141d // mova z29.s, p5/M, za0h.s[x12]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082343a // mova z26.s, p5/M, za0h.s[x13, #1]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + ".inst 0xc0821439 // mova z25.s, p5/M, za0h.s[x12, #1]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc0827457 // mova z23.s, p5/M, za0h.s[XZR, #2]\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + ".inst 0xc0823456 // mova z22.s, p5/M, za0h.s[x13, #2]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc0821455 // mova z21.s, p5/M, za0h.s[x12, #2]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + ".inst 0xc0825454 // mova z20.s, p5/M, za0h.s[x14, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + ".inst 0xc0827473 // mova z19.s, p5/M, za0h.s[XZR, #3]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + ".inst 0xc0823472 // mova z18.s, p5/M, za0h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + ".inst 0xc0821471 // mova z17.s, p5/M, za0h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + ".inst 0xc0825470 // mova z16.s, p5/M, za0h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "st1w { z31.s }, p0, [%x[output], x25, LSL #2]\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z27.s }, p0, [x8, x25, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x25, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x25, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x25, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x25, LSL #2]\n" + "whilelt p0.s, x24, %x[n_channels]\n" + "b.none 3f\n" + ".inst 0xc082749f // mova z31.s, p5/M, za1h.s[XZR]\n" + ".inst 0xc082349e // mova z30.s, p5/M, za1h.s[x13]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082149d // mova z29.s, p5/M, za1h.s[x12]\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082549c // mova z28.s, p5/M, za1h.s[x14]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + ".inst 0xc08274bb // mova z27.s, p5/M, za1h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc08234ba // mova z26.s, p5/M, za1h.s[x13, #1]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + ".inst 0xc08214b9 // mova z25.s, p5/M, za1h.s[x12, #1]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc08254b8 // mova z24.s, p5/M, za1h.s[x14, #1]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + ".inst 0xc08274d7 // mova z23.s, p5/M, za1h.s[XZR, #2]\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc08234d6 // mova z22.s, p5/M, za1h.s[x13, #2]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc08214d5 // mova z21.s, p5/M, za1h.s[x12, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0xc08254d4 // mova z20.s, p5/M, za1h.s[x14, #2]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0xc08274f3 // mova z19.s, p5/M, za1h.s[XZR, #3]\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + ".inst 0xc08234f2 // mova z18.s, p5/M, za1h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + ".inst 0xc08214f1 // mova z17.s, p5/M, za1h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + ".inst 0xc08254f0 // mova z16.s, p5/M, za1h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "st1w { z31.s }, p0, [%x[output], x24, LSL #2]\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z27.s }, p0, [x8, x24, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x24, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x24, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x24, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x24, LSL #2]\n" + "whilelt p0.s, x23, %x[n_channels]\n" + "b.none 3f\n" + ".inst 0xc082751f // mova z31.s, p5/M, za2h.s[XZR]\n" + ".inst 0xc082351e // mova z30.s, p5/M, za2h.s[x13]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082151d // mova z29.s, p5/M, za2h.s[x12]\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082551c // mova z28.s, p5/M, za2h.s[x14]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + ".inst 0xc082753b // mova z27.s, p5/M, za2h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc082353a // mova z26.s, p5/M, za2h.s[x13, #1]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + ".inst 0xc0821539 // mova z25.s, p5/M, za2h.s[x12, #1]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc0825538 // mova z24.s, p5/M, za2h.s[x14, #1]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + ".inst 0xc0827557 // mova z23.s, p5/M, za2h.s[XZR, #2]\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc0823556 // mova z22.s, p5/M, za2h.s[x13, #2]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc0821555 // mova z21.s, p5/M, za2h.s[x12, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0xc0825554 // mova z20.s, p5/M, za2h.s[x14, #2]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0xc0827573 // mova z19.s, p5/M, za2h.s[XZR, #3]\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + ".inst 0xc0823572 // mova z18.s, p5/M, za2h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + ".inst 0xc0821571 // mova z17.s, p5/M, za2h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + ".inst 0xc0825570 // mova z16.s, p5/M, za2h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "st1w { z31.s }, p0, [%x[output], x23, LSL #2]\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z27.s }, p0, [x8, x23, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x23, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x23, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x23, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x23, LSL #2]\n" + "whilelt p0.s, x22, %x[n_channels]\n" + "b.none 3f\n" + "fmov z1.s, #1.0\n" + ".inst 0xc082759f // mova z31.s, p5/M, za3h.s[XZR]\n" + ".inst 0xc082359e // mova z30.s, p5/M, za3h.s[x13]\n" + "fmin z31.s, p5/M, z31.s, z10.s\n" + ".inst 0xc082159d // mova z29.s, p5/M, za3h.s[x12]\n" + "fmin z30.s, p5/M, z30.s, z10.s\n" + ".inst 0xc082559c // mova z28.s, p5/M, za3h.s[x14]\n" + "fmin z29.s, p5/M, z29.s, z10.s\n" + ".inst 0xc08275bb // mova z27.s, p5/M, za3h.s[XZR, #1]\n" + "fmin z28.s, p5/M, z28.s, z10.s\n" + ".inst 0xc08235ba // mova z26.s, p5/M, za3h.s[x13, #1]\n" + "fmax z31.s, p5/M, z31.s, z12.s\n" + "fmin z27.s, p5/M, z27.s, z10.s\n" + ".inst 0xc08215b9 // mova z25.s, p5/M, za3h.s[x12, #1]\n" + "fmax z30.s, p5/M, z30.s, z12.s\n" + "fmin z26.s, p5/M, z26.s, z10.s\n" + ".inst 0xc08255b8 // mova z24.s, p5/M, za3h.s[x14, #1]\n" + "fmax z29.s, p5/M, z29.s, z12.s\n" + "fmin z25.s, p5/M, z25.s, z10.s\n" + ".inst 0xc08275d7 // mova z23.s, p5/M, za3h.s[XZR, #2]\n" + "fmax z28.s, p5/M, z28.s, z12.s\n" + "fmin z24.s, p5/M, z24.s, z10.s\n" + ".inst 0xc08235d6 // mova z22.s, p5/M, za3h.s[x13, #2]\n" + "fmax z27.s, p5/M, z27.s, z12.s\n" + "fmin z23.s, p5/M, z23.s, z10.s\n" + ".inst 0xc08215d5 // mova z21.s, p5/M, za3h.s[x12, #2]\n" + "fmax z26.s, p5/M, z26.s, z12.s\n" + "fmin z22.s, p5/M, z22.s, z10.s\n" + ".inst 0xc08255d4 // mova z20.s, p5/M, za3h.s[x14, #2]\n" + "fmax z25.s, p5/M, z25.s, z12.s\n" + "fmin z21.s, p5/M, z21.s, z10.s\n" + "add x20, %x[output], %x[output_col_stride], LSL #2\n" + ".inst 0xc08275f3 // mova z19.s, p5/M, za3h.s[XZR, #3]\n" + "fmax z24.s, p5/M, z24.s, z12.s\n" + "fmin z20.s, p5/M, z20.s, z10.s\n" + "add x8, %x[output], %x[output_row_stride], LSL #2\n" + ".inst 0xc08235f2 // mova z18.s, p5/M, za3h.s[x13, #3]\n" + "fmax z23.s, p5/M, z23.s, z12.s\n" + "fmin z19.s, p5/M, z19.s, z10.s\n" + "incw x25, ALL, MUL #4\n" + ".inst 0xc08215f1 // mova z17.s, p5/M, za3h.s[x12, #3]\n" + "fmax z22.s, p5/M, z22.s, z12.s\n" + "fmin z18.s, p5/M, z18.s, z10.s\n" + "incw x24, ALL, MUL #4\n" + ".inst 0xc08255f0 // mova z16.s, p5/M, za3h.s[x14, #3]\n" + "fmax z21.s, p5/M, z21.s, z12.s\n" + "fmin z17.s, p5/M, z17.s, z10.s\n" + "incw x23, ALL, MUL #4\n" + ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" + "fmax z20.s, p5/M, z20.s, z12.s\n" + "fmin z16.s, p5/M, z16.s, z10.s\n" + "add x21, %x[inptr], %x[matrix_stride], LSL #2\n" + "fmax z19.s, p5/M, z19.s, z12.s\n" + "st1w { z31.s }, p0, [%x[output], x22, LSL #2]\n" + "fmax z18.s, p5/M, z18.s, z12.s\n" + "st1w { z30.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z17.s, p5/M, z17.s, z12.s\n" + "st1w { z29.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "fmax z16.s, p5/M, z16.s, z12.s\n" + "st1w { z28.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z27.s }, p0, [x8, x22, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z26.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z25.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z24.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z23.s }, p0, [x8, x22, LSL #2]\n" + "add x8, x8, %x[output_row_stride], LSL #2\n" + "st1w { z22.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z21.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z20.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x8, %x[output_col_stride], LSL #2\n" + "st1w { z19.s }, p0, [x8, x22, LSL #2]\n" + "st1w { z18.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z17.s }, p0, [x20, x22, LSL #2]\n" + "add x20, x20, %x[output_col_stride], LSL #2\n" + "st1w { z16.s }, p0, [x20, x22, LSL #2]\n" + "incw x22, ALL, MUL #4\n" + "whilelt p1.s, x22, %x[n_channels]\n" + "ld1w { z28.s }, p1/Z, [%x[inptr], x22, LSL #2]\n" + "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" + "whilelt p2.s, x23, %x[n_channels]\n" + "whilelt p3.s, x24, %x[n_channels]\n" + "ld1w { z30.s }, p3/Z, [%x[inptr], x24, LSL #2]\n" + "whilelt p4.s, x25, %x[n_channels]\n" + "ld1w { z31.s }, p4/Z, [%x[inptr], x25, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p4.b\n" + "ld1w { z29.s }, p2/Z, [%x[inptr], x23, LSL #2]\n" + "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" + "add x21, x21, %x[matrix_stride], LSL #2\n" + "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" + "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" + "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x25, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p3.b\n" + ".inst 0x8080b420 // fmopa za0.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x24, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p2.b\n" + ".inst 0x8080b421 // fmopa za1.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x23, LSL #2]\n" + "and p0.b, p5/Z, p8.b, p1.b\n" + ".inst 0x8080b422 // fmopa za2.s, p5/M, p5/M, z1.s, z0.s\n" + "ld1w { z0.s }, p0/Z, [%x[bptr], x22, LSL #2]\n" + ".inst 0x8080b423 // fmopa za3.s, p5/M, p5/M, z1.s, z0.s\n" + "b.any 2b\n" + "3:" // End + ".inst 0xd503467f // SMSTOP\n" + : + : [bptr] "r" (bptr), [inptr] "r" (inptr), [matrix_stride] "r" (matrix_stride), [n_channels] "r" (n_channels), [offsetof_Params_act_max] "I" (offsetof(Params, act_max)), [offsetof_Params_act_min] "I" (offsetof(Params, act_min)), [offsetof_Params_inner_terms] "I" (offsetof(Params, inner_terms)), [offsetof_Params_outer_terms] "I" (offsetof(Params, outer_terms)), [output] "r" (output), [output_col_stride] "r" (output_col_stride), [output_row_stride] "r" (output_row_stride), [params] "r" (¶ms) + : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p8", "x12", "x13", "x14", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + ); +} + +} // namespace output_transform +} // namespace winograd +} // namespace arm_conv + +#endif // defined(ARM_COMPUTE_ENABLE_SME) |