/* * Copyright (c) 2022-2024 Arm Limited. * * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #if defined(ARM_COMPUTE_ENABLE_SME) #include namespace arm_conv { namespace winograd { namespace output_transform { void sme_fp32_mopa_4x4_3x3( unsigned int n_channels, const float* inptr, size_t matrix_stride, const float* bptr, float* const output, size_t output_row_stride, size_t output_col_stride, float output_min, float output_max ) { // The below assembler uses the Kronecker product and the "vec trick" to // implement the Winograd output transform (y = AT Y A) using the SME // array. This code REQUIRES that the vectors are 512b long (or longer, if // we add some predication). // // The "vec trick" uses the identity $vec(AT Y A) = (AT (x) AT) vec(Y)$ to // convert the chain of matrix multiplications into a matrix-vector // product. We then stack multiple channels of vec(Y) together to allow us // to perform multiple channels of the transformation simultaneously. // // Since the complete matrix (AT (x) AT) is quite big [16 x 36], we compute // it on the fly. To do so, we store two representations of the matrix AT. // The first representation (the outer terms) contains, within each quad, // four coefficients of the matrix AT. const float outer_terms[32] = { 1, 1, 1, 1, 0, 1, -1, 2, 0, 1, 1, 4, 0, 1, -1, 8, // The following rows are continuations of the first four rows, and each // contains two columns of padding values which aren't used in the // computation but are there to ensure that the coefficients end up in // the right quads of the vector into which they're read. 1, 0, 0, 0, -2, 0, 0, 0, 4, 0, 0, 0, -8, 1, 0, 0 }; // This should be loaded completely into two Z registers. // // We can then use by-element FMLA to construct columns of (AT (x) AT) by // multiplying elements of the outer terms against the following inner // terms (again split into quads, but expected to be loaded replicated such // that each of the six required Z registers contains a repeated quad of // the values). const float inner_terms[24] = { 1, 0, 0, 0, 1, 1, 1, 1, 1, -1, 1, -1, 1, 2, 4, 8, 1, -2, 4, -8, 0, 0, 0, 1 }; struct Params { const float *outer_terms; const float *inner_terms; float act_min; float act_max; Params(const float *outer_terms, const float *inner_terms, float act_min, float act_max) : outer_terms(outer_terms), inner_terms(inner_terms), act_min(act_min), act_max(act_max) { } }; Params params(outer_terms, inner_terms, output_min, output_max); __asm__ __volatile__( "ldr x20, [%x[params], %[offsetof_Params_outer_terms]]\n" ".inst 0xd503477f // SMSTART ZA\n" "ptrue p5.b\n" "ld1rw { z12.s }, p5/Z, [%x[params], %[offsetof_Params_act_min]]\n" "ld1rw { z10.s }, p5/Z, [%x[params], %[offsetof_Params_act_max]]\n" "pfalse p8.b\n" "ldr x8, [%x[params], %[offsetof_Params_inner_terms]]\n" "ld1w { z6.s }, p5/Z, [x20]\n" "ld1w { z7.s }, p5/Z, [x20, #1, MUL VL]\n" "ld1rqw { z9.s }, p5/Z, [x8]\n" "ld1rqw { z8.s }, p5/Z, [x8, #16]\n" "ld1rqw { z15.s }, p5/Z, [x8, #32]\n" "fmul z11.s, z9.s, z6.s[0]\n" "fmul z5.s, z9.s, z6.s[1]\n" "ld1rqw { z4.s }, p5/Z, [x8, #48]\n" "ld1rqw { z3.s }, p5/Z, [x8, #64]\n" "ld1rqw { z2.s }, p5/Z, [x8, #80]\n" "cbz %x[bptr], 1f\n" "ptrue p8.s\n" "1:" // Set bias predicate: Done ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "fmov z1.s, #1.0\n" "mov x25, #0x0\n" "cntw x24\n" "cntw x23, ALL, MUL #2\n" "cntw x22, ALL, MUL #3\n" "whilelt p4.s, x25, %x[n_channels]\n" "whilelt p3.s, x24, %x[n_channels]\n" "ld1w { z31.s }, p4/Z, [%x[inptr], x25, LSL #2]\n" "ld1w { z30.s }, p3/Z, [%x[inptr], x24, LSL #2]\n" "whilelt p2.s, x23, %x[n_channels]\n" "whilelt p1.s, x22, %x[n_channels]\n" "ld1w { z29.s }, p2/Z, [%x[inptr], x23, LSL #2]\n" "add x21, %x[inptr], %x[matrix_stride], LSL #2\n" "and p0.b, p5/Z, p8.b, p4.b\n" "ld1w { z28.s }, p1/Z, [%x[inptr], x22, LSL #2]\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x25, LSL #2]\n" "and p0.b, p5/Z, p8.b, p3.b\n" ".inst 0x8080b420 // fmopa za0.s, p5/M, p5/M, z1.s, z0.s\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x24, LSL #2]\n" "and p0.b, p5/Z, p8.b, p2.b\n" ".inst 0x8080b421 // fmopa za1.s, p5/M, p5/M, z1.s, z0.s\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x23, LSL #2]\n" "and p0.b, p5/Z, p8.b, p1.b\n" ".inst 0x8080b422 // fmopa za2.s, p5/M, p5/M, z1.s, z0.s\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x22, LSL #2]\n" ".inst 0x8080b423 // fmopa za3.s, p5/M, p5/M, z1.s, z0.s\n" "2:" // Loop ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "add x21, x21, %x[matrix_stride], LSL #2\n" "mov x14, #0xc\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" "whilelt p0.s, x25, %x[n_channels]\n" "add x20, %x[output], %x[output_col_stride], LSL #2\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z9.s, z6.s[2]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z9.s, z6.s[3]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z9.s, z7.s[0]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z9.s, z7.s[1]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z8.s, z6.s[0]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z8.s, z6.s[1]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z8.s, z6.s[2]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z8.s, z6.s[3]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z8.s, z7.s[0]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z8.s, z7.s[1]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z15.s, z6.s[0]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z15.s, z6.s[1]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z15.s, z6.s[2]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z15.s, z6.s[3]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z15.s, z7.s[0]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z15.s, z7.s[1]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z4.s, z6.s[0]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z4.s, z6.s[1]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z4.s, z6.s[2]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z4.s, z6.s[3]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z4.s, z7.s[0]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z4.s, z7.s[1]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z3.s, z6.s[0]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z3.s, z6.s[1]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z3.s, z6.s[2]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z3.s, z6.s[3]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z3.s, z7.s[0]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z3.s, z7.s[1]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" "ld1w { z31.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" "ld1w { z30.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z2.s, z6.s[0]\n" "ld1w { z29.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" "ld1w { z28.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z2.s, z6.s[1]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z2.s, z6.s[2]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z2.s, z6.s[3]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" ".inst 0x809fb560 // fmopa za0.s, p5/M, p5/M, z11.s, z31.s\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" ".inst 0x809eb561 // fmopa za1.s, p5/M, p5/M, z11.s, z30.s\n" ".inst 0x809db562 // fmopa za2.s, p5/M, p5/M, z11.s, z29.s\n" ".inst 0x809cb563 // fmopa za3.s, p5/M, p5/M, z11.s, z28.s\n" "fmul z11.s, z2.s, z7.s[0]\n" ".inst 0x809bb4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z27.s\n" ".inst 0x809ab4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z26.s\n" ".inst 0x8099b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z25.s\n" ".inst 0x8098b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z24.s\n" "fmul z5.s, z2.s, z7.s[1]\n" ".inst 0x8097b560 // fmopa za0.s, p5/M, p5/M, z11.s, z23.s\n" ".inst 0x8096b561 // fmopa za1.s, p5/M, p5/M, z11.s, z22.s\n" ".inst 0x8095b562 // fmopa za2.s, p5/M, p5/M, z11.s, z21.s\n" ".inst 0x8094b563 // fmopa za3.s, p5/M, p5/M, z11.s, z20.s\n" "fmul z11.s, z9.s, z6.s[0]\n" ".inst 0x8093b4a0 // fmopa za0.s, p5/M, p5/M, z5.s, z19.s\n" ".inst 0x8092b4a1 // fmopa za1.s, p5/M, p5/M, z5.s, z18.s\n" ".inst 0x8091b4a2 // fmopa za2.s, p5/M, p5/M, z5.s, z17.s\n" ".inst 0x8090b4a3 // fmopa za3.s, p5/M, p5/M, z5.s, z16.s\n" "fmul z5.s, z9.s, z6.s[1]\n" ".inst 0xc082741f // mova z31.s, p5/M, za0h.s[XZR]\n" ".inst 0xc082541c // mova z28.s, p5/M, za0h.s[x14]\n" "fmin z31.s, p5/M, z31.s, z10.s\n" ".inst 0xc082743b // mova z27.s, p5/M, za0h.s[XZR, #1]\n" "fmin z28.s, p5/M, z28.s, z10.s\n" ".inst 0xc0825438 // mova z24.s, p5/M, za0h.s[x14, #1]\n" "fmin z27.s, p5/M, z27.s, z10.s\n" "mov x13, #0x4\n" "mov x12, #0x8\n" ".inst 0xc082341e // mova z30.s, p5/M, za0h.s[x13]\n" "fmin z24.s, p5/M, z24.s, z10.s\n" ".inst 0xc082141d // mova z29.s, p5/M, za0h.s[x12]\n" "fmax z31.s, p5/M, z31.s, z12.s\n" "fmin z30.s, p5/M, z30.s, z10.s\n" ".inst 0xc082343a // mova z26.s, p5/M, za0h.s[x13, #1]\n" "fmin z29.s, p5/M, z29.s, z10.s\n" "fmax z28.s, p5/M, z28.s, z12.s\n" ".inst 0xc0821439 // mova z25.s, p5/M, za0h.s[x12, #1]\n" "fmax z27.s, p5/M, z27.s, z12.s\n" "fmin z26.s, p5/M, z26.s, z10.s\n" ".inst 0xc0827457 // mova z23.s, p5/M, za0h.s[XZR, #2]\n" "fmin z25.s, p5/M, z25.s, z10.s\n" "fmax z24.s, p5/M, z24.s, z12.s\n" ".inst 0xc0823456 // mova z22.s, p5/M, za0h.s[x13, #2]\n" "fmax z30.s, p5/M, z30.s, z12.s\n" "fmin z23.s, p5/M, z23.s, z10.s\n" ".inst 0xc0821455 // mova z21.s, p5/M, za0h.s[x12, #2]\n" "fmax z29.s, p5/M, z29.s, z12.s\n" "fmin z22.s, p5/M, z22.s, z10.s\n" ".inst 0xc0825454 // mova z20.s, p5/M, za0h.s[x14, #2]\n" "fmax z26.s, p5/M, z26.s, z12.s\n" "fmin z21.s, p5/M, z21.s, z10.s\n" ".inst 0xc0827473 // mova z19.s, p5/M, za0h.s[XZR, #3]\n" "fmax z25.s, p5/M, z25.s, z12.s\n" "fmin z20.s, p5/M, z20.s, z10.s\n" ".inst 0xc0823472 // mova z18.s, p5/M, za0h.s[x13, #3]\n" "fmax z23.s, p5/M, z23.s, z12.s\n" "fmin z19.s, p5/M, z19.s, z10.s\n" ".inst 0xc0821471 // mova z17.s, p5/M, za0h.s[x12, #3]\n" "fmax z22.s, p5/M, z22.s, z12.s\n" "fmin z18.s, p5/M, z18.s, z10.s\n" ".inst 0xc0825470 // mova z16.s, p5/M, za0h.s[x14, #3]\n" "fmax z21.s, p5/M, z21.s, z12.s\n" "fmin z17.s, p5/M, z17.s, z10.s\n" "fmax z20.s, p5/M, z20.s, z12.s\n" "fmin z16.s, p5/M, z16.s, z10.s\n" "st1w { z31.s }, p0, [%x[output], x25, LSL #2]\n" "fmax z19.s, p5/M, z19.s, z12.s\n" "st1w { z30.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z18.s, p5/M, z18.s, z12.s\n" "st1w { z29.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z17.s, p5/M, z17.s, z12.s\n" "st1w { z28.s }, p0, [x20, x25, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" "st1w { z27.s }, p0, [x8, x25, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x25, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z23.s }, p0, [x8, x25, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x25, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z19.s }, p0, [x8, x25, LSL #2]\n" "st1w { z18.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x25, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z16.s }, p0, [x20, x25, LSL #2]\n" "whilelt p0.s, x24, %x[n_channels]\n" "b.none 3f\n" ".inst 0xc082749f // mova z31.s, p5/M, za1h.s[XZR]\n" ".inst 0xc082349e // mova z30.s, p5/M, za1h.s[x13]\n" "fmin z31.s, p5/M, z31.s, z10.s\n" ".inst 0xc082149d // mova z29.s, p5/M, za1h.s[x12]\n" "fmin z30.s, p5/M, z30.s, z10.s\n" ".inst 0xc082549c // mova z28.s, p5/M, za1h.s[x14]\n" "fmin z29.s, p5/M, z29.s, z10.s\n" ".inst 0xc08274bb // mova z27.s, p5/M, za1h.s[XZR, #1]\n" "fmin z28.s, p5/M, z28.s, z10.s\n" ".inst 0xc08234ba // mova z26.s, p5/M, za1h.s[x13, #1]\n" "fmax z31.s, p5/M, z31.s, z12.s\n" "fmin z27.s, p5/M, z27.s, z10.s\n" ".inst 0xc08214b9 // mova z25.s, p5/M, za1h.s[x12, #1]\n" "fmax z30.s, p5/M, z30.s, z12.s\n" "fmin z26.s, p5/M, z26.s, z10.s\n" ".inst 0xc08254b8 // mova z24.s, p5/M, za1h.s[x14, #1]\n" "fmax z29.s, p5/M, z29.s, z12.s\n" "fmin z25.s, p5/M, z25.s, z10.s\n" ".inst 0xc08274d7 // mova z23.s, p5/M, za1h.s[XZR, #2]\n" "fmax z28.s, p5/M, z28.s, z12.s\n" "fmin z24.s, p5/M, z24.s, z10.s\n" ".inst 0xc08234d6 // mova z22.s, p5/M, za1h.s[x13, #2]\n" "fmax z27.s, p5/M, z27.s, z12.s\n" "fmin z23.s, p5/M, z23.s, z10.s\n" ".inst 0xc08214d5 // mova z21.s, p5/M, za1h.s[x12, #2]\n" "fmax z26.s, p5/M, z26.s, z12.s\n" "fmin z22.s, p5/M, z22.s, z10.s\n" "add x20, %x[output], %x[output_col_stride], LSL #2\n" ".inst 0xc08254d4 // mova z20.s, p5/M, za1h.s[x14, #2]\n" "fmax z25.s, p5/M, z25.s, z12.s\n" "fmin z21.s, p5/M, z21.s, z10.s\n" "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0xc08274f3 // mova z19.s, p5/M, za1h.s[XZR, #3]\n" "fmax z24.s, p5/M, z24.s, z12.s\n" "fmin z20.s, p5/M, z20.s, z10.s\n" ".inst 0xc08234f2 // mova z18.s, p5/M, za1h.s[x13, #3]\n" "fmax z23.s, p5/M, z23.s, z12.s\n" "fmin z19.s, p5/M, z19.s, z10.s\n" ".inst 0xc08214f1 // mova z17.s, p5/M, za1h.s[x12, #3]\n" "fmax z22.s, p5/M, z22.s, z12.s\n" "fmin z18.s, p5/M, z18.s, z10.s\n" ".inst 0xc08254f0 // mova z16.s, p5/M, za1h.s[x14, #3]\n" "fmax z21.s, p5/M, z21.s, z12.s\n" "fmin z17.s, p5/M, z17.s, z10.s\n" "fmax z20.s, p5/M, z20.s, z12.s\n" "fmin z16.s, p5/M, z16.s, z10.s\n" "st1w { z31.s }, p0, [%x[output], x24, LSL #2]\n" "fmax z19.s, p5/M, z19.s, z12.s\n" "st1w { z30.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z18.s, p5/M, z18.s, z12.s\n" "st1w { z29.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z17.s, p5/M, z17.s, z12.s\n" "st1w { z28.s }, p0, [x20, x24, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" "st1w { z27.s }, p0, [x8, x24, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x24, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z23.s }, p0, [x8, x24, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x24, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z19.s }, p0, [x8, x24, LSL #2]\n" "st1w { z18.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x24, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z16.s }, p0, [x20, x24, LSL #2]\n" "whilelt p0.s, x23, %x[n_channels]\n" "b.none 3f\n" ".inst 0xc082751f // mova z31.s, p5/M, za2h.s[XZR]\n" ".inst 0xc082351e // mova z30.s, p5/M, za2h.s[x13]\n" "fmin z31.s, p5/M, z31.s, z10.s\n" ".inst 0xc082151d // mova z29.s, p5/M, za2h.s[x12]\n" "fmin z30.s, p5/M, z30.s, z10.s\n" ".inst 0xc082551c // mova z28.s, p5/M, za2h.s[x14]\n" "fmin z29.s, p5/M, z29.s, z10.s\n" ".inst 0xc082753b // mova z27.s, p5/M, za2h.s[XZR, #1]\n" "fmin z28.s, p5/M, z28.s, z10.s\n" ".inst 0xc082353a // mova z26.s, p5/M, za2h.s[x13, #1]\n" "fmax z31.s, p5/M, z31.s, z12.s\n" "fmin z27.s, p5/M, z27.s, z10.s\n" ".inst 0xc0821539 // mova z25.s, p5/M, za2h.s[x12, #1]\n" "fmax z30.s, p5/M, z30.s, z12.s\n" "fmin z26.s, p5/M, z26.s, z10.s\n" ".inst 0xc0825538 // mova z24.s, p5/M, za2h.s[x14, #1]\n" "fmax z29.s, p5/M, z29.s, z12.s\n" "fmin z25.s, p5/M, z25.s, z10.s\n" ".inst 0xc0827557 // mova z23.s, p5/M, za2h.s[XZR, #2]\n" "fmax z28.s, p5/M, z28.s, z12.s\n" "fmin z24.s, p5/M, z24.s, z10.s\n" ".inst 0xc0823556 // mova z22.s, p5/M, za2h.s[x13, #2]\n" "fmax z27.s, p5/M, z27.s, z12.s\n" "fmin z23.s, p5/M, z23.s, z10.s\n" ".inst 0xc0821555 // mova z21.s, p5/M, za2h.s[x12, #2]\n" "fmax z26.s, p5/M, z26.s, z12.s\n" "fmin z22.s, p5/M, z22.s, z10.s\n" "add x20, %x[output], %x[output_col_stride], LSL #2\n" ".inst 0xc0825554 // mova z20.s, p5/M, za2h.s[x14, #2]\n" "fmax z25.s, p5/M, z25.s, z12.s\n" "fmin z21.s, p5/M, z21.s, z10.s\n" "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0xc0827573 // mova z19.s, p5/M, za2h.s[XZR, #3]\n" "fmax z24.s, p5/M, z24.s, z12.s\n" "fmin z20.s, p5/M, z20.s, z10.s\n" ".inst 0xc0823572 // mova z18.s, p5/M, za2h.s[x13, #3]\n" "fmax z23.s, p5/M, z23.s, z12.s\n" "fmin z19.s, p5/M, z19.s, z10.s\n" ".inst 0xc0821571 // mova z17.s, p5/M, za2h.s[x12, #3]\n" "fmax z22.s, p5/M, z22.s, z12.s\n" "fmin z18.s, p5/M, z18.s, z10.s\n" ".inst 0xc0825570 // mova z16.s, p5/M, za2h.s[x14, #3]\n" "fmax z21.s, p5/M, z21.s, z12.s\n" "fmin z17.s, p5/M, z17.s, z10.s\n" "fmax z20.s, p5/M, z20.s, z12.s\n" "fmin z16.s, p5/M, z16.s, z10.s\n" "st1w { z31.s }, p0, [%x[output], x23, LSL #2]\n" "fmax z19.s, p5/M, z19.s, z12.s\n" "st1w { z30.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z18.s, p5/M, z18.s, z12.s\n" "st1w { z29.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z17.s, p5/M, z17.s, z12.s\n" "st1w { z28.s }, p0, [x20, x23, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" "st1w { z27.s }, p0, [x8, x23, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x23, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z23.s }, p0, [x8, x23, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x23, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z19.s }, p0, [x8, x23, LSL #2]\n" "st1w { z18.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x23, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z16.s }, p0, [x20, x23, LSL #2]\n" "whilelt p0.s, x22, %x[n_channels]\n" "b.none 3f\n" "fmov z1.s, #1.0\n" ".inst 0xc082759f // mova z31.s, p5/M, za3h.s[XZR]\n" ".inst 0xc082359e // mova z30.s, p5/M, za3h.s[x13]\n" "fmin z31.s, p5/M, z31.s, z10.s\n" ".inst 0xc082159d // mova z29.s, p5/M, za3h.s[x12]\n" "fmin z30.s, p5/M, z30.s, z10.s\n" ".inst 0xc082559c // mova z28.s, p5/M, za3h.s[x14]\n" "fmin z29.s, p5/M, z29.s, z10.s\n" ".inst 0xc08275bb // mova z27.s, p5/M, za3h.s[XZR, #1]\n" "fmin z28.s, p5/M, z28.s, z10.s\n" ".inst 0xc08235ba // mova z26.s, p5/M, za3h.s[x13, #1]\n" "fmax z31.s, p5/M, z31.s, z12.s\n" "fmin z27.s, p5/M, z27.s, z10.s\n" ".inst 0xc08215b9 // mova z25.s, p5/M, za3h.s[x12, #1]\n" "fmax z30.s, p5/M, z30.s, z12.s\n" "fmin z26.s, p5/M, z26.s, z10.s\n" ".inst 0xc08255b8 // mova z24.s, p5/M, za3h.s[x14, #1]\n" "fmax z29.s, p5/M, z29.s, z12.s\n" "fmin z25.s, p5/M, z25.s, z10.s\n" ".inst 0xc08275d7 // mova z23.s, p5/M, za3h.s[XZR, #2]\n" "fmax z28.s, p5/M, z28.s, z12.s\n" "fmin z24.s, p5/M, z24.s, z10.s\n" ".inst 0xc08235d6 // mova z22.s, p5/M, za3h.s[x13, #2]\n" "fmax z27.s, p5/M, z27.s, z12.s\n" "fmin z23.s, p5/M, z23.s, z10.s\n" ".inst 0xc08215d5 // mova z21.s, p5/M, za3h.s[x12, #2]\n" "fmax z26.s, p5/M, z26.s, z12.s\n" "fmin z22.s, p5/M, z22.s, z10.s\n" ".inst 0xc08255d4 // mova z20.s, p5/M, za3h.s[x14, #2]\n" "fmax z25.s, p5/M, z25.s, z12.s\n" "fmin z21.s, p5/M, z21.s, z10.s\n" "add x20, %x[output], %x[output_col_stride], LSL #2\n" ".inst 0xc08275f3 // mova z19.s, p5/M, za3h.s[XZR, #3]\n" "fmax z24.s, p5/M, z24.s, z12.s\n" "fmin z20.s, p5/M, z20.s, z10.s\n" "add x8, %x[output], %x[output_row_stride], LSL #2\n" ".inst 0xc08235f2 // mova z18.s, p5/M, za3h.s[x13, #3]\n" "fmax z23.s, p5/M, z23.s, z12.s\n" "fmin z19.s, p5/M, z19.s, z10.s\n" "incw x25, ALL, MUL #4\n" ".inst 0xc08215f1 // mova z17.s, p5/M, za3h.s[x12, #3]\n" "fmax z22.s, p5/M, z22.s, z12.s\n" "fmin z18.s, p5/M, z18.s, z10.s\n" "incw x24, ALL, MUL #4\n" ".inst 0xc08255f0 // mova z16.s, p5/M, za3h.s[x14, #3]\n" "fmax z21.s, p5/M, z21.s, z12.s\n" "fmin z17.s, p5/M, z17.s, z10.s\n" "incw x23, ALL, MUL #4\n" ".inst 0xc00800ff // zero { zad0, zad1, zad2, zad3, zad4, zad5, zad6, zad7 }\n" "fmax z20.s, p5/M, z20.s, z12.s\n" "fmin z16.s, p5/M, z16.s, z10.s\n" "add x21, %x[inptr], %x[matrix_stride], LSL #2\n" "fmax z19.s, p5/M, z19.s, z12.s\n" "st1w { z31.s }, p0, [%x[output], x22, LSL #2]\n" "fmax z18.s, p5/M, z18.s, z12.s\n" "st1w { z30.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z17.s, p5/M, z17.s, z12.s\n" "st1w { z29.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "fmax z16.s, p5/M, z16.s, z12.s\n" "st1w { z28.s }, p0, [x20, x22, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z27.s }, p0, [x8, x22, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z26.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z25.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z24.s }, p0, [x20, x22, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z23.s }, p0, [x8, x22, LSL #2]\n" "add x8, x8, %x[output_row_stride], LSL #2\n" "st1w { z22.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z21.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z20.s }, p0, [x20, x22, LSL #2]\n" "add x20, x8, %x[output_col_stride], LSL #2\n" "st1w { z19.s }, p0, [x8, x22, LSL #2]\n" "st1w { z18.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z17.s }, p0, [x20, x22, LSL #2]\n" "add x20, x20, %x[output_col_stride], LSL #2\n" "st1w { z16.s }, p0, [x20, x22, LSL #2]\n" "incw x22, ALL, MUL #4\n" "whilelt p1.s, x22, %x[n_channels]\n" "ld1w { z28.s }, p1/Z, [%x[inptr], x22, LSL #2]\n" "ld1w { z24.s }, p1/Z, [x21, x22, LSL #2]\n" "whilelt p2.s, x23, %x[n_channels]\n" "whilelt p3.s, x24, %x[n_channels]\n" "ld1w { z30.s }, p3/Z, [%x[inptr], x24, LSL #2]\n" "whilelt p4.s, x25, %x[n_channels]\n" "ld1w { z31.s }, p4/Z, [%x[inptr], x25, LSL #2]\n" "and p0.b, p5/Z, p8.b, p4.b\n" "ld1w { z29.s }, p2/Z, [%x[inptr], x23, LSL #2]\n" "ld1w { z27.s }, p4/Z, [x21, x25, LSL #2]\n" "ld1w { z26.s }, p3/Z, [x21, x24, LSL #2]\n" "ld1w { z25.s }, p2/Z, [x21, x23, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" "ld1w { z23.s }, p4/Z, [x21, x25, LSL #2]\n" "ld1w { z22.s }, p3/Z, [x21, x24, LSL #2]\n" "ld1w { z21.s }, p2/Z, [x21, x23, LSL #2]\n" "ld1w { z20.s }, p1/Z, [x21, x22, LSL #2]\n" "add x21, x21, %x[matrix_stride], LSL #2\n" "ld1w { z19.s }, p4/Z, [x21, x25, LSL #2]\n" "ld1w { z18.s }, p3/Z, [x21, x24, LSL #2]\n" "ld1w { z17.s }, p2/Z, [x21, x23, LSL #2]\n" "ld1w { z16.s }, p1/Z, [x21, x22, LSL #2]\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x25, LSL #2]\n" "and p0.b, p5/Z, p8.b, p3.b\n" ".inst 0x8080b420 // fmopa za0.s, p5/M, p5/M, z1.s, z0.s\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x24, LSL #2]\n" "and p0.b, p5/Z, p8.b, p2.b\n" ".inst 0x8080b421 // fmopa za1.s, p5/M, p5/M, z1.s, z0.s\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x23, LSL #2]\n" "and p0.b, p5/Z, p8.b, p1.b\n" ".inst 0x8080b422 // fmopa za2.s, p5/M, p5/M, z1.s, z0.s\n" "ld1w { z0.s }, p0/Z, [%x[bptr], x22, LSL #2]\n" ".inst 0x8080b423 // fmopa za3.s, p5/M, p5/M, z1.s, z0.s\n" "b.any 2b\n" "3:" // End ".inst 0xd503467f // SMSTOP\n" : : [bptr] "r" (bptr), [inptr] "r" (inptr), [matrix_stride] "r" (matrix_stride), [n_channels] "r" (n_channels), [offsetof_Params_act_max] "I" (offsetof(Params, act_max)), [offsetof_Params_act_min] "I" (offsetof(Params, act_min)), [offsetof_Params_inner_terms] "I" (offsetof(Params, inner_terms)), [offsetof_Params_outer_terms] "I" (offsetof(Params, outer_terms)), [output] "r" (output), [output_col_stride] "r" (output_col_stride), [output_row_stride] "r" (output_row_stride), [params] "r" (¶ms) : "cc", "memory", "p0", "p1", "p2", "p3", "p4", "p5", "p8", "x12", "x13", "x14", "x8", "x20", "x21", "x22", "x23", "x24", "x25", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace output_transform } // namespace winograd } // namespace arm_conv #endif // defined(ARM_COMPUTE_ENABLE_SME)