/* * Copyright (c) 2018-2019 Arm Limited. * * SPDX-License-Identifier: MIT * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to * deal in the Software without restriction, including without limitation the * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or * sell copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in all * copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. */ #include "impl_fp32_fp32.hpp" namespace depthwise { using namespace neon_convolution_kernels; using Conv = DepthwiseConvolution<2, 2, 3, 3, 2, 2, float, float, float>; #ifdef __aarch64__ template <> template <> void Conv::execute_tile( int n_channels, const void *weight_bias_ptr, const float *input, const unsigned int input_row_stride, const unsigned int input_col_stride, float *output, const unsigned int output_row_stride, const unsigned int output_col_stride ) { __asm __volatile( "add x23, %[inptr0], %[input_row_stride]\n" "add x19, %[input_col_stride1], %[input_col_stride1]\n" "add x22, %[outptr0], %[output_row_stride]\n" "add x24, x23, %[input_row_stride]\n" "add x20, x19, %[input_col_stride1]\n" "and x27, %[n_channels], #3\n" "add x25, x24, %[input_row_stride]\n" "add x21, x20, %[input_col_stride1]\n" "lsr x28, %[n_channels], #2\n" "add x26, x25, %[input_row_stride]\n" "cbz x28, 4f\n" "1:\n" "ldr q14, [%[wbptr]]\n" "subs x28, x28, #1\n" "mov v12.16b, v14.16b\n" "ldr q8, [%[wbptr], #16]\n" "mov v10.16b, v14.16b\n" "ldr q7, [%[wbptr], #32]\n" "mov v11.16b, v14.16b\n" "ldr q6, [%[wbptr], #48]\n" "mov v9.16b, v14.16b\n" "ldr q5, [%[wbptr], #64]\n" "ldr q4, [%[wbptr], #80]\n" "ldr q3, [%[wbptr], #96]\n" "ldr q2, [%[wbptr], #112]\n" "ldr q1, [%[wbptr], #128]\n" "ldr q0, [%[wbptr], #144]\n" "ldr q15, [%[inptr0]]\n" "fmla v12.4s, v15.4s, v8.4s\n" "ldr q20, [x23]\n" "ldr q13, [%[inptr0], %[input_col_stride1]]\n" "ldr q17, [x24]\n" "fmla v10.4s, v17.4s, v8.4s\n" "ldr q16, [x23, %[input_col_stride1]]\n" "fmla v12.4s, v20.4s, v5.4s\n" "ldr q18, [%[inptr0], x19]\n" "ldr q14, [x25]\n" "ldr q15, [x24, %[input_col_stride1]]\n" "fmla v12.4s, v13.4s, v7.4s\n" "fmla v12.4s, v17.4s, v2.4s\n" "fmla v12.4s, v16.4s, v4.4s\n" "fmla v12.4s, v18.4s, v6.4s\n" "beq 3f\n" "2:\n" "fmla v11.4s, v18.4s, v8.4s\n" "ldr q19, [x23, x19]\n" "fmla v10.4s, v14.4s, v5.4s\n" "ldr q20, [%[inptr0], x20]\n" "fmla v12.4s, v15.4s, v1.4s\n" "ldr q14, [x26]\n" "fmla v11.4s, v19.4s, v5.4s\n" "ldr q13, [x25, %[input_col_stride1]]\n" "fmla v10.4s, v15.4s, v7.4s\n" "ldr q17, [x24, x19]\n" "fmla v12.4s, v19.4s, v3.4s\n" "ldr q19, [x23, x20]\n" "fmla v11.4s, v20.4s, v7.4s\n" "ldr q18, [%[inptr0], x21]\n" "fmla v10.4s, v14.4s, v2.4s\n" "ldr q16, [x26, %[input_col_stride1]]\n" "fmla v12.4s, v17.4s, v0.4s\n" "ldr q14, [x25, x19]\n" "fmla v11.4s, v17.4s, v2.4s\n" "ldr q15, [x24, x20]\n" "fmla v10.4s, v13.4s, v4.4s\n" "ldr q13, [x23, x21]\n" "str q12, [%[outptr0]]\n" "fmla v9.4s, v17.4s, v8.4s\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr q12, [x26, x19]\n" "fmla v10.4s, v17.4s, v6.4s\n" "ldr q20, [x25, x20]\n" "fmla v9.4s, v14.4s, v5.4s\n" "ldr q17, [x24, x21]\n" "fmla v11.4s, v18.4s, v6.4s\n" "ldr q19, [x26, x20]\n" "fmla v10.4s, v16.4s, v1.4s\n" "ldr q18, [x25, x21]\n" "fmla v9.4s, v15.4s, v7.4s\n" "ldr q16, [x26, x21]\n" "fmla v11.4s, v15.4s, v1.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v10.4s, v14.4s, v3.4s\n" "ldr q14, [%[wbptr]]\n" "fmla v9.4s, v12.4s, v2.4s\n" "ldr q8, [%[wbptr], #16]\n" "fmla v11.4s, v13.4s, v3.4s\n" "ldr q7, [%[wbptr], #32]\n" "fmla v10.4s, v12.4s, v0.4s\n" "ldr q5, [%[wbptr], #64]\n" "fmla v9.4s, v20.4s, v4.4s\n" "ldr q2, [%[wbptr], #112]\n" "fmla v11.4s, v17.4s, v0.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "str q10, [x22]\n" "mov v12.16b, v14.16b\n" "fmla v9.4s, v17.4s, v6.4s\n" "ldr q4, [%[wbptr], #80]\n" "str q11, [%[outptr0], %[output_col_stride1]]\n" "mov v10.16b, v14.16b\n" "mov v11.16b, v14.16b\n" "add %[inptr0], %[inptr0], #16\n" "fmla v9.4s, v19.4s, v1.4s\n" "ldr q6, [%[wbptr], #48]\n" "ldr q15, [%[inptr0]]\n" "add x23, x23, #16\n" "fmla v12.4s, v15.4s, v8.4s\n" "ldr q20, [x23]\n" "fmla v9.4s, v18.4s, v3.4s\n" "ldr q1, [%[wbptr], #128]\n" "ldr q13, [%[inptr0], %[input_col_stride1]]\n" "add x24, x24, #16\n" "fmla v12.4s, v20.4s, v5.4s\n" "ldr q17, [x24]\n" "fmla v9.4s, v16.4s, v0.4s\n" "ldr q3, [%[wbptr], #96]\n" "fmla v10.4s, v17.4s, v8.4s\n" "ldr q16, [x23, %[input_col_stride1]]\n" "fmla v12.4s, v13.4s, v7.4s\n" "ldr q18, [%[inptr0], x19]\n" "str q9, [x22, %[output_col_stride1]]\n" "add x25, x25, #16\n" "mov v9.16b, v14.16b\n" "ldr q0, [%[wbptr], #144]\n" "fmla v12.4s, v17.4s, v2.4s\n" "ldr q14, [x25]\n" "ldr q15, [x24, %[input_col_stride1]]\n" "add x26, x26, #16\n" "add %[outptr0], %[outptr0], #16\n" "add x22, x22, #16\n" "subs x28, x28, #1\n" "fmla v12.4s, v16.4s, v4.4s\n" "fmla v12.4s, v18.4s, v6.4s\n" "bne 2b\n" "3:\n" "fmla v11.4s, v18.4s, v8.4s\n" "ldr q19, [x23, x19]\n" "fmla v10.4s, v14.4s, v5.4s\n" "ldr q20, [%[inptr0], x20]\n" "fmla v12.4s, v15.4s, v1.4s\n" "ldr q14, [x26]\n" "fmla v11.4s, v19.4s, v5.4s\n" "ldr q13, [x25, %[input_col_stride1]]\n" "fmla v10.4s, v15.4s, v7.4s\n" "ldr q17, [x24, x19]\n" "fmla v12.4s, v19.4s, v3.4s\n" "ldr q19, [x23, x20]\n" "fmla v11.4s, v20.4s, v7.4s\n" "ldr q18, [%[inptr0], x21]\n" "fmla v10.4s, v14.4s, v2.4s\n" "ldr q16, [x26, %[input_col_stride1]]\n" "fmla v12.4s, v17.4s, v0.4s\n" "ldr q14, [x25, x19]\n" "fmla v11.4s, v17.4s, v2.4s\n" "ldr q15, [x24, x20]\n" "fmla v10.4s, v13.4s, v4.4s\n" "ldr q13, [x23, x21]\n" "str q12, [%[outptr0]]\n" "fmla v9.4s, v17.4s, v8.4s\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr q12, [x26, x19]\n" "fmla v10.4s, v17.4s, v6.4s\n" "ldr q20, [x25, x20]\n" "fmla v9.4s, v14.4s, v5.4s\n" "ldr q17, [x24, x21]\n" "fmla v11.4s, v18.4s, v6.4s\n" "ldr q19, [x26, x20]\n" "fmla v10.4s, v16.4s, v1.4s\n" "ldr q18, [x25, x21]\n" "fmla v9.4s, v15.4s, v7.4s\n" "ldr q16, [x26, x21]\n" "fmla v11.4s, v15.4s, v1.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v10.4s, v14.4s, v3.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v9.4s, v12.4s, v2.4s\n" "add %[inptr0], %[inptr0], #16\n" "fmla v11.4s, v13.4s, v3.4s\n" "add x23, x23, #16\n" "fmla v10.4s, v12.4s, v0.4s\n" "add x24, x24, #16\n" "fmla v9.4s, v20.4s, v4.4s\n" "add x25, x25, #16\n" "fmla v11.4s, v17.4s, v0.4s\n" "add x26, x26, #16\n" "str q10, [x22]\n" "fmla v9.4s, v17.4s, v6.4s\n" "str q11, [%[outptr0], %[output_col_stride1]]\n" "add %[outptr0], %[outptr0], #16\n" "fmla v9.4s, v19.4s, v1.4s\n" "fmla v9.4s, v18.4s, v3.4s\n" "fmla v9.4s, v16.4s, v0.4s\n" "str q9, [x22, %[output_col_stride1]]\n" "add x22, x22, #16\n" "4:\n" "cbz x27, 7f\n" "ldr s14, [%[wbptr]]\n" "mov v12.16b, v14.16b\n" "ldr s8, [%[wbptr], #4]\n" "mov v10.16b, v14.16b\n" "ldr s7, [%[wbptr], #8]\n" "mov v11.16b, v14.16b\n" "ldr s6, [%[wbptr], #12]\n" "mov v9.16b, v14.16b\n" "ldr s5, [%[wbptr], #16]\n" "ldr s4, [%[wbptr], #20]\n" "subs x27, x27, #1\n" "ldr s3, [%[wbptr], #24]\n" "ldr s2, [%[wbptr], #28]\n" "ldr s1, [%[wbptr], #32]\n" "ldr s0, [%[wbptr], #36]\n" "ldr s15, [%[inptr0]]\n" "ldr s20, [x23]\n" "fmla v12.4s, v15.4s, v8.4s\n" "ldr s13, [%[inptr0], %[input_col_stride1]]\n" "ldr s17, [x24]\n" "ldr s16, [x23, %[input_col_stride1]]\n" "fmla v10.4s, v17.4s, v8.4s\n" "ldr s18, [%[inptr0], x19]\n" "fmla v12.4s, v20.4s, v5.4s\n" "ldr s14, [x25]\n" "ldr s15, [x24, %[input_col_stride1]]\n" "fmla v12.4s, v13.4s, v7.4s\n" "fmla v12.4s, v17.4s, v2.4s\n" "fmla v12.4s, v16.4s, v4.4s\n" "fmla v12.4s, v18.4s, v6.4s\n" "beq 6f\n" "5:\n" "fmla v11.4s, v18.4s, v8.4s\n" "ldr s19, [x23, x19]\n" "fmla v10.4s, v14.4s, v5.4s\n" "ldr s20, [%[inptr0], x20]\n" "fmla v12.4s, v15.4s, v1.4s\n" "ldr s14, [x26]\n" "fmla v11.4s, v19.4s, v5.4s\n" "ldr s13, [x25, %[input_col_stride1]]\n" "fmla v10.4s, v15.4s, v7.4s\n" "ldr s17, [x24, x19]\n" "fmla v12.4s, v19.4s, v3.4s\n" "ldr s19, [x23, x20]\n" "fmla v11.4s, v20.4s, v7.4s\n" "ldr s18, [%[inptr0], x21]\n" "fmla v10.4s, v14.4s, v2.4s\n" "ldr s16, [x26, %[input_col_stride1]]\n" "fmla v12.4s, v17.4s, v0.4s\n" "ldr s14, [x25, x19]\n" "fmla v11.4s, v17.4s, v2.4s\n" "ldr s15, [x24, x20]\n" "fmla v10.4s, v13.4s, v4.4s\n" "ldr s13, [x23, x21]\n" "str s12, [%[outptr0]]\n" "fmla v9.4s, v17.4s, v8.4s\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr s12, [x26, x19]\n" "fmla v10.4s, v17.4s, v6.4s\n" "ldr s20, [x25, x20]\n" "fmla v9.4s, v14.4s, v5.4s\n" "ldr s17, [x24, x21]\n" "fmla v11.4s, v18.4s, v6.4s\n" "ldr s19, [x26, x20]\n" "fmla v10.4s, v16.4s, v1.4s\n" "ldr s18, [x25, x21]\n" "fmla v9.4s, v15.4s, v7.4s\n" "ldr s16, [x26, x21]\n" "fmla v11.4s, v15.4s, v1.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v10.4s, v14.4s, v3.4s\n" "ldr s14, [%[wbptr]]\n" "fmla v9.4s, v12.4s, v2.4s\n" "ldr s8, [%[wbptr], #4]\n" "fmla v11.4s, v13.4s, v3.4s\n" "ldr s7, [%[wbptr], #8]\n" "fmla v10.4s, v12.4s, v0.4s\n" "ldr s5, [%[wbptr], #16]\n" "fmla v9.4s, v20.4s, v4.4s\n" "ldr s2, [%[wbptr], #28]\n" "fmla v11.4s, v17.4s, v0.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "str s10, [x22]\n" "mov v12.16b, v14.16b\n" "fmla v9.4s, v17.4s, v6.4s\n" "ldr s4, [%[wbptr], #20]\n" "str s11, [%[outptr0], %[output_col_stride1]]\n" "mov v10.16b, v14.16b\n" "mov v11.16b, v14.16b\n" "add %[inptr0], %[inptr0], #4\n" "fmla v9.4s, v19.4s, v1.4s\n" "ldr s6, [%[wbptr], #12]\n" "ldr s15, [%[inptr0]]\n" "add x23, x23, #4\n" "fmla v12.4s, v15.4s, v8.4s\n" "ldr s20, [x23]\n" "fmla v9.4s, v18.4s, v3.4s\n" "ldr s1, [%[wbptr], #32]\n" "ldr s13, [%[inptr0], %[input_col_stride1]]\n" "add x24, x24, #4\n" "fmla v12.4s, v20.4s, v5.4s\n" "ldr s17, [x24]\n" "fmla v9.4s, v16.4s, v0.4s\n" "ldr s3, [%[wbptr], #24]\n" "fmla v10.4s, v17.4s, v8.4s\n" "ldr s16, [x23, %[input_col_stride1]]\n" "fmla v12.4s, v13.4s, v7.4s\n" "ldr s18, [%[inptr0], x19]\n" "str s9, [x22, %[output_col_stride1]]\n" "add x25, x25, #4\n" "mov v9.16b, v14.16b\n" "ldr s0, [%[wbptr], #36]\n" "fmla v12.4s, v17.4s, v2.4s\n" "ldr s14, [x25]\n" "ldr s15, [x24, %[input_col_stride1]]\n" "add x26, x26, #4\n" "add %[outptr0], %[outptr0], #4\n" "add x22, x22, #4\n" "subs x27, x27, #1\n" "fmla v12.4s, v16.4s, v4.4s\n" "fmla v12.4s, v18.4s, v6.4s\n" "bne 5b\n" "6:\n" "fmla v11.4s, v18.4s, v8.4s\n" "ldr s19, [x23, x19]\n" "fmla v10.4s, v14.4s, v5.4s\n" "ldr s20, [%[inptr0], x20]\n" "fmla v12.4s, v15.4s, v1.4s\n" "ldr s14, [x26]\n" "fmla v11.4s, v19.4s, v5.4s\n" "ldr s13, [x25, %[input_col_stride1]]\n" "fmla v10.4s, v15.4s, v7.4s\n" "ldr s17, [x24, x19]\n" "fmla v12.4s, v19.4s, v3.4s\n" "ldr s19, [x23, x20]\n" "fmla v11.4s, v20.4s, v7.4s\n" "ldr s18, [%[inptr0], x21]\n" "fmla v10.4s, v14.4s, v2.4s\n" "ldr s16, [x26, %[input_col_stride1]]\n" "fmla v12.4s, v17.4s, v0.4s\n" "ldr s14, [x25, x19]\n" "fmla v11.4s, v17.4s, v2.4s\n" "ldr s15, [x24, x20]\n" "fmla v10.4s, v13.4s, v4.4s\n" "ldr s13, [x23, x21]\n" "str s12, [%[outptr0]]\n" "fmla v9.4s, v17.4s, v8.4s\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr s12, [x26, x19]\n" "fmla v10.4s, v17.4s, v6.4s\n" "ldr s20, [x25, x20]\n" "fmla v9.4s, v14.4s, v5.4s\n" "ldr s17, [x24, x21]\n" "fmla v11.4s, v18.4s, v6.4s\n" "ldr s19, [x26, x20]\n" "fmla v10.4s, v16.4s, v1.4s\n" "ldr s18, [x25, x21]\n" "fmla v9.4s, v15.4s, v7.4s\n" "ldr s16, [x26, x21]\n" "fmla v11.4s, v15.4s, v1.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v10.4s, v14.4s, v3.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v9.4s, v12.4s, v2.4s\n" "add %[inptr0], %[inptr0], #4\n" "fmla v11.4s, v13.4s, v3.4s\n" "add x23, x23, #4\n" "fmla v10.4s, v12.4s, v0.4s\n" "add x24, x24, #4\n" "fmla v9.4s, v20.4s, v4.4s\n" "add x25, x25, #4\n" "fmla v11.4s, v17.4s, v0.4s\n" "add x26, x26, #4\n" "str s10, [x22]\n" "fmla v9.4s, v17.4s, v6.4s\n" "str s11, [%[outptr0], %[output_col_stride1]]\n" "add %[outptr0], %[outptr0], #4\n" "fmla v9.4s, v19.4s, v1.4s\n" "fmla v9.4s, v18.4s, v3.4s\n" "fmla v9.4s, v16.4s, v0.4s\n" "str s9, [x22, %[output_col_stride1]]\n" "add x22, x22, #4\n" "7:\n" : [outptr0] "+r" (output), [inptr0] "+r" (input), [wbptr] "+r" (weight_bias_ptr) : [n_channels] "r" ((long) n_channels), [output_row_stride] "r" (output_row_stride * sizeof(float)), [output_col_stride1] "r" (output_col_stride * sizeof(float)), [input_row_stride] "r" (input_row_stride * sizeof(float)), [input_col_stride1] "r" (input_col_stride * sizeof(float)) : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" ); } template <> template <> void Conv::execute_tile( int n_channels, const void *weight_bias_ptr, const float *inptrs[Base::inner_tile_rows][Base::inner_tile_cols], float *outptrs[Base::output_tile_rows][Base::output_tile_cols] ) { __asm __volatile( "mov x23, xzr\n" "mov x24, xzr\n" "and x25, %[n_channels], #3\n" "lsr x26, %[n_channels], #2\n" "cbz x26, 4f\n" "1:\n" "ldr q13, [%[wbptr]]\n" "ldr x19, [%[inptrs], 0]\n" "mov v10.16b, v13.16b\n" "ldr q12, [%[wbptr], #16]\n" "mov v8.16b, v13.16b\n" "ldr q6, [%[wbptr], #32]\n" "mov v9.16b, v13.16b\n" "ldr q5, [%[wbptr], #48]\n" "mov v7.16b, v13.16b\n" "ldr q11, [%[wbptr], #64]\n" "ldr q4, [%[wbptr], #80]\n" "ldr x20, [%[inptrs], 40]\n" "ldr q3, [%[wbptr], #96]\n" "ldr x21, [%[inptrs], 80]\n" "ldr q2, [%[wbptr], #112]\n" "ldr x27, [%[inptrs], 120]\n" "ldr q1, [%[wbptr], #128]\n" "subs x26, x26, #1\n" "ldr q0, [%[wbptr], #144]\n" "ldr q14, [x19, x23]\n" "fmla v10.4s, v14.4s, v12.4s\n" "ldr q18, [x20, x23]\n" "ldr q14, [x21, x23]\n" "ldr x19, [%[inptrs], 8]\n" "ldr q16, [x27, x23]\n" "ldr x20, [%[inptrs], 48]\n" "ldr q19, [x19, x23]\n" "ldr x21, [%[inptrs], 88]\n" "fmla v10.4s, v18.4s, v11.4s\n" "ldr q15, [x20, x23]\n" "ldr q18, [x21, x23]\n" "ldr x19, [%[inptrs], 16]\n" "ldr q13, [x19, x23]\n" "fmla v10.4s, v19.4s, v6.4s\n" "fmla v10.4s, v14.4s, v2.4s\n" "beq 3f\n" "2:\n" "fmla v8.4s, v14.4s, v12.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v10.4s, v15.4s, v4.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v9.4s, v13.4s, v12.4s\n" "ldr q14, [x20, x23]\n" "ldr q17, [x19, x23]\n" "ldr x22, [%[inptrs], 160]\n" "fmla v8.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 128]\n" "fmla v10.4s, v13.4s, v5.4s\n" "ldr q15, [x22, x23]\n" "fmla v9.4s, v14.4s, v11.4s\n" "ldr q19, [x27, x23]\n" "ldr x21, [%[inptrs], 96]\n" "ldr x20, [%[inptrs], 64]\n" "ldr x19, [%[inptrs], 32]\n" "fmla v8.4s, v18.4s, v6.4s\n" "ldr x22, [%[inptrs], 168]\n" "fmla v10.4s, v18.4s, v1.4s\n" "ldr q13, [x21, x23]\n" "fmla v9.4s, v17.4s, v6.4s\n" "ldr q18, [x20, x23]\n" "fmla v7.4s, v13.4s, v12.4s\n" "ldr q17, [x19, x23]\n" "fmla v8.4s, v15.4s, v2.4s\n" "ldr q15, [x22, x23]\n" "fmla v10.4s, v14.4s, v3.4s\n" "ldr x27, [%[inptrs], 136]\n" "fmla v9.4s, v13.4s, v2.4s\n" "ldr x21, [%[inptrs], 104]\n" "ldr q16, [x27, x23]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v8.4s, v19.4s, v4.4s\n" "ldr q19, [x21, x23]\n" "fmla v10.4s, v13.4s, v0.4s\n" "ldr q12, [x20, x23]\n" "fmla v9.4s, v18.4s, v4.4s\n" "ldr x22, [%[inptrs], 176]\n" "fmla v7.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 144]\n" "fmla v8.4s, v13.4s, v5.4s\n" "ldr q11, [x22, x23]\n" "ldr q13, [x27, x23]\n" "ldr x21, [%[inptrs], 112]\n" "fmla v9.4s, v17.4s, v5.4s\n" "ldr x22, [%[inptrs], 184]\n" "fmla v7.4s, v19.4s, v6.4s\n" "ldr q14, [x21, x23]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr q17, [x22, x23]\n" "ldr x27, [%[inptrs], 152]\n" "ldr x22, [%[inptrs], 192]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v9.4s, v19.4s, v1.4s\n" "ldr x28, [%[outptrs], 16]\n" "str q10, [x21, x24]\n" "fmla v7.4s, v11.4s, v2.4s\n" "fmla v8.4s, v16.4s, v3.4s\n" "ldr q16, [x27, x23]\n" "ldr q15, [x22, x23]\n" "ldr x21, [%[outptrs], 8]\n" "fmla v9.4s, v12.4s, v3.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v7.4s, v13.4s, v4.4s\n" "ldr q13, [%[wbptr]]\n" "fmla v8.4s, v11.4s, v0.4s\n" "ldr q12, [%[wbptr], #16]\n" "mov v10.16b, v13.16b\n" "ldr q6, [%[wbptr], #32]\n" "fmla v9.4s, v14.4s, v0.4s\n" "ldr q11, [%[wbptr], #64]\n" "fmla v7.4s, v14.4s, v5.4s\n" "ldr q4, [%[wbptr], #80]\n" "str q8, [x28, x24]\n" "add x23, x23, #16\n" "mov v8.16b, v13.16b\n" "ldr q2, [%[wbptr], #112]\n" "str q9, [x21, x24]\n" "ldr x28, [%[outptrs], 24]\n" "fmla v7.4s, v17.4s, v1.4s\n" "ldr q5, [%[wbptr], #48]\n" "mov v9.16b, v13.16b\n" "prfm pldl1keep, [%[wbptr], #64]\n" "ldr x19, [%[inptrs], 0]\n" "ldr x20, [%[inptrs], 40]\n" "ldr x21, [%[inptrs], 80]\n" "ldr x27, [%[inptrs], 120]\n" "subs x26, x26, #1\n" "fmla v7.4s, v16.4s, v3.4s\n" "ldr q1, [%[wbptr], #128]\n" "ldr q14, [x19, x23]\n" "fmla v10.4s, v14.4s, v12.4s\n" "ldr q18, [x20, x23]\n" "ldr q14, [x21, x23]\n" "ldr x19, [%[inptrs], 8]\n" "fmla v7.4s, v15.4s, v0.4s\n" "ldr q3, [%[wbptr], #96]\n" "ldr q19, [x19, x23]\n" "ldr x20, [%[inptrs], 48]\n" "fmla v10.4s, v18.4s, v11.4s\n" "ldr q16, [x27, x23]\n" "ldr q15, [x20, x23]\n" "ldr x19, [%[inptrs], 16]\n" "str q7, [x28, x24]\n" "ldr x21, [%[inptrs], 88]\n" "mov v7.16b, v13.16b\n" "ldr q0, [%[wbptr], #144]\n" "fmla v10.4s, v19.4s, v6.4s\n" "ldr q13, [x19, x23]\n" "ldr q18, [x21, x23]\n" "add x24, x24, #16\n" "fmla v10.4s, v14.4s, v2.4s\n" "bne 2b\n" "3:\n" "fmla v8.4s, v14.4s, v12.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v10.4s, v15.4s, v4.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v9.4s, v13.4s, v12.4s\n" "ldr q14, [x20, x23]\n" "ldr q17, [x19, x23]\n" "ldr x22, [%[inptrs], 160]\n" "fmla v8.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 128]\n" "fmla v10.4s, v13.4s, v5.4s\n" "ldr q15, [x22, x23]\n" "fmla v9.4s, v14.4s, v11.4s\n" "ldr q19, [x27, x23]\n" "ldr x21, [%[inptrs], 96]\n" "ldr x20, [%[inptrs], 64]\n" "ldr x19, [%[inptrs], 32]\n" "fmla v8.4s, v18.4s, v6.4s\n" "ldr x22, [%[inptrs], 168]\n" "fmla v10.4s, v18.4s, v1.4s\n" "ldr q13, [x21, x23]\n" "fmla v9.4s, v17.4s, v6.4s\n" "ldr q18, [x20, x23]\n" "fmla v7.4s, v13.4s, v12.4s\n" "ldr q17, [x19, x23]\n" "fmla v8.4s, v15.4s, v2.4s\n" "ldr q15, [x22, x23]\n" "fmla v10.4s, v14.4s, v3.4s\n" "ldr x27, [%[inptrs], 136]\n" "fmla v9.4s, v13.4s, v2.4s\n" "ldr x21, [%[inptrs], 104]\n" "ldr q16, [x27, x23]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v8.4s, v19.4s, v4.4s\n" "ldr q19, [x21, x23]\n" "fmla v10.4s, v13.4s, v0.4s\n" "ldr q12, [x20, x23]\n" "fmla v9.4s, v18.4s, v4.4s\n" "ldr x22, [%[inptrs], 176]\n" "fmla v7.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 144]\n" "fmla v8.4s, v13.4s, v5.4s\n" "ldr q11, [x22, x23]\n" "ldr q13, [x27, x23]\n" "ldr x21, [%[inptrs], 112]\n" "fmla v9.4s, v17.4s, v5.4s\n" "ldr x22, [%[inptrs], 184]\n" "fmla v7.4s, v19.4s, v6.4s\n" "ldr q14, [x21, x23]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr q17, [x22, x23]\n" "ldr x27, [%[inptrs], 152]\n" "ldr x22, [%[inptrs], 192]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v9.4s, v19.4s, v1.4s\n" "ldr x28, [%[outptrs], 16]\n" "str q10, [x21, x24]\n" "fmla v7.4s, v11.4s, v2.4s\n" "fmla v8.4s, v16.4s, v3.4s\n" "ldr q16, [x27, x23]\n" "ldr q15, [x22, x23]\n" "ldr x21, [%[outptrs], 8]\n" "fmla v9.4s, v12.4s, v3.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v7.4s, v13.4s, v4.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v8.4s, v11.4s, v0.4s\n" "add x23, x23, #16\n" "fmla v9.4s, v14.4s, v0.4s\n" "fmla v7.4s, v14.4s, v5.4s\n" "str q8, [x28, x24]\n" "ldr x28, [%[outptrs], 24]\n" "str q9, [x21, x24]\n" "fmla v7.4s, v17.4s, v1.4s\n" "fmla v7.4s, v16.4s, v3.4s\n" "fmla v7.4s, v15.4s, v0.4s\n" "str q7, [x28, x24]\n" "add x24, x24, #16\n" "4:\n" "cbz x25, 7f\n" "ldr s13, [%[wbptr]]\n" "mov v10.16b, v13.16b\n" "ldr s12, [%[wbptr], #4]\n" "mov v8.16b, v13.16b\n" "ldr s6, [%[wbptr], #8]\n" "mov v9.16b, v13.16b\n" "ldr s5, [%[wbptr], #12]\n" "mov v7.16b, v13.16b\n" "ldr s11, [%[wbptr], #16]\n" "ldr s4, [%[wbptr], #20]\n" "ldr x19, [%[inptrs], 0]\n" "ldr s3, [%[wbptr], #24]\n" "ldr x20, [%[inptrs], 40]\n" "ldr s2, [%[wbptr], #28]\n" "ldr x21, [%[inptrs], 80]\n" "ldr s1, [%[wbptr], #32]\n" "ldr x27, [%[inptrs], 120]\n" "ldr s0, [%[wbptr], #36]\n" "subs x25, x25, #1\n" "ldr s14, [x19, x23]\n" "ldr s18, [x20, x23]\n" "fmla v10.4s, v14.4s, v12.4s\n" "ldr s14, [x21, x23]\n" "ldr s16, [x27, x23]\n" "ldr x19, [%[inptrs], 8]\n" "ldr x20, [%[inptrs], 48]\n" "ldr x21, [%[inptrs], 88]\n" "ldr s19, [x19, x23]\n" "fmla v10.4s, v18.4s, v11.4s\n" "ldr s15, [x20, x23]\n" "ldr s18, [x21, x23]\n" "ldr x19, [%[inptrs], 16]\n" "ldr s13, [x19, x23]\n" "fmla v10.4s, v19.4s, v6.4s\n" "fmla v10.4s, v14.4s, v2.4s\n" "beq 6f\n" "5:\n" "fmla v8.4s, v14.4s, v12.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v10.4s, v15.4s, v4.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v9.4s, v13.4s, v12.4s\n" "ldr s14, [x20, x23]\n" "ldr s17, [x19, x23]\n" "ldr x22, [%[inptrs], 160]\n" "fmla v8.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 128]\n" "fmla v10.4s, v13.4s, v5.4s\n" "ldr s15, [x22, x23]\n" "fmla v9.4s, v14.4s, v11.4s\n" "ldr s19, [x27, x23]\n" "ldr x21, [%[inptrs], 96]\n" "ldr x20, [%[inptrs], 64]\n" "ldr x19, [%[inptrs], 32]\n" "fmla v8.4s, v18.4s, v6.4s\n" "ldr x22, [%[inptrs], 168]\n" "fmla v10.4s, v18.4s, v1.4s\n" "ldr s13, [x21, x23]\n" "fmla v9.4s, v17.4s, v6.4s\n" "ldr s18, [x20, x23]\n" "fmla v7.4s, v13.4s, v12.4s\n" "ldr s17, [x19, x23]\n" "fmla v8.4s, v15.4s, v2.4s\n" "ldr s15, [x22, x23]\n" "fmla v10.4s, v14.4s, v3.4s\n" "ldr x27, [%[inptrs], 136]\n" "fmla v9.4s, v13.4s, v2.4s\n" "ldr x21, [%[inptrs], 104]\n" "ldr s16, [x27, x23]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v8.4s, v19.4s, v4.4s\n" "ldr s19, [x21, x23]\n" "fmla v10.4s, v13.4s, v0.4s\n" "ldr s12, [x20, x23]\n" "fmla v9.4s, v18.4s, v4.4s\n" "ldr x22, [%[inptrs], 176]\n" "fmla v7.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 144]\n" "fmla v8.4s, v13.4s, v5.4s\n" "ldr s11, [x22, x23]\n" "ldr s13, [x27, x23]\n" "ldr x21, [%[inptrs], 112]\n" "fmla v9.4s, v17.4s, v5.4s\n" "ldr x22, [%[inptrs], 184]\n" "fmla v7.4s, v19.4s, v6.4s\n" "ldr s14, [x21, x23]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr s17, [x22, x23]\n" "ldr x27, [%[inptrs], 152]\n" "ldr x22, [%[inptrs], 192]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v9.4s, v19.4s, v1.4s\n" "ldr x28, [%[outptrs], 16]\n" "str s10, [x21, x24]\n" "fmla v7.4s, v11.4s, v2.4s\n" "fmla v8.4s, v16.4s, v3.4s\n" "ldr s16, [x27, x23]\n" "ldr s15, [x22, x23]\n" "ldr x21, [%[outptrs], 8]\n" "fmla v9.4s, v12.4s, v3.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v7.4s, v13.4s, v4.4s\n" "ldr s13, [%[wbptr]]\n" "fmla v8.4s, v11.4s, v0.4s\n" "ldr s12, [%[wbptr], #4]\n" "mov v10.16b, v13.16b\n" "ldr s6, [%[wbptr], #8]\n" "fmla v9.4s, v14.4s, v0.4s\n" "ldr s11, [%[wbptr], #16]\n" "fmla v7.4s, v14.4s, v5.4s\n" "ldr s4, [%[wbptr], #20]\n" "str s8, [x28, x24]\n" "add x23, x23, #4\n" "mov v8.16b, v13.16b\n" "ldr s2, [%[wbptr], #28]\n" "str s9, [x21, x24]\n" "ldr x28, [%[outptrs], 24]\n" "fmla v7.4s, v17.4s, v1.4s\n" "ldr s5, [%[wbptr], #12]\n" "mov v9.16b, v13.16b\n" "prfm pldl1keep, [%[wbptr], #64]\n" "ldr x19, [%[inptrs], 0]\n" "ldr x20, [%[inptrs], 40]\n" "ldr x21, [%[inptrs], 80]\n" "ldr x27, [%[inptrs], 120]\n" "subs x25, x25, #1\n" "fmla v7.4s, v16.4s, v3.4s\n" "ldr s1, [%[wbptr], #32]\n" "ldr s14, [x19, x23]\n" "fmla v10.4s, v14.4s, v12.4s\n" "ldr s18, [x20, x23]\n" "ldr s14, [x21, x23]\n" "ldr x19, [%[inptrs], 8]\n" "fmla v7.4s, v15.4s, v0.4s\n" "ldr s3, [%[wbptr], #24]\n" "ldr s19, [x19, x23]\n" "ldr x20, [%[inptrs], 48]\n" "fmla v10.4s, v18.4s, v11.4s\n" "ldr s16, [x27, x23]\n" "ldr s15, [x20, x23]\n" "ldr x19, [%[inptrs], 16]\n" "str s7, [x28, x24]\n" "ldr x21, [%[inptrs], 88]\n" "mov v7.16b, v13.16b\n" "ldr s0, [%[wbptr], #36]\n" "fmla v10.4s, v19.4s, v6.4s\n" "ldr s13, [x19, x23]\n" "ldr s18, [x21, x23]\n" "add x24, x24, #4\n" "fmla v10.4s, v14.4s, v2.4s\n" "bne 5b\n" "6:\n" "fmla v8.4s, v14.4s, v12.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v10.4s, v15.4s, v4.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v9.4s, v13.4s, v12.4s\n" "ldr s14, [x20, x23]\n" "ldr s17, [x19, x23]\n" "ldr x22, [%[inptrs], 160]\n" "fmla v8.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 128]\n" "fmla v10.4s, v13.4s, v5.4s\n" "ldr s15, [x22, x23]\n" "fmla v9.4s, v14.4s, v11.4s\n" "ldr s19, [x27, x23]\n" "ldr x21, [%[inptrs], 96]\n" "ldr x20, [%[inptrs], 64]\n" "ldr x19, [%[inptrs], 32]\n" "fmla v8.4s, v18.4s, v6.4s\n" "ldr x22, [%[inptrs], 168]\n" "fmla v10.4s, v18.4s, v1.4s\n" "ldr s13, [x21, x23]\n" "fmla v9.4s, v17.4s, v6.4s\n" "ldr s18, [x20, x23]\n" "fmla v7.4s, v13.4s, v12.4s\n" "ldr s17, [x19, x23]\n" "fmla v8.4s, v15.4s, v2.4s\n" "ldr s15, [x22, x23]\n" "fmla v10.4s, v14.4s, v3.4s\n" "ldr x27, [%[inptrs], 136]\n" "fmla v9.4s, v13.4s, v2.4s\n" "ldr x21, [%[inptrs], 104]\n" "ldr s16, [x27, x23]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v8.4s, v19.4s, v4.4s\n" "ldr s19, [x21, x23]\n" "fmla v10.4s, v13.4s, v0.4s\n" "ldr s12, [x20, x23]\n" "fmla v9.4s, v18.4s, v4.4s\n" "ldr x22, [%[inptrs], 176]\n" "fmla v7.4s, v16.4s, v11.4s\n" "ldr x27, [%[inptrs], 144]\n" "fmla v8.4s, v13.4s, v5.4s\n" "ldr s11, [x22, x23]\n" "ldr s13, [x27, x23]\n" "ldr x21, [%[inptrs], 112]\n" "fmla v9.4s, v17.4s, v5.4s\n" "ldr x22, [%[inptrs], 184]\n" "fmla v7.4s, v19.4s, v6.4s\n" "ldr s14, [x21, x23]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr s17, [x22, x23]\n" "ldr x27, [%[inptrs], 152]\n" "ldr x22, [%[inptrs], 192]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v9.4s, v19.4s, v1.4s\n" "ldr x28, [%[outptrs], 16]\n" "str s10, [x21, x24]\n" "fmla v7.4s, v11.4s, v2.4s\n" "fmla v8.4s, v16.4s, v3.4s\n" "ldr s16, [x27, x23]\n" "ldr s15, [x22, x23]\n" "ldr x21, [%[outptrs], 8]\n" "fmla v9.4s, v12.4s, v3.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v7.4s, v13.4s, v4.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v8.4s, v11.4s, v0.4s\n" "add x23, x23, #4\n" "fmla v9.4s, v14.4s, v0.4s\n" "fmla v7.4s, v14.4s, v5.4s\n" "str s8, [x28, x24]\n" "ldr x28, [%[outptrs], 24]\n" "str s9, [x21, x24]\n" "fmla v7.4s, v17.4s, v1.4s\n" "fmla v7.4s, v16.4s, v3.4s\n" "fmla v7.4s, v15.4s, v0.4s\n" "str s7, [x28, x24]\n" "add x24, x24, #4\n" "7:\n" : [wbptr] "+r" (weight_bias_ptr) : [inptrs] "r" (inptrs), [n_channels] "r" ((long) n_channels), [outptrs] "r" (outptrs) : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" ); } template <> template <> void Conv::execute_tile( int n_channels, const void *weight_bias_ptr, const float *input, const unsigned int input_row_stride, const unsigned int input_col_stride, float *output, const unsigned int output_row_stride, const unsigned int output_col_stride ) { __asm __volatile( "add x24, %[inptr0], %[input_row_stride]\n" "add x27, %[input_col_stride1], %[input_col_stride1]\n" "add x19, %[outptr0], %[output_row_stride]\n" "add x25, x24, %[input_row_stride]\n" "add x23, x27, %[input_col_stride1]\n" "and x20, %[n_channels], #3\n" "add x28, x25, %[input_row_stride]\n" "add x22, x23, %[input_col_stride1]\n" "lsr x21, %[n_channels], #2\n" "add x26, x28, %[input_row_stride]\n" "cbz x21, 4f\n" "1:\n" "ldr q16, [%[wbptr]]\n" "subs x21, x21, #1\n" "mov v3.16b, v16.16b\n" "ldr q4, [%[wbptr], #16]\n" "mov v1.16b, v16.16b\n" "ldr q5, [%[wbptr], #32]\n" "mov v2.16b, v16.16b\n" "ldr q12, [%[wbptr], #48]\n" "mov v0.16b, v16.16b\n" "ldr q11, [%[wbptr], #64]\n" "ldr q10, [%[wbptr], #80]\n" "ldr q6, [%[wbptr], #96]\n" "ldr q9, [%[wbptr], #112]\n" "ldr q8, [%[wbptr], #128]\n" "ldr q7, [%[wbptr], #144]\n" "ldr q21, [%[inptr0]]\n" "fmla v3.4s, v21.4s, v4.4s\n" "ldr q23, [x24]\n" "ldr q19, [%[inptr0], %[input_col_stride1]]\n" "ldr q14, [x25]\n" "fmla v1.4s, v14.4s, v4.4s\n" "ldr q13, [x24, %[input_col_stride1]]\n" "fmla v3.4s, v23.4s, v11.4s\n" "ldr q18, [%[inptr0], x27]\n" "ldr q15, [x28]\n" "ldr q22, [x25, %[input_col_stride1]]\n" "fmla v3.4s, v19.4s, v5.4s\n" "fmla v3.4s, v14.4s, v9.4s\n" "beq 3f\n" "2:\n" "fmla v3.4s, v13.4s, v10.4s\n" "ldr q17, [x24, x27]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr q20, [%[inptr0], x23]\n" "fmla v1.4s, v15.4s, v11.4s\n" "ldr q19, [x26]\n" "fmla v3.4s, v18.4s, v12.4s\n" "ldr q13, [x28, %[input_col_stride1]]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr q14, [x25, x27]\n" "fmla v1.4s, v22.4s, v5.4s\n" "ldr q15, [x24, x23]\n" "fmla v3.4s, v22.4s, v8.4s\n" "ldr q16, [%[inptr0], x22]\n" "fmla v2.4s, v20.4s, v5.4s\n" "ldr q20, [x26, %[input_col_stride1]]\n" "fmla v1.4s, v19.4s, v9.4s\n" "ldr q19, [x28, x27]\n" "fmla v3.4s, v17.4s, v6.4s\n" "ldr q21, [x25, x23]\n" "fmla v2.4s, v14.4s, v9.4s\n" "ldr q22, [x24, x22]\n" "fmla v1.4s, v13.4s, v10.4s\n" "ldr q23, [x26, x27]\n" "fmla v3.4s, v14.4s, v7.4s\n" "ldr q18, [x28, x23]\n" "fmla v0.4s, v14.4s, v4.4s\n" "ldr q13, [x25, x22]\n" "fmla v1.4s, v14.4s, v12.4s\n" "ldr q14, [x26, x23]\n" "fmla v2.4s, v15.4s, v10.4s\n" "ldr q17, [x28, x22]\n" "fmla v0.4s, v19.4s, v11.4s\n" "ldr q15, [x26, x22]\n" "fmla v1.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v2.4s, v16.4s, v12.4s\n" "ldr q16, [%[wbptr]]\n" "fmla v0.4s, v21.4s, v5.4s\n" "ldr q4, [%[wbptr], #16]\n" "fmla v1.4s, v19.4s, v6.4s\n" "ldr q11, [%[wbptr], #64]\n" "fmla v2.4s, v21.4s, v8.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v0.4s, v23.4s, v9.4s\n" "ldr q5, [%[wbptr], #32]\n" "fmla v1.4s, v23.4s, v7.4s\n" "add %[inptr0], %[inptr0], #16\n" "fmla v2.4s, v22.4s, v6.4s\n" "ldr q21, [%[inptr0]]\n" "fmla v0.4s, v18.4s, v10.4s\n" "ldr q9, [%[wbptr], #112]\n" "movi v20.16b, #0\n" "ldr q19, [%[inptr0], %[input_col_stride1]]\n" "fmla v2.4s, v13.4s, v7.4s\n" "ldr q18, [%[inptr0], x27]\n" "fmla v0.4s, v13.4s, v12.4s\n" "ldr q10, [%[wbptr], #80]\n" "fmax v3.4s, v3.4s, v20.4s\n" "add x24, x24, #16\n" "fmax v2.4s, v2.4s, v20.4s\n" "ldr q23, [x24]\n" "str q3, [%[outptr0]]\n" "fmla v0.4s, v14.4s, v8.4s\n" "str q2, [%[outptr0], %[output_col_stride1]]\n" "fmax v1.4s, v1.4s, v20.4s\n" "mov v3.16b, v16.16b\n" "ldr q12, [%[wbptr], #48]\n" "str q1, [x19]\n" "fmla v0.4s, v17.4s, v6.4s\n" "mov v1.16b, v16.16b\n" "ldr q8, [%[wbptr], #128]\n" "mov v2.16b, v16.16b\n" "ldr q13, [x24, %[input_col_stride1]]\n" "fmla v0.4s, v15.4s, v7.4s\n" "ldr q6, [%[wbptr], #96]\n" "fmla v3.4s, v21.4s, v4.4s\n" "add x25, x25, #16\n" "ldr q14, [x25]\n" "add x28, x28, #16\n" "fmax v0.4s, v0.4s, v20.4s\n" "ldr q7, [%[wbptr], #144]\n" "fmla v3.4s, v23.4s, v11.4s\n" "ldr q15, [x28]\n" "str q0, [x19, %[output_col_stride1]]\n" "fmla v1.4s, v14.4s, v4.4s\n" "mov v0.16b, v16.16b\n" "ldr q22, [x25, %[input_col_stride1]]\n" "fmla v3.4s, v19.4s, v5.4s\n" "add x26, x26, #16\n" "add %[outptr0], %[outptr0], #16\n" "add x19, x19, #16\n" "subs x21, x21, #1\n" "fmla v3.4s, v14.4s, v9.4s\n" "bne 2b\n" "3:\n" "fmla v3.4s, v13.4s, v10.4s\n" "ldr q17, [x24, x27]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr q20, [%[inptr0], x23]\n" "fmla v1.4s, v15.4s, v11.4s\n" "ldr q19, [x26]\n" "fmla v3.4s, v18.4s, v12.4s\n" "ldr q13, [x28, %[input_col_stride1]]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr q14, [x25, x27]\n" "fmla v1.4s, v22.4s, v5.4s\n" "ldr q15, [x24, x23]\n" "fmla v3.4s, v22.4s, v8.4s\n" "ldr q16, [%[inptr0], x22]\n" "fmla v2.4s, v20.4s, v5.4s\n" "ldr q20, [x26, %[input_col_stride1]]\n" "fmla v1.4s, v19.4s, v9.4s\n" "ldr q19, [x28, x27]\n" "fmla v3.4s, v17.4s, v6.4s\n" "ldr q21, [x25, x23]\n" "fmla v2.4s, v14.4s, v9.4s\n" "ldr q22, [x24, x22]\n" "fmla v1.4s, v13.4s, v10.4s\n" "ldr q23, [x26, x27]\n" "fmla v3.4s, v14.4s, v7.4s\n" "ldr q18, [x28, x23]\n" "fmla v0.4s, v14.4s, v4.4s\n" "ldr q13, [x25, x22]\n" "fmla v1.4s, v14.4s, v12.4s\n" "ldr q14, [x26, x23]\n" "fmla v2.4s, v15.4s, v10.4s\n" "ldr q17, [x28, x22]\n" "fmla v0.4s, v19.4s, v11.4s\n" "ldr q15, [x26, x22]\n" "fmla v1.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v2.4s, v16.4s, v12.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v0.4s, v21.4s, v5.4s\n" "add %[inptr0], %[inptr0], #16\n" "fmla v1.4s, v19.4s, v6.4s\n" "add x24, x24, #16\n" "fmla v2.4s, v21.4s, v8.4s\n" "add x25, x25, #16\n" "fmla v0.4s, v23.4s, v9.4s\n" "add x28, x28, #16\n" "fmla v1.4s, v23.4s, v7.4s\n" "add x26, x26, #16\n" "fmla v2.4s, v22.4s, v6.4s\n" "movi v20.16b, #0\n" "fmla v0.4s, v18.4s, v10.4s\n" "fmax v3.4s, v3.4s, v20.4s\n" "fmla v2.4s, v13.4s, v7.4s\n" "fmax v1.4s, v1.4s, v20.4s\n" "str q3, [%[outptr0]]\n" "fmla v0.4s, v13.4s, v12.4s\n" "str q1, [x19]\n" "fmax v2.4s, v2.4s, v20.4s\n" "fmla v0.4s, v14.4s, v8.4s\n" "str q2, [%[outptr0], %[output_col_stride1]]\n" "add %[outptr0], %[outptr0], #16\n" "fmla v0.4s, v17.4s, v6.4s\n" "fmla v0.4s, v15.4s, v7.4s\n" "fmax v0.4s, v0.4s, v20.4s\n" "str q0, [x19, %[output_col_stride1]]\n" "add x19, x19, #16\n" "4:\n" "cbz x20, 7f\n" "ldr s16, [%[wbptr]]\n" "mov v3.16b, v16.16b\n" "ldr s4, [%[wbptr], #4]\n" "mov v1.16b, v16.16b\n" "ldr s5, [%[wbptr], #8]\n" "mov v2.16b, v16.16b\n" "ldr s12, [%[wbptr], #12]\n" "mov v0.16b, v16.16b\n" "ldr s11, [%[wbptr], #16]\n" "ldr s10, [%[wbptr], #20]\n" "subs x20, x20, #1\n" "ldr s6, [%[wbptr], #24]\n" "ldr s9, [%[wbptr], #28]\n" "ldr s8, [%[wbptr], #32]\n" "ldr s7, [%[wbptr], #36]\n" "ldr s21, [%[inptr0]]\n" "ldr s23, [x24]\n" "fmla v3.4s, v21.4s, v4.4s\n" "ldr s19, [%[inptr0], %[input_col_stride1]]\n" "ldr s14, [x25]\n" "ldr s13, [x24, %[input_col_stride1]]\n" "fmla v1.4s, v14.4s, v4.4s\n" "ldr s18, [%[inptr0], x27]\n" "fmla v3.4s, v23.4s, v11.4s\n" "ldr s15, [x28]\n" "ldr s22, [x25, %[input_col_stride1]]\n" "fmla v3.4s, v19.4s, v5.4s\n" "fmla v3.4s, v14.4s, v9.4s\n" "beq 6f\n" "5:\n" "fmla v3.4s, v13.4s, v10.4s\n" "ldr s17, [x24, x27]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr s20, [%[inptr0], x23]\n" "fmla v1.4s, v15.4s, v11.4s\n" "ldr s19, [x26]\n" "fmla v3.4s, v18.4s, v12.4s\n" "ldr s13, [x28, %[input_col_stride1]]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr s14, [x25, x27]\n" "fmla v1.4s, v22.4s, v5.4s\n" "ldr s15, [x24, x23]\n" "fmla v3.4s, v22.4s, v8.4s\n" "ldr s16, [%[inptr0], x22]\n" "fmla v2.4s, v20.4s, v5.4s\n" "ldr s20, [x26, %[input_col_stride1]]\n" "fmla v1.4s, v19.4s, v9.4s\n" "ldr s19, [x28, x27]\n" "fmla v3.4s, v17.4s, v6.4s\n" "ldr s21, [x25, x23]\n" "fmla v2.4s, v14.4s, v9.4s\n" "ldr s22, [x24, x22]\n" "fmla v1.4s, v13.4s, v10.4s\n" "ldr s23, [x26, x27]\n" "fmla v3.4s, v14.4s, v7.4s\n" "ldr s18, [x28, x23]\n" "fmla v0.4s, v14.4s, v4.4s\n" "ldr s13, [x25, x22]\n" "fmla v1.4s, v14.4s, v12.4s\n" "ldr s14, [x26, x23]\n" "fmla v2.4s, v15.4s, v10.4s\n" "ldr s17, [x28, x22]\n" "fmla v0.4s, v19.4s, v11.4s\n" "ldr s15, [x26, x22]\n" "fmla v1.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v2.4s, v16.4s, v12.4s\n" "ldr s16, [%[wbptr]]\n" "fmla v0.4s, v21.4s, v5.4s\n" "ldr s4, [%[wbptr], #4]\n" "fmla v1.4s, v19.4s, v6.4s\n" "ldr s11, [%[wbptr], #16]\n" "fmla v2.4s, v21.4s, v8.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v0.4s, v23.4s, v9.4s\n" "ldr s5, [%[wbptr], #8]\n" "fmla v1.4s, v23.4s, v7.4s\n" "add %[inptr0], %[inptr0], #4\n" "fmla v2.4s, v22.4s, v6.4s\n" "ldr s21, [%[inptr0]]\n" "fmla v0.4s, v18.4s, v10.4s\n" "ldr s9, [%[wbptr], #28]\n" "movi v20.16b, #0\n" "ldr s19, [%[inptr0], %[input_col_stride1]]\n" "fmla v2.4s, v13.4s, v7.4s\n" "ldr s18, [%[inptr0], x27]\n" "fmla v0.4s, v13.4s, v12.4s\n" "ldr s10, [%[wbptr], #20]\n" "fmax v3.4s, v3.4s, v20.4s\n" "add x24, x24, #4\n" "fmax v2.4s, v2.4s, v20.4s\n" "ldr s23, [x24]\n" "str s3, [%[outptr0]]\n" "fmla v0.4s, v14.4s, v8.4s\n" "str s2, [%[outptr0], %[output_col_stride1]]\n" "fmax v1.4s, v1.4s, v20.4s\n" "mov v3.16b, v16.16b\n" "ldr s12, [%[wbptr], #12]\n" "str s1, [x19]\n" "fmla v0.4s, v17.4s, v6.4s\n" "mov v1.16b, v16.16b\n" "ldr s8, [%[wbptr], #32]\n" "mov v2.16b, v16.16b\n" "ldr s13, [x24, %[input_col_stride1]]\n" "fmla v0.4s, v15.4s, v7.4s\n" "ldr s6, [%[wbptr], #24]\n" "fmla v3.4s, v21.4s, v4.4s\n" "add x25, x25, #4\n" "ldr s14, [x25]\n" "add x28, x28, #4\n" "fmax v0.4s, v0.4s, v20.4s\n" "ldr s7, [%[wbptr], #36]\n" "fmla v3.4s, v23.4s, v11.4s\n" "ldr s15, [x28]\n" "str s0, [x19, %[output_col_stride1]]\n" "fmla v1.4s, v14.4s, v4.4s\n" "mov v0.16b, v16.16b\n" "ldr s22, [x25, %[input_col_stride1]]\n" "fmla v3.4s, v19.4s, v5.4s\n" "add x26, x26, #4\n" "add %[outptr0], %[outptr0], #4\n" "add x19, x19, #4\n" "subs x20, x20, #1\n" "fmla v3.4s, v14.4s, v9.4s\n" "bne 5b\n" "6:\n" "fmla v3.4s, v13.4s, v10.4s\n" "ldr s17, [x24, x27]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr s20, [%[inptr0], x23]\n" "fmla v1.4s, v15.4s, v11.4s\n" "ldr s19, [x26]\n" "fmla v3.4s, v18.4s, v12.4s\n" "ldr s13, [x28, %[input_col_stride1]]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr s14, [x25, x27]\n" "fmla v1.4s, v22.4s, v5.4s\n" "ldr s15, [x24, x23]\n" "fmla v3.4s, v22.4s, v8.4s\n" "ldr s16, [%[inptr0], x22]\n" "fmla v2.4s, v20.4s, v5.4s\n" "ldr s20, [x26, %[input_col_stride1]]\n" "fmla v1.4s, v19.4s, v9.4s\n" "ldr s19, [x28, x27]\n" "fmla v3.4s, v17.4s, v6.4s\n" "ldr s21, [x25, x23]\n" "fmla v2.4s, v14.4s, v9.4s\n" "ldr s22, [x24, x22]\n" "fmla v1.4s, v13.4s, v10.4s\n" "ldr s23, [x26, x27]\n" "fmla v3.4s, v14.4s, v7.4s\n" "ldr s18, [x28, x23]\n" "fmla v0.4s, v14.4s, v4.4s\n" "ldr s13, [x25, x22]\n" "fmla v1.4s, v14.4s, v12.4s\n" "ldr s14, [x26, x23]\n" "fmla v2.4s, v15.4s, v10.4s\n" "ldr s17, [x28, x22]\n" "fmla v0.4s, v19.4s, v11.4s\n" "ldr s15, [x26, x22]\n" "fmla v1.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v2.4s, v16.4s, v12.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v0.4s, v21.4s, v5.4s\n" "add %[inptr0], %[inptr0], #4\n" "fmla v1.4s, v19.4s, v6.4s\n" "add x24, x24, #4\n" "fmla v2.4s, v21.4s, v8.4s\n" "add x25, x25, #4\n" "fmla v0.4s, v23.4s, v9.4s\n" "add x28, x28, #4\n" "fmla v1.4s, v23.4s, v7.4s\n" "add x26, x26, #4\n" "fmla v2.4s, v22.4s, v6.4s\n" "movi v20.16b, #0\n" "fmla v0.4s, v18.4s, v10.4s\n" "fmax v3.4s, v3.4s, v20.4s\n" "fmla v2.4s, v13.4s, v7.4s\n" "fmax v1.4s, v1.4s, v20.4s\n" "str s3, [%[outptr0]]\n" "fmla v0.4s, v13.4s, v12.4s\n" "str s1, [x19]\n" "fmax v2.4s, v2.4s, v20.4s\n" "fmla v0.4s, v14.4s, v8.4s\n" "str s2, [%[outptr0], %[output_col_stride1]]\n" "add %[outptr0], %[outptr0], #4\n" "fmla v0.4s, v17.4s, v6.4s\n" "fmla v0.4s, v15.4s, v7.4s\n" "fmax v0.4s, v0.4s, v20.4s\n" "str s0, [x19, %[output_col_stride1]]\n" "add x19, x19, #4\n" "7:\n" : [wbptr] "+r" (weight_bias_ptr), [outptr0] "+r" (output), [inptr0] "+r" (input) : [n_channels] "r" ((long) n_channels), [output_col_stride1] "r" (output_col_stride * sizeof(float)), [input_col_stride1] "r" (input_col_stride * sizeof(float)), [input_row_stride] "r" (input_row_stride * sizeof(float)), [output_row_stride] "r" (output_row_stride * sizeof(float)) : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" ); } template <> template <> void Conv::execute_tile( int n_channels, const void *weight_bias_ptr, const float *inptrs[Base::inner_tile_rows][Base::inner_tile_cols], float *outptrs[Base::output_tile_rows][Base::output_tile_cols] ) { __asm __volatile( "mov x22, xzr\n" "mov x26, xzr\n" "and x23, %[n_channels], #3\n" "lsr x24, %[n_channels], #2\n" "cbz x24, 4f\n" "1:\n" "ldr q14, [%[wbptr]]\n" "ldr x19, [%[inptrs], 0]\n" "mov v3.16b, v14.16b\n" "ldr q13, [%[wbptr], #16]\n" "mov v1.16b, v14.16b\n" "ldr q11, [%[wbptr], #32]\n" "mov v2.16b, v14.16b\n" "ldr q4, [%[wbptr], #48]\n" "mov v0.16b, v14.16b\n" "ldr q12, [%[wbptr], #64]\n" "ldr q9, [%[wbptr], #80]\n" "ldr x20, [%[inptrs], 40]\n" "ldr q8, [%[wbptr], #96]\n" "ldr x21, [%[inptrs], 80]\n" "ldr q7, [%[wbptr], #112]\n" "ldr x25, [%[inptrs], 120]\n" "ldr q6, [%[wbptr], #128]\n" "subs x24, x24, #1\n" "ldr q5, [%[wbptr], #144]\n" "ldr q15, [x19, x22]\n" "fmla v3.4s, v15.4s, v13.4s\n" "ldr q17, [x20, x22]\n" "ldr q16, [x21, x22]\n" "ldr x19, [%[inptrs], 8]\n" "ldr q15, [x25, x22]\n" "ldr x20, [%[inptrs], 48]\n" "ldr q10, [x19, x22]\n" "ldr x21, [%[inptrs], 88]\n" "fmla v3.4s, v17.4s, v12.4s\n" "ldr q17, [x20, x22]\n" "ldr q14, [x21, x22]\n" "ldr x19, [%[inptrs], 16]\n" "ldr q18, [x19, x22]\n" "fmla v3.4s, v10.4s, v11.4s\n" "fmla v3.4s, v16.4s, v7.4s\n" "beq 3f\n" "2:\n" "fmla v1.4s, v16.4s, v13.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v3.4s, v17.4s, v9.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v2.4s, v18.4s, v13.4s\n" "ldr q16, [x20, x22]\n" "movi v10.16b, #0\n" "ldr q17, [x19, x22]\n" "fmla v1.4s, v15.4s, v12.4s\n" "ldr x27, [%[inptrs], 160]\n" "fmla v3.4s, v18.4s, v4.4s\n" "ldr x25, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v12.4s\n" "ldr q18, [x27, x22]\n" "ldr q15, [x25, x22]\n" "ldr x21, [%[inptrs], 96]\n" "fmla v1.4s, v14.4s, v11.4s\n" "ldr x20, [%[inptrs], 64]\n" "fmla v3.4s, v14.4s, v6.4s\n" "ldr q14, [x21, x22]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr q17, [x20, x22]\n" "fmla v0.4s, v14.4s, v13.4s\n" "ldr x19, [%[inptrs], 32]\n" "fmla v1.4s, v18.4s, v7.4s\n" "ldr x27, [%[inptrs], 168]\n" "fmla v3.4s, v16.4s, v8.4s\n" "ldr q18, [x19, x22]\n" "fmla v2.4s, v14.4s, v7.4s\n" "ldr q13, [x27, x22]\n" "ldr x25, [%[inptrs], 136]\n" "ldr x21, [%[inptrs], 104]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v1.4s, v15.4s, v9.4s\n" "ldr x27, [%[inptrs], 176]\n" "fmla v3.4s, v14.4s, v5.4s\n" "ldr q16, [x25, x22]\n" "fmla v2.4s, v17.4s, v9.4s\n" "ldr q17, [x21, x22]\n" "fmla v0.4s, v16.4s, v12.4s\n" "ldr q12, [x20, x22]\n" "fmla v1.4s, v14.4s, v4.4s\n" "ldr q15, [x27, x22]\n" "fmax v3.4s, v3.4s, v10.4s\n" "ldr x25, [%[inptrs], 144]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr x21, [%[inptrs], 112]\n" "fmla v0.4s, v17.4s, v11.4s\n" "ldr q14, [x25, x22]\n" "fmla v1.4s, v13.4s, v6.4s\n" "ldr q11, [x21, x22]\n" "ldr x27, [%[inptrs], 184]\n" "ldr x25, [%[inptrs], 152]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr x28, [%[outptrs], 16]\n" "str q3, [x21, x26]\n" "fmla v0.4s, v15.4s, v7.4s\n" "fmla v1.4s, v16.4s, v8.4s\n" "ldr q18, [x27, x22]\n" "ldr q17, [x25, x22]\n" "ldr x27, [%[inptrs], 192]\n" "fmla v2.4s, v12.4s, v8.4s\n" "ldr x21, [%[outptrs], 8]\n" "fmla v0.4s, v14.4s, v9.4s\n" "ldr q16, [x27, x22]\n" "fmla v1.4s, v15.4s, v5.4s\n" "add %[wbptr], %[wbptr], #160\n" "ldr q14, [%[wbptr]]\n" "add x22, x22, #16\n" "fmla v2.4s, v11.4s, v5.4s\n" "ldr q13, [%[wbptr], #16]\n" "fmla v0.4s, v11.4s, v4.4s\n" "ldr q11, [%[wbptr], #32]\n" "fmax v1.4s, v1.4s, v10.4s\n" "ldr q12, [%[wbptr], #64]\n" "mov v3.16b, v14.16b\n" "ldr q9, [%[wbptr], #80]\n" "fmax v2.4s, v2.4s, v10.4s\n" "ldr q7, [%[wbptr], #112]\n" "str q1, [x28, x26]\n" "fmla v0.4s, v18.4s, v6.4s\n" "mov v1.16b, v14.16b\n" "ldr q4, [%[wbptr], #48]\n" "str q2, [x21, x26]\n" "ldr x28, [%[outptrs], 24]\n" "mov v2.16b, v14.16b\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v0.4s, v17.4s, v8.4s\n" "ldr q6, [%[wbptr], #128]\n" "ldr x19, [%[inptrs], 0]\n" "ldr x20, [%[inptrs], 40]\n" "ldr x21, [%[inptrs], 80]\n" "ldr x25, [%[inptrs], 120]\n" "subs x24, x24, #1\n" "ldr q15, [x19, x22]\n" "fmla v0.4s, v16.4s, v5.4s\n" "ldr q8, [%[wbptr], #96]\n" "fmla v3.4s, v15.4s, v13.4s\n" "ldr q17, [x20, x22]\n" "ldr q16, [x21, x22]\n" "ldr x19, [%[inptrs], 8]\n" "ldr q15, [x25, x22]\n" "ldr x20, [%[inptrs], 48]\n" "fmax v0.4s, v0.4s, v10.4s\n" "ldr q5, [%[wbptr], #144]\n" "fmla v3.4s, v17.4s, v12.4s\n" "ldr q10, [x19, x22]\n" "ldr q17, [x20, x22]\n" "ldr x19, [%[inptrs], 16]\n" "str q0, [x28, x26]\n" "ldr x21, [%[inptrs], 88]\n" "mov v0.16b, v14.16b\n" "ldr q18, [x19, x22]\n" "fmla v3.4s, v10.4s, v11.4s\n" "ldr q14, [x21, x22]\n" "add x26, x26, #16\n" "fmla v3.4s, v16.4s, v7.4s\n" "bne 2b\n" "3:\n" "fmla v1.4s, v16.4s, v13.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v3.4s, v17.4s, v9.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v2.4s, v18.4s, v13.4s\n" "ldr q16, [x20, x22]\n" "movi v10.16b, #0\n" "ldr q17, [x19, x22]\n" "fmla v1.4s, v15.4s, v12.4s\n" "ldr x27, [%[inptrs], 160]\n" "fmla v3.4s, v18.4s, v4.4s\n" "ldr x25, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v12.4s\n" "ldr q18, [x27, x22]\n" "ldr q15, [x25, x22]\n" "ldr x21, [%[inptrs], 96]\n" "fmla v1.4s, v14.4s, v11.4s\n" "ldr x20, [%[inptrs], 64]\n" "fmla v3.4s, v14.4s, v6.4s\n" "ldr q14, [x21, x22]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr q17, [x20, x22]\n" "fmla v0.4s, v14.4s, v13.4s\n" "ldr x19, [%[inptrs], 32]\n" "fmla v1.4s, v18.4s, v7.4s\n" "ldr x27, [%[inptrs], 168]\n" "fmla v3.4s, v16.4s, v8.4s\n" "ldr q18, [x19, x22]\n" "fmla v2.4s, v14.4s, v7.4s\n" "ldr q13, [x27, x22]\n" "ldr x25, [%[inptrs], 136]\n" "ldr x21, [%[inptrs], 104]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v1.4s, v15.4s, v9.4s\n" "ldr x27, [%[inptrs], 176]\n" "fmla v3.4s, v14.4s, v5.4s\n" "ldr q16, [x25, x22]\n" "fmla v2.4s, v17.4s, v9.4s\n" "ldr q17, [x21, x22]\n" "fmla v0.4s, v16.4s, v12.4s\n" "ldr q12, [x20, x22]\n" "fmla v1.4s, v14.4s, v4.4s\n" "ldr q15, [x27, x22]\n" "fmax v3.4s, v3.4s, v10.4s\n" "ldr x25, [%[inptrs], 144]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr x21, [%[inptrs], 112]\n" "fmla v0.4s, v17.4s, v11.4s\n" "ldr q14, [x25, x22]\n" "fmla v1.4s, v13.4s, v6.4s\n" "ldr q11, [x21, x22]\n" "ldr x27, [%[inptrs], 184]\n" "ldr x25, [%[inptrs], 152]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr x28, [%[outptrs], 16]\n" "str q3, [x21, x26]\n" "fmla v0.4s, v15.4s, v7.4s\n" "fmla v1.4s, v16.4s, v8.4s\n" "ldr q18, [x27, x22]\n" "ldr q17, [x25, x22]\n" "ldr x27, [%[inptrs], 192]\n" "fmla v2.4s, v12.4s, v8.4s\n" "ldr x21, [%[outptrs], 8]\n" "fmla v0.4s, v14.4s, v9.4s\n" "ldr q16, [x27, x22]\n" "fmla v1.4s, v15.4s, v5.4s\n" "add %[wbptr], %[wbptr], #160\n" "prfm pldl1keep, [%[wbptr], #64]\n" "add x22, x22, #16\n" "fmla v2.4s, v11.4s, v5.4s\n" "fmla v0.4s, v11.4s, v4.4s\n" "fmax v1.4s, v1.4s, v10.4s\n" "fmax v2.4s, v2.4s, v10.4s\n" "str q1, [x28, x26]\n" "fmla v0.4s, v18.4s, v6.4s\n" "ldr x28, [%[outptrs], 24]\n" "str q2, [x21, x26]\n" "fmla v0.4s, v17.4s, v8.4s\n" "fmla v0.4s, v16.4s, v5.4s\n" "fmax v0.4s, v0.4s, v10.4s\n" "str q0, [x28, x26]\n" "add x26, x26, #16\n" "4:\n" "cbz x23, 7f\n" "ldr s14, [%[wbptr]]\n" "mov v3.16b, v14.16b\n" "ldr s13, [%[wbptr], #4]\n" "mov v1.16b, v14.16b\n" "ldr s11, [%[wbptr], #8]\n" "mov v2.16b, v14.16b\n" "ldr s4, [%[wbptr], #12]\n" "mov v0.16b, v14.16b\n" "ldr s12, [%[wbptr], #16]\n" "ldr s9, [%[wbptr], #20]\n" "ldr x19, [%[inptrs], 0]\n" "ldr s8, [%[wbptr], #24]\n" "ldr x20, [%[inptrs], 40]\n" "ldr s7, [%[wbptr], #28]\n" "ldr x21, [%[inptrs], 80]\n" "ldr s6, [%[wbptr], #32]\n" "ldr x25, [%[inptrs], 120]\n" "ldr s5, [%[wbptr], #36]\n" "subs x23, x23, #1\n" "ldr s15, [x19, x22]\n" "ldr s17, [x20, x22]\n" "fmla v3.4s, v15.4s, v13.4s\n" "ldr s16, [x21, x22]\n" "ldr s15, [x25, x22]\n" "ldr x19, [%[inptrs], 8]\n" "ldr x20, [%[inptrs], 48]\n" "ldr x21, [%[inptrs], 88]\n" "ldr s10, [x19, x22]\n" "fmla v3.4s, v17.4s, v12.4s\n" "ldr s17, [x20, x22]\n" "ldr s14, [x21, x22]\n" "ldr x19, [%[inptrs], 16]\n" "ldr s18, [x19, x22]\n" "fmla v3.4s, v10.4s, v11.4s\n" "fmla v3.4s, v16.4s, v7.4s\n" "beq 6f\n" "5:\n" "fmla v1.4s, v16.4s, v13.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v3.4s, v17.4s, v9.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v2.4s, v18.4s, v13.4s\n" "ldr s16, [x20, x22]\n" "movi v10.16b, #0\n" "ldr s17, [x19, x22]\n" "fmla v1.4s, v15.4s, v12.4s\n" "ldr x27, [%[inptrs], 160]\n" "fmla v3.4s, v18.4s, v4.4s\n" "ldr x25, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v12.4s\n" "ldr s18, [x27, x22]\n" "ldr s15, [x25, x22]\n" "ldr x21, [%[inptrs], 96]\n" "fmla v1.4s, v14.4s, v11.4s\n" "ldr x20, [%[inptrs], 64]\n" "fmla v3.4s, v14.4s, v6.4s\n" "ldr s14, [x21, x22]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr s17, [x20, x22]\n" "fmla v0.4s, v14.4s, v13.4s\n" "ldr x19, [%[inptrs], 32]\n" "fmla v1.4s, v18.4s, v7.4s\n" "ldr x27, [%[inptrs], 168]\n" "fmla v3.4s, v16.4s, v8.4s\n" "ldr s18, [x19, x22]\n" "fmla v2.4s, v14.4s, v7.4s\n" "ldr s13, [x27, x22]\n" "ldr x25, [%[inptrs], 136]\n" "ldr x21, [%[inptrs], 104]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v1.4s, v15.4s, v9.4s\n" "ldr x27, [%[inptrs], 176]\n" "fmla v3.4s, v14.4s, v5.4s\n" "ldr s16, [x25, x22]\n" "fmla v2.4s, v17.4s, v9.4s\n" "ldr s17, [x21, x22]\n" "fmla v0.4s, v16.4s, v12.4s\n" "ldr s12, [x20, x22]\n" "fmla v1.4s, v14.4s, v4.4s\n" "ldr s15, [x27, x22]\n" "fmax v3.4s, v3.4s, v10.4s\n" "ldr x25, [%[inptrs], 144]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr x21, [%[inptrs], 112]\n" "fmla v0.4s, v17.4s, v11.4s\n" "ldr s14, [x25, x22]\n" "fmla v1.4s, v13.4s, v6.4s\n" "ldr s11, [x21, x22]\n" "ldr x27, [%[inptrs], 184]\n" "ldr x25, [%[inptrs], 152]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr x28, [%[outptrs], 16]\n" "str s3, [x21, x26]\n" "fmla v0.4s, v15.4s, v7.4s\n" "fmla v1.4s, v16.4s, v8.4s\n" "ldr s18, [x27, x22]\n" "ldr s17, [x25, x22]\n" "ldr x27, [%[inptrs], 192]\n" "fmla v2.4s, v12.4s, v8.4s\n" "ldr x21, [%[outptrs], 8]\n" "fmla v0.4s, v14.4s, v9.4s\n" "ldr s16, [x27, x22]\n" "fmla v1.4s, v15.4s, v5.4s\n" "add %[wbptr], %[wbptr], #40\n" "ldr s14, [%[wbptr]]\n" "add x22, x22, #4\n" "fmla v2.4s, v11.4s, v5.4s\n" "ldr s13, [%[wbptr], #4]\n" "fmla v0.4s, v11.4s, v4.4s\n" "ldr s11, [%[wbptr], #8]\n" "fmax v1.4s, v1.4s, v10.4s\n" "ldr s12, [%[wbptr], #16]\n" "mov v3.16b, v14.16b\n" "ldr s9, [%[wbptr], #20]\n" "fmax v2.4s, v2.4s, v10.4s\n" "ldr s7, [%[wbptr], #28]\n" "str s1, [x28, x26]\n" "fmla v0.4s, v18.4s, v6.4s\n" "mov v1.16b, v14.16b\n" "ldr s4, [%[wbptr], #12]\n" "str s2, [x21, x26]\n" "ldr x28, [%[outptrs], 24]\n" "mov v2.16b, v14.16b\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v0.4s, v17.4s, v8.4s\n" "ldr s6, [%[wbptr], #32]\n" "ldr x19, [%[inptrs], 0]\n" "ldr x20, [%[inptrs], 40]\n" "ldr x21, [%[inptrs], 80]\n" "ldr x25, [%[inptrs], 120]\n" "subs x23, x23, #1\n" "ldr s15, [x19, x22]\n" "fmla v0.4s, v16.4s, v5.4s\n" "ldr s8, [%[wbptr], #24]\n" "fmla v3.4s, v15.4s, v13.4s\n" "ldr s17, [x20, x22]\n" "ldr s16, [x21, x22]\n" "ldr x19, [%[inptrs], 8]\n" "ldr s15, [x25, x22]\n" "ldr x20, [%[inptrs], 48]\n" "fmax v0.4s, v0.4s, v10.4s\n" "ldr s5, [%[wbptr], #36]\n" "fmla v3.4s, v17.4s, v12.4s\n" "ldr s10, [x19, x22]\n" "ldr s17, [x20, x22]\n" "ldr x19, [%[inptrs], 16]\n" "str s0, [x28, x26]\n" "ldr x21, [%[inptrs], 88]\n" "mov v0.16b, v14.16b\n" "ldr s18, [x19, x22]\n" "fmla v3.4s, v10.4s, v11.4s\n" "ldr s14, [x21, x22]\n" "add x26, x26, #4\n" "fmla v3.4s, v16.4s, v7.4s\n" "bne 5b\n" "6:\n" "fmla v1.4s, v16.4s, v13.4s\n" "ldr x20, [%[inptrs], 56]\n" "fmla v3.4s, v17.4s, v9.4s\n" "ldr x19, [%[inptrs], 24]\n" "fmla v2.4s, v18.4s, v13.4s\n" "ldr s16, [x20, x22]\n" "movi v10.16b, #0\n" "ldr s17, [x19, x22]\n" "fmla v1.4s, v15.4s, v12.4s\n" "ldr x27, [%[inptrs], 160]\n" "fmla v3.4s, v18.4s, v4.4s\n" "ldr x25, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v12.4s\n" "ldr s18, [x27, x22]\n" "ldr s15, [x25, x22]\n" "ldr x21, [%[inptrs], 96]\n" "fmla v1.4s, v14.4s, v11.4s\n" "ldr x20, [%[inptrs], 64]\n" "fmla v3.4s, v14.4s, v6.4s\n" "ldr s14, [x21, x22]\n" "fmla v2.4s, v17.4s, v11.4s\n" "ldr s17, [x20, x22]\n" "fmla v0.4s, v14.4s, v13.4s\n" "ldr x19, [%[inptrs], 32]\n" "fmla v1.4s, v18.4s, v7.4s\n" "ldr x27, [%[inptrs], 168]\n" "fmla v3.4s, v16.4s, v8.4s\n" "ldr s18, [x19, x22]\n" "fmla v2.4s, v14.4s, v7.4s\n" "ldr s13, [x27, x22]\n" "ldr x25, [%[inptrs], 136]\n" "ldr x21, [%[inptrs], 104]\n" "ldr x20, [%[inptrs], 72]\n" "fmla v1.4s, v15.4s, v9.4s\n" "ldr x27, [%[inptrs], 176]\n" "fmla v3.4s, v14.4s, v5.4s\n" "ldr s16, [x25, x22]\n" "fmla v2.4s, v17.4s, v9.4s\n" "ldr s17, [x21, x22]\n" "fmla v0.4s, v16.4s, v12.4s\n" "ldr s12, [x20, x22]\n" "fmla v1.4s, v14.4s, v4.4s\n" "ldr s15, [x27, x22]\n" "fmax v3.4s, v3.4s, v10.4s\n" "ldr x25, [%[inptrs], 144]\n" "fmla v2.4s, v18.4s, v4.4s\n" "ldr x21, [%[inptrs], 112]\n" "fmla v0.4s, v17.4s, v11.4s\n" "ldr s14, [x25, x22]\n" "fmla v1.4s, v13.4s, v6.4s\n" "ldr s11, [x21, x22]\n" "ldr x27, [%[inptrs], 184]\n" "ldr x25, [%[inptrs], 152]\n" "ldr x21, [%[outptrs], 0]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr x28, [%[outptrs], 16]\n" "str s3, [x21, x26]\n" "fmla v0.4s, v15.4s, v7.4s\n" "fmla v1.4s, v16.4s, v8.4s\n" "ldr s18, [x27, x22]\n" "ldr s17, [x25, x22]\n" "ldr x27, [%[inptrs], 192]\n" "fmla v2.4s, v12.4s, v8.4s\n" "ldr x21, [%[outptrs], 8]\n" "fmla v0.4s, v14.4s, v9.4s\n" "ldr s16, [x27, x22]\n" "fmla v1.4s, v15.4s, v5.4s\n" "add %[wbptr], %[wbptr], #40\n" "prfm pldl1keep, [%[wbptr], #64]\n" "add x22, x22, #4\n" "fmla v2.4s, v11.4s, v5.4s\n" "fmla v0.4s, v11.4s, v4.4s\n" "fmax v1.4s, v1.4s, v10.4s\n" "fmax v2.4s, v2.4s, v10.4s\n" "str s1, [x28, x26]\n" "fmla v0.4s, v18.4s, v6.4s\n" "ldr x28, [%[outptrs], 24]\n" "str s2, [x21, x26]\n" "fmla v0.4s, v17.4s, v8.4s\n" "fmla v0.4s, v16.4s, v5.4s\n" "fmax v0.4s, v0.4s, v10.4s\n" "str s0, [x28, x26]\n" "add x26, x26, #4\n" "7:\n" : [wbptr] "+r" (weight_bias_ptr) : [inptrs] "r" (inptrs), [n_channels] "r" ((long) n_channels), [outptrs] "r" (outptrs) : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" ); } template <> template <> void Conv::execute_tile( int n_channels, const void *weight_bias_ptr, const float *input, const unsigned int input_row_stride, const unsigned int input_col_stride, float *output, const unsigned int output_row_stride, const unsigned int output_col_stride ) { __asm __volatile( "add x21, %[inptr0], %[input_row_stride]\n" "add x23, %[input_col_stride1], %[input_col_stride1]\n" "add x24, %[outptr0], %[output_row_stride]\n" "add x28, x21, %[input_row_stride]\n" "add x26, x23, %[input_col_stride1]\n" "and x19, %[n_channels], #3\n" "add x27, x28, %[input_row_stride]\n" "add x25, x26, %[input_col_stride1]\n" "lsr x20, %[n_channels], #2\n" "add x22, x27, %[input_row_stride]\n" "cbz x20, 4f\n" "1:\n" "ldr q14, [%[wbptr]]\n" "subs x20, x20, #1\n" "mov v5.16b, v14.16b\n" "ldr q0, [%[wbptr], #16]\n" "mov v11.16b, v14.16b\n" "ldr q1, [%[wbptr], #32]\n" "mov v12.16b, v14.16b\n" "ldr q2, [%[wbptr], #48]\n" "mov v10.16b, v14.16b\n" "ldr q6, [%[wbptr], #64]\n" "ldr q3, [%[wbptr], #80]\n" "ldr q7, [%[wbptr], #96]\n" "ldr q4, [%[wbptr], #112]\n" "ldr q8, [%[wbptr], #128]\n" "ldr q9, [%[wbptr], #144]\n" "ldr q19, [%[inptr0]]\n" "fmla v5.4s, v19.4s, v0.4s\n" "ldr q15, [x21]\n" "ldr q21, [%[inptr0], %[input_col_stride1]]\n" "ldr q16, [x28]\n" "fmla v11.4s, v16.4s, v0.4s\n" "ldr q23, [x21, %[input_col_stride1]]\n" "fmla v5.4s, v15.4s, v6.4s\n" "ldr q18, [%[inptr0], x23]\n" "ldr q17, [x27]\n" "ldr q13, [x28, %[input_col_stride1]]\n" "fmla v5.4s, v21.4s, v1.4s\n" "fmla v5.4s, v16.4s, v4.4s\n" "beq 3f\n" "2:\n" "fmla v5.4s, v23.4s, v3.4s\n" "ldr q21, [x21, x23]\n" "fmla v12.4s, v18.4s, v0.4s\n" "ldr q20, [%[inptr0], x26]\n" "fmla v11.4s, v17.4s, v6.4s\n" "ldr q19, [x22]\n" "fmla v5.4s, v18.4s, v2.4s\n" "ldr q15, [x27, %[input_col_stride1]]\n" "fmla v12.4s, v21.4s, v6.4s\n" "ldr q16, [x28, x23]\n" "fmla v11.4s, v13.4s, v1.4s\n" "ldr q17, [x21, x26]\n" "fmla v5.4s, v13.4s, v8.4s\n" "ldr q14, [%[inptr0], x25]\n" "fmla v12.4s, v20.4s, v1.4s\n" "ldr q20, [x22, %[input_col_stride1]]\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr q19, [x27, x23]\n" "fmla v5.4s, v21.4s, v7.4s\n" "ldr q22, [x28, x26]\n" "fmla v12.4s, v16.4s, v4.4s\n" "ldr q21, [x21, x25]\n" "fmla v11.4s, v15.4s, v3.4s\n" "ldr q23, [x22, x23]\n" "fmla v5.4s, v16.4s, v9.4s\n" "ldr q18, [x27, x26]\n" "fmla v10.4s, v16.4s, v0.4s\n" "ldr q15, [x28, x25]\n" "fmla v11.4s, v16.4s, v2.4s\n" "ldr q16, [x22, x26]\n" "fmla v12.4s, v17.4s, v3.4s\n" "ldr q17, [x27, x25]\n" "fmla v10.4s, v19.4s, v6.4s\n" "ldr q13, [x22, x25]\n" "fmla v11.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v12.4s, v14.4s, v2.4s\n" "ldr q14, [%[wbptr]]\n" "fmla v10.4s, v22.4s, v1.4s\n" "ldr q0, [%[wbptr], #16]\n" "fmla v11.4s, v19.4s, v7.4s\n" "ldr q6, [%[wbptr], #64]\n" "fmla v12.4s, v22.4s, v8.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v10.4s, v23.4s, v4.4s\n" "ldr q1, [%[wbptr], #32]\n" "fmla v11.4s, v23.4s, v9.4s\n" "add %[inptr0], %[inptr0], #16\n" "fmla v12.4s, v21.4s, v7.4s\n" "ldr q19, [%[inptr0]]\n" "fmla v10.4s, v18.4s, v3.4s\n" "ldr q4, [%[wbptr], #112]\n" "movi v20.16b, #0\n" "ldr q21, [%[inptr0], %[input_col_stride1]]\n" "fmla v12.4s, v15.4s, v9.4s\n" "ldr q18, [%[inptr0], x23]\n" "fmla v10.4s, v15.4s, v2.4s\n" "ldr q3, [%[wbptr], #80]\n" "fmov v22.4s, #6.0\n" "add x21, x21, #16\n" "fmax v5.4s, v5.4s, v20.4s\n" "ldr q15, [x21]\n" "fmla v10.4s, v16.4s, v8.4s\n" "ldr q2, [%[wbptr], #48]\n" "fmin v5.4s, v5.4s, v22.4s\n" "ldr q23, [x21, %[input_col_stride1]]\n" "fmax v12.4s, v12.4s, v20.4s\n" "add x28, x28, #16\n" "str q5, [%[outptr0]]\n" "fmla v10.4s, v17.4s, v7.4s\n" "fmin v12.4s, v12.4s, v22.4s\n" "ldr q8, [%[wbptr], #128]\n" "fmax v11.4s, v11.4s, v20.4s\n" "ldr q16, [x28]\n" "str q12, [%[outptr0], %[output_col_stride1]]\n" "fmla v10.4s, v13.4s, v9.4s\n" "fmin v11.4s, v11.4s, v22.4s\n" "ldr q7, [%[wbptr], #96]\n" "mov v5.16b, v14.16b\n" "ldr q13, [x28, %[input_col_stride1]]\n" "str q11, [x24]\n" "fmax v10.4s, v10.4s, v20.4s\n" "mov v11.16b, v14.16b\n" "ldr q9, [%[wbptr], #144]\n" "fmin v10.4s, v10.4s, v22.4s\n" "add x27, x27, #16\n" "mov v12.16b, v14.16b\n" "ldr q17, [x27]\n" "str q10, [x24, %[output_col_stride1]]\n" "fmla v5.4s, v19.4s, v0.4s\n" "mov v10.16b, v14.16b\n" "add x22, x22, #16\n" "fmla v11.4s, v16.4s, v0.4s\n" "add %[outptr0], %[outptr0], #16\n" "fmla v5.4s, v15.4s, v6.4s\n" "add x24, x24, #16\n" "subs x20, x20, #1\n" "fmla v5.4s, v21.4s, v1.4s\n" "fmla v5.4s, v16.4s, v4.4s\n" "bne 2b\n" "3:\n" "fmla v5.4s, v23.4s, v3.4s\n" "ldr q21, [x21, x23]\n" "fmla v12.4s, v18.4s, v0.4s\n" "ldr q20, [%[inptr0], x26]\n" "fmla v11.4s, v17.4s, v6.4s\n" "ldr q19, [x22]\n" "fmla v5.4s, v18.4s, v2.4s\n" "ldr q15, [x27, %[input_col_stride1]]\n" "fmla v12.4s, v21.4s, v6.4s\n" "ldr q16, [x28, x23]\n" "fmla v11.4s, v13.4s, v1.4s\n" "ldr q17, [x21, x26]\n" "fmla v5.4s, v13.4s, v8.4s\n" "ldr q14, [%[inptr0], x25]\n" "fmla v12.4s, v20.4s, v1.4s\n" "ldr q20, [x22, %[input_col_stride1]]\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr q19, [x27, x23]\n" "fmla v5.4s, v21.4s, v7.4s\n" "ldr q22, [x28, x26]\n" "fmla v12.4s, v16.4s, v4.4s\n" "ldr q21, [x21, x25]\n" "fmla v11.4s, v15.4s, v3.4s\n" "ldr q23, [x22, x23]\n" "fmla v5.4s, v16.4s, v9.4s\n" "ldr q18, [x27, x26]\n" "fmla v10.4s, v16.4s, v0.4s\n" "ldr q15, [x28, x25]\n" "fmla v11.4s, v16.4s, v2.4s\n" "ldr q16, [x22, x26]\n" "fmla v12.4s, v17.4s, v3.4s\n" "ldr q17, [x27, x25]\n" "fmla v10.4s, v19.4s, v6.4s\n" "ldr q13, [x22, x25]\n" "fmla v11.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #160\n" "fmla v12.4s, v14.4s, v2.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v10.4s, v22.4s, v1.4s\n" "add %[inptr0], %[inptr0], #16\n" "fmla v11.4s, v19.4s, v7.4s\n" "add x21, x21, #16\n" "fmla v12.4s, v22.4s, v8.4s\n" "add x28, x28, #16\n" "fmla v10.4s, v23.4s, v4.4s\n" "add x27, x27, #16\n" "fmla v11.4s, v23.4s, v9.4s\n" "add x22, x22, #16\n" "fmla v12.4s, v21.4s, v7.4s\n" "movi v20.16b, #0\n" "fmla v10.4s, v18.4s, v3.4s\n" "fmov v22.4s, #6.0\n" "fmax v5.4s, v5.4s, v20.4s\n" "fmax v11.4s, v11.4s, v20.4s\n" "fmla v12.4s, v15.4s, v9.4s\n" "fmla v10.4s, v15.4s, v2.4s\n" "fmin v5.4s, v5.4s, v22.4s\n" "fmin v11.4s, v11.4s, v22.4s\n" "fmax v12.4s, v12.4s, v20.4s\n" "str q5, [%[outptr0]]\n" "str q11, [x24]\n" "fmla v10.4s, v16.4s, v8.4s\n" "fmin v12.4s, v12.4s, v22.4s\n" "str q12, [%[outptr0], %[output_col_stride1]]\n" "fmla v10.4s, v17.4s, v7.4s\n" "add %[outptr0], %[outptr0], #16\n" "fmla v10.4s, v13.4s, v9.4s\n" "fmax v10.4s, v10.4s, v20.4s\n" "fmin v10.4s, v10.4s, v22.4s\n" "str q10, [x24, %[output_col_stride1]]\n" "add x24, x24, #16\n" "4:\n" "cbz x19, 7f\n" "ldr s14, [%[wbptr]]\n" "mov v5.16b, v14.16b\n" "ldr s0, [%[wbptr], #4]\n" "mov v11.16b, v14.16b\n" "ldr s1, [%[wbptr], #8]\n" "mov v12.16b, v14.16b\n" "ldr s2, [%[wbptr], #12]\n" "mov v10.16b, v14.16b\n" "ldr s6, [%[wbptr], #16]\n" "ldr s3, [%[wbptr], #20]\n" "subs x19, x19, #1\n" "ldr s7, [%[wbptr], #24]\n" "ldr s4, [%[wbptr], #28]\n" "ldr s8, [%[wbptr], #32]\n" "ldr s9, [%[wbptr], #36]\n" "ldr s19, [%[inptr0]]\n" "ldr s15, [x21]\n" "fmla v5.4s, v19.4s, v0.4s\n" "ldr s21, [%[inptr0], %[input_col_stride1]]\n" "ldr s16, [x28]\n" "ldr s23, [x21, %[input_col_stride1]]\n" "fmla v11.4s, v16.4s, v0.4s\n" "ldr s18, [%[inptr0], x23]\n" "fmla v5.4s, v15.4s, v6.4s\n" "ldr s17, [x27]\n" "ldr s13, [x28, %[input_col_stride1]]\n" "fmla v5.4s, v21.4s, v1.4s\n" "fmla v5.4s, v16.4s, v4.4s\n" "beq 6f\n" "5:\n" "fmla v5.4s, v23.4s, v3.4s\n" "ldr s21, [x21, x23]\n" "fmla v12.4s, v18.4s, v0.4s\n" "ldr s20, [%[inptr0], x26]\n" "fmla v11.4s, v17.4s, v6.4s\n" "ldr s19, [x22]\n" "fmla v5.4s, v18.4s, v2.4s\n" "ldr s15, [x27, %[input_col_stride1]]\n" "fmla v12.4s, v21.4s, v6.4s\n" "ldr s16, [x28, x23]\n" "fmla v11.4s, v13.4s, v1.4s\n" "ldr s17, [x21, x26]\n" "fmla v5.4s, v13.4s, v8.4s\n" "ldr s14, [%[inptr0], x25]\n" "fmla v12.4s, v20.4s, v1.4s\n" "ldr s20, [x22, %[input_col_stride1]]\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr s19, [x27, x23]\n" "fmla v5.4s, v21.4s, v7.4s\n" "ldr s22, [x28, x26]\n" "fmla v12.4s, v16.4s, v4.4s\n" "ldr s21, [x21, x25]\n" "fmla v11.4s, v15.4s, v3.4s\n" "ldr s23, [x22, x23]\n" "fmla v5.4s, v16.4s, v9.4s\n" "ldr s18, [x27, x26]\n" "fmla v10.4s, v16.4s, v0.4s\n" "ldr s15, [x28, x25]\n" "fmla v11.4s, v16.4s, v2.4s\n" "ldr s16, [x22, x26]\n" "fmla v12.4s, v17.4s, v3.4s\n" "ldr s17, [x27, x25]\n" "fmla v10.4s, v19.4s, v6.4s\n" "ldr s13, [x22, x25]\n" "fmla v11.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v12.4s, v14.4s, v2.4s\n" "ldr s14, [%[wbptr]]\n" "fmla v10.4s, v22.4s, v1.4s\n" "ldr s0, [%[wbptr], #4]\n" "fmla v11.4s, v19.4s, v7.4s\n" "ldr s6, [%[wbptr], #16]\n" "fmla v12.4s, v22.4s, v8.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v10.4s, v23.4s, v4.4s\n" "ldr s1, [%[wbptr], #8]\n" "fmla v11.4s, v23.4s, v9.4s\n" "add %[inptr0], %[inptr0], #4\n" "fmla v12.4s, v21.4s, v7.4s\n" "ldr s19, [%[inptr0]]\n" "fmla v10.4s, v18.4s, v3.4s\n" "ldr s4, [%[wbptr], #28]\n" "movi v20.16b, #0\n" "ldr s21, [%[inptr0], %[input_col_stride1]]\n" "fmla v12.4s, v15.4s, v9.4s\n" "ldr s18, [%[inptr0], x23]\n" "fmla v10.4s, v15.4s, v2.4s\n" "ldr s3, [%[wbptr], #20]\n" "fmov v22.4s, #6.0\n" "add x21, x21, #4\n" "fmax v5.4s, v5.4s, v20.4s\n" "ldr s15, [x21]\n" "fmla v10.4s, v16.4s, v8.4s\n" "ldr s2, [%[wbptr], #12]\n" "fmin v5.4s, v5.4s, v22.4s\n" "ldr s23, [x21, %[input_col_stride1]]\n" "fmax v12.4s, v12.4s, v20.4s\n" "add x28, x28, #4\n" "str s5, [%[outptr0]]\n" "fmla v10.4s, v17.4s, v7.4s\n" "fmin v12.4s, v12.4s, v22.4s\n" "ldr s8, [%[wbptr], #32]\n" "fmax v11.4s, v11.4s, v20.4s\n" "ldr s16, [x28]\n" "str s12, [%[outptr0], %[output_col_stride1]]\n" "fmla v10.4s, v13.4s, v9.4s\n" "fmin v11.4s, v11.4s, v22.4s\n" "ldr s7, [%[wbptr], #24]\n" "mov v5.16b, v14.16b\n" "ldr s13, [x28, %[input_col_stride1]]\n" "str s11, [x24]\n" "fmax v10.4s, v10.4s, v20.4s\n" "mov v11.16b, v14.16b\n" "ldr s9, [%[wbptr], #36]\n" "fmin v10.4s, v10.4s, v22.4s\n" "add x27, x27, #4\n" "mov v12.16b, v14.16b\n" "ldr s17, [x27]\n" "str s10, [x24, %[output_col_stride1]]\n" "fmla v5.4s, v19.4s, v0.4s\n" "mov v10.16b, v14.16b\n" "add x22, x22, #4\n" "fmla v11.4s, v16.4s, v0.4s\n" "add %[outptr0], %[outptr0], #4\n" "fmla v5.4s, v15.4s, v6.4s\n" "add x24, x24, #4\n" "subs x19, x19, #1\n" "fmla v5.4s, v21.4s, v1.4s\n" "fmla v5.4s, v16.4s, v4.4s\n" "bne 5b\n" "6:\n" "fmla v5.4s, v23.4s, v3.4s\n" "ldr s21, [x21, x23]\n" "fmla v12.4s, v18.4s, v0.4s\n" "ldr s20, [%[inptr0], x26]\n" "fmla v11.4s, v17.4s, v6.4s\n" "ldr s19, [x22]\n" "fmla v5.4s, v18.4s, v2.4s\n" "ldr s15, [x27, %[input_col_stride1]]\n" "fmla v12.4s, v21.4s, v6.4s\n" "ldr s16, [x28, x23]\n" "fmla v11.4s, v13.4s, v1.4s\n" "ldr s17, [x21, x26]\n" "fmla v5.4s, v13.4s, v8.4s\n" "ldr s14, [%[inptr0], x25]\n" "fmla v12.4s, v20.4s, v1.4s\n" "ldr s20, [x22, %[input_col_stride1]]\n" "fmla v11.4s, v19.4s, v4.4s\n" "ldr s19, [x27, x23]\n" "fmla v5.4s, v21.4s, v7.4s\n" "ldr s22, [x28, x26]\n" "fmla v12.4s, v16.4s, v4.4s\n" "ldr s21, [x21, x25]\n" "fmla v11.4s, v15.4s, v3.4s\n" "ldr s23, [x22, x23]\n" "fmla v5.4s, v16.4s, v9.4s\n" "ldr s18, [x27, x26]\n" "fmla v10.4s, v16.4s, v0.4s\n" "ldr s15, [x28, x25]\n" "fmla v11.4s, v16.4s, v2.4s\n" "ldr s16, [x22, x26]\n" "fmla v12.4s, v17.4s, v3.4s\n" "ldr s17, [x27, x25]\n" "fmla v10.4s, v19.4s, v6.4s\n" "ldr s13, [x22, x25]\n" "fmla v11.4s, v20.4s, v8.4s\n" "add %[wbptr], %[wbptr], #40\n" "fmla v12.4s, v14.4s, v2.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmla v10.4s, v22.4s, v1.4s\n" "add %[inptr0], %[inptr0], #4\n" "fmla v11.4s, v19.4s, v7.4s\n" "add x21, x21, #4\n" "fmla v12.4s, v22.4s, v8.4s\n" "add x28, x28, #4\n" "fmla v10.4s, v23.4s, v4.4s\n" "add x27, x27, #4\n" "fmla v11.4s, v23.4s, v9.4s\n" "add x22, x22, #4\n" "fmla v12.4s, v21.4s, v7.4s\n" "movi v20.16b, #0\n" "fmla v10.4s, v18.4s, v3.4s\n" "fmov v22.4s, #6.0\n" "fmax v5.4s, v5.4s, v20.4s\n" "fmax v11.4s, v11.4s, v20.4s\n" "fmla v12.4s, v15.4s, v9.4s\n" "fmla v10.4s, v15.4s, v2.4s\n" "fmin v5.4s, v5.4s, v22.4s\n" "fmin v11.4s, v11.4s, v22.4s\n" "fmax v12.4s, v12.4s, v20.4s\n" "str s5, [%[outptr0]]\n" "str s11, [x24]\n" "fmla v10.4s, v16.4s, v8.4s\n" "fmin v12.4s, v12.4s, v22.4s\n" "str s12, [%[outptr0], %[output_col_stride1]]\n" "fmla v10.4s, v17.4s, v7.4s\n" "add %[outptr0], %[outptr0], #4\n" "fmla v10.4s, v13.4s, v9.4s\n" "fmax v10.4s, v10.4s, v20.4s\n" "fmin v10.4s, v10.4s, v22.4s\n" "str s10, [x24, %[output_col_stride1]]\n" "add x24, x24, #4\n" "7:\n" : [outptr0] "+r" (output), [inptr0] "+r" (input), [wbptr] "+r" (weight_bias_ptr) : [input_row_stride] "r" (input_row_stride * sizeof(float)), [output_row_stride] "r" (output_row_stride * sizeof(float)), [output_col_stride1] "r" (output_col_stride * sizeof(float)), [n_channels] "r" ((long) n_channels), [input_col_stride1] "r" (input_col_stride * sizeof(float)) : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" ); } template <> template <> void Conv::execute_tile( int n_channels, const void *weight_bias_ptr, const float *inptrs[Base::inner_tile_rows][Base::inner_tile_cols], float *outptrs[Base::output_tile_rows][Base::output_tile_cols] ) { __asm __volatile( "mov x27, xzr\n" "mov x28, xzr\n" "and x26, %[n_channels], #3\n" "lsr x25, %[n_channels], #2\n" "cbz x25, 4f\n" "1:\n" "ldr q15, [%[wbptr]]\n" "ldr x21, [%[inptrs], 0]\n" "mov v8.16b, v15.16b\n" "ldr q14, [%[wbptr], #16]\n" "mov v3.16b, v15.16b\n" "ldr q10, [%[wbptr], #32]\n" "mov v2.16b, v15.16b\n" "ldr q7, [%[wbptr], #48]\n" "mov v4.16b, v15.16b\n" "ldr q13, [%[wbptr], #64]\n" "ldr q5, [%[wbptr], #80]\n" "ldr x22, [%[inptrs], 40]\n" "ldr q0, [%[wbptr], #96]\n" "ldr x20, [%[inptrs], 80]\n" "ldr q9, [%[wbptr], #112]\n" "ldr x23, [%[inptrs], 120]\n" "ldr q6, [%[wbptr], #128]\n" "subs x25, x25, #1\n" "ldr q1, [%[wbptr], #144]\n" "ldr q17, [x21, x27]\n" "fmla v8.4s, v17.4s, v14.4s\n" "ldr q18, [x22, x27]\n" "ldr q16, [x20, x27]\n" "ldr x21, [%[inptrs], 8]\n" "ldr q17, [x23, x27]\n" "ldr x22, [%[inptrs], 48]\n" "ldr q11, [x21, x27]\n" "ldr x20, [%[inptrs], 88]\n" "fmla v8.4s, v18.4s, v13.4s\n" "ldr q19, [x22, x27]\n" "ldr q15, [x20, x27]\n" "ldr x21, [%[inptrs], 16]\n" "ldr q12, [x21, x27]\n" "fmla v8.4s, v11.4s, v10.4s\n" "fmla v8.4s, v16.4s, v9.4s\n" "beq 3f\n" "2:\n" "fmla v3.4s, v16.4s, v14.4s\n" "ldr x22, [%[inptrs], 56]\n" "fmla v8.4s, v19.4s, v5.4s\n" "ldr x21, [%[inptrs], 24]\n" "fmla v2.4s, v12.4s, v14.4s\n" "ldr q16, [x22, x27]\n" "movi v11.16b, #0\n" "ldr q18, [x21, x27]\n" "fmla v3.4s, v17.4s, v13.4s\n" "ldr x20, [%[inptrs], 160]\n" "fmla v8.4s, v12.4s, v7.4s\n" "ldr x23, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v13.4s\n" "ldr q19, [x20, x27]\n" "fmov v12.4s, #6.0\n" "ldr q17, [x23, x27]\n" "fmla v3.4s, v15.4s, v10.4s\n" "ldr x20, [%[inptrs], 96]\n" "fmla v8.4s, v15.4s, v6.4s\n" "ldr x22, [%[inptrs], 64]\n" "fmla v2.4s, v18.4s, v10.4s\n" "ldr q15, [x20, x27]\n" "fmla v4.4s, v15.4s, v14.4s\n" "ldr q18, [x22, x27]\n" "fmla v3.4s, v19.4s, v9.4s\n" "ldr x21, [%[inptrs], 32]\n" "fmla v8.4s, v16.4s, v0.4s\n" "ldr x20, [%[inptrs], 168]\n" "fmla v2.4s, v15.4s, v9.4s\n" "ldr q19, [x21, x27]\n" "ldr q16, [x20, x27]\n" "ldr x23, [%[inptrs], 136]\n" "fmla v3.4s, v17.4s, v5.4s\n" "ldr x20, [%[inptrs], 104]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr q14, [x23, x27]\n" "fmla v2.4s, v18.4s, v5.4s\n" "ldr q17, [x20, x27]\n" "fmla v4.4s, v14.4s, v13.4s\n" "ldr x22, [%[inptrs], 72]\n" "fmla v3.4s, v15.4s, v7.4s\n" "ldr x20, [%[inptrs], 176]\n" "fmax v8.4s, v8.4s, v11.4s\n" "ldr q18, [x22, x27]\n" "fmla v2.4s, v19.4s, v7.4s\n" "ldr q13, [x20, x27]\n" "fmla v4.4s, v17.4s, v10.4s\n" "ldr x23, [%[inptrs], 144]\n" "fmla v3.4s, v16.4s, v6.4s\n" "ldr x20, [%[inptrs], 112]\n" "fmin v8.4s, v8.4s, v12.4s\n" "ldr q10, [x23, x27]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr q15, [x20, x27]\n" "fmla v4.4s, v13.4s, v9.4s\n" "ldr x20, [%[inptrs], 184]\n" "fmla v3.4s, v14.4s, v0.4s\n" "ldr x23, [%[inptrs], 152]\n" "ldr q9, [x20, x27]\n" "ldr x22, [%[outptrs], 0]\n" "fmla v2.4s, v18.4s, v0.4s\n" "ldr q19, [x23, x27]\n" "str q8, [x22, x28]\n" "fmla v4.4s, v10.4s, v5.4s\n" "fmla v3.4s, v13.4s, v1.4s\n" "ldr x20, [%[inptrs], 192]\n" "ldr x22, [%[outptrs], 8]\n" "ldr x24, [%[outptrs], 16]\n" "add %[wbptr], %[wbptr], #160\n" "fmla v2.4s, v15.4s, v1.4s\n" "ldr q16, [x20, x27]\n" "fmla v4.4s, v15.4s, v7.4s\n" "ldr q15, [%[wbptr]]\n" "fmax v3.4s, v3.4s, v11.4s\n" "ldr q14, [%[wbptr], #16]\n" "mov v8.16b, v15.16b\n" "ldr q10, [%[wbptr], #32]\n" "fmax v2.4s, v2.4s, v11.4s\n" "ldr q13, [%[wbptr], #64]\n" "fmla v4.4s, v9.4s, v6.4s\n" "ldr q7, [%[wbptr], #48]\n" "fmin v3.4s, v3.4s, v12.4s\n" "ldr q5, [%[wbptr], #80]\n" "fmin v2.4s, v2.4s, v12.4s\n" "ldr q9, [%[wbptr], #112]\n" "prfm pldl1keep, [%[wbptr], #64]\n" "add x27, x27, #16\n" "str q3, [x24, x28]\n" "fmla v4.4s, v19.4s, v0.4s\n" "str q2, [x22, x28]\n" "mov v3.16b, v15.16b\n" "mov v2.16b, v15.16b\n" "ldr q6, [%[wbptr], #128]\n" "ldr x24, [%[outptrs], 24]\n" "ldr x21, [%[inptrs], 0]\n" "ldr x22, [%[inptrs], 40]\n" "fmla v4.4s, v16.4s, v1.4s\n" "ldr q0, [%[wbptr], #96]\n" "ldr q17, [x21, x27]\n" "ldr x20, [%[inptrs], 80]\n" "fmla v8.4s, v17.4s, v14.4s\n" "ldr q18, [x22, x27]\n" "ldr q16, [x20, x27]\n" "ldr x21, [%[inptrs], 8]\n" "fmax v4.4s, v4.4s, v11.4s\n" "ldr q1, [%[wbptr], #144]\n" "ldr q11, [x21, x27]\n" "ldr x22, [%[inptrs], 48]\n" "fmla v8.4s, v18.4s, v13.4s\n" "ldr x21, [%[inptrs], 16]\n" "fmin v4.4s, v4.4s, v12.4s\n" "ldr q19, [x22, x27]\n" "ldr q12, [x21, x27]\n" "ldr x23, [%[inptrs], 120]\n" "ldr x20, [%[inptrs], 88]\n" "subs x25, x25, #1\n" "str q4, [x24, x28]\n" "mov v4.16b, v15.16b\n" "ldr q17, [x23, x27]\n" "fmla v8.4s, v11.4s, v10.4s\n" "ldr q15, [x20, x27]\n" "add x28, x28, #16\n" "fmla v8.4s, v16.4s, v9.4s\n" "bne 2b\n" "3:\n" "fmla v3.4s, v16.4s, v14.4s\n" "ldr x22, [%[inptrs], 56]\n" "fmla v8.4s, v19.4s, v5.4s\n" "ldr x21, [%[inptrs], 24]\n" "fmla v2.4s, v12.4s, v14.4s\n" "ldr q16, [x22, x27]\n" "movi v11.16b, #0\n" "ldr q18, [x21, x27]\n" "fmla v3.4s, v17.4s, v13.4s\n" "ldr x20, [%[inptrs], 160]\n" "fmla v8.4s, v12.4s, v7.4s\n" "ldr x23, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v13.4s\n" "ldr q19, [x20, x27]\n" "fmov v12.4s, #6.0\n" "ldr q17, [x23, x27]\n" "fmla v3.4s, v15.4s, v10.4s\n" "ldr x20, [%[inptrs], 96]\n" "fmla v8.4s, v15.4s, v6.4s\n" "ldr x22, [%[inptrs], 64]\n" "fmla v2.4s, v18.4s, v10.4s\n" "ldr q15, [x20, x27]\n" "fmla v4.4s, v15.4s, v14.4s\n" "ldr q18, [x22, x27]\n" "fmla v3.4s, v19.4s, v9.4s\n" "ldr x21, [%[inptrs], 32]\n" "fmla v8.4s, v16.4s, v0.4s\n" "ldr x20, [%[inptrs], 168]\n" "fmla v2.4s, v15.4s, v9.4s\n" "ldr q19, [x21, x27]\n" "ldr q16, [x20, x27]\n" "ldr x23, [%[inptrs], 136]\n" "fmla v3.4s, v17.4s, v5.4s\n" "ldr x20, [%[inptrs], 104]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr q14, [x23, x27]\n" "fmla v2.4s, v18.4s, v5.4s\n" "ldr q17, [x20, x27]\n" "fmla v4.4s, v14.4s, v13.4s\n" "ldr x22, [%[inptrs], 72]\n" "fmla v3.4s, v15.4s, v7.4s\n" "ldr x20, [%[inptrs], 176]\n" "fmax v8.4s, v8.4s, v11.4s\n" "ldr q18, [x22, x27]\n" "fmla v2.4s, v19.4s, v7.4s\n" "ldr q13, [x20, x27]\n" "fmla v4.4s, v17.4s, v10.4s\n" "ldr x23, [%[inptrs], 144]\n" "fmla v3.4s, v16.4s, v6.4s\n" "ldr x20, [%[inptrs], 112]\n" "fmin v8.4s, v8.4s, v12.4s\n" "ldr q10, [x23, x27]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr q15, [x20, x27]\n" "fmla v4.4s, v13.4s, v9.4s\n" "ldr x20, [%[inptrs], 184]\n" "fmla v3.4s, v14.4s, v0.4s\n" "ldr x23, [%[inptrs], 152]\n" "ldr q9, [x20, x27]\n" "ldr x22, [%[outptrs], 0]\n" "fmla v2.4s, v18.4s, v0.4s\n" "ldr q19, [x23, x27]\n" "str q8, [x22, x28]\n" "fmla v4.4s, v10.4s, v5.4s\n" "fmla v3.4s, v13.4s, v1.4s\n" "ldr x20, [%[inptrs], 192]\n" "ldr x22, [%[outptrs], 8]\n" "ldr x24, [%[outptrs], 16]\n" "add %[wbptr], %[wbptr], #160\n" "fmla v2.4s, v15.4s, v1.4s\n" "ldr q16, [x20, x27]\n" "fmla v4.4s, v15.4s, v7.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmax v3.4s, v3.4s, v11.4s\n" "add x27, x27, #16\n" "fmax v2.4s, v2.4s, v11.4s\n" "fmla v4.4s, v9.4s, v6.4s\n" "fmin v3.4s, v3.4s, v12.4s\n" "fmin v2.4s, v2.4s, v12.4s\n" "str q3, [x24, x28]\n" "fmla v4.4s, v19.4s, v0.4s\n" "str q2, [x22, x28]\n" "ldr x24, [%[outptrs], 24]\n" "fmla v4.4s, v16.4s, v1.4s\n" "fmax v4.4s, v4.4s, v11.4s\n" "fmin v4.4s, v4.4s, v12.4s\n" "str q4, [x24, x28]\n" "add x28, x28, #16\n" "4:\n" "cbz x26, 7f\n" "ldr s15, [%[wbptr]]\n" "mov v8.16b, v15.16b\n" "ldr s14, [%[wbptr], #4]\n" "mov v3.16b, v15.16b\n" "ldr s10, [%[wbptr], #8]\n" "mov v2.16b, v15.16b\n" "ldr s7, [%[wbptr], #12]\n" "mov v4.16b, v15.16b\n" "ldr s13, [%[wbptr], #16]\n" "ldr s5, [%[wbptr], #20]\n" "ldr x21, [%[inptrs], 0]\n" "ldr s0, [%[wbptr], #24]\n" "ldr x22, [%[inptrs], 40]\n" "ldr s9, [%[wbptr], #28]\n" "ldr x20, [%[inptrs], 80]\n" "ldr s6, [%[wbptr], #32]\n" "ldr x23, [%[inptrs], 120]\n" "ldr s1, [%[wbptr], #36]\n" "subs x26, x26, #1\n" "ldr s17, [x21, x27]\n" "ldr s18, [x22, x27]\n" "fmla v8.4s, v17.4s, v14.4s\n" "ldr s16, [x20, x27]\n" "ldr s17, [x23, x27]\n" "ldr x21, [%[inptrs], 8]\n" "ldr x22, [%[inptrs], 48]\n" "ldr x20, [%[inptrs], 88]\n" "ldr s11, [x21, x27]\n" "fmla v8.4s, v18.4s, v13.4s\n" "ldr s19, [x22, x27]\n" "ldr s15, [x20, x27]\n" "ldr x21, [%[inptrs], 16]\n" "ldr s12, [x21, x27]\n" "fmla v8.4s, v11.4s, v10.4s\n" "fmla v8.4s, v16.4s, v9.4s\n" "beq 6f\n" "5:\n" "fmla v3.4s, v16.4s, v14.4s\n" "ldr x22, [%[inptrs], 56]\n" "fmla v8.4s, v19.4s, v5.4s\n" "ldr x21, [%[inptrs], 24]\n" "fmla v2.4s, v12.4s, v14.4s\n" "ldr s16, [x22, x27]\n" "movi v11.16b, #0\n" "ldr s18, [x21, x27]\n" "fmla v3.4s, v17.4s, v13.4s\n" "ldr x20, [%[inptrs], 160]\n" "fmla v8.4s, v12.4s, v7.4s\n" "ldr x23, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v13.4s\n" "ldr s19, [x20, x27]\n" "fmov v12.4s, #6.0\n" "ldr s17, [x23, x27]\n" "fmla v3.4s, v15.4s, v10.4s\n" "ldr x20, [%[inptrs], 96]\n" "fmla v8.4s, v15.4s, v6.4s\n" "ldr x22, [%[inptrs], 64]\n" "fmla v2.4s, v18.4s, v10.4s\n" "ldr s15, [x20, x27]\n" "fmla v4.4s, v15.4s, v14.4s\n" "ldr s18, [x22, x27]\n" "fmla v3.4s, v19.4s, v9.4s\n" "ldr x21, [%[inptrs], 32]\n" "fmla v8.4s, v16.4s, v0.4s\n" "ldr x20, [%[inptrs], 168]\n" "fmla v2.4s, v15.4s, v9.4s\n" "ldr s19, [x21, x27]\n" "ldr s16, [x20, x27]\n" "ldr x23, [%[inptrs], 136]\n" "fmla v3.4s, v17.4s, v5.4s\n" "ldr x20, [%[inptrs], 104]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr s14, [x23, x27]\n" "fmla v2.4s, v18.4s, v5.4s\n" "ldr s17, [x20, x27]\n" "fmla v4.4s, v14.4s, v13.4s\n" "ldr x22, [%[inptrs], 72]\n" "fmla v3.4s, v15.4s, v7.4s\n" "ldr x20, [%[inptrs], 176]\n" "fmax v8.4s, v8.4s, v11.4s\n" "ldr s18, [x22, x27]\n" "fmla v2.4s, v19.4s, v7.4s\n" "ldr s13, [x20, x27]\n" "fmla v4.4s, v17.4s, v10.4s\n" "ldr x23, [%[inptrs], 144]\n" "fmla v3.4s, v16.4s, v6.4s\n" "ldr x20, [%[inptrs], 112]\n" "fmin v8.4s, v8.4s, v12.4s\n" "ldr s10, [x23, x27]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr s15, [x20, x27]\n" "fmla v4.4s, v13.4s, v9.4s\n" "ldr x20, [%[inptrs], 184]\n" "fmla v3.4s, v14.4s, v0.4s\n" "ldr x23, [%[inptrs], 152]\n" "ldr s9, [x20, x27]\n" "ldr x22, [%[outptrs], 0]\n" "fmla v2.4s, v18.4s, v0.4s\n" "ldr s19, [x23, x27]\n" "str s8, [x22, x28]\n" "fmla v4.4s, v10.4s, v5.4s\n" "fmla v3.4s, v13.4s, v1.4s\n" "ldr x20, [%[inptrs], 192]\n" "ldr x22, [%[outptrs], 8]\n" "ldr x24, [%[outptrs], 16]\n" "add %[wbptr], %[wbptr], #40\n" "fmla v2.4s, v15.4s, v1.4s\n" "ldr s16, [x20, x27]\n" "fmla v4.4s, v15.4s, v7.4s\n" "ldr s15, [%[wbptr]]\n" "fmax v3.4s, v3.4s, v11.4s\n" "ldr s14, [%[wbptr], #4]\n" "mov v8.16b, v15.16b\n" "ldr s10, [%[wbptr], #8]\n" "fmax v2.4s, v2.4s, v11.4s\n" "ldr s13, [%[wbptr], #16]\n" "fmla v4.4s, v9.4s, v6.4s\n" "ldr s7, [%[wbptr], #12]\n" "fmin v3.4s, v3.4s, v12.4s\n" "ldr s5, [%[wbptr], #20]\n" "fmin v2.4s, v2.4s, v12.4s\n" "ldr s9, [%[wbptr], #28]\n" "prfm pldl1keep, [%[wbptr], #64]\n" "add x27, x27, #4\n" "str s3, [x24, x28]\n" "fmla v4.4s, v19.4s, v0.4s\n" "str s2, [x22, x28]\n" "mov v3.16b, v15.16b\n" "mov v2.16b, v15.16b\n" "ldr s6, [%[wbptr], #32]\n" "ldr x24, [%[outptrs], 24]\n" "ldr x21, [%[inptrs], 0]\n" "ldr x22, [%[inptrs], 40]\n" "fmla v4.4s, v16.4s, v1.4s\n" "ldr s0, [%[wbptr], #24]\n" "ldr s17, [x21, x27]\n" "ldr x20, [%[inptrs], 80]\n" "fmla v8.4s, v17.4s, v14.4s\n" "ldr s18, [x22, x27]\n" "ldr s16, [x20, x27]\n" "ldr x21, [%[inptrs], 8]\n" "fmax v4.4s, v4.4s, v11.4s\n" "ldr s1, [%[wbptr], #36]\n" "ldr s11, [x21, x27]\n" "ldr x22, [%[inptrs], 48]\n" "fmla v8.4s, v18.4s, v13.4s\n" "ldr x21, [%[inptrs], 16]\n" "fmin v4.4s, v4.4s, v12.4s\n" "ldr s19, [x22, x27]\n" "ldr s12, [x21, x27]\n" "ldr x23, [%[inptrs], 120]\n" "ldr x20, [%[inptrs], 88]\n" "subs x26, x26, #1\n" "str s4, [x24, x28]\n" "mov v4.16b, v15.16b\n" "ldr s17, [x23, x27]\n" "fmla v8.4s, v11.4s, v10.4s\n" "ldr s15, [x20, x27]\n" "add x28, x28, #4\n" "fmla v8.4s, v16.4s, v9.4s\n" "bne 5b\n" "6:\n" "fmla v3.4s, v16.4s, v14.4s\n" "ldr x22, [%[inptrs], 56]\n" "fmla v8.4s, v19.4s, v5.4s\n" "ldr x21, [%[inptrs], 24]\n" "fmla v2.4s, v12.4s, v14.4s\n" "ldr s16, [x22, x27]\n" "movi v11.16b, #0\n" "ldr s18, [x21, x27]\n" "fmla v3.4s, v17.4s, v13.4s\n" "ldr x20, [%[inptrs], 160]\n" "fmla v8.4s, v12.4s, v7.4s\n" "ldr x23, [%[inptrs], 128]\n" "fmla v2.4s, v16.4s, v13.4s\n" "ldr s19, [x20, x27]\n" "fmov v12.4s, #6.0\n" "ldr s17, [x23, x27]\n" "fmla v3.4s, v15.4s, v10.4s\n" "ldr x20, [%[inptrs], 96]\n" "fmla v8.4s, v15.4s, v6.4s\n" "ldr x22, [%[inptrs], 64]\n" "fmla v2.4s, v18.4s, v10.4s\n" "ldr s15, [x20, x27]\n" "fmla v4.4s, v15.4s, v14.4s\n" "ldr s18, [x22, x27]\n" "fmla v3.4s, v19.4s, v9.4s\n" "ldr x21, [%[inptrs], 32]\n" "fmla v8.4s, v16.4s, v0.4s\n" "ldr x20, [%[inptrs], 168]\n" "fmla v2.4s, v15.4s, v9.4s\n" "ldr s19, [x21, x27]\n" "ldr s16, [x20, x27]\n" "ldr x23, [%[inptrs], 136]\n" "fmla v3.4s, v17.4s, v5.4s\n" "ldr x20, [%[inptrs], 104]\n" "fmla v8.4s, v15.4s, v1.4s\n" "ldr s14, [x23, x27]\n" "fmla v2.4s, v18.4s, v5.4s\n" "ldr s17, [x20, x27]\n" "fmla v4.4s, v14.4s, v13.4s\n" "ldr x22, [%[inptrs], 72]\n" "fmla v3.4s, v15.4s, v7.4s\n" "ldr x20, [%[inptrs], 176]\n" "fmax v8.4s, v8.4s, v11.4s\n" "ldr s18, [x22, x27]\n" "fmla v2.4s, v19.4s, v7.4s\n" "ldr s13, [x20, x27]\n" "fmla v4.4s, v17.4s, v10.4s\n" "ldr x23, [%[inptrs], 144]\n" "fmla v3.4s, v16.4s, v6.4s\n" "ldr x20, [%[inptrs], 112]\n" "fmin v8.4s, v8.4s, v12.4s\n" "ldr s10, [x23, x27]\n" "fmla v2.4s, v17.4s, v6.4s\n" "ldr s15, [x20, x27]\n" "fmla v4.4s, v13.4s, v9.4s\n" "ldr x20, [%[inptrs], 184]\n" "fmla v3.4s, v14.4s, v0.4s\n" "ldr x23, [%[inptrs], 152]\n" "ldr s9, [x20, x27]\n" "ldr x22, [%[outptrs], 0]\n" "fmla v2.4s, v18.4s, v0.4s\n" "ldr s19, [x23, x27]\n" "str s8, [x22, x28]\n" "fmla v4.4s, v10.4s, v5.4s\n" "fmla v3.4s, v13.4s, v1.4s\n" "ldr x20, [%[inptrs], 192]\n" "ldr x22, [%[outptrs], 8]\n" "ldr x24, [%[outptrs], 16]\n" "add %[wbptr], %[wbptr], #40\n" "fmla v2.4s, v15.4s, v1.4s\n" "ldr s16, [x20, x27]\n" "fmla v4.4s, v15.4s, v7.4s\n" "prfm pldl1keep, [%[wbptr], #64]\n" "fmax v3.4s, v3.4s, v11.4s\n" "add x27, x27, #4\n" "fmax v2.4s, v2.4s, v11.4s\n" "fmla v4.4s, v9.4s, v6.4s\n" "fmin v3.4s, v3.4s, v12.4s\n" "fmin v2.4s, v2.4s, v12.4s\n" "str s3, [x24, x28]\n" "fmla v4.4s, v19.4s, v0.4s\n" "str s2, [x22, x28]\n" "ldr x24, [%[outptrs], 24]\n" "fmla v4.4s, v16.4s, v1.4s\n" "fmax v4.4s, v4.4s, v11.4s\n" "fmin v4.4s, v4.4s, v12.4s\n" "str s4, [x24, x28]\n" "add x28, x28, #4\n" "7:\n" : [wbptr] "+r" (weight_bias_ptr) : [inptrs] "r" (inptrs), [outptrs] "r" (outptrs), [n_channels] "r" ((long) n_channels) : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" ); } #endif // __aarch64__ template class DepthwiseConvolution<2, 2, 3, 3, 2, 2, float, float, float>; } // namespace depthwise