diff options
Diffstat (limited to 'src/core/NEON/kernels/convolution/depthwise/depthwise_3x3_3x3_1x1_fp32_fp32.cpp')
| -rw-r--r-- | src/core/NEON/kernels/convolution/depthwise/depthwise_3x3_3x3_1x1_fp32_fp32.cpp | 3210 |
1 files changed, 2298 insertions, 912 deletions
diff --git a/src/core/NEON/kernels/convolution/depthwise/depthwise_3x3_3x3_1x1_fp32_fp32.cpp b/src/core/NEON/kernels/convolution/depthwise/depthwise_3x3_3x3_1x1_fp32_fp32.cpp index 21e8f04774..2508ec7aeb 100644 --- a/src/core/NEON/kernels/convolution/depthwise/depthwise_3x3_3x3_1x1_fp32_fp32.cpp +++ b/src/core/NEON/kernels/convolution/depthwise/depthwise_3x3_3x3_1x1_fp32_fp32.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 ARM Limited. + * Copyright (c) 2018-2019 ARM Limited. * * SPDX-License-Identifier: MIT * @@ -25,931 +25,2317 @@ namespace depthwise { -using Conv = DepthwiseConvolution<3, 3, 3, 3, 1, 1, float, float>; -using ConvImpl = DepthwiseConvolutionImpl<3, 3, 3, 3, 1, 1, float, float>; -#ifdef __aarch64__ +using namespace neon_convolution_kernels; +using Conv = DepthwiseConvolution<3, 3, 3, 3, 1, 1, float, float, float>; +#ifdef __aarch64__ template <> template <> -void ConvImpl::process_tile<true, 0, 0, 0, 0, 0, 0>( - const int n_channels, - const float* const weights, - const int weight_row_stride, - const int weight_col_stride, - const float* const inptr, - const int in_row_stride, - const int in_col_stride, - float* const outptr, - const int out_row_stride, - const int out_col_stride, - const int, const int, const int, const int, const int, const int, const int, const int +void Conv::execute_tile<ActivationFunction::None>( + int n_channels, + const void *weight_bias_ptr, + const float *input, + const unsigned int input_row_stride, + const unsigned int input_col_stride, + float *output, + const unsigned int output_row_stride, + const unsigned int output_col_stride ) { - // Copy pointers - const float *uptr0 = inptr; - const float *wptr0 = weights; - float *vptr0 = outptr; - - int channels_remaining = n_channels; - if (channels_remaining >= 4) - { - // Process blocks of 4 channels at a time - int n_iters = ((channels_remaining / 4) + 1)/2 - 1; - const bool odd_tail = (channels_remaining / 4) & 1; - channels_remaining %= 4; - - asm volatile ( - "qU22B .req q0\n" "qU23B .req q0\n" "qW22A .req q0\n" - "vU22B .req v0\n" "vU23B .req v0\n" "vW22A .req v0\n" - "qV12A .req q1\n" "qW11B .req q1\n" - "vV12A .req v1\n" "vW11B .req v1\n" - "qU41A .req q2\n" "qU32B .req q2\n" "qU33A .req q2\n" "qV13B .req q2\n" - "vU41A .req v2\n" "vU32B .req v2\n" "vU33A .req v2\n" "vV13B .req v2\n" - "qU42B .req q3\n" "qU13B .req q3\n" "qU44B .req q3\n" "qU55A .req q3\n" - "vU42B .req v3\n" "vU13B .req v3\n" "vU44B .req v3\n" "vU55A .req v3\n" - "qU34B .req q4\n" "qU15A .req q4\n" "qU42A .req q4\n" "qU44A .req q4\n" "qU12B .req q4\n" - "vU34B .req v4\n" "vU15A .req v4\n" "vU42A .req v4\n" "vU44A .req v4\n" "vU12B .req v4\n" - "qU33B .req q5\n" "qU52A .req q5\n" "qW23A .req q5\n" - "vU33B .req v5\n" "vU52A .req v5\n" "vW23A .req v5\n" - "qV31A .req q6\n" "qU13A .req q6\n" "qV12B .req q6\n" - "vV31A .req v6\n" "vU13A .req v6\n" "vV12B .req v6\n" - "qU35B .req q7\n" "qU51B .req q7\n" "qV11A .req q7\n" "qU53B .req q7\n" - "vU35B .req v7\n" "vU51B .req v7\n" "vV11A .req v7\n" "vU53B .req v7\n" - "qW21A .req q8\n" "qV22B .req q8\n" - "vW21A .req v8\n" "vV22B .req v8\n" - "qV33B .req q9\n" "qU14A .req q9\n" "qV23A .req q9\n" "qU25B .req q9\n" - "vV33B .req v9\n" "vU14A .req v9\n" "vV23A .req v9\n" "vU25B .req v9\n" - "qW21B .req q10\n" "qV32A .req q10\n" "qU35A .req q10\n" - "vW21B .req v10\n" "vV32A .req v10\n" "vU35A .req v10\n" - "qV11B .req q11\n" "qU15B .req q11\n" "qV33A .req q11\n" - "vV11B .req v11\n" "vU15B .req v11\n" "vV33A .req v11\n" - "qU11B .req q12\n" "qW23B .req q12\n" "qU45A .req q12\n" - "vU11B .req v12\n" "vW23B .req v12\n" "vU45A .req v12\n" - "qW11A .req q13\n" "qU45B .req q13\n" "qU52B .req q13\n" - "vW11A .req v13\n" "vU45B .req v13\n" "vU52B .req v13\n" - "qU55B .req q14\n" "qU25A .req q14\n" "qV21A .req q14\n" - "vU55B .req v14\n" "vU25A .req v14\n" "vV21A .req v14\n" - "qU53A .req q15\n" "qV21B .req q15\n" "qU31A .req q15\n" - "vU53A .req v15\n" "vV21B .req v15\n" "vU31A .req v15\n" - "qW13B .req q16\n" "qU23A .req q16\n" - "vW13B .req v16\n" "vU23A .req v16\n" - "qW33B .req q17\n" "qW33A .req q17\n" - "vW33B .req v17\n" "vW33A .req v17\n" - "qU24B .req q18\n" "qU32A .req q18\n" "qV31B .req q18\n" "qV13A .req q18\n" - "vU24B .req v18\n" "vU32A .req v18\n" "vV31B .req v18\n" "vV13A .req v18\n" - "qU31B .req q19\n" "qU11A .req q19\n" "qU54B .req q19\n" "qU43A .req q19\n" - "vU31B .req v19\n" "vU11A .req v19\n" "vU54B .req v19\n" "vU43A .req v19\n" - "qU24A .req q20\n" "qW12B .req q20\n" "qU54A .req q20\n" - "vU24A .req v20\n" "vW12B .req v20\n" "vU54A .req v20\n" - "qV23B .req q21\n" "qW12A .req q21\n" - "vV23B .req v21\n" "vW12A .req v21\n" - "qW32A .req q22\n" "qU43B .req q22\n" - "vW32A .req v22\n" "vU43B .req v22\n" - "qW31A .req q23\n" "qV32B .req q23\n" - "vW31A .req v23\n" "vV32B .req v23\n" - "qU22A .req q24\n" "qW31B .req q24\n" - "vU22A .req v24\n" "vW31B .req v24\n" - "qU21B .req q25\n" "qV22A .req q25\n" - "vU21B .req v25\n" "vV22A .req v25\n" - "qU34A .req q26\n" "qW22B .req q26\n" "qU12A .req q26\n" - "vU34A .req v26\n" "vW22B .req v26\n" "vU12A .req v26\n" - "qW13A .req q27\n" "qU51A .req q27\n" - "vW13A .req v27\n" "vU51A .req v27\n" - "qW32B .req q28\n" - "vW32B .req v28\n" - "qU41B .req q29\n" "qU14B .req q29\n" - "vU41B .req v29\n" "vU14B .req v29\n" - "qU21A .req q30\n" - "vU21A .req v30\n" - - "uptr1 .req x0\n" - "uptr2 .req x1\n" - "uptr3 .req x2\n" - "uptr4 .req x3\n" - - "u_col_stride1 .req %x[u_col_stride]\n" - "u_col_stride2 .req x4\n" - "u_col_stride3 .req x5\n" - "u_col_stride4 .req x6\n" - - "wptr1 .req x7\n" - "wptr2 .req x8\n" - "w_col_stride1 .req %x[w_col_stride]\n" - "w_col_stride2 .req x9\n" - - "vptr1 .req x10\n" - "vptr2 .req x11\n" - "v_col_stride1 .req %x[v_col_stride]\n" - "v_col_stride2 .req x12\n" - - // Prepare strides and pointers - "add uptr1, %x[uptr0], %x[u_row_stride]\n" - "add uptr2, uptr1 , %x[u_row_stride]\n" - "add uptr3, uptr2 , %x[u_row_stride]\n" - "add uptr4, uptr3 , %x[u_row_stride]\n" - "add u_col_stride2, u_col_stride1, u_col_stride1\n" - "add u_col_stride3, u_col_stride2, u_col_stride1\n" - "add u_col_stride4, u_col_stride3, u_col_stride1\n" - - "add wptr1, %x[wptr0], %x[w_row_stride]\n" - "add wptr2, wptr1 , %x[w_row_stride]\n" - "add w_col_stride2, w_col_stride1, w_col_stride1\n" - - "add vptr1, %x[vptr0], %x[v_row_stride]\n" - "add vptr2, vptr1 , %x[v_row_stride]\n" - "add v_col_stride2, v_col_stride1, v_col_stride1\n" - - // Pre-load for A - "ldr qW13A, [%x[wptr0], w_col_stride2]\n" - "ldr qW23A, [wptr1, w_col_stride2]\n" - "ldr qW33A, [wptr2, w_col_stride2]\n" - "ldr qW12A, [%x[wptr0], w_col_stride1]\n" - "ldr qU15A, [%x[uptr0], u_col_stride4]\n" - "ldr qW22A, [wptr1, w_col_stride1]\n" - "ldr qU14A, [%x[uptr0], u_col_stride3]\n" - "ldr qW32A, [wptr2, w_col_stride1]\n" - "ldr qU13A, [%x[uptr0], u_col_stride2]\n" - "ldr qU25A, [uptr1, u_col_stride4]\n" - "ldr qU24A, [uptr1, u_col_stride3]\n" - "ldr qW11A, [%x[wptr0]], #0x10\n" - "ldr qU23A, [uptr1, u_col_stride2]\n" - "ldr qW21A, [wptr1], #0x10\n" - "ldr qW31A, [wptr2], #0x10\n" - "ldr qU34A, [uptr2, u_col_stride3]\n" - "ldr qU35A, [uptr2, u_col_stride4]\n" - - // First part of A - "fmul vV13A.4s, vU15A.4s, vW13A.4s\n" - "ldr qU33A, [uptr2, u_col_stride2]\n" - "fmul vV12A.4s, vU14A.4s, vW13A.4s\n" - "cbz %x[n_iters], 2f\n" // Jump to tail if not looping - - "1:" // Main loop, double unrolled - // A Part - "fmla vV13A.4s, vU14A.4s, vW12A.4s\n" - "ldr qU45A, [uptr3, u_col_stride4]\n" - "fmul vV11A.4s, vU13A.4s, vW13A.4s\n" - "fmla vV12A.4s, vU13A.4s, vW12A.4s\n" - "fmla vV13A.4s, vU13A.4s, vW11A.4s\n" - "ldr qU44A, [uptr3, u_col_stride3]\n" - "fmla vV13A.4s, vU25A.4s, vW23A.4s\n" - "fmul vV23A.4s, vU25A.4s, vW13A.4s\n" - "ldr qU43A, [uptr3, u_col_stride2]\n" - "fmla vV12A.4s, vU24A.4s, vW23A.4s\n" - "fmla vV13A.4s, vU24A.4s, vW22A.4s\n" - "fmul vV22A.4s, vU24A.4s, vW13A.4s\n" - "fmla vV23A.4s, vU24A.4s, vW12A.4s\n" - "ldr qU55A, [uptr4, u_col_stride4]\n" - "fmla vV11A.4s, vU23A.4s, vW23A.4s\n" - "fmla vV12A.4s, vU23A.4s, vW22A.4s\n" - "fmla vV13A.4s, vU23A.4s, vW21A.4s\n" - "fmul vV21A.4s, vU23A.4s, vW13A.4s\n" - "fmla vV22A.4s, vU23A.4s, vW12A.4s\n" - "fmla vV23A.4s, vU23A.4s, vW11A.4s\n" - "ldr qU54A, [uptr4, u_col_stride3]\n" - "fmla vV13A.4s, vU35A.4s, vW33A.4s\n" - "fmla vV23A.4s, vU35A.4s, vW23A.4s\n" - "fmul vV33A.4s, vU35A.4s, vW13A.4s\n" - "ldr qU53A, [uptr4, u_col_stride2]\n" - "fmla vV12A.4s, vU34A.4s, vW33A.4s\n" - "fmla vV13A.4s, vU34A.4s, vW32A.4s\n" - "fmla vV22A.4s, vU34A.4s, vW23A.4s\n" - "fmla vV23A.4s, vU34A.4s, vW22A.4s\n" - "fmul vV32A.4s, vU34A.4s, vW13A.4s\n" - "fmla vV33A.4s, vU34A.4s, vW12A.4s\n" - "ldr qU12A, [%x[uptr0], u_col_stride1]\n" - "fmla vV11A.4s, vU33A.4s, vW33A.4s\n" - "fmla vV12A.4s, vU33A.4s, vW32A.4s\n" - "fmla vV13A.4s, vU33A.4s, vW31A.4s\n" - "str qV13A, [%x[vptr0], v_col_stride2]\n" - "fmla vV21A.4s, vU33A.4s, vW23A.4s\n" - "fmla vV22A.4s, vU33A.4s, vW22A.4s\n" - "fmla vV23A.4s, vU33A.4s, vW21A.4s\n" - "fmul vV31A.4s, vU33A.4s, vW13A.4s\n" - "ldr qW13B, [%x[wptr0], w_col_stride2]\n" - "fmla vV32A.4s, vU33A.4s, vW12A.4s\n" - "fmla vV33A.4s, vU33A.4s, vW11A.4s\n" - "ldr qU22A, [uptr1, u_col_stride1]\n" - "fmla vV23A.4s, vU45A.4s, vW33A.4s\n" - "fmla vV33A.4s, vU45A.4s, vW23A.4s\n" - "ldr qU32A, [uptr2, u_col_stride1]\n" - "fmla vV22A.4s, vU44A.4s, vW33A.4s\n" - "fmla vV23A.4s, vU44A.4s, vW32A.4s\n" - "fmla vV32A.4s, vU44A.4s, vW23A.4s\n" - "fmla vV33A.4s, vU44A.4s, vW22A.4s\n" - "ldr qU42A, [uptr3, u_col_stride1]\n" - "fmla vV21A.4s, vU43A.4s, vW33A.4s\n" - "fmla vV22A.4s, vU43A.4s, vW32A.4s\n" - "fmla vV23A.4s, vU43A.4s, vW31A.4s\n" - "str qV23A, [vptr1, v_col_stride2]\n" - "fmla vV31A.4s, vU43A.4s, vW23A.4s\n" - "ldr qW23B, [wptr1, w_col_stride2]\n" - "fmla vV32A.4s, vU43A.4s, vW22A.4s\n" - "fmla vV33A.4s, vU43A.4s, vW21A.4s\n" - "ldr qU52A, [uptr4, u_col_stride1]\n" - "fmla vV33A.4s, vU55A.4s, vW33A.4s\n" - "ldr qU11A, [%x[uptr0]], #0x10\n" - "fmla vV32A.4s, vU54A.4s, vW33A.4s\n" - "fmla vV33A.4s, vU54A.4s, vW32A.4s\n" - "ldr qU21A, [uptr1], #0x10\n" - "fmla vV31A.4s, vU53A.4s, vW33A.4s\n" - "ldr qW33B, [wptr2, w_col_stride2]\n" - "fmla vV32A.4s, vU53A.4s, vW32A.4s\n" - "fmla vV33A.4s, vU53A.4s, vW31A.4s\n" - "str qV33A, [vptr2, v_col_stride2]\n" - "fmla vV11A.4s, vU12A.4s, vW12A.4s\n" - "ldr qU31A, [uptr2], #0x10\n" - "fmla vV12A.4s, vU12A.4s, vW11A.4s\n" - "ldr qU41A, [uptr3], #0x10\n" - "fmla vV11A.4s, vU22A.4s, vW22A.4s\n" - "ldr qU51A, [uptr4], #0x10\n" - "fmla vV12A.4s, vU22A.4s, vW21A.4s\n" - "ldr qW12B, [%x[wptr0], w_col_stride1]\n" - "fmla vV21A.4s, vU22A.4s, vW12A.4s\n" - "ldr qU15B, [%x[uptr0], u_col_stride4]\n" - "fmla vV22A.4s, vU22A.4s, vW11A.4s\n" - "ldr qW22B, [wptr1, w_col_stride1]\n" - "fmla vV11A.4s, vU32A.4s, vW32A.4s\n" - "ldr qU14B, [%x[uptr0], u_col_stride3]\n" - "fmla vV12A.4s, vU32A.4s, vW31A.4s\n" - "str qV12A, [%x[vptr0], v_col_stride1]\n" - "fmla vV21A.4s, vU32A.4s, vW22A.4s\n" - "ldr qW32B, [wptr2, w_col_stride1]\n" - "fmla vV22A.4s, vU32A.4s, vW21A.4s\n" - "ldr qU13B, [%x[uptr0], u_col_stride2]\n" - "fmla vV31A.4s, vU32A.4s, vW12A.4s\n" - "ldr qU25B, [uptr1, u_col_stride4]\n" - "fmla vV32A.4s, vU32A.4s, vW11A.4s\n" - "ldr qU24B, [uptr1, u_col_stride3]\n" - "fmla vV21A.4s, vU42A.4s, vW32A.4s\n" - "fmla vV22A.4s, vU42A.4s, vW31A.4s\n" - "str qV22A, [vptr1, v_col_stride1]\n" - "fmla vV31A.4s, vU42A.4s, vW22A.4s\n" - "fmla vV32A.4s, vU42A.4s, vW21A.4s\n" - "fmla vV31A.4s, vU52A.4s, vW32A.4s\n" - "fmla vV32A.4s, vU52A.4s, vW31A.4s\n" - "str qV32A, [vptr2, v_col_stride1]\n" - "fmla vV11A.4s, vU11A.4s, vW11A.4s\n" - "ldr qW11B, [%x[wptr0]], #0x10\n" - "fmla vV11A.4s, vU21A.4s, vW21A.4s\n" - "ldr qU23B, [uptr1, u_col_stride2]\n" - "fmla vV21A.4s, vU21A.4s, vW11A.4s\n" - "ldr qW21B, [wptr1], #0x10\n" - "fmla vV11A.4s, vU31A.4s, vW31A.4s\n" - "str qV11A, [%x[vptr0]], #0x10\n" - "fmla vV21A.4s, vU31A.4s, vW21A.4s\n" - "ldr qW31B, [wptr2], #0x10\n" - "fmla vV31A.4s, vU31A.4s, vW11A.4s\n" - "ldr qU34B, [uptr2, u_col_stride3]\n" - "fmla vV21A.4s, vU41A.4s, vW31A.4s\n" - "str qV21A, [vptr1], #0x10\n" - "fmla vV31A.4s, vU41A.4s, vW21A.4s\n" - "ldr qU35B, [uptr2, u_col_stride4]\n" - "fmla vV31A.4s, vU51A.4s, vW31A.4s\n" - "str qV31A, [vptr2], #0x10\n" - - // B Part - "fmul vV13B.4s, vU15B.4s, vW13B.4s\n" - "ldr qU33B, [uptr2, u_col_stride2]\n" - "fmul vV12B.4s, vU14B.4s, vW13B.4s\n" - "fmla vV13B.4s, vU14B.4s, vW12B.4s\n" - "ldr qU45B, [uptr3, u_col_stride4]\n" - "fmul vV11B.4s, vU13B.4s, vW13B.4s\n" - "fmla vV12B.4s, vU13B.4s, vW12B.4s\n" - "fmla vV13B.4s, vU13B.4s, vW11B.4s\n" - "ldr qU44B, [uptr3, u_col_stride3]\n" - "fmla vV13B.4s, vU25B.4s, vW23B.4s\n" - "fmul vV23B.4s, vU25B.4s, vW13B.4s\n" - "ldr qU43B, [uptr3, u_col_stride2]\n" - "fmla vV12B.4s, vU24B.4s, vW23B.4s\n" - "fmla vV13B.4s, vU24B.4s, vW22B.4s\n" - "fmul vV22B.4s, vU24B.4s, vW13B.4s\n" - "fmla vV23B.4s, vU24B.4s, vW12B.4s\n" - "ldr qU55B, [uptr4, u_col_stride4]\n" - "fmla vV11B.4s, vU23B.4s, vW23B.4s\n" - "fmla vV12B.4s, vU23B.4s, vW22B.4s\n" - "fmla vV13B.4s, vU23B.4s, vW21B.4s\n" - "fmul vV21B.4s, vU23B.4s, vW13B.4s\n" - "fmla vV22B.4s, vU23B.4s, vW12B.4s\n" - "fmla vV23B.4s, vU23B.4s, vW11B.4s\n" - "ldr qU54B, [uptr4, u_col_stride3]\n" - "fmla vV13B.4s, vU35B.4s, vW33B.4s\n" - "fmla vV23B.4s, vU35B.4s, vW23B.4s\n" - "fmul vV33B.4s, vU35B.4s, vW13B.4s\n" - "ldr qU53B, [uptr4, u_col_stride2]\n" - "fmla vV12B.4s, vU34B.4s, vW33B.4s\n" - "fmla vV13B.4s, vU34B.4s, vW32B.4s\n" - "fmla vV22B.4s, vU34B.4s, vW23B.4s\n" - "fmla vV23B.4s, vU34B.4s, vW22B.4s\n" - "fmul vV32B.4s, vU34B.4s, vW13B.4s\n" - "fmla vV33B.4s, vU34B.4s, vW12B.4s\n" - "ldr qU12B, [%x[uptr0], u_col_stride1]\n" - "fmla vV11B.4s, vU33B.4s, vW33B.4s\n" - "fmla vV12B.4s, vU33B.4s, vW32B.4s\n" - "fmla vV13B.4s, vU33B.4s, vW31B.4s\n" - "str qV13B, [%x[vptr0], v_col_stride2]\n" - "fmla vV21B.4s, vU33B.4s, vW23B.4s\n" - "fmla vV22B.4s, vU33B.4s, vW22B.4s\n" - "fmla vV23B.4s, vU33B.4s, vW21B.4s\n" - "fmul vV31B.4s, vU33B.4s, vW13B.4s\n" - "ldr qW13A, [%x[wptr0], w_col_stride2]\n" - "fmla vV32B.4s, vU33B.4s, vW12B.4s\n" - "fmla vV33B.4s, vU33B.4s, vW11B.4s\n" - "ldr qU22B, [uptr1, u_col_stride1]\n" - "fmla vV23B.4s, vU45B.4s, vW33B.4s\n" - "fmla vV33B.4s, vU45B.4s, vW23B.4s\n" - "ldr qU32B, [uptr2, u_col_stride1]\n" - "fmla vV22B.4s, vU44B.4s, vW33B.4s\n" - "fmla vV23B.4s, vU44B.4s, vW32B.4s\n" - "fmla vV32B.4s, vU44B.4s, vW23B.4s\n" - "fmla vV33B.4s, vU44B.4s, vW22B.4s\n" - "ldr qU42B, [uptr3, u_col_stride1]\n" - "fmla vV21B.4s, vU43B.4s, vW33B.4s\n" - "fmla vV22B.4s, vU43B.4s, vW32B.4s\n" - "fmla vV23B.4s, vU43B.4s, vW31B.4s\n" - "str qV23B, [vptr1, v_col_stride2]\n" - "fmla vV31B.4s, vU43B.4s, vW23B.4s\n" - "ldr qW23A, [wptr1, w_col_stride2]\n" - "fmla vV32B.4s, vU43B.4s, vW22B.4s\n" - "fmla vV33B.4s, vU43B.4s, vW21B.4s\n" - "ldr qU52B, [uptr4, u_col_stride1]\n" - "fmla vV33B.4s, vU55B.4s, vW33B.4s\n" - "ldr qU11B, [%x[uptr0]], #0x10\n" - "fmla vV32B.4s, vU54B.4s, vW33B.4s\n" - "fmla vV33B.4s, vU54B.4s, vW32B.4s\n" - "ldr qU21B, [uptr1], #0x10\n" - "fmla vV31B.4s, vU53B.4s, vW33B.4s\n" - "ldr qW33A, [wptr2, w_col_stride2]\n" - "fmla vV32B.4s, vU53B.4s, vW32B.4s\n" - "fmla vV33B.4s, vU53B.4s, vW31B.4s\n" - "str qV33B, [vptr2, v_col_stride2]\n" - "fmla vV11B.4s, vU12B.4s, vW12B.4s\n" - "ldr qU31B, [uptr2], #0x10\n" - "fmla vV12B.4s, vU12B.4s, vW11B.4s\n" - "ldr qU41B, [uptr3], #0x10\n" - "fmla vV11B.4s, vU22B.4s, vW22B.4s\n" - "ldr qU51B, [uptr4], #0x10\n" - "fmla vV12B.4s, vU22B.4s, vW21B.4s\n" - "ldr qW12A, [%x[wptr0], w_col_stride1]\n" - "fmla vV21B.4s, vU22B.4s, vW12B.4s\n" - "ldr qU15A, [%x[uptr0], u_col_stride4]\n" - "fmla vV22B.4s, vU22B.4s, vW11B.4s\n" - "ldr qW22A, [wptr1, w_col_stride1]\n" - "fmla vV11B.4s, vU32B.4s, vW32B.4s\n" - "ldr qU14A, [%x[uptr0], u_col_stride3]\n" - "fmla vV12B.4s, vU32B.4s, vW31B.4s\n" - "str qV12B, [%x[vptr0], v_col_stride1]\n" - "fmla vV21B.4s, vU32B.4s, vW22B.4s\n" - "ldr qW32A, [wptr2, w_col_stride1]\n" - "fmla vV22B.4s, vU32B.4s, vW21B.4s\n" - "ldr qU13A, [%x[uptr0], u_col_stride2]\n" - "fmla vV31B.4s, vU32B.4s, vW12B.4s\n" - "ldr qU25A, [uptr1, u_col_stride4]\n" - "fmla vV32B.4s, vU32B.4s, vW11B.4s\n" - "ldr qU24A, [uptr1, u_col_stride3]\n" - "fmla vV21B.4s, vU42B.4s, vW32B.4s\n" - "fmla vV22B.4s, vU42B.4s, vW31B.4s\n" - "str qV22B, [vptr1, v_col_stride1]\n" - "fmla vV31B.4s, vU42B.4s, vW22B.4s\n" - "fmla vV32B.4s, vU42B.4s, vW21B.4s\n" - "fmla vV31B.4s, vU52B.4s, vW32B.4s\n" - "subs %x[n_iters], %x[n_iters], #1\n" - "fmla vV32B.4s, vU52B.4s, vW31B.4s\n" - "str qV32B, [vptr2, v_col_stride1]\n" - "fmla vV11B.4s, vU11B.4s, vW11B.4s\n" - "ldr qW11A, [%x[wptr0]], #0x10\n" - "fmla vV11B.4s, vU21B.4s, vW21B.4s\n" - "ldr qU23A, [uptr1, u_col_stride2]\n" - "fmla vV21B.4s, vU21B.4s, vW11B.4s\n" - "ldr qW21A, [wptr1], #0x10\n" - "fmla vV11B.4s, vU31B.4s, vW31B.4s\n" - "str qV11B, [%x[vptr0]], #0x10\n" - "fmla vV21B.4s, vU31B.4s, vW21B.4s\n" - "ldr qW31A, [wptr2], #0x10\n" - "fmla vV31B.4s, vU31B.4s, vW11B.4s\n" - "ldr qU34A, [uptr2, u_col_stride3]\n" - "fmla vV21B.4s, vU41B.4s, vW31B.4s\n" - "str qV21B, [vptr1], #0x10\n" - "fmla vV31B.4s, vU41B.4s, vW21B.4s\n" - "ldr qU35A, [uptr2, u_col_stride4]\n" - "fmla vV31B.4s, vU51B.4s, vW31B.4s\n" - "str qV31B, [vptr2], #0x10\n" - - // First part of A - "fmul vV13A.4s, vU15A.4s, vW13A.4s\n" - "ldr qU33A, [uptr2, u_col_stride2]\n" - "fmul vV12A.4s, vU14A.4s, vW13A.4s\n" - "bne 1b\n" // Loop - - "2:" // Tail dispatch - "cbnz %w[odd_tail], 3f\n" - - // Even tail - // A Part - "fmla vV13A.4s, vU14A.4s, vW12A.4s\n" - "ldr qU45A, [uptr3, u_col_stride4]\n" - "fmul vV11A.4s, vU13A.4s, vW13A.4s\n" - "fmla vV12A.4s, vU13A.4s, vW12A.4s\n" - "fmla vV13A.4s, vU13A.4s, vW11A.4s\n" - "ldr qU44A, [uptr3, u_col_stride3]\n" - "fmla vV13A.4s, vU25A.4s, vW23A.4s\n" - "fmul vV23A.4s, vU25A.4s, vW13A.4s\n" - "ldr qU43A, [uptr3, u_col_stride2]\n" - "fmla vV12A.4s, vU24A.4s, vW23A.4s\n" - "fmla vV13A.4s, vU24A.4s, vW22A.4s\n" - "fmul vV22A.4s, vU24A.4s, vW13A.4s\n" - "fmla vV23A.4s, vU24A.4s, vW12A.4s\n" - "ldr qU55A, [uptr4, u_col_stride4]\n" - "fmla vV11A.4s, vU23A.4s, vW23A.4s\n" - "fmla vV12A.4s, vU23A.4s, vW22A.4s\n" - "fmla vV13A.4s, vU23A.4s, vW21A.4s\n" - "fmul vV21A.4s, vU23A.4s, vW13A.4s\n" - "fmla vV22A.4s, vU23A.4s, vW12A.4s\n" - "fmla vV23A.4s, vU23A.4s, vW11A.4s\n" - "ldr qU54A, [uptr4, u_col_stride3]\n" - "fmla vV13A.4s, vU35A.4s, vW33A.4s\n" - "fmla vV23A.4s, vU35A.4s, vW23A.4s\n" - "fmul vV33A.4s, vU35A.4s, vW13A.4s\n" - "ldr qU53A, [uptr4, u_col_stride2]\n" - "fmla vV12A.4s, vU34A.4s, vW33A.4s\n" - "fmla vV13A.4s, vU34A.4s, vW32A.4s\n" - "fmla vV22A.4s, vU34A.4s, vW23A.4s\n" - "fmla vV23A.4s, vU34A.4s, vW22A.4s\n" - "fmul vV32A.4s, vU34A.4s, vW13A.4s\n" - "fmla vV33A.4s, vU34A.4s, vW12A.4s\n" - "ldr qU12A, [%x[uptr0], u_col_stride1]\n" - "fmla vV11A.4s, vU33A.4s, vW33A.4s\n" - "fmla vV12A.4s, vU33A.4s, vW32A.4s\n" - "fmla vV13A.4s, vU33A.4s, vW31A.4s\n" - "str qV13A, [%x[vptr0], v_col_stride2]\n" - "fmla vV21A.4s, vU33A.4s, vW23A.4s\n" - "fmla vV22A.4s, vU33A.4s, vW22A.4s\n" - "fmla vV23A.4s, vU33A.4s, vW21A.4s\n" - "fmul vV31A.4s, vU33A.4s, vW13A.4s\n" - "ldr qW13B, [%x[wptr0], w_col_stride2]\n" - "fmla vV32A.4s, vU33A.4s, vW12A.4s\n" - "fmla vV33A.4s, vU33A.4s, vW11A.4s\n" - "ldr qU22A, [uptr1, u_col_stride1]\n" - "fmla vV23A.4s, vU45A.4s, vW33A.4s\n" - "fmla vV33A.4s, vU45A.4s, vW23A.4s\n" - "ldr qU32A, [uptr2, u_col_stride1]\n" - "fmla vV22A.4s, vU44A.4s, vW33A.4s\n" - "fmla vV23A.4s, vU44A.4s, vW32A.4s\n" - "fmla vV32A.4s, vU44A.4s, vW23A.4s\n" - "fmla vV33A.4s, vU44A.4s, vW22A.4s\n" - "ldr qU42A, [uptr3, u_col_stride1]\n" - "fmla vV21A.4s, vU43A.4s, vW33A.4s\n" - "fmla vV22A.4s, vU43A.4s, vW32A.4s\n" - "fmla vV23A.4s, vU43A.4s, vW31A.4s\n" - "str qV23A, [vptr1, v_col_stride2]\n" - "fmla vV31A.4s, vU43A.4s, vW23A.4s\n" - "ldr qW23B, [wptr1, w_col_stride2]\n" - "fmla vV32A.4s, vU43A.4s, vW22A.4s\n" - "fmla vV33A.4s, vU43A.4s, vW21A.4s\n" - "ldr qU52A, [uptr4, u_col_stride1]\n" - "fmla vV33A.4s, vU55A.4s, vW33A.4s\n" - "ldr qU11A, [%x[uptr0]], #0x10\n" - "fmla vV32A.4s, vU54A.4s, vW33A.4s\n" - "fmla vV33A.4s, vU54A.4s, vW32A.4s\n" - "ldr qU21A, [uptr1], #0x10\n" - "fmla vV31A.4s, vU53A.4s, vW33A.4s\n" - "ldr qW33B, [wptr2, w_col_stride2]\n" - "fmla vV32A.4s, vU53A.4s, vW32A.4s\n" - "fmla vV33A.4s, vU53A.4s, vW31A.4s\n" - "str qV33A, [vptr2, v_col_stride2]\n" - "fmla vV11A.4s, vU12A.4s, vW12A.4s\n" - "ldr qU31A, [uptr2], #0x10\n" - "fmla vV12A.4s, vU12A.4s, vW11A.4s\n" - "ldr qU41A, [uptr3], #0x10\n" - "fmla vV11A.4s, vU22A.4s, vW22A.4s\n" - "ldr qU51A, [uptr4], #0x10\n" - "fmla vV12A.4s, vU22A.4s, vW21A.4s\n" - "ldr qW12B, [%x[wptr0], w_col_stride1]\n" - "fmla vV21A.4s, vU22A.4s, vW12A.4s\n" - "ldr qU15B, [%x[uptr0], u_col_stride4]\n" - "fmla vV22A.4s, vU22A.4s, vW11A.4s\n" - "ldr qW22B, [wptr1, w_col_stride1]\n" - "fmla vV11A.4s, vU32A.4s, vW32A.4s\n" - "ldr qU14B, [%x[uptr0], u_col_stride3]\n" - "fmla vV12A.4s, vU32A.4s, vW31A.4s\n" - "str qV12A, [%x[vptr0], v_col_stride1]\n" - "fmla vV21A.4s, vU32A.4s, vW22A.4s\n" - "ldr qW32B, [wptr2, w_col_stride1]\n" - "fmla vV22A.4s, vU32A.4s, vW21A.4s\n" - "ldr qU13B, [%x[uptr0], u_col_stride2]\n" - "fmla vV31A.4s, vU32A.4s, vW12A.4s\n" - "ldr qU25B, [uptr1, u_col_stride4]\n" - "fmla vV32A.4s, vU32A.4s, vW11A.4s\n" - "ldr qU24B, [uptr1, u_col_stride3]\n" - "fmla vV21A.4s, vU42A.4s, vW32A.4s\n" - "fmla vV22A.4s, vU42A.4s, vW31A.4s\n" - "str qV22A, [vptr1, v_col_stride1]\n" - "fmla vV31A.4s, vU42A.4s, vW22A.4s\n" - "fmla vV32A.4s, vU42A.4s, vW21A.4s\n" - "fmla vV31A.4s, vU52A.4s, vW32A.4s\n" - "fmla vV32A.4s, vU52A.4s, vW31A.4s\n" - "str qV32A, [vptr2, v_col_stride1]\n" - "fmla vV11A.4s, vU11A.4s, vW11A.4s\n" - "ldr qW11B, [%x[wptr0]], #0x10\n" - "fmla vV11A.4s, vU21A.4s, vW21A.4s\n" - "ldr qU23B, [uptr1, u_col_stride2]\n" - "fmla vV21A.4s, vU21A.4s, vW11A.4s\n" - "ldr qW21B, [wptr1], #0x10\n" - "fmla vV11A.4s, vU31A.4s, vW31A.4s\n" - "str qV11A, [%x[vptr0]], #0x10\n" - "fmla vV21A.4s, vU31A.4s, vW21A.4s\n" - "ldr qW31B, [wptr2], #0x10\n" - "fmla vV31A.4s, vU31A.4s, vW11A.4s\n" - "ldr qU34B, [uptr2, u_col_stride3]\n" - "fmla vV21A.4s, vU41A.4s, vW31A.4s\n" - "str qV21A, [vptr1], #0x10\n" - "fmla vV31A.4s, vU41A.4s, vW21A.4s\n" - "ldr qU35B, [uptr2, u_col_stride4]\n" - "fmla vV31A.4s, vU51A.4s, vW31A.4s\n" - "str qV31A, [vptr2], #0x10\n" - - // B Part - "fmul vV13B.4s, vU15B.4s, vW13B.4s\n" - "ldr qU33B, [uptr2, u_col_stride2]\n" - "fmul vV12B.4s, vU14B.4s, vW13B.4s\n" - "fmla vV13B.4s, vU14B.4s, vW12B.4s\n" - "ldr qU45B, [uptr3, u_col_stride4]\n" - "fmul vV11B.4s, vU13B.4s, vW13B.4s\n" - "fmla vV12B.4s, vU13B.4s, vW12B.4s\n" - "fmla vV13B.4s, vU13B.4s, vW11B.4s\n" - "ldr qU44B, [uptr3, u_col_stride3]\n" - "fmla vV13B.4s, vU25B.4s, vW23B.4s\n" - "fmul vV23B.4s, vU25B.4s, vW13B.4s\n" - "ldr qU43B, [uptr3, u_col_stride2]\n" - "fmla vV12B.4s, vU24B.4s, vW23B.4s\n" - "fmla vV13B.4s, vU24B.4s, vW22B.4s\n" - "fmul vV22B.4s, vU24B.4s, vW13B.4s\n" - "fmla vV23B.4s, vU24B.4s, vW12B.4s\n" - "ldr qU55B, [uptr4, u_col_stride4]\n" - "fmla vV11B.4s, vU23B.4s, vW23B.4s\n" - "fmla vV12B.4s, vU23B.4s, vW22B.4s\n" - "fmla vV13B.4s, vU23B.4s, vW21B.4s\n" - "fmul vV21B.4s, vU23B.4s, vW13B.4s\n" - "fmla vV22B.4s, vU23B.4s, vW12B.4s\n" - "fmla vV23B.4s, vU23B.4s, vW11B.4s\n" - "ldr qU54B, [uptr4, u_col_stride3]\n" - "fmla vV13B.4s, vU35B.4s, vW33B.4s\n" - "fmla vV23B.4s, vU35B.4s, vW23B.4s\n" - "fmul vV33B.4s, vU35B.4s, vW13B.4s\n" - "ldr qU53B, [uptr4, u_col_stride2]\n" - "fmla vV12B.4s, vU34B.4s, vW33B.4s\n" - "fmla vV13B.4s, vU34B.4s, vW32B.4s\n" - "fmla vV22B.4s, vU34B.4s, vW23B.4s\n" - "fmla vV23B.4s, vU34B.4s, vW22B.4s\n" - "fmul vV32B.4s, vU34B.4s, vW13B.4s\n" - "fmla vV33B.4s, vU34B.4s, vW12B.4s\n" - "ldr qU12B, [%x[uptr0], u_col_stride1]\n" - "fmla vV11B.4s, vU33B.4s, vW33B.4s\n" - "fmla vV12B.4s, vU33B.4s, vW32B.4s\n" - "fmla vV13B.4s, vU33B.4s, vW31B.4s\n" - "str qV13B, [%x[vptr0], v_col_stride2]\n" - "fmla vV21B.4s, vU33B.4s, vW23B.4s\n" - "fmla vV22B.4s, vU33B.4s, vW22B.4s\n" - "fmla vV23B.4s, vU33B.4s, vW21B.4s\n" - "fmul vV31B.4s, vU33B.4s, vW13B.4s\n" - "fmla vV32B.4s, vU33B.4s, vW12B.4s\n" - "fmla vV33B.4s, vU33B.4s, vW11B.4s\n" - "ldr qU22B, [uptr1, u_col_stride1]\n" - "fmla vV23B.4s, vU45B.4s, vW33B.4s\n" - "fmla vV33B.4s, vU45B.4s, vW23B.4s\n" - "ldr qU32B, [uptr2, u_col_stride1]\n" - "fmla vV22B.4s, vU44B.4s, vW33B.4s\n" - "fmla vV23B.4s, vU44B.4s, vW32B.4s\n" - "fmla vV32B.4s, vU44B.4s, vW23B.4s\n" - "fmla vV33B.4s, vU44B.4s, vW22B.4s\n" - "ldr qU42B, [uptr3, u_col_stride1]\n" - "fmla vV21B.4s, vU43B.4s, vW33B.4s\n" - "fmla vV22B.4s, vU43B.4s, vW32B.4s\n" - "fmla vV23B.4s, vU43B.4s, vW31B.4s\n" - "str qV23B, [vptr1, v_col_stride2]\n" - "fmla vV31B.4s, vU43B.4s, vW23B.4s\n" - "fmla vV32B.4s, vU43B.4s, vW22B.4s\n" - "fmla vV33B.4s, vU43B.4s, vW21B.4s\n" - "ldr qU52B, [uptr4, u_col_stride1]\n" - "fmla vV33B.4s, vU55B.4s, vW33B.4s\n" - "ldr qU11B, [%x[uptr0]], #0x10\n" - "fmla vV32B.4s, vU54B.4s, vW33B.4s\n" - "fmla vV33B.4s, vU54B.4s, vW32B.4s\n" - "ldr qU21B, [uptr1], #0x10\n" - "fmla vV31B.4s, vU53B.4s, vW33B.4s\n" - "fmla vV32B.4s, vU53B.4s, vW32B.4s\n" - "fmla vV33B.4s, vU53B.4s, vW31B.4s\n" - "str qV33B, [vptr2, v_col_stride2]\n" - "fmla vV11B.4s, vU12B.4s, vW12B.4s\n" - "ldr qU31B, [uptr2], #0x10\n" - "fmla vV12B.4s, vU12B.4s, vW11B.4s\n" - "ldr qU41B, [uptr3], #0x10\n" - "fmla vV11B.4s, vU22B.4s, vW22B.4s\n" - "ldr qU51B, [uptr4], #0x10\n" - "fmla vV12B.4s, vU22B.4s, vW21B.4s\n" - "fmla vV21B.4s, vU22B.4s, vW12B.4s\n" - "fmla vV22B.4s, vU22B.4s, vW11B.4s\n" - "fmla vV11B.4s, vU32B.4s, vW32B.4s\n" - "fmla vV12B.4s, vU32B.4s, vW31B.4s\n" - "str qV12B, [%x[vptr0], v_col_stride1]\n" - "fmla vV21B.4s, vU32B.4s, vW22B.4s\n" - "fmla vV22B.4s, vU32B.4s, vW21B.4s\n" - "fmla vV31B.4s, vU32B.4s, vW12B.4s\n" - "fmla vV32B.4s, vU32B.4s, vW11B.4s\n" - "fmla vV21B.4s, vU42B.4s, vW32B.4s\n" - "fmla vV22B.4s, vU42B.4s, vW31B.4s\n" - "str qV22B, [vptr1, v_col_stride1]\n" - "fmla vV31B.4s, vU42B.4s, vW22B.4s\n" - "fmla vV32B.4s, vU42B.4s, vW21B.4s\n" - "fmla vV31B.4s, vU52B.4s, vW32B.4s\n" - "subs %x[n_iters], %x[n_iters], #1\n" - "fmla vV32B.4s, vU52B.4s, vW31B.4s\n" - "str qV32B, [vptr2, v_col_stride1]\n" - "fmla vV11B.4s, vU11B.4s, vW11B.4s\n" - "fmla vV11B.4s, vU21B.4s, vW21B.4s\n" - "fmla vV21B.4s, vU21B.4s, vW11B.4s\n" - "fmla vV11B.4s, vU31B.4s, vW31B.4s\n" - "str qV11B, [%x[vptr0]], #0x10\n" - "fmla vV21B.4s, vU31B.4s, vW21B.4s\n" - "fmla vV31B.4s, vU31B.4s, vW11B.4s\n" - "fmla vV21B.4s, vU41B.4s, vW31B.4s\n" - "str qV21B, [vptr1], #0x10\n" - "fmla vV31B.4s, vU41B.4s, vW21B.4s\n" - "fmla vV31B.4s, vU51B.4s, vW31B.4s\n" - "str qV31B, [vptr2], #0x10\n" - - "b 4f\n" // Branch to end of method - - "3:" // Odd tail, finish off A - "fmla vV13A.4s, vU14A.4s, vW12A.4s\n" - "ldr qU45A, [uptr3, u_col_stride4]\n" - "fmul vV11A.4s, vU13A.4s, vW13A.4s\n" - "fmla vV12A.4s, vU13A.4s, vW12A.4s\n" - "fmla vV13A.4s, vU13A.4s, vW11A.4s\n" - "ldr qU44A, [uptr3, u_col_stride3]\n" - "fmla vV13A.4s, vU25A.4s, vW23A.4s\n" - "fmul vV23A.4s, vU25A.4s, vW13A.4s\n" - "ldr qU43A, [uptr3, u_col_stride2]\n" - "fmla vV12A.4s, vU24A.4s, vW23A.4s\n" - "fmla vV13A.4s, vU24A.4s, vW22A.4s\n" - "fmul vV22A.4s, vU24A.4s, vW13A.4s\n" - "fmla vV23A.4s, vU24A.4s, vW12A.4s\n" - "ldr qU55A, [uptr4, u_col_stride4]\n" - "fmla vV11A.4s, vU23A.4s, vW23A.4s\n" - "fmla vV12A.4s, vU23A.4s, vW22A.4s\n" - "fmla vV13A.4s, vU23A.4s, vW21A.4s\n" - "fmul vV21A.4s, vU23A.4s, vW13A.4s\n" - "fmla vV22A.4s, vU23A.4s, vW12A.4s\n" - "fmla vV23A.4s, vU23A.4s, vW11A.4s\n" - "ldr qU54A, [uptr4, u_col_stride3]\n" - "fmla vV13A.4s, vU35A.4s, vW33A.4s\n" - "fmla vV23A.4s, vU35A.4s, vW23A.4s\n" - "fmul vV33A.4s, vU35A.4s, vW13A.4s\n" - "ldr qU53A, [uptr4, u_col_stride2]\n" - "fmla vV12A.4s, vU34A.4s, vW33A.4s\n" - "fmla vV13A.4s, vU34A.4s, vW32A.4s\n" - "fmla vV22A.4s, vU34A.4s, vW23A.4s\n" - "fmla vV23A.4s, vU34A.4s, vW22A.4s\n" - "fmul vV32A.4s, vU34A.4s, vW13A.4s\n" - "fmla vV33A.4s, vU34A.4s, vW12A.4s\n" - "ldr qU12A, [%x[uptr0], u_col_stride1]\n" - "fmla vV11A.4s, vU33A.4s, vW33A.4s\n" - "fmla vV12A.4s, vU33A.4s, vW32A.4s\n" - "fmla vV13A.4s, vU33A.4s, vW31A.4s\n" - "str qV13A, [%x[vptr0], v_col_stride2]\n" - "fmla vV21A.4s, vU33A.4s, vW23A.4s\n" - "fmla vV22A.4s, vU33A.4s, vW22A.4s\n" - "fmla vV23A.4s, vU33A.4s, vW21A.4s\n" - "fmul vV31A.4s, vU33A.4s, vW13A.4s\n" - "fmla vV32A.4s, vU33A.4s, vW12A.4s\n" - "fmla vV33A.4s, vU33A.4s, vW11A.4s\n" - "ldr qU22A, [uptr1, u_col_stride1]\n" - "fmla vV23A.4s, vU45A.4s, vW33A.4s\n" - "fmla vV33A.4s, vU45A.4s, vW23A.4s\n" - "ldr qU32A, [uptr2, u_col_stride1]\n" - "fmla vV22A.4s, vU44A.4s, vW33A.4s\n" - "fmla vV23A.4s, vU44A.4s, vW32A.4s\n" - "fmla vV32A.4s, vU44A.4s, vW23A.4s\n" - "fmla vV33A.4s, vU44A.4s, vW22A.4s\n" - "ldr qU42A, [uptr3, u_col_stride1]\n" - "fmla vV21A.4s, vU43A.4s, vW33A.4s\n" - "fmla vV22A.4s, vU43A.4s, vW32A.4s\n" - "fmla vV23A.4s, vU43A.4s, vW31A.4s\n" - "str qV23A, [vptr1, v_col_stride2]\n" - "fmla vV31A.4s, vU43A.4s, vW23A.4s\n" - "fmla vV32A.4s, vU43A.4s, vW22A.4s\n" - "fmla vV33A.4s, vU43A.4s, vW21A.4s\n" - "ldr qU52A, [uptr4, u_col_stride1]\n" - "fmla vV33A.4s, vU55A.4s, vW33A.4s\n" - "ldr qU11A, [%x[uptr0]], #0x10\n" - "fmla vV32A.4s, vU54A.4s, vW33A.4s\n" - "fmla vV33A.4s, vU54A.4s, vW32A.4s\n" - "ldr qU21A, [uptr1], #0x10\n" - "fmla vV31A.4s, vU53A.4s, vW33A.4s\n" - "fmla vV32A.4s, vU53A.4s, vW32A.4s\n" - "fmla vV33A.4s, vU53A.4s, vW31A.4s\n" - "str qV33A, [vptr2, v_col_stride2]\n" - "fmla vV11A.4s, vU12A.4s, vW12A.4s\n" - "ldr qU31A, [uptr2], #0x10\n" - "fmla vV12A.4s, vU12A.4s, vW11A.4s\n" - "ldr qU41A, [uptr3], #0x10\n" - "fmla vV11A.4s, vU22A.4s, vW22A.4s\n" - "ldr qU51A, [uptr4], #0x10\n" - "fmla vV12A.4s, vU22A.4s, vW21A.4s\n" - "fmla vV21A.4s, vU22A.4s, vW12A.4s\n" - "fmla vV22A.4s, vU22A.4s, vW11A.4s\n" - "fmla vV11A.4s, vU32A.4s, vW32A.4s\n" - "fmla vV12A.4s, vU32A.4s, vW31A.4s\n" - "str qV12A, [%x[vptr0], v_col_stride1]\n" - "fmla vV21A.4s, vU32A.4s, vW22A.4s\n" - "fmla vV22A.4s, vU32A.4s, vW21A.4s\n" - "fmla vV31A.4s, vU32A.4s, vW12A.4s\n" - "fmla vV32A.4s, vU32A.4s, vW11A.4s\n" - "fmla vV21A.4s, vU42A.4s, vW32A.4s\n" - "fmla vV22A.4s, vU42A.4s, vW31A.4s\n" - "str qV22A, [vptr1, v_col_stride1]\n" - "fmla vV31A.4s, vU42A.4s, vW22A.4s\n" - "fmla vV32A.4s, vU42A.4s, vW21A.4s\n" - "fmla vV31A.4s, vU52A.4s, vW32A.4s\n" - "fmla vV32A.4s, vU52A.4s, vW31A.4s\n" - "str qV32A, [vptr2, v_col_stride1]\n" - "fmla vV11A.4s, vU11A.4s, vW11A.4s\n" - "fmla vV11A.4s, vU21A.4s, vW21A.4s\n" - "fmla vV21A.4s, vU21A.4s, vW11A.4s\n" - "fmla vV11A.4s, vU31A.4s, vW31A.4s\n" - "str qV11A, [%x[vptr0]], #0x10\n" - "fmla vV21A.4s, vU31A.4s, vW21A.4s\n" - "fmla vV31A.4s, vU31A.4s, vW11A.4s\n" - "fmla vV21A.4s, vU41A.4s, vW31A.4s\n" - "str qV21A, [vptr1], #0x10\n" - "fmla vV31A.4s, vU41A.4s, vW21A.4s\n" - "fmla vV31A.4s, vU51A.4s, vW31A.4s\n" - "str qV31A, [vptr2], #0x10\n" - - "4:" // End of method - ".unreq uptr1\n" ".unreq uptr2\n" ".unreq uptr3\n" ".unreq uptr4\n" - ".unreq u_col_stride1\n" ".unreq u_col_stride2\n" - ".unreq u_col_stride3\n" ".unreq u_col_stride4\n" - ".unreq wptr1\n" ".unreq wptr2\n" - ".unreq w_col_stride1\n" ".unreq w_col_stride2\n" - ".unreq vptr1\n" ".unreq vptr2\n" - ".unreq v_col_stride1\n" ".unreq v_col_stride2\n" - - ".unreq qU22B\n" ".unreq qW13B\n" ".unreq qW13A\n" ".unreq qU51B\n" - ".unreq qU54B\n" ".unreq qU45A\n" ".unreq qU15A\n" ".unreq qU41B\n" - ".unreq qU24B\n" ".unreq qU21A\n" - ".unreq qV11B\n" ".unreq qU51A\n" ".unreq qU35A\n" ".unreq qU12A\n" - ".unreq qU42B\n" ".unreq qU44B\n" ".unreq qU13B\n" ".unreq qW33A\n" - ".unreq qV31B\n" ".unreq qV23A\n" ".unreq qU31A\n" ".unreq qU35B\n" ".unreq qU13A\n" - ".unreq qV23B\n" ".unreq qU11A\n" ".unreq qU25A\n" ".unreq qU43A\n" ".unreq qU52B\n" - ".unreq qU24A\n" ".unreq qU23B\n" ".unreq qV21A\n" ".unreq qV32B\n" - ".unreq qV33B\n" ".unreq qW11A\n" ".unreq qU31B\n" - ".unreq qW12B\n" ".unreq qU33A\n" ".unreq qU14A\n" ".unreq qU22A\n" - ".unreq qU25B\n" ".unreq qU53B\n" ".unreq qU42A\n" ".unreq qU44A\n" - ".unreq qU43B\n" ".unreq qW31A\n" ".unreq qU11B\n" - ".unreq qW11B\n" ".unreq qW32A\n" - ".unreq qU12B\n" ".unreq qU34B\n" ".unreq qW21A\n" - ".unreq qU14B\n" ".unreq qV21B\n" ".unreq qW22A\n" - ".unreq qW23B\n" ".unreq qW23A\n" ".unreq qU21B\n" - ".unreq qU32B\n" ".unreq qU34A\n" ".unreq qU45B\n" ".unreq qV31A\n" - ".unreq qW12A\n" ".unreq qU33B\n" ".unreq qU15B\n" - ".unreq qW33B\n" ".unreq qU54A\n" ".unreq qU23A\n" - ".unreq qW32B\n" ".unreq qV33A\n" ".unreq qW31B\n" ".unreq qV12A\n" - ".unreq qV12B\n" ".unreq qU41A\n" ".unreq qU53A\n" - ".unreq qV13A\n" ".unreq qU32A\n" ".unreq qW22B\n" - ".unreq qV22B\n" ".unreq qU52A\n" ".unreq qV13B\n" ".unreq qV32A\n" - ".unreq qU55A\n" ".unreq qU55B\n" ".unreq qV22A\n" ".unreq qW21B\n" - ".unreq qV11A\n" - ".unreq vU22B\n" ".unreq vW13B\n" ".unreq vW13A\n" ".unreq vU51B\n" - ".unreq vU54B\n" ".unreq vU45A\n" ".unreq vU15A\n" ".unreq vU41B\n" - ".unreq vU24B\n" ".unreq vU21A\n" - ".unreq vV11B\n" ".unreq vU51A\n" ".unreq vU35A\n" ".unreq vU12A\n" - ".unreq vU42B\n" ".unreq vU44B\n" ".unreq vU13B\n" ".unreq vW33A\n" - ".unreq vV31B\n" ".unreq vV23A\n" ".unreq vU31A\n" ".unreq vU35B\n" ".unreq vU13A\n" - ".unreq vV23B\n" ".unreq vU11A\n" ".unreq vU25A\n" ".unreq vU43A\n" ".unreq vU52B\n" - ".unreq vU24A\n" ".unreq vU23B\n" ".unreq vV21A\n" ".unreq vV32B\n" - ".unreq vV33B\n" ".unreq vW11A\n" ".unreq vU31B\n" - ".unreq vW12B\n" ".unreq vU33A\n" ".unreq vU14A\n" ".unreq vU22A\n" - ".unreq vU25B\n" ".unreq vU53B\n" ".unreq vU42A\n" ".unreq vU44A\n" - ".unreq vU43B\n" ".unreq vW31A\n" ".unreq vU11B\n" - ".unreq vW11B\n" ".unreq vW32A\n" - ".unreq vU12B\n" ".unreq vU34B\n" ".unreq vW21A\n" - ".unreq vU14B\n" ".unreq vV21B\n" ".unreq vW22A\n" - ".unreq vW23B\n" ".unreq vW23A\n" ".unreq vU21B\n" - ".unreq vU32B\n" ".unreq vU34A\n" ".unreq vU45B\n" ".unreq vV31A\n" - ".unreq vW12A\n" ".unreq vU33B\n" ".unreq vU15B\n" - ".unreq vW33B\n" ".unreq vU54A\n" ".unreq vU23A\n" - ".unreq vW32B\n" ".unreq vV33A\n" ".unreq vW31B\n" ".unreq vV12A\n" - ".unreq vV12B\n" ".unreq vU41A\n" ".unreq vU53A\n" - ".unreq vV13A\n" ".unreq vU32A\n" ".unreq vW22B\n" - ".unreq vV22B\n" ".unreq vU52A\n" ".unreq vV13B\n" ".unreq vV32A\n" - ".unreq vU55A\n" ".unreq vU55B\n" ".unreq vV22A\n" ".unreq vW21B\n" - ".unreq vV11A\n" - : [uptr0] "+r" (uptr0), [wptr0] "+r" (wptr0), [vptr0] "+r" (vptr0), - [n_iters] "+r" (n_iters) - : [u_row_stride] "r" (in_row_stride * sizeof(float)), - [u_col_stride] "r" (in_col_stride * sizeof(float)), - [w_row_stride] "r" (weight_row_stride * sizeof(float)), - [w_col_stride] "r" (weight_col_stride * sizeof(float)), - [v_row_stride] "r" (out_row_stride * sizeof(float)), - [v_col_stride] "r" (out_col_stride * sizeof(float)), - [odd_tail] "r" (odd_tail) - : "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", - "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", "x0", - "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", - "x12", "cc", "memory" - ); - } - if (channels_remaining) - { - // Fall back on the unoptimised version to clean up the tail - ConvImpl::process_tile<false>( - channels_remaining, - wptr0, weight_row_stride, weight_col_stride, - uptr0, in_row_stride, in_col_stride, - vptr0, out_row_stride, out_col_stride, - 0, 0, 0, 0, 0, 0 - ); - } + __asm __volatile( + "add x20, %[inptr0], %[input_row_stride]\n" + "add x13, %[input_col_stride1], %[input_col_stride1]\n" + "add x24, %[outptr0], %[output_row_stride]\n" + "add x21, x20, %[input_row_stride]\n" + "add x14, x13, #64\n" + "add x15, x13, %[input_col_stride1]\n" + "add x22, x21, %[input_row_stride]\n" + "add x16, x15, #64\n" + "add x17, x15, %[input_col_stride1]\n" + "add x23, x22, %[input_row_stride]\n" + "add x18, x17, #64\n" + "add x25, x24, %[output_row_stride]\n" + "add x26, %[output_col_stride1], %[output_col_stride1]\n" + "and x27, %[n_channels], #3\n" + "lsr x28, %[n_channels], #2\n" + "cbz x28, 4f\n" + "1:\n" + "ldr q25, [%[wbptr]]\n" + "subs x28, x28, #1\n" + "mov v17.16b, v25.16b\n" + "ldr q16, [%[wbptr], #16]\n" + "mov v13.16b, v25.16b\n" + "ldr q7, [%[wbptr], #32]\n" + "mov v15.16b, v25.16b\n" + "ldr q6, [%[wbptr], #48]\n" + "mov v10.16b, v25.16b\n" + "ldr q5, [%[wbptr], #64]\n" + "mov v12.16b, v25.16b\n" + "ldr q4, [%[wbptr], #80]\n" + "mov v14.16b, v25.16b\n" + "ldr q3, [%[wbptr], #96]\n" + "mov v9.16b, v25.16b\n" + "ldr q2, [%[wbptr], #112]\n" + "mov v11.16b, v25.16b\n" + "ldr q1, [%[wbptr], #128]\n" + "mov v8.16b, v25.16b\n" + "ldr q0, [%[wbptr], #144]\n" + "ldr q26, [%[inptr0]]\n" + "ldr q28, [x20]\n" + "fmla v17.4s, v26.4s, v16.4s\n" + "ldr q29, [%[inptr0], %[input_col_stride1]]\n" + "fmla v13.4s, v28.4s, v16.4s\n" + "ldr q27, [x21]\n" + "fmla v15.4s, v29.4s, v16.4s\n" + "ldr q21, [x20, %[input_col_stride1]]\n" + "fmla v17.4s, v28.4s, v5.4s\n" + "ldr q20, [%[inptr0], x13]\n" + "ldr q23, [x22]\n" + "ldr q19, [x21, %[input_col_stride1]]\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "prfm pldl1keep, [x20, #64]\n" + "fmla v17.4s, v29.4s, v7.4s\n" + "prfm pldl1keep, [%[inptr0], x19]\n" + "prfm pldl1keep, [x21, #64]\n" + "prfm pldl1keep, [x20, x19]\n" + "prfm pldl1keep, [%[inptr0], x14]\n" + "prfm pldl1keep, [x22, #64]\n" + "prfm pldl1keep, [x21, x19]\n" + "beq 3f\n" + "2:\n" + "fmla v17.4s, v27.4s, v2.4s\n" + "ldr q30, [x20, x13]\n" + "fmla v13.4s, v27.4s, v5.4s\n" + "ldr q29, [%[inptr0], x15]\n" + "fmla v10.4s, v27.4s, v16.4s\n" + "ldr q28, [x23]\n" + "fmla v17.4s, v21.4s, v4.4s\n" + "ldr q24, [x22, %[input_col_stride1]]\n" + "fmla v13.4s, v21.4s, v7.4s\n" + "ldr q18, [x21, x13]\n" + "fmla v15.4s, v21.4s, v5.4s\n" + "prfm pldl1keep, [x20, x14]\n" + "fmla v12.4s, v21.4s, v16.4s\n" + "ldr q22, [x20, x15]\n" + "fmla v17.4s, v20.4s, v6.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v15.4s, v20.4s, v7.4s\n" + "prfm pldl1keep, [x23, #64]\n" + "fmla v14.4s, v20.4s, v16.4s\n" + "ldr q25, [%[inptr0], x17]\n" + "fmla v13.4s, v23.4s, v2.4s\n" + "prfm pldl1keep, [x22, x19]\n" + "fmla v10.4s, v23.4s, v5.4s\n" + "ldr q26, [x23, %[input_col_stride1]]\n" + "fmla v17.4s, v19.4s, v1.4s\n" + "prfm pldl1keep, [x21, x14]\n" + "fmla v13.4s, v19.4s, v4.4s\n" + "prfm pldl1keep, [x20, x16]\n" + "fmla v15.4s, v19.4s, v2.4s\n" + "prfm pldl1keep, [%[inptr0], x18]\n" + "fmla v10.4s, v19.4s, v7.4s\n" + "prfm pldl1keep, [x23, x19]\n" + "fmla v12.4s, v19.4s, v5.4s\n" + "prfm pldl1keep, [x22, x14]\n" + "fmla v9.4s, v19.4s, v16.4s\n" + "ldr q27, [x22, x13]\n" + "fmla v17.4s, v30.4s, v3.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v13.4s, v30.4s, v6.4s\n" + "prfm pldl1keep, [x20, x18]\n" + "fmla v15.4s, v30.4s, v4.4s\n" + "prfm pldl1keep, [x23, x14]\n" + "fmla v12.4s, v30.4s, v7.4s\n" + "prfm pldl1keep, [x22, x16]\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "prfm pldl1keep, [x21, x18]\n" + "fmla v11.4s, v30.4s, v16.4s\n" + "ldr q21, [x21, x15]\n" + "fmla v15.4s, v29.4s, v6.4s\n" + "prfm pldl1keep, [x23, x16]\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "ldr q20, [x20, x17]\n" + "fmla v10.4s, v28.4s, v2.4s\n" + "ldr q19, [x23, x13]\n" + "fmla v13.4s, v24.4s, v1.4s\n" + "prfm pldl1keep, [x22, x18]\n" + "fmla v12.4s, v24.4s, v2.4s\n" + "prfm pldl1keep, [x23, x18]\n" + "fmla v10.4s, v24.4s, v4.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v9.4s, v24.4s, v5.4s\n" + "ldr q23, [x22, x15]\n" + "fmla v17.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v13.4s, v18.4s, v3.4s\n" + "add %[inptr0], %[inptr0], #16\n" + "fmla v15.4s, v18.4s, v1.4s\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "str q17, [%[outptr0]]\n" + "fmla v10.4s, v18.4s, v6.4s\n" + "fmla v12.4s, v18.4s, v4.4s\n" + "ldr q17, [x21, x17]\n" + "fmla v14.4s, v18.4s, v2.4s\n" + "prfm pldl1keep, [%[inptr0], x19]\n" + "fmla v9.4s, v18.4s, v7.4s\n" + "prfm pldl1keep, [%[inptr0], x14]\n" + "fmla v11.4s, v18.4s, v5.4s\n" + "add x20, x20, #16\n" + "fmla v8.4s, v18.4s, v16.4s\n" + "ldr q24, [x23, x15]\n" + "fmla v15.4s, v22.4s, v3.4s\n" + "ldr q18, [x22, x17]\n" + "fmla v12.4s, v22.4s, v6.4s\n" + "prfm pldl1keep, [x20, #64]\n" + "fmla v14.4s, v22.4s, v4.4s\n" + "prfm pldl1keep, [x20, x19]\n" + "fmla v11.4s, v22.4s, v7.4s\n" + "ldr q22, [x23, x17]\n" + "fmla v10.4s, v26.4s, v1.4s\n" + "add x21, x21, #16\n" + "fmla v14.4s, v25.4s, v6.4s\n" + "ldr q25, [%[wbptr]]\n" + "fmla v9.4s, v26.4s, v2.4s\n" + "ldr q16, [%[wbptr], #16]\n" + "fmla v13.4s, v27.4s, v0.4s\n" + "prfm pldl1keep, [x21, #64]\n" + "fmla v10.4s, v27.4s, v3.4s\n" + "prfm pldl1keep, [x21, x19]\n" + "fmla v12.4s, v27.4s, v1.4s\n" + "add x22, x22, #16\n" + "str q13, [x24]\n" + "fmla v9.4s, v27.4s, v4.4s\n" + "fmla v11.4s, v27.4s, v2.4s\n" + "ldr q26, [%[inptr0]]\n" + "fmla v8.4s, v27.4s, v5.4s\n" + "ldr q28, [x20]\n" + "fmla v15.4s, v21.4s, v0.4s\n" + "ldr q29, [%[inptr0], %[input_col_stride1]]\n" + "fmla v12.4s, v21.4s, v3.4s\n" + "prfm pldl1keep, [x22, #64]\n" + "fmla v14.4s, v21.4s, v1.4s\n" + "add x23, x23, #16\n" + "str q15, [%[outptr0], %[output_col_stride1]]\n" + "fmla v9.4s, v21.4s, v6.4s\n" + "fmla v11.4s, v21.4s, v4.4s\n" + "ldr q5, [%[wbptr], #64]\n" + "fmla v8.4s, v21.4s, v7.4s\n" + "ldr q27, [x21]\n" + "fmla v14.4s, v20.4s, v3.4s\n" + "ldr q21, [x20, %[input_col_stride1]]\n" + "fmla v11.4s, v20.4s, v6.4s\n" + "ldr q20, [%[inptr0], x13]\n" + "fmla v10.4s, v19.4s, v0.4s\n" + "subs x28, x28, #1\n" + "fmla v9.4s, v19.4s, v1.4s\n" + "fmla v8.4s, v19.4s, v2.4s\n" + "fmla v12.4s, v23.4s, v0.4s\n" + "ldr q7, [%[wbptr], #32]\n" + "str q10, [x25]\n" + "fmla v11.4s, v23.4s, v1.4s\n" + "fmla v9.4s, v23.4s, v3.4s\n" + "ldr q2, [%[wbptr], #112]\n" + "str q12, [x24, %[output_col_stride1]]\n" + "fmla v8.4s, v23.4s, v4.4s\n" + "fmla v14.4s, v17.4s, v0.4s\n" + "ldr q23, [x22]\n" + "fmla v11.4s, v17.4s, v3.4s\n" + "ldr q19, [x21, %[input_col_stride1]]\n" + "fmla v8.4s, v17.4s, v6.4s\n" + "ldr q4, [%[wbptr], #80]\n" + "str q14, [%[outptr0], x26]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "fmla v11.4s, v18.4s, v0.4s\n" + "add %[outptr0], %[outptr0], #16\n" + "fmla v8.4s, v24.4s, v1.4s\n" + "ldr q6, [%[wbptr], #48]\n" + "str q9, [x25, %[output_col_stride1]]\n" + "mov v17.16b, v25.16b\n" + "str q11, [x24, x26]\n" + "mov v13.16b, v25.16b\n" + "fmla v8.4s, v18.4s, v3.4s\n" + "ldr q1, [%[wbptr], #128]\n" + "mov v15.16b, v25.16b\n" + "add x24, x24, #16\n" + "mov v10.16b, v25.16b\n" + "mov v12.16b, v25.16b\n" + "fmla v8.4s, v22.4s, v0.4s\n" + "ldr q3, [%[wbptr], #96]\n" + "mov v14.16b, v25.16b\n" + "mov v9.16b, v25.16b\n" + "mov v11.16b, v25.16b\n" + "fmla v17.4s, v26.4s, v16.4s\n" + "str q8, [x25, x26]\n" + "fmla v13.4s, v28.4s, v16.4s\n" + "mov v8.16b, v25.16b\n" + "ldr q0, [%[wbptr], #144]\n" + "fmla v17.4s, v28.4s, v5.4s\n" + "fmla v15.4s, v29.4s, v16.4s\n" + "add x25, x25, #16\n" + "fmla v17.4s, v29.4s, v7.4s\n" + "bne 2b\n" + "3:\n" + "fmla v17.4s, v27.4s, v2.4s\n" + "ldr q30, [x20, x13]\n" + "fmla v13.4s, v27.4s, v5.4s\n" + "ldr q29, [%[inptr0], x15]\n" + "fmla v10.4s, v27.4s, v16.4s\n" + "ldr q28, [x23]\n" + "fmla v17.4s, v21.4s, v4.4s\n" + "ldr q24, [x22, %[input_col_stride1]]\n" + "fmla v13.4s, v21.4s, v7.4s\n" + "ldr q18, [x21, x13]\n" + "fmla v15.4s, v21.4s, v5.4s\n" + "prfm pldl1keep, [x20, x14]\n" + "fmla v12.4s, v21.4s, v16.4s\n" + "ldr q22, [x20, x15]\n" + "fmla v17.4s, v20.4s, v6.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v15.4s, v20.4s, v7.4s\n" + "prfm pldl1keep, [x23, #64]\n" + "fmla v14.4s, v20.4s, v16.4s\n" + "ldr q25, [%[inptr0], x17]\n" + "fmla v13.4s, v23.4s, v2.4s\n" + "prfm pldl1keep, [x22, x19]\n" + "fmla v10.4s, v23.4s, v5.4s\n" + "ldr q26, [x23, %[input_col_stride1]]\n" + "fmla v17.4s, v19.4s, v1.4s\n" + "prfm pldl1keep, [x21, x14]\n" + "fmla v13.4s, v19.4s, v4.4s\n" + "prfm pldl1keep, [x20, x16]\n" + "fmla v15.4s, v19.4s, v2.4s\n" + "prfm pldl1keep, [%[inptr0], x18]\n" + "fmla v10.4s, v19.4s, v7.4s\n" + "prfm pldl1keep, [x23, x19]\n" + "fmla v12.4s, v19.4s, v5.4s\n" + "prfm pldl1keep, [x22, x14]\n" + "fmla v9.4s, v19.4s, v16.4s\n" + "ldr q27, [x22, x13]\n" + "fmla v17.4s, v30.4s, v3.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v13.4s, v30.4s, v6.4s\n" + "prfm pldl1keep, [x20, x18]\n" + "fmla v15.4s, v30.4s, v4.4s\n" + "prfm pldl1keep, [x23, x14]\n" + "fmla v12.4s, v30.4s, v7.4s\n" + "prfm pldl1keep, [x22, x16]\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "prfm pldl1keep, [x21, x18]\n" + "fmla v11.4s, v30.4s, v16.4s\n" + "ldr q21, [x21, x15]\n" + "fmla v15.4s, v29.4s, v6.4s\n" + "prfm pldl1keep, [x23, x16]\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "ldr q20, [x20, x17]\n" + "fmla v10.4s, v28.4s, v2.4s\n" + "ldr q19, [x23, x13]\n" + "fmla v13.4s, v24.4s, v1.4s\n" + "prfm pldl1keep, [x22, x18]\n" + "fmla v12.4s, v24.4s, v2.4s\n" + "prfm pldl1keep, [x23, x18]\n" + "fmla v10.4s, v24.4s, v4.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v9.4s, v24.4s, v5.4s\n" + "ldr q23, [x22, x15]\n" + "fmla v17.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v13.4s, v18.4s, v3.4s\n" + "add %[inptr0], %[inptr0], #16\n" + "fmla v15.4s, v18.4s, v1.4s\n" + "add x20, x20, #16\n" + "str q17, [%[outptr0]]\n" + "fmla v10.4s, v18.4s, v6.4s\n" + "fmla v12.4s, v18.4s, v4.4s\n" + "ldr q17, [x21, x17]\n" + "fmla v14.4s, v18.4s, v2.4s\n" + "add x21, x21, #16\n" + "fmla v9.4s, v18.4s, v7.4s\n" + "fmla v11.4s, v18.4s, v5.4s\n" + "fmla v8.4s, v18.4s, v16.4s\n" + "ldr q24, [x23, x15]\n" + "fmla v15.4s, v22.4s, v3.4s\n" + "ldr q18, [x22, x17]\n" + "fmla v12.4s, v22.4s, v6.4s\n" + "add x22, x22, #16\n" + "fmla v14.4s, v22.4s, v4.4s\n" + "fmla v11.4s, v22.4s, v7.4s\n" + "fmla v10.4s, v26.4s, v1.4s\n" + "ldr q22, [x23, x17]\n" + "fmla v9.4s, v26.4s, v2.4s\n" + "add x23, x23, #16\n" + "fmla v14.4s, v25.4s, v6.4s\n" + "fmla v13.4s, v27.4s, v0.4s\n" + "fmla v10.4s, v27.4s, v3.4s\n" + "fmla v12.4s, v27.4s, v1.4s\n" + "fmla v9.4s, v27.4s, v4.4s\n" + "fmla v11.4s, v27.4s, v2.4s\n" + "str q13, [x24]\n" + "fmla v8.4s, v27.4s, v5.4s\n" + "fmla v15.4s, v21.4s, v0.4s\n" + "fmla v12.4s, v21.4s, v3.4s\n" + "fmla v14.4s, v21.4s, v1.4s\n" + "fmla v9.4s, v21.4s, v6.4s\n" + "fmla v11.4s, v21.4s, v4.4s\n" + "fmla v8.4s, v21.4s, v7.4s\n" + "str q15, [%[outptr0], %[output_col_stride1]]\n" + "fmla v10.4s, v19.4s, v0.4s\n" + "fmla v14.4s, v20.4s, v3.4s\n" + "fmla v9.4s, v19.4s, v1.4s\n" + "fmla v11.4s, v20.4s, v6.4s\n" + "fmla v8.4s, v19.4s, v2.4s\n" + "str q10, [x25]\n" + "fmla v12.4s, v23.4s, v0.4s\n" + "fmla v9.4s, v23.4s, v3.4s\n" + "fmla v14.4s, v17.4s, v0.4s\n" + "fmla v11.4s, v23.4s, v1.4s\n" + "fmla v8.4s, v23.4s, v4.4s\n" + "str q12, [x24, %[output_col_stride1]]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "str q14, [%[outptr0], x26]\n" + "fmla v11.4s, v17.4s, v3.4s\n" + "fmla v8.4s, v17.4s, v6.4s\n" + "add %[outptr0], %[outptr0], #16\n" + "str q9, [x25, %[output_col_stride1]]\n" + "fmla v11.4s, v18.4s, v0.4s\n" + "fmla v8.4s, v24.4s, v1.4s\n" + "str q11, [x24, x26]\n" + "fmla v8.4s, v18.4s, v3.4s\n" + "add x24, x24, #16\n" + "fmla v8.4s, v22.4s, v0.4s\n" + "str q8, [x25, x26]\n" + "add x25, x25, #16\n" + "4:\n" + "cbz x27, 7f\n" + "ldr s25, [%[wbptr]]\n" + "mov v17.16b, v25.16b\n" + "ldr s16, [%[wbptr], #4]\n" + "mov v13.16b, v25.16b\n" + "ldr s7, [%[wbptr], #8]\n" + "mov v15.16b, v25.16b\n" + "ldr s6, [%[wbptr], #12]\n" + "mov v10.16b, v25.16b\n" + "ldr s5, [%[wbptr], #16]\n" + "mov v12.16b, v25.16b\n" + "ldr s4, [%[wbptr], #20]\n" + "mov v14.16b, v25.16b\n" + "ldr s3, [%[wbptr], #24]\n" + "mov v9.16b, v25.16b\n" + "ldr s2, [%[wbptr], #28]\n" + "mov v11.16b, v25.16b\n" + "ldr s1, [%[wbptr], #32]\n" + "mov v8.16b, v25.16b\n" + "ldr s0, [%[wbptr], #36]\n" + "ldr s26, [%[inptr0]]\n" + "subs x27, x27, #1\n" + "fmla v17.4s, v26.4s, v16.4s\n" + "ldr s28, [x20]\n" + "fmla v13.4s, v28.4s, v16.4s\n" + "ldr s29, [%[inptr0], %[input_col_stride1]]\n" + "fmla v15.4s, v29.4s, v16.4s\n" + "ldr s27, [x21]\n" + "fmla v17.4s, v28.4s, v5.4s\n" + "ldr s21, [x20, %[input_col_stride1]]\n" + "ldr s20, [%[inptr0], x13]\n" + "ldr s23, [x22]\n" + "ldr s19, [x21, %[input_col_stride1]]\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "fmla v17.4s, v29.4s, v7.4s\n" + "prfm pldl1keep, [x20, #64]\n" + "prfm pldl1keep, [%[inptr0], x19]\n" + "prfm pldl1keep, [x21, #64]\n" + "prfm pldl1keep, [x20, x19]\n" + "prfm pldl1keep, [%[inptr0], x14]\n" + "prfm pldl1keep, [x22, #64]\n" + "prfm pldl1keep, [x21, x19]\n" + "beq 6f\n" + "5:\n" + "fmla v17.4s, v27.4s, v2.4s\n" + "ldr s30, [x20, x13]\n" + "fmla v13.4s, v27.4s, v5.4s\n" + "ldr s29, [%[inptr0], x15]\n" + "fmla v10.4s, v27.4s, v16.4s\n" + "ldr s28, [x23]\n" + "fmla v17.4s, v21.4s, v4.4s\n" + "ldr s24, [x22, %[input_col_stride1]]\n" + "fmla v13.4s, v21.4s, v7.4s\n" + "ldr s18, [x21, x13]\n" + "fmla v15.4s, v21.4s, v5.4s\n" + "prfm pldl1keep, [x20, x14]\n" + "fmla v12.4s, v21.4s, v16.4s\n" + "ldr s22, [x20, x15]\n" + "fmla v17.4s, v20.4s, v6.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v15.4s, v20.4s, v7.4s\n" + "prfm pldl1keep, [x23, #64]\n" + "fmla v14.4s, v20.4s, v16.4s\n" + "ldr s25, [%[inptr0], x17]\n" + "fmla v13.4s, v23.4s, v2.4s\n" + "prfm pldl1keep, [x22, x19]\n" + "fmla v10.4s, v23.4s, v5.4s\n" + "ldr s26, [x23, %[input_col_stride1]]\n" + "fmla v17.4s, v19.4s, v1.4s\n" + "prfm pldl1keep, [x21, x14]\n" + "fmla v13.4s, v19.4s, v4.4s\n" + "prfm pldl1keep, [x20, x16]\n" + "fmla v15.4s, v19.4s, v2.4s\n" + "prfm pldl1keep, [%[inptr0], x18]\n" + "fmla v10.4s, v19.4s, v7.4s\n" + "prfm pldl1keep, [x23, x19]\n" + "fmla v12.4s, v19.4s, v5.4s\n" + "prfm pldl1keep, [x22, x14]\n" + "fmla v9.4s, v19.4s, v16.4s\n" + "ldr s27, [x22, x13]\n" + "fmla v17.4s, v30.4s, v3.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v13.4s, v30.4s, v6.4s\n" + "prfm pldl1keep, [x20, x18]\n" + "fmla v15.4s, v30.4s, v4.4s\n" + "prfm pldl1keep, [x23, x14]\n" + "fmla v12.4s, v30.4s, v7.4s\n" + "prfm pldl1keep, [x22, x16]\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "prfm pldl1keep, [x21, x18]\n" + "fmla v11.4s, v30.4s, v16.4s\n" + "ldr s21, [x21, x15]\n" + "fmla v15.4s, v29.4s, v6.4s\n" + "prfm pldl1keep, [x23, x16]\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "ldr s20, [x20, x17]\n" + "fmla v10.4s, v28.4s, v2.4s\n" + "ldr s19, [x23, x13]\n" + "fmla v13.4s, v24.4s, v1.4s\n" + "prfm pldl1keep, [x22, x18]\n" + "fmla v12.4s, v24.4s, v2.4s\n" + "prfm pldl1keep, [x23, x18]\n" + "fmla v10.4s, v24.4s, v4.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v9.4s, v24.4s, v5.4s\n" + "ldr s23, [x22, x15]\n" + "fmla v17.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v13.4s, v18.4s, v3.4s\n" + "add %[inptr0], %[inptr0], #4\n" + "fmla v15.4s, v18.4s, v1.4s\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "str s17, [%[outptr0]]\n" + "fmla v10.4s, v18.4s, v6.4s\n" + "fmla v12.4s, v18.4s, v4.4s\n" + "ldr s17, [x21, x17]\n" + "fmla v14.4s, v18.4s, v2.4s\n" + "prfm pldl1keep, [%[inptr0], x19]\n" + "fmla v9.4s, v18.4s, v7.4s\n" + "prfm pldl1keep, [%[inptr0], x14]\n" + "fmla v11.4s, v18.4s, v5.4s\n" + "add x20, x20, #4\n" + "fmla v8.4s, v18.4s, v16.4s\n" + "ldr s24, [x23, x15]\n" + "fmla v15.4s, v22.4s, v3.4s\n" + "ldr s18, [x22, x17]\n" + "fmla v12.4s, v22.4s, v6.4s\n" + "prfm pldl1keep, [x20, #64]\n" + "fmla v14.4s, v22.4s, v4.4s\n" + "prfm pldl1keep, [x20, x19]\n" + "fmla v11.4s, v22.4s, v7.4s\n" + "ldr s22, [x23, x17]\n" + "fmla v10.4s, v26.4s, v1.4s\n" + "add x21, x21, #4\n" + "fmla v14.4s, v25.4s, v6.4s\n" + "ldr s25, [%[wbptr]]\n" + "fmla v9.4s, v26.4s, v2.4s\n" + "ldr s16, [%[wbptr], #4]\n" + "fmla v13.4s, v27.4s, v0.4s\n" + "prfm pldl1keep, [x21, #64]\n" + "fmla v10.4s, v27.4s, v3.4s\n" + "prfm pldl1keep, [x21, x19]\n" + "fmla v12.4s, v27.4s, v1.4s\n" + "add x22, x22, #4\n" + "str s13, [x24]\n" + "fmla v9.4s, v27.4s, v4.4s\n" + "fmla v11.4s, v27.4s, v2.4s\n" + "ldr s26, [%[inptr0]]\n" + "fmla v8.4s, v27.4s, v5.4s\n" + "ldr s28, [x20]\n" + "fmla v15.4s, v21.4s, v0.4s\n" + "ldr s29, [%[inptr0], %[input_col_stride1]]\n" + "fmla v12.4s, v21.4s, v3.4s\n" + "prfm pldl1keep, [x22, #64]\n" + "fmla v14.4s, v21.4s, v1.4s\n" + "add x23, x23, #4\n" + "str s15, [%[outptr0], %[output_col_stride1]]\n" + "fmla v9.4s, v21.4s, v6.4s\n" + "fmla v11.4s, v21.4s, v4.4s\n" + "ldr s5, [%[wbptr], #16]\n" + "fmla v8.4s, v21.4s, v7.4s\n" + "ldr s27, [x21]\n" + "fmla v14.4s, v20.4s, v3.4s\n" + "ldr s21, [x20, %[input_col_stride1]]\n" + "fmla v11.4s, v20.4s, v6.4s\n" + "ldr s20, [%[inptr0], x13]\n" + "fmla v10.4s, v19.4s, v0.4s\n" + "subs x27, x27, #1\n" + "fmla v9.4s, v19.4s, v1.4s\n" + "fmla v8.4s, v19.4s, v2.4s\n" + "fmla v12.4s, v23.4s, v0.4s\n" + "ldr s7, [%[wbptr], #8]\n" + "str s10, [x25]\n" + "fmla v11.4s, v23.4s, v1.4s\n" + "fmla v9.4s, v23.4s, v3.4s\n" + "ldr s2, [%[wbptr], #28]\n" + "str s12, [x24, %[output_col_stride1]]\n" + "fmla v8.4s, v23.4s, v4.4s\n" + "fmla v14.4s, v17.4s, v0.4s\n" + "ldr s23, [x22]\n" + "fmla v11.4s, v17.4s, v3.4s\n" + "ldr s19, [x21, %[input_col_stride1]]\n" + "fmla v8.4s, v17.4s, v6.4s\n" + "ldr s4, [%[wbptr], #20]\n" + "str s14, [%[outptr0], x26]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "fmla v11.4s, v18.4s, v0.4s\n" + "add %[outptr0], %[outptr0], #4\n" + "fmla v8.4s, v24.4s, v1.4s\n" + "ldr s6, [%[wbptr], #12]\n" + "str s9, [x25, %[output_col_stride1]]\n" + "mov v17.16b, v25.16b\n" + "str s11, [x24, x26]\n" + "mov v13.16b, v25.16b\n" + "fmla v8.4s, v18.4s, v3.4s\n" + "ldr s1, [%[wbptr], #32]\n" + "mov v15.16b, v25.16b\n" + "add x24, x24, #4\n" + "mov v10.16b, v25.16b\n" + "mov v12.16b, v25.16b\n" + "fmla v8.4s, v22.4s, v0.4s\n" + "ldr s3, [%[wbptr], #24]\n" + "mov v14.16b, v25.16b\n" + "mov v9.16b, v25.16b\n" + "mov v11.16b, v25.16b\n" + "fmla v17.4s, v26.4s, v16.4s\n" + "str s8, [x25, x26]\n" + "fmla v13.4s, v28.4s, v16.4s\n" + "mov v8.16b, v25.16b\n" + "ldr s0, [%[wbptr], #36]\n" + "fmla v17.4s, v28.4s, v5.4s\n" + "fmla v15.4s, v29.4s, v16.4s\n" + "add x25, x25, #4\n" + "fmla v17.4s, v29.4s, v7.4s\n" + "bne 5b\n" + "6:\n" + "fmla v17.4s, v27.4s, v2.4s\n" + "ldr s30, [x20, x13]\n" + "fmla v13.4s, v27.4s, v5.4s\n" + "ldr s29, [%[inptr0], x15]\n" + "fmla v10.4s, v27.4s, v16.4s\n" + "ldr s28, [x23]\n" + "fmla v17.4s, v21.4s, v4.4s\n" + "ldr s24, [x22, %[input_col_stride1]]\n" + "fmla v13.4s, v21.4s, v7.4s\n" + "ldr s18, [x21, x13]\n" + "fmla v15.4s, v21.4s, v5.4s\n" + "prfm pldl1keep, [x20, x14]\n" + "fmla v12.4s, v21.4s, v16.4s\n" + "ldr s22, [x20, x15]\n" + "fmla v17.4s, v20.4s, v6.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v15.4s, v20.4s, v7.4s\n" + "prfm pldl1keep, [x23, #64]\n" + "fmla v14.4s, v20.4s, v16.4s\n" + "ldr s25, [%[inptr0], x17]\n" + "fmla v13.4s, v23.4s, v2.4s\n" + "prfm pldl1keep, [x22, x19]\n" + "fmla v10.4s, v23.4s, v5.4s\n" + "ldr s26, [x23, %[input_col_stride1]]\n" + "fmla v17.4s, v19.4s, v1.4s\n" + "prfm pldl1keep, [x21, x14]\n" + "fmla v13.4s, v19.4s, v4.4s\n" + "prfm pldl1keep, [x20, x16]\n" + "fmla v15.4s, v19.4s, v2.4s\n" + "prfm pldl1keep, [%[inptr0], x18]\n" + "fmla v10.4s, v19.4s, v7.4s\n" + "prfm pldl1keep, [x23, x19]\n" + "fmla v12.4s, v19.4s, v5.4s\n" + "prfm pldl1keep, [x22, x14]\n" + "fmla v9.4s, v19.4s, v16.4s\n" + "ldr s27, [x22, x13]\n" + "fmla v17.4s, v30.4s, v3.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v13.4s, v30.4s, v6.4s\n" + "prfm pldl1keep, [x20, x18]\n" + "fmla v15.4s, v30.4s, v4.4s\n" + "prfm pldl1keep, [x23, x14]\n" + "fmla v12.4s, v30.4s, v7.4s\n" + "prfm pldl1keep, [x22, x16]\n" + "fmla v14.4s, v30.4s, v5.4s\n" + "prfm pldl1keep, [x21, x18]\n" + "fmla v11.4s, v30.4s, v16.4s\n" + "ldr s21, [x21, x15]\n" + "fmla v15.4s, v29.4s, v6.4s\n" + "prfm pldl1keep, [x23, x16]\n" + "fmla v14.4s, v29.4s, v7.4s\n" + "ldr s20, [x20, x17]\n" + "fmla v10.4s, v28.4s, v2.4s\n" + "ldr s19, [x23, x13]\n" + "fmla v13.4s, v24.4s, v1.4s\n" + "prfm pldl1keep, [x22, x18]\n" + "fmla v12.4s, v24.4s, v2.4s\n" + "prfm pldl1keep, [x23, x18]\n" + "fmla v10.4s, v24.4s, v4.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v9.4s, v24.4s, v5.4s\n" + "ldr s23, [x22, x15]\n" + "fmla v17.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v13.4s, v18.4s, v3.4s\n" + "add %[inptr0], %[inptr0], #4\n" + "fmla v15.4s, v18.4s, v1.4s\n" + "add x20, x20, #4\n" + "str s17, [%[outptr0]]\n" + "fmla v10.4s, v18.4s, v6.4s\n" + "fmla v12.4s, v18.4s, v4.4s\n" + "ldr s17, [x21, x17]\n" + "fmla v14.4s, v18.4s, v2.4s\n" + "add x21, x21, #4\n" + "fmla v9.4s, v18.4s, v7.4s\n" + "fmla v11.4s, v18.4s, v5.4s\n" + "fmla v8.4s, v18.4s, v16.4s\n" + "ldr s24, [x23, x15]\n" + "fmla v15.4s, v22.4s, v3.4s\n" + "ldr s18, [x22, x17]\n" + "fmla v12.4s, v22.4s, v6.4s\n" + "add x22, x22, #4\n" + "fmla v14.4s, v22.4s, v4.4s\n" + "fmla v11.4s, v22.4s, v7.4s\n" + "fmla v10.4s, v26.4s, v1.4s\n" + "ldr s22, [x23, x17]\n" + "fmla v9.4s, v26.4s, v2.4s\n" + "add x23, x23, #4\n" + "fmla v14.4s, v25.4s, v6.4s\n" + "fmla v13.4s, v27.4s, v0.4s\n" + "fmla v10.4s, v27.4s, v3.4s\n" + "fmla v12.4s, v27.4s, v1.4s\n" + "fmla v9.4s, v27.4s, v4.4s\n" + "fmla v11.4s, v27.4s, v2.4s\n" + "str s13, [x24]\n" + "fmla v8.4s, v27.4s, v5.4s\n" + "fmla v15.4s, v21.4s, v0.4s\n" + "fmla v12.4s, v21.4s, v3.4s\n" + "fmla v14.4s, v21.4s, v1.4s\n" + "fmla v9.4s, v21.4s, v6.4s\n" + "fmla v11.4s, v21.4s, v4.4s\n" + "fmla v8.4s, v21.4s, v7.4s\n" + "str s15, [%[outptr0], %[output_col_stride1]]\n" + "fmla v10.4s, v19.4s, v0.4s\n" + "fmla v14.4s, v20.4s, v3.4s\n" + "fmla v9.4s, v19.4s, v1.4s\n" + "fmla v11.4s, v20.4s, v6.4s\n" + "fmla v8.4s, v19.4s, v2.4s\n" + "str s10, [x25]\n" + "fmla v12.4s, v23.4s, v0.4s\n" + "fmla v9.4s, v23.4s, v3.4s\n" + "fmla v14.4s, v17.4s, v0.4s\n" + "fmla v11.4s, v23.4s, v1.4s\n" + "fmla v8.4s, v23.4s, v4.4s\n" + "str s12, [x24, %[output_col_stride1]]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "str s14, [%[outptr0], x26]\n" + "fmla v11.4s, v17.4s, v3.4s\n" + "fmla v8.4s, v17.4s, v6.4s\n" + "add %[outptr0], %[outptr0], #4\n" + "str s9, [x25, %[output_col_stride1]]\n" + "fmla v11.4s, v18.4s, v0.4s\n" + "fmla v8.4s, v24.4s, v1.4s\n" + "str s11, [x24, x26]\n" + "fmla v8.4s, v18.4s, v3.4s\n" + "add x24, x24, #4\n" + "fmla v8.4s, v22.4s, v0.4s\n" + "str s8, [x25, x26]\n" + "add x25, x25, #4\n" + "7:\n" + : [outptr0] "+r" (output), [inptr0] "+r" (input), [wbptr] "+r" (weight_bias_ptr) + : [output_row_stride] "r" (output_row_stride * sizeof(float)), [output_col_stride1] "r" (output_col_stride * sizeof(float)), [n_channels] "r" ((long) n_channels), [input_row_stride] "r" (input_row_stride * sizeof(float)), [input_col_stride1] "r" (input_col_stride * sizeof(float)) + : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v3", "v30", "v4", "v5", "v6", "v7", "v8", "v9", "x13", "x14", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" + ); } -#endif // __aarch64__ - template <> -const Conv::TileFn Conv::tilefn_unpadded = ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 0>; - template <> -const Conv::TileFn Conv::tilefn_top[n_in_pad_top_fns] = { - ConvImpl::template process_tile<true, 1, 0, 0, 0, 0, 0>, -}; +void Conv::execute_tile<ActivationFunction::ReLU>( + int n_channels, + const void *weight_bias_ptr, + const float *input, + const unsigned int input_row_stride, + const unsigned int input_col_stride, + float *output, + const unsigned int output_row_stride, + const unsigned int output_col_stride +) +{ + __asm __volatile( + "add x25, %[inptr0], %[input_row_stride]\n" + "add x16, %[input_col_stride1], %[input_col_stride1]\n" + "add x21, %[outptr0], %[output_row_stride]\n" + "add x22, x25, %[input_row_stride]\n" + "add x23, x16, #64\n" + "add x26, x16, %[input_col_stride1]\n" + "add x13, x22, %[input_row_stride]\n" + "add x20, x26, #64\n" + "add x18, x26, %[input_col_stride1]\n" + "add x24, x13, %[input_row_stride]\n" + "add x15, x18, #64\n" + "add x14, x21, %[output_row_stride]\n" + "add x19, %[output_col_stride1], %[output_col_stride1]\n" + "and x27, %[n_channels], #3\n" + "lsr x28, %[n_channels], #2\n" + "cbz x28, 4f\n" + "1:\n" + "ldr q20, [%[wbptr]]\n" + "subs x28, x28, #1\n" + "mov v4.16b, v20.16b\n" + "ldr q15, [%[wbptr], #16]\n" + "mov v1.16b, v20.16b\n" + "ldr q0, [%[wbptr], #32]\n" + "mov v3.16b, v20.16b\n" + "ldr q13, [%[wbptr], #48]\n" + "mov v7.16b, v20.16b\n" + "ldr q16, [%[wbptr], #64]\n" + "mov v9.16b, v20.16b\n" + "ldr q12, [%[wbptr], #80]\n" + "mov v2.16b, v20.16b\n" + "ldr q17, [%[wbptr], #96]\n" + "mov v6.16b, v20.16b\n" + "ldr q11, [%[wbptr], #112]\n" + "mov v8.16b, v20.16b\n" + "ldr q10, [%[wbptr], #128]\n" + "mov v5.16b, v20.16b\n" + "ldr q14, [%[wbptr], #144]\n" + "ldr q27, [%[inptr0]]\n" + "ldr q24, [x25]\n" + "fmla v4.4s, v27.4s, v15.4s\n" + "ldr q22, [%[inptr0], %[input_col_stride1]]\n" + "ldr q21, [x22]\n" + "ldr q19, [x25, %[input_col_stride1]]\n" + "ldr q31, [%[inptr0], x16]\n" + "ldr q28, [x13]\n" + "fmla v4.4s, v24.4s, v16.4s\n" + "ldr q18, [x22, %[input_col_stride1]]\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "prfm pldl1keep, [x25, #64]\n" + "prfm pldl1keep, [%[inptr0], x17]\n" + "prfm pldl1keep, [x22, #64]\n" + "prfm pldl1keep, [x25, x17]\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "prfm pldl1keep, [x13, #64]\n" + "prfm pldl1keep, [x22, x17]\n" + "beq 3f\n" + "2:\n" + "fmla v1.4s, v24.4s, v15.4s\n" + "ldr q24, [x25, x16]\n" + "fmla v4.4s, v22.4s, v0.4s\n" + "ldr q29, [%[inptr0], x26]\n" + "fmla v3.4s, v22.4s, v15.4s\n" + "ldr q30, [x24]\n" + "fmla v1.4s, v21.4s, v16.4s\n" + "ldr q25, [x13, %[input_col_stride1]]\n" + "fmla v4.4s, v21.4s, v11.4s\n" + "prfm pldl1keep, [x25, x23]\n" + "fmla v7.4s, v21.4s, v15.4s\n" + "ldr q26, [x22, x16]\n" + "fmla v1.4s, v19.4s, v0.4s\n" + "prfm pldl1keep, [%[inptr0], x20]\n" + "fmla v4.4s, v19.4s, v12.4s\n" + "prfm pldl1keep, [x24, #64]\n" + "fmla v3.4s, v19.4s, v16.4s\n" + "prfm pldl1keep, [x13, x17]\n" + "fmla v9.4s, v19.4s, v15.4s\n" + "ldr q23, [x25, x26]\n" + "fmla v4.4s, v31.4s, v13.4s\n" + "prfm pldl1keep, [x22, x23]\n" + "fmla v3.4s, v31.4s, v0.4s\n" + "prfm pldl1keep, [x25, x20]\n" + "fmla v2.4s, v31.4s, v15.4s\n" + "ldr q20, [%[inptr0], x18]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x15]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "ldr q28, [x24, %[input_col_stride1]]\n" + "fmla v4.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x24, x17]\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "prfm pldl1keep, [x13, x23]\n" + "fmla v3.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x22, x20]\n" + "fmla v7.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [x25, x15]\n" + "fmla v9.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x24, x23]\n" + "fmla v6.4s, v18.4s, v15.4s\n" + "ldr q27, [x13, x16]\n" + "fmla v4.4s, v24.4s, v17.4s\n" + "prfm pldl1keep, [x13, x20]\n" + "fmla v1.4s, v24.4s, v13.4s\n" + "prfm pldl1keep, [x22, x15]\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "prfm pldl1keep, [x24, x20]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "prfm pldl1keep, [x13, x15]\n" + "fmla v2.4s, v24.4s, v16.4s\n" + "prfm pldl1keep, [x24, x15]\n" + "fmla v8.4s, v24.4s, v15.4s\n" + "ldr q24, [x22, x26]\n" + "fmla v3.4s, v29.4s, v13.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v2.4s, v29.4s, v0.4s\n" + "ldr q22, [x25, x18]\n" + "fmla v7.4s, v30.4s, v11.4s\n" + "ldr q21, [x24, x16]\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v9.4s, v25.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #16\n" + "fmla v7.4s, v25.4s, v12.4s\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "fmla v6.4s, v25.4s, v16.4s\n" + "ldr q19, [x13, x26]\n" + "fmla v4.4s, v26.4s, v14.4s\n" + "prfm pldl1keep, [%[inptr0], x17]\n" + "fmla v1.4s, v26.4s, v17.4s\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "fmla v3.4s, v26.4s, v10.4s\n" + "add x25, x25, #16\n" + "fmla v7.4s, v26.4s, v13.4s\n" + "prfm pldl1keep, [x25, #64]\n" + "fmla v9.4s, v26.4s, v12.4s\n" + "prfm pldl1keep, [x25, x17]\n" + "fmla v2.4s, v26.4s, v11.4s\n" + "subs x28, x28, #1\n" + "fmla v6.4s, v26.4s, v0.4s\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "fmla v5.4s, v26.4s, v15.4s\n" + "ldr q26, [x22, x18]\n" + "fmla v3.4s, v23.4s, v17.4s\n" + "ldr q18, [x24, x26]\n" + "fmla v9.4s, v23.4s, v13.4s\n" + "add x22, x22, #16\n" + "fmla v2.4s, v23.4s, v12.4s\n" + "prfm pldl1keep, [x22, #64]\n" + "fmla v8.4s, v23.4s, v0.4s\n" + "ldr q23, [x13, x18]\n" + "fmla v7.4s, v28.4s, v10.4s\n" + "prfm pldl1keep, [x22, x17]\n" + "fmla v2.4s, v20.4s, v13.4s\n" + "ldr q25, [x24, x18]\n" + "fmla v6.4s, v28.4s, v11.4s\n" + "ldr q20, [%[wbptr]]\n" + "fmla v1.4s, v27.4s, v14.4s\n" + "add x13, x13, #16\n" + "fmla v7.4s, v27.4s, v17.4s\n" + "prfm pldl1keep, [x13, #64]\n" + "fmla v9.4s, v27.4s, v10.4s\n" + "add x24, x24, #16\n" + "fmla v6.4s, v27.4s, v12.4s\n" + "fmla v8.4s, v27.4s, v11.4s\n" + "fmla v5.4s, v27.4s, v16.4s\n" + "ldr q15, [%[wbptr], #16]\n" + "fmla v3.4s, v24.4s, v14.4s\n" + "ldr q27, [%[inptr0]]\n" + "fmla v9.4s, v24.4s, v17.4s\n" + "fmla v2.4s, v24.4s, v10.4s\n" + "fmla v6.4s, v24.4s, v13.4s\n" + "fmla v8.4s, v24.4s, v12.4s\n" + "fmla v5.4s, v24.4s, v0.4s\n" + "ldr q16, [%[wbptr], #64]\n" + "fmla v2.4s, v22.4s, v17.4s\n" + "ldr q24, [x25]\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "ldr q22, [%[inptr0], %[input_col_stride1]]\n" + "fmla v7.4s, v21.4s, v14.4s\n" + "fmla v6.4s, v21.4s, v10.4s\n" + "fmla v5.4s, v21.4s, v11.4s\n" + "ldr q0, [%[wbptr], #32]\n" + "fmla v9.4s, v19.4s, v14.4s\n" + "ldr q21, [x22]\n" + "fmla v6.4s, v19.4s, v17.4s\n" + "fmla v8.4s, v19.4s, v10.4s\n" + "fmla v5.4s, v19.4s, v12.4s\n" + "ldr q11, [%[wbptr], #112]\n" + "fmla v2.4s, v26.4s, v14.4s\n" + "movi v29.16b, #0\n" + "fmla v8.4s, v26.4s, v17.4s\n" + "fmla v6.4s, v18.4s, v14.4s\n" + "fmla v5.4s, v26.4s, v13.4s\n" + "ldr q12, [%[wbptr], #80]\n" + "fmax v4.4s, v4.4s, v29.4s\n" + "ldr q19, [x25, %[input_col_stride1]]\n" + "fmla v8.4s, v23.4s, v14.4s\n" + "fmax v3.4s, v3.4s, v29.4s\n" + "str q4, [%[outptr0]]\n" + "fmla v5.4s, v18.4s, v10.4s\n" + "str q3, [%[outptr0], %[output_col_stride1]]\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "ldr q13, [%[wbptr], #48]\n" + "str q2, [%[outptr0], x19]\n" + "fmla v5.4s, v23.4s, v17.4s\n" + "str q1, [x21]\n" + "fmax v9.4s, v9.4s, v29.4s\n" + "fmax v8.4s, v8.4s, v29.4s\n" + "ldr q10, [%[wbptr], #128]\n" + "str q9, [x21, %[output_col_stride1]]\n" + "fmla v5.4s, v25.4s, v14.4s\n" + "str q8, [x21, x19]\n" + "fmax v7.4s, v7.4s, v29.4s\n" + "fmax v6.4s, v6.4s, v29.4s\n" + "ldr q17, [%[wbptr], #96]\n" + "str q7, [x14]\n" + "fmax v5.4s, v5.4s, v29.4s\n" + "str q6, [x14, %[output_col_stride1]]\n" + "mov v4.16b, v20.16b\n" + "str q5, [x14, x19]\n" + "mov v1.16b, v20.16b\n" + "mov v3.16b, v20.16b\n" + "ldr q14, [%[wbptr], #144]\n" + "mov v7.16b, v20.16b\n" + "ldr q31, [%[inptr0], x16]\n" + "mov v9.16b, v20.16b\n" + "ldr q28, [x13]\n" + "mov v2.16b, v20.16b\n" + "ldr q18, [x22, %[input_col_stride1]]\n" + "mov v6.16b, v20.16b\n" + "add %[outptr0], %[outptr0], #16\n" + "mov v8.16b, v20.16b\n" + "add x21, x21, #16\n" + "mov v5.16b, v20.16b\n" + "add x14, x14, #16\n" + "fmla v4.4s, v27.4s, v15.4s\n" + "fmla v4.4s, v24.4s, v16.4s\n" + "bne 2b\n" + "3:\n" + "fmla v1.4s, v24.4s, v15.4s\n" + "ldr q24, [x25, x16]\n" + "fmla v4.4s, v22.4s, v0.4s\n" + "ldr q29, [%[inptr0], x26]\n" + "fmla v3.4s, v22.4s, v15.4s\n" + "ldr q30, [x24]\n" + "fmla v1.4s, v21.4s, v16.4s\n" + "ldr q25, [x13, %[input_col_stride1]]\n" + "fmla v4.4s, v21.4s, v11.4s\n" + "prfm pldl1keep, [x25, x23]\n" + "fmla v7.4s, v21.4s, v15.4s\n" + "ldr q26, [x22, x16]\n" + "fmla v1.4s, v19.4s, v0.4s\n" + "prfm pldl1keep, [%[inptr0], x20]\n" + "fmla v4.4s, v19.4s, v12.4s\n" + "prfm pldl1keep, [x24, #64]\n" + "fmla v3.4s, v19.4s, v16.4s\n" + "prfm pldl1keep, [x13, x17]\n" + "fmla v9.4s, v19.4s, v15.4s\n" + "ldr q23, [x25, x26]\n" + "fmla v4.4s, v31.4s, v13.4s\n" + "prfm pldl1keep, [x22, x23]\n" + "fmla v3.4s, v31.4s, v0.4s\n" + "prfm pldl1keep, [x25, x20]\n" + "fmla v2.4s, v31.4s, v15.4s\n" + "ldr q20, [%[inptr0], x18]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x15]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "ldr q28, [x24, %[input_col_stride1]]\n" + "fmla v4.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x24, x17]\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "prfm pldl1keep, [x13, x23]\n" + "fmla v3.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x22, x20]\n" + "fmla v7.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [x25, x15]\n" + "fmla v9.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x24, x23]\n" + "fmla v6.4s, v18.4s, v15.4s\n" + "ldr q27, [x13, x16]\n" + "fmla v4.4s, v24.4s, v17.4s\n" + "prfm pldl1keep, [x13, x20]\n" + "fmla v1.4s, v24.4s, v13.4s\n" + "prfm pldl1keep, [x22, x15]\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "prfm pldl1keep, [x24, x20]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "prfm pldl1keep, [x13, x15]\n" + "fmla v2.4s, v24.4s, v16.4s\n" + "prfm pldl1keep, [x24, x15]\n" + "fmla v8.4s, v24.4s, v15.4s\n" + "ldr q24, [x22, x26]\n" + "fmla v3.4s, v29.4s, v13.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v2.4s, v29.4s, v0.4s\n" + "ldr q22, [x25, x18]\n" + "fmla v7.4s, v30.4s, v11.4s\n" + "ldr q21, [x24, x16]\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v9.4s, v25.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #16\n" + "fmla v7.4s, v25.4s, v12.4s\n" + "add x25, x25, #16\n" + "fmla v6.4s, v25.4s, v16.4s\n" + "ldr q19, [x13, x26]\n" + "fmla v4.4s, v26.4s, v14.4s\n" + "fmla v1.4s, v26.4s, v17.4s\n" + "fmla v3.4s, v26.4s, v10.4s\n" + "fmla v7.4s, v26.4s, v13.4s\n" + "fmla v9.4s, v26.4s, v12.4s\n" + "fmla v2.4s, v26.4s, v11.4s\n" + "fmla v6.4s, v26.4s, v0.4s\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "fmla v5.4s, v26.4s, v15.4s\n" + "ldr q26, [x22, x18]\n" + "fmla v3.4s, v23.4s, v17.4s\n" + "ldr q18, [x24, x26]\n" + "fmla v9.4s, v23.4s, v13.4s\n" + "add x22, x22, #16\n" + "fmla v2.4s, v23.4s, v12.4s\n" + "fmla v8.4s, v23.4s, v0.4s\n" + "fmla v7.4s, v28.4s, v10.4s\n" + "ldr q23, [x13, x18]\n" + "fmla v6.4s, v28.4s, v11.4s\n" + "ldr q25, [x24, x18]\n" + "fmla v2.4s, v20.4s, v13.4s\n" + "add x13, x13, #16\n" + "fmla v1.4s, v27.4s, v14.4s\n" + "add x24, x24, #16\n" + "fmla v7.4s, v27.4s, v17.4s\n" + "fmla v9.4s, v27.4s, v10.4s\n" + "fmla v6.4s, v27.4s, v12.4s\n" + "fmla v8.4s, v27.4s, v11.4s\n" + "fmla v5.4s, v27.4s, v16.4s\n" + "fmla v3.4s, v24.4s, v14.4s\n" + "fmla v9.4s, v24.4s, v17.4s\n" + "fmla v2.4s, v24.4s, v10.4s\n" + "fmla v6.4s, v24.4s, v13.4s\n" + "fmla v8.4s, v24.4s, v12.4s\n" + "fmla v5.4s, v24.4s, v0.4s\n" + "fmla v7.4s, v21.4s, v14.4s\n" + "fmla v2.4s, v22.4s, v17.4s\n" + "fmla v9.4s, v19.4s, v14.4s\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "fmla v6.4s, v21.4s, v10.4s\n" + "fmla v5.4s, v21.4s, v11.4s\n" + "movi v29.16b, #0\n" + "fmla v2.4s, v26.4s, v14.4s\n" + "fmla v6.4s, v19.4s, v17.4s\n" + "fmla v8.4s, v19.4s, v10.4s\n" + "fmla v5.4s, v19.4s, v12.4s\n" + "fmax v4.4s, v4.4s, v29.4s\n" + "fmax v3.4s, v3.4s, v29.4s\n" + "fmla v6.4s, v18.4s, v14.4s\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "str q4, [%[outptr0]]\n" + "fmla v8.4s, v26.4s, v17.4s\n" + "str q3, [%[outptr0], %[output_col_stride1]]\n" + "fmla v5.4s, v26.4s, v13.4s\n" + "str q2, [%[outptr0], x19]\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "fmla v8.4s, v23.4s, v14.4s\n" + "add %[outptr0], %[outptr0], #16\n" + "str q1, [x21]\n" + "fmla v5.4s, v18.4s, v10.4s\n" + "fmax v9.4s, v9.4s, v29.4s\n" + "fmax v7.4s, v7.4s, v29.4s\n" + "fmax v8.4s, v8.4s, v29.4s\n" + "fmax v6.4s, v6.4s, v29.4s\n" + "str q9, [x21, %[output_col_stride1]]\n" + "fmla v5.4s, v23.4s, v17.4s\n" + "str q8, [x21, x19]\n" + "str q7, [x14]\n" + "str q6, [x14, %[output_col_stride1]]\n" + "add x21, x21, #16\n" + "fmla v5.4s, v25.4s, v14.4s\n" + "fmax v5.4s, v5.4s, v29.4s\n" + "str q5, [x14, x19]\n" + "add x14, x14, #16\n" + "4:\n" + "cbz x27, 7f\n" + "ldr s20, [%[wbptr]]\n" + "mov v4.16b, v20.16b\n" + "ldr s15, [%[wbptr], #4]\n" + "mov v1.16b, v20.16b\n" + "ldr s0, [%[wbptr], #8]\n" + "mov v3.16b, v20.16b\n" + "ldr s13, [%[wbptr], #12]\n" + "mov v7.16b, v20.16b\n" + "ldr s16, [%[wbptr], #16]\n" + "mov v9.16b, v20.16b\n" + "ldr s12, [%[wbptr], #20]\n" + "mov v2.16b, v20.16b\n" + "ldr s17, [%[wbptr], #24]\n" + "mov v6.16b, v20.16b\n" + "ldr s11, [%[wbptr], #28]\n" + "mov v8.16b, v20.16b\n" + "ldr s10, [%[wbptr], #32]\n" + "mov v5.16b, v20.16b\n" + "ldr s14, [%[wbptr], #36]\n" + "ldr s27, [%[inptr0]]\n" + "subs x27, x27, #1\n" + "fmla v4.4s, v27.4s, v15.4s\n" + "ldr s24, [x25]\n" + "ldr s22, [%[inptr0], %[input_col_stride1]]\n" + "ldr s21, [x22]\n" + "ldr s19, [x25, %[input_col_stride1]]\n" + "ldr s31, [%[inptr0], x16]\n" + "fmla v4.4s, v24.4s, v16.4s\n" + "ldr s28, [x13]\n" + "ldr s18, [x22, %[input_col_stride1]]\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "prfm pldl1keep, [x25, #64]\n" + "prfm pldl1keep, [%[inptr0], x17]\n" + "prfm pldl1keep, [x22, #64]\n" + "prfm pldl1keep, [x25, x17]\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "prfm pldl1keep, [x13, #64]\n" + "prfm pldl1keep, [x22, x17]\n" + "beq 6f\n" + "5:\n" + "fmla v1.4s, v24.4s, v15.4s\n" + "ldr s24, [x25, x16]\n" + "fmla v4.4s, v22.4s, v0.4s\n" + "ldr s29, [%[inptr0], x26]\n" + "fmla v3.4s, v22.4s, v15.4s\n" + "ldr s30, [x24]\n" + "fmla v1.4s, v21.4s, v16.4s\n" + "ldr s25, [x13, %[input_col_stride1]]\n" + "fmla v4.4s, v21.4s, v11.4s\n" + "prfm pldl1keep, [x25, x23]\n" + "fmla v7.4s, v21.4s, v15.4s\n" + "ldr s26, [x22, x16]\n" + "fmla v1.4s, v19.4s, v0.4s\n" + "prfm pldl1keep, [%[inptr0], x20]\n" + "fmla v4.4s, v19.4s, v12.4s\n" + "prfm pldl1keep, [x24, #64]\n" + "fmla v3.4s, v19.4s, v16.4s\n" + "prfm pldl1keep, [x13, x17]\n" + "fmla v9.4s, v19.4s, v15.4s\n" + "ldr s23, [x25, x26]\n" + "fmla v4.4s, v31.4s, v13.4s\n" + "prfm pldl1keep, [x22, x23]\n" + "fmla v3.4s, v31.4s, v0.4s\n" + "prfm pldl1keep, [x25, x20]\n" + "fmla v2.4s, v31.4s, v15.4s\n" + "ldr s20, [%[inptr0], x18]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x15]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "ldr s28, [x24, %[input_col_stride1]]\n" + "fmla v4.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x24, x17]\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "prfm pldl1keep, [x13, x23]\n" + "fmla v3.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x22, x20]\n" + "fmla v7.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [x25, x15]\n" + "fmla v9.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x24, x23]\n" + "fmla v6.4s, v18.4s, v15.4s\n" + "ldr s27, [x13, x16]\n" + "fmla v4.4s, v24.4s, v17.4s\n" + "prfm pldl1keep, [x13, x20]\n" + "fmla v1.4s, v24.4s, v13.4s\n" + "prfm pldl1keep, [x22, x15]\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "prfm pldl1keep, [x24, x20]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "prfm pldl1keep, [x13, x15]\n" + "fmla v2.4s, v24.4s, v16.4s\n" + "prfm pldl1keep, [x24, x15]\n" + "fmla v8.4s, v24.4s, v15.4s\n" + "ldr s24, [x22, x26]\n" + "fmla v3.4s, v29.4s, v13.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v2.4s, v29.4s, v0.4s\n" + "ldr s22, [x25, x18]\n" + "fmla v7.4s, v30.4s, v11.4s\n" + "ldr s21, [x24, x16]\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v9.4s, v25.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #4\n" + "fmla v7.4s, v25.4s, v12.4s\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "fmla v6.4s, v25.4s, v16.4s\n" + "ldr s19, [x13, x26]\n" + "fmla v4.4s, v26.4s, v14.4s\n" + "prfm pldl1keep, [%[inptr0], x17]\n" + "fmla v1.4s, v26.4s, v17.4s\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "fmla v3.4s, v26.4s, v10.4s\n" + "add x25, x25, #4\n" + "fmla v7.4s, v26.4s, v13.4s\n" + "prfm pldl1keep, [x25, #64]\n" + "fmla v9.4s, v26.4s, v12.4s\n" + "prfm pldl1keep, [x25, x17]\n" + "fmla v2.4s, v26.4s, v11.4s\n" + "subs x27, x27, #1\n" + "fmla v6.4s, v26.4s, v0.4s\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "fmla v5.4s, v26.4s, v15.4s\n" + "ldr s26, [x22, x18]\n" + "fmla v3.4s, v23.4s, v17.4s\n" + "ldr s18, [x24, x26]\n" + "fmla v9.4s, v23.4s, v13.4s\n" + "add x22, x22, #4\n" + "fmla v2.4s, v23.4s, v12.4s\n" + "prfm pldl1keep, [x22, #64]\n" + "fmla v8.4s, v23.4s, v0.4s\n" + "ldr s23, [x13, x18]\n" + "fmla v7.4s, v28.4s, v10.4s\n" + "prfm pldl1keep, [x22, x17]\n" + "fmla v2.4s, v20.4s, v13.4s\n" + "ldr s25, [x24, x18]\n" + "fmla v6.4s, v28.4s, v11.4s\n" + "ldr s20, [%[wbptr]]\n" + "fmla v1.4s, v27.4s, v14.4s\n" + "add x13, x13, #4\n" + "fmla v7.4s, v27.4s, v17.4s\n" + "prfm pldl1keep, [x13, #64]\n" + "fmla v9.4s, v27.4s, v10.4s\n" + "add x24, x24, #4\n" + "fmla v6.4s, v27.4s, v12.4s\n" + "fmla v8.4s, v27.4s, v11.4s\n" + "fmla v5.4s, v27.4s, v16.4s\n" + "ldr s15, [%[wbptr], #4]\n" + "fmla v3.4s, v24.4s, v14.4s\n" + "ldr s27, [%[inptr0]]\n" + "fmla v9.4s, v24.4s, v17.4s\n" + "fmla v2.4s, v24.4s, v10.4s\n" + "fmla v6.4s, v24.4s, v13.4s\n" + "fmla v8.4s, v24.4s, v12.4s\n" + "fmla v5.4s, v24.4s, v0.4s\n" + "ldr s16, [%[wbptr], #16]\n" + "fmla v2.4s, v22.4s, v17.4s\n" + "ldr s24, [x25]\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "ldr s22, [%[inptr0], %[input_col_stride1]]\n" + "fmla v7.4s, v21.4s, v14.4s\n" + "fmla v6.4s, v21.4s, v10.4s\n" + "fmla v5.4s, v21.4s, v11.4s\n" + "ldr s0, [%[wbptr], #8]\n" + "fmla v9.4s, v19.4s, v14.4s\n" + "ldr s21, [x22]\n" + "fmla v6.4s, v19.4s, v17.4s\n" + "fmla v8.4s, v19.4s, v10.4s\n" + "fmla v5.4s, v19.4s, v12.4s\n" + "ldr s11, [%[wbptr], #28]\n" + "fmla v2.4s, v26.4s, v14.4s\n" + "movi v29.16b, #0\n" + "fmla v8.4s, v26.4s, v17.4s\n" + "fmla v6.4s, v18.4s, v14.4s\n" + "fmla v5.4s, v26.4s, v13.4s\n" + "ldr s12, [%[wbptr], #20]\n" + "fmax v4.4s, v4.4s, v29.4s\n" + "ldr s19, [x25, %[input_col_stride1]]\n" + "fmla v8.4s, v23.4s, v14.4s\n" + "fmax v3.4s, v3.4s, v29.4s\n" + "str s4, [%[outptr0]]\n" + "fmla v5.4s, v18.4s, v10.4s\n" + "str s3, [%[outptr0], %[output_col_stride1]]\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "ldr s13, [%[wbptr], #12]\n" + "str s2, [%[outptr0], x19]\n" + "fmla v5.4s, v23.4s, v17.4s\n" + "str s1, [x21]\n" + "fmax v9.4s, v9.4s, v29.4s\n" + "fmax v8.4s, v8.4s, v29.4s\n" + "ldr s10, [%[wbptr], #32]\n" + "str s9, [x21, %[output_col_stride1]]\n" + "fmla v5.4s, v25.4s, v14.4s\n" + "str s8, [x21, x19]\n" + "fmax v7.4s, v7.4s, v29.4s\n" + "fmax v6.4s, v6.4s, v29.4s\n" + "ldr s17, [%[wbptr], #24]\n" + "str s7, [x14]\n" + "fmax v5.4s, v5.4s, v29.4s\n" + "str s6, [x14, %[output_col_stride1]]\n" + "mov v4.16b, v20.16b\n" + "str s5, [x14, x19]\n" + "mov v1.16b, v20.16b\n" + "mov v3.16b, v20.16b\n" + "ldr s14, [%[wbptr], #36]\n" + "mov v7.16b, v20.16b\n" + "ldr s31, [%[inptr0], x16]\n" + "mov v9.16b, v20.16b\n" + "ldr s28, [x13]\n" + "mov v2.16b, v20.16b\n" + "ldr s18, [x22, %[input_col_stride1]]\n" + "mov v6.16b, v20.16b\n" + "add %[outptr0], %[outptr0], #4\n" + "mov v8.16b, v20.16b\n" + "add x21, x21, #4\n" + "mov v5.16b, v20.16b\n" + "add x14, x14, #4\n" + "fmla v4.4s, v27.4s, v15.4s\n" + "fmla v4.4s, v24.4s, v16.4s\n" + "bne 5b\n" + "6:\n" + "fmla v1.4s, v24.4s, v15.4s\n" + "ldr s24, [x25, x16]\n" + "fmla v4.4s, v22.4s, v0.4s\n" + "ldr s29, [%[inptr0], x26]\n" + "fmla v3.4s, v22.4s, v15.4s\n" + "ldr s30, [x24]\n" + "fmla v1.4s, v21.4s, v16.4s\n" + "ldr s25, [x13, %[input_col_stride1]]\n" + "fmla v4.4s, v21.4s, v11.4s\n" + "prfm pldl1keep, [x25, x23]\n" + "fmla v7.4s, v21.4s, v15.4s\n" + "ldr s26, [x22, x16]\n" + "fmla v1.4s, v19.4s, v0.4s\n" + "prfm pldl1keep, [%[inptr0], x20]\n" + "fmla v4.4s, v19.4s, v12.4s\n" + "prfm pldl1keep, [x24, #64]\n" + "fmla v3.4s, v19.4s, v16.4s\n" + "prfm pldl1keep, [x13, x17]\n" + "fmla v9.4s, v19.4s, v15.4s\n" + "ldr s23, [x25, x26]\n" + "fmla v4.4s, v31.4s, v13.4s\n" + "prfm pldl1keep, [x22, x23]\n" + "fmla v3.4s, v31.4s, v0.4s\n" + "prfm pldl1keep, [x25, x20]\n" + "fmla v2.4s, v31.4s, v15.4s\n" + "ldr s20, [%[inptr0], x18]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x15]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "ldr s28, [x24, %[input_col_stride1]]\n" + "fmla v4.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x24, x17]\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "prfm pldl1keep, [x13, x23]\n" + "fmla v3.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x22, x20]\n" + "fmla v7.4s, v18.4s, v0.4s\n" + "prfm pldl1keep, [x25, x15]\n" + "fmla v9.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x24, x23]\n" + "fmla v6.4s, v18.4s, v15.4s\n" + "ldr s27, [x13, x16]\n" + "fmla v4.4s, v24.4s, v17.4s\n" + "prfm pldl1keep, [x13, x20]\n" + "fmla v1.4s, v24.4s, v13.4s\n" + "prfm pldl1keep, [x22, x15]\n" + "fmla v3.4s, v24.4s, v12.4s\n" + "prfm pldl1keep, [x24, x20]\n" + "fmla v9.4s, v24.4s, v0.4s\n" + "prfm pldl1keep, [x13, x15]\n" + "fmla v2.4s, v24.4s, v16.4s\n" + "prfm pldl1keep, [x24, x15]\n" + "fmla v8.4s, v24.4s, v15.4s\n" + "ldr s24, [x22, x26]\n" + "fmla v3.4s, v29.4s, v13.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v2.4s, v29.4s, v0.4s\n" + "ldr s22, [x25, x18]\n" + "fmla v7.4s, v30.4s, v11.4s\n" + "ldr s21, [x24, x16]\n" + "fmla v1.4s, v25.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v9.4s, v25.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #4\n" + "fmla v7.4s, v25.4s, v12.4s\n" + "add x25, x25, #4\n" + "fmla v6.4s, v25.4s, v16.4s\n" + "ldr s19, [x13, x26]\n" + "fmla v4.4s, v26.4s, v14.4s\n" + "fmla v1.4s, v26.4s, v17.4s\n" + "fmla v3.4s, v26.4s, v10.4s\n" + "fmla v7.4s, v26.4s, v13.4s\n" + "fmla v9.4s, v26.4s, v12.4s\n" + "fmla v2.4s, v26.4s, v11.4s\n" + "fmla v6.4s, v26.4s, v0.4s\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "fmla v5.4s, v26.4s, v15.4s\n" + "ldr s26, [x22, x18]\n" + "fmla v3.4s, v23.4s, v17.4s\n" + "ldr s18, [x24, x26]\n" + "fmla v9.4s, v23.4s, v13.4s\n" + "add x22, x22, #4\n" + "fmla v2.4s, v23.4s, v12.4s\n" + "fmla v8.4s, v23.4s, v0.4s\n" + "fmla v7.4s, v28.4s, v10.4s\n" + "ldr s23, [x13, x18]\n" + "fmla v6.4s, v28.4s, v11.4s\n" + "ldr s25, [x24, x18]\n" + "fmla v2.4s, v20.4s, v13.4s\n" + "add x13, x13, #4\n" + "fmla v1.4s, v27.4s, v14.4s\n" + "add x24, x24, #4\n" + "fmla v7.4s, v27.4s, v17.4s\n" + "fmla v9.4s, v27.4s, v10.4s\n" + "fmla v6.4s, v27.4s, v12.4s\n" + "fmla v8.4s, v27.4s, v11.4s\n" + "fmla v5.4s, v27.4s, v16.4s\n" + "fmla v3.4s, v24.4s, v14.4s\n" + "fmla v9.4s, v24.4s, v17.4s\n" + "fmla v2.4s, v24.4s, v10.4s\n" + "fmla v6.4s, v24.4s, v13.4s\n" + "fmla v8.4s, v24.4s, v12.4s\n" + "fmla v5.4s, v24.4s, v0.4s\n" + "fmla v7.4s, v21.4s, v14.4s\n" + "fmla v2.4s, v22.4s, v17.4s\n" + "fmla v9.4s, v19.4s, v14.4s\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "fmla v6.4s, v21.4s, v10.4s\n" + "fmla v5.4s, v21.4s, v11.4s\n" + "movi v29.16b, #0\n" + "fmla v2.4s, v26.4s, v14.4s\n" + "fmla v6.4s, v19.4s, v17.4s\n" + "fmla v8.4s, v19.4s, v10.4s\n" + "fmla v5.4s, v19.4s, v12.4s\n" + "fmax v4.4s, v4.4s, v29.4s\n" + "fmax v3.4s, v3.4s, v29.4s\n" + "fmla v6.4s, v18.4s, v14.4s\n" + "fmax v2.4s, v2.4s, v29.4s\n" + "str s4, [%[outptr0]]\n" + "fmla v8.4s, v26.4s, v17.4s\n" + "str s3, [%[outptr0], %[output_col_stride1]]\n" + "fmla v5.4s, v26.4s, v13.4s\n" + "str s2, [%[outptr0], x19]\n" + "fmax v1.4s, v1.4s, v29.4s\n" + "fmla v8.4s, v23.4s, v14.4s\n" + "add %[outptr0], %[outptr0], #4\n" + "str s1, [x21]\n" + "fmla v5.4s, v18.4s, v10.4s\n" + "fmax v9.4s, v9.4s, v29.4s\n" + "fmax v7.4s, v7.4s, v29.4s\n" + "fmax v8.4s, v8.4s, v29.4s\n" + "fmax v6.4s, v6.4s, v29.4s\n" + "str s9, [x21, %[output_col_stride1]]\n" + "fmla v5.4s, v23.4s, v17.4s\n" + "str s8, [x21, x19]\n" + "str s7, [x14]\n" + "str s6, [x14, %[output_col_stride1]]\n" + "add x21, x21, #4\n" + "fmla v5.4s, v25.4s, v14.4s\n" + "fmax v5.4s, v5.4s, v29.4s\n" + "str s5, [x14, x19]\n" + "add x14, x14, #4\n" + "7:\n" + : [wbptr] "+r" (weight_bias_ptr), [outptr0] "+r" (output), [inptr0] "+r" (input) + : [output_row_stride] "r" (output_row_stride * sizeof(float)), [n_channels] "r" ((long) n_channels), [output_col_stride1] "r" (output_col_stride * sizeof(float)), [input_row_stride] "r" (input_row_stride * sizeof(float)), [input_col_stride1] "r" (input_col_stride * sizeof(float)) + : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v3", "v30", "v31", "v4", "v5", "v6", "v7", "v8", "v9", "x13", "x14", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" + ); +} template <> -const Conv::TileFn Conv::tilefn_left[n_in_pad_left_fns] = { - ConvImpl::template process_tile<true, 0, 1, 0, 0, 0, 0>, -}; - template <> -const Conv::TileFn Conv::tilefn_bottom[n_in_pad_bottom_fns][n_out_pad_bottom_fns] = { - { - ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 0, 0, 1, 0>, - ConvImpl::template process_tile<true, 0, 0, 0, 0, 2, 0>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 1, 0, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 1, 0, 1, 0>, - ConvImpl::template process_tile<true, 0, 0, 1, 0, 2, 0>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 2, 0, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 2, 0, 1, 0>, - ConvImpl::template process_tile<true, 0, 0, 2, 0, 2, 0>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 3, 0, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 3, 0, 1, 0>, - ConvImpl::template process_tile<true, 0, 0, 3, 0, 2, 0>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 4, 0, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 4, 0, 1, 0>, - ConvImpl::template process_tile<true, 0, 0, 4, 0, 2, 0>, - }, -}; +void Conv::execute_tile<ActivationFunction::ReLU6>( + int n_channels, + const void *weight_bias_ptr, + const float *input, + const unsigned int input_row_stride, + const unsigned int input_col_stride, + float *output, + const unsigned int output_row_stride, + const unsigned int output_col_stride +) +{ + __asm __volatile( + "add x17, %[inptr0], %[input_row_stride]\n" + "add x18, %[input_col_stride1], %[input_col_stride1]\n" + "add x25, %[outptr0], %[output_row_stride]\n" + "add x14, x17, %[input_row_stride]\n" + "add x22, x18, #64\n" + "add x15, x18, %[input_col_stride1]\n" + "add x21, x14, %[input_row_stride]\n" + "add x16, x15, #64\n" + "add x24, x15, %[input_col_stride1]\n" + "add x26, x21, %[input_row_stride]\n" + "add x23, x24, #64\n" + "add x13, x25, %[output_row_stride]\n" + "add x27, %[output_col_stride1], %[output_col_stride1]\n" + "and x19, %[n_channels], #3\n" + "lsr x20, %[n_channels], #2\n" + "cbz x20, 4f\n" + "1:\n" + "ldr q19, [%[wbptr]]\n" + "subs x20, x20, #1\n" + "mov v8.16b, v19.16b\n" + "ldr q17, [%[wbptr], #16]\n" + "mov v5.16b, v19.16b\n" + "ldr q16, [%[wbptr], #32]\n" + "mov v7.16b, v19.16b\n" + "ldr q15, [%[wbptr], #48]\n" + "mov v2.16b, v19.16b\n" + "ldr q14, [%[wbptr], #64]\n" + "mov v4.16b, v19.16b\n" + "ldr q13, [%[wbptr], #80]\n" + "mov v6.16b, v19.16b\n" + "ldr q12, [%[wbptr], #96]\n" + "mov v1.16b, v19.16b\n" + "ldr q11, [%[wbptr], #112]\n" + "mov v3.16b, v19.16b\n" + "ldr q10, [%[wbptr], #128]\n" + "mov v0.16b, v19.16b\n" + "ldr q9, [%[wbptr], #144]\n" + "ldr q25, [%[inptr0]]\n" + "ldr q27, [x17]\n" + "fmla v8.4s, v25.4s, v17.4s\n" + "ldr q26, [%[inptr0], %[input_col_stride1]]\n" + "ldr q20, [x14]\n" + "ldr q22, [x17, %[input_col_stride1]]\n" + "ldr q28, [%[inptr0], x18]\n" + "ldr q23, [x21]\n" + "fmla v8.4s, v27.4s, v14.4s\n" + "ldr q18, [x14, %[input_col_stride1]]\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "prfm pldl1keep, [x17, #64]\n" + "prfm pldl1keep, [%[inptr0], x28]\n" + "prfm pldl1keep, [x14, #64]\n" + "prfm pldl1keep, [x17, x28]\n" + "prfm pldl1keep, [%[inptr0], x22]\n" + "prfm pldl1keep, [x21, #64]\n" + "prfm pldl1keep, [x14, x28]\n" + "beq 3f\n" + "2:\n" + "fmla v5.4s, v27.4s, v17.4s\n" + "ldr q27, [x17, x18]\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "ldr q30, [%[inptr0], x15]\n" + "fmla v7.4s, v26.4s, v17.4s\n" + "ldr q31, [x26]\n" + "fmla v5.4s, v20.4s, v14.4s\n" + "ldr q24, [x21, %[input_col_stride1]]\n" + "fmla v8.4s, v20.4s, v11.4s\n" + "prfm pldl1keep, [x17, x22]\n" + "fmla v2.4s, v20.4s, v17.4s\n" + "ldr q29, [x14, x18]\n" + "fmla v5.4s, v22.4s, v16.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "prfm pldl1keep, [x26, #64]\n" + "fmla v7.4s, v22.4s, v14.4s\n" + "prfm pldl1keep, [x21, x28]\n" + "fmla v4.4s, v22.4s, v17.4s\n" + "ldr q21, [x17, x15]\n" + "fmla v8.4s, v28.4s, v15.4s\n" + "prfm pldl1keep, [x14, x22]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "prfm pldl1keep, [x17, x16]\n" + "fmla v6.4s, v28.4s, v17.4s\n" + "ldr q19, [%[inptr0], x24]\n" + "fmla v5.4s, v23.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "fmla v2.4s, v23.4s, v14.4s\n" + "ldr q28, [x26, %[input_col_stride1]]\n" + "fmla v8.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x26, x28]\n" + "fmla v5.4s, v18.4s, v13.4s\n" + "prfm pldl1keep, [x21, x22]\n" + "fmla v7.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x14, x16]\n" + "fmla v2.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x17, x23]\n" + "fmla v4.4s, v18.4s, v14.4s\n" + "prfm pldl1keep, [x26, x22]\n" + "fmla v1.4s, v18.4s, v17.4s\n" + "ldr q25, [x21, x18]\n" + "fmla v8.4s, v27.4s, v12.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v5.4s, v27.4s, v15.4s\n" + "prfm pldl1keep, [x14, x23]\n" + "fmla v7.4s, v27.4s, v13.4s\n" + "prfm pldl1keep, [x26, x16]\n" + "fmla v4.4s, v27.4s, v16.4s\n" + "prfm pldl1keep, [x21, x23]\n" + "fmla v6.4s, v27.4s, v14.4s\n" + "prfm pldl1keep, [x26, x23]\n" + "fmla v3.4s, v27.4s, v17.4s\n" + "ldr q27, [x14, x15]\n" + "fmla v7.4s, v30.4s, v15.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v6.4s, v30.4s, v16.4s\n" + "ldr q26, [x17, x24]\n" + "fmla v2.4s, v31.4s, v11.4s\n" + "ldr q20, [x26, x18]\n" + "fmla v5.4s, v24.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v4.4s, v24.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #16\n" + "fmla v2.4s, v24.4s, v13.4s\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "fmla v1.4s, v24.4s, v14.4s\n" + "ldr q18, [x21, x15]\n" + "fmla v8.4s, v29.4s, v9.4s\n" + "prfm pldl1keep, [%[inptr0], x28]\n" + "fmla v5.4s, v29.4s, v12.4s\n" + "prfm pldl1keep, [%[inptr0], x22]\n" + "fmla v7.4s, v29.4s, v10.4s\n" + "add x17, x17, #16\n" + "fmla v2.4s, v29.4s, v15.4s\n" + "prfm pldl1keep, [x17, #64]\n" + "fmla v4.4s, v29.4s, v13.4s\n" + "prfm pldl1keep, [x17, x28]\n" + "fmla v6.4s, v29.4s, v11.4s\n" + "subs x20, x20, #1\n" + "fmla v1.4s, v29.4s, v16.4s\n" + "fmla v3.4s, v29.4s, v14.4s\n" + "fmla v0.4s, v29.4s, v17.4s\n" + "ldr q22, [x14, x24]\n" + "fmla v7.4s, v21.4s, v12.4s\n" + "ldr q23, [x26, x15]\n" + "fmla v4.4s, v21.4s, v15.4s\n" + "add x14, x14, #16\n" + "fmla v6.4s, v21.4s, v13.4s\n" + "prfm pldl1keep, [x14, #64]\n" + "fmla v3.4s, v21.4s, v16.4s\n" + "ldr q24, [x21, x24]\n" + "fmla v2.4s, v28.4s, v10.4s\n" + "prfm pldl1keep, [x14, x28]\n" + "fmla v6.4s, v19.4s, v15.4s\n" + "ldr q21, [x26, x24]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "ldr q19, [%[wbptr]]\n" + "fmla v5.4s, v25.4s, v9.4s\n" + "add x21, x21, #16\n" + "fmla v2.4s, v25.4s, v12.4s\n" + "prfm pldl1keep, [x21, #64]\n" + "fmla v4.4s, v25.4s, v10.4s\n" + "add x26, x26, #16\n" + "fmla v1.4s, v25.4s, v13.4s\n" + "fmla v3.4s, v25.4s, v11.4s\n" + "fmla v0.4s, v25.4s, v14.4s\n" + "ldr q17, [%[wbptr], #16]\n" + "fmla v7.4s, v27.4s, v9.4s\n" + "ldr q25, [%[inptr0]]\n" + "fmla v4.4s, v27.4s, v12.4s\n" + "fmla v6.4s, v27.4s, v10.4s\n" + "fmla v1.4s, v27.4s, v15.4s\n" + "fmla v3.4s, v27.4s, v13.4s\n" + "fmla v0.4s, v27.4s, v16.4s\n" + "ldr q14, [%[wbptr], #64]\n" + "fmla v6.4s, v26.4s, v12.4s\n" + "ldr q27, [x17]\n" + "fmla v3.4s, v26.4s, v15.4s\n" + "ldr q26, [%[inptr0], %[input_col_stride1]]\n" + "fmla v2.4s, v20.4s, v9.4s\n" + "fmla v1.4s, v20.4s, v10.4s\n" + "fmla v0.4s, v20.4s, v11.4s\n" + "ldr q16, [%[wbptr], #32]\n" + "fmla v4.4s, v18.4s, v9.4s\n" + "ldr q20, [x14]\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "fmla v3.4s, v18.4s, v10.4s\n" + "fmla v0.4s, v18.4s, v13.4s\n" + "ldr q11, [%[wbptr], #112]\n" + "fmla v6.4s, v22.4s, v9.4s\n" + "movi v30.16b, #0\n" + "fmla v3.4s, v22.4s, v12.4s\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "fmla v0.4s, v22.4s, v15.4s\n" + "ldr q13, [%[wbptr], #80]\n" + "fmov v29.4s, #6.0\n" + "fmax v8.4s, v8.4s, v30.4s\n" + "fmla v3.4s, v24.4s, v9.4s\n" + "fmax v7.4s, v7.4s, v30.4s\n" + "fmla v0.4s, v23.4s, v10.4s\n" + "ldr q15, [%[wbptr], #48]\n" + "fmin v8.4s, v8.4s, v29.4s\n" + "ldr q22, [x17, %[input_col_stride1]]\n" + "fmin v7.4s, v7.4s, v29.4s\n" + "fmax v6.4s, v6.4s, v30.4s\n" + "str q8, [%[outptr0]]\n" + "fmla v0.4s, v24.4s, v12.4s\n" + "str q7, [%[outptr0], %[output_col_stride1]]\n" + "fmin v6.4s, v6.4s, v29.4s\n" + "fmax v5.4s, v5.4s, v30.4s\n" + "ldr q10, [%[wbptr], #128]\n" + "str q6, [%[outptr0], x27]\n" + "fmla v0.4s, v21.4s, v9.4s\n" + "fmin v5.4s, v5.4s, v29.4s\n" + "ldr q12, [%[wbptr], #96]\n" + "fmax v4.4s, v4.4s, v30.4s\n" + "ldr q28, [%[inptr0], x18]\n" + "str q5, [x25]\n" + "fmax v3.4s, v3.4s, v30.4s\n" + "fmin v4.4s, v4.4s, v29.4s\n" + "ldr q9, [%[wbptr], #144]\n" + "fmin v3.4s, v3.4s, v29.4s\n" + "ldr q23, [x21]\n" + "str q4, [x25, %[output_col_stride1]]\n" + "fmax v2.4s, v2.4s, v30.4s\n" + "str q3, [x25, x27]\n" + "fmax v1.4s, v1.4s, v30.4s\n" + "fmin v2.4s, v2.4s, v29.4s\n" + "ldr q18, [x14, %[input_col_stride1]]\n" + "fmin v1.4s, v1.4s, v29.4s\n" + "add %[outptr0], %[outptr0], #16\n" + "str q2, [x13]\n" + "fmax v0.4s, v0.4s, v30.4s\n" + "str q1, [x13, %[output_col_stride1]]\n" + "mov v8.16b, v19.16b\n" + "fmin v0.4s, v0.4s, v29.4s\n" + "add x25, x25, #16\n" + "mov v5.16b, v19.16b\n" + "mov v7.16b, v19.16b\n" + "str q0, [x13, x27]\n" + "mov v2.16b, v19.16b\n" + "mov v4.16b, v19.16b\n" + "add x13, x13, #16\n" + "mov v6.16b, v19.16b\n" + "mov v1.16b, v19.16b\n" + "mov v3.16b, v19.16b\n" + "mov v0.16b, v19.16b\n" + "fmla v8.4s, v25.4s, v17.4s\n" + "fmla v8.4s, v27.4s, v14.4s\n" + "bne 2b\n" + "3:\n" + "fmla v5.4s, v27.4s, v17.4s\n" + "ldr q27, [x17, x18]\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "ldr q30, [%[inptr0], x15]\n" + "fmla v7.4s, v26.4s, v17.4s\n" + "ldr q31, [x26]\n" + "fmla v5.4s, v20.4s, v14.4s\n" + "ldr q24, [x21, %[input_col_stride1]]\n" + "fmla v8.4s, v20.4s, v11.4s\n" + "prfm pldl1keep, [x17, x22]\n" + "fmla v2.4s, v20.4s, v17.4s\n" + "ldr q29, [x14, x18]\n" + "fmla v5.4s, v22.4s, v16.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "prfm pldl1keep, [x26, #64]\n" + "fmla v7.4s, v22.4s, v14.4s\n" + "prfm pldl1keep, [x21, x28]\n" + "fmla v4.4s, v22.4s, v17.4s\n" + "ldr q21, [x17, x15]\n" + "fmla v8.4s, v28.4s, v15.4s\n" + "prfm pldl1keep, [x14, x22]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "prfm pldl1keep, [x17, x16]\n" + "fmla v6.4s, v28.4s, v17.4s\n" + "ldr q19, [%[inptr0], x24]\n" + "fmla v5.4s, v23.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "fmla v2.4s, v23.4s, v14.4s\n" + "ldr q28, [x26, %[input_col_stride1]]\n" + "fmla v8.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x26, x28]\n" + "fmla v5.4s, v18.4s, v13.4s\n" + "prfm pldl1keep, [x21, x22]\n" + "fmla v7.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x14, x16]\n" + "fmla v2.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x17, x23]\n" + "fmla v4.4s, v18.4s, v14.4s\n" + "prfm pldl1keep, [x26, x22]\n" + "fmla v1.4s, v18.4s, v17.4s\n" + "ldr q25, [x21, x18]\n" + "fmla v8.4s, v27.4s, v12.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v5.4s, v27.4s, v15.4s\n" + "prfm pldl1keep, [x14, x23]\n" + "fmla v7.4s, v27.4s, v13.4s\n" + "prfm pldl1keep, [x26, x16]\n" + "fmla v4.4s, v27.4s, v16.4s\n" + "prfm pldl1keep, [x21, x23]\n" + "fmla v6.4s, v27.4s, v14.4s\n" + "prfm pldl1keep, [x26, x23]\n" + "fmla v3.4s, v27.4s, v17.4s\n" + "ldr q27, [x14, x15]\n" + "fmla v7.4s, v30.4s, v15.4s\n" + "add %[wbptr], %[wbptr], #160\n" + "fmla v6.4s, v30.4s, v16.4s\n" + "ldr q26, [x17, x24]\n" + "fmla v2.4s, v31.4s, v11.4s\n" + "ldr q20, [x26, x18]\n" + "fmla v5.4s, v24.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v4.4s, v24.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #16\n" + "fmla v2.4s, v24.4s, v13.4s\n" + "add x17, x17, #16\n" + "fmla v1.4s, v24.4s, v14.4s\n" + "ldr q18, [x21, x15]\n" + "fmla v8.4s, v29.4s, v9.4s\n" + "fmla v5.4s, v29.4s, v12.4s\n" + "fmla v7.4s, v29.4s, v10.4s\n" + "fmla v2.4s, v29.4s, v15.4s\n" + "fmla v4.4s, v29.4s, v13.4s\n" + "fmla v6.4s, v29.4s, v11.4s\n" + "fmla v1.4s, v29.4s, v16.4s\n" + "fmla v3.4s, v29.4s, v14.4s\n" + "fmla v0.4s, v29.4s, v17.4s\n" + "ldr q22, [x14, x24]\n" + "fmla v7.4s, v21.4s, v12.4s\n" + "ldr q23, [x26, x15]\n" + "fmla v4.4s, v21.4s, v15.4s\n" + "add x14, x14, #16\n" + "fmla v6.4s, v21.4s, v13.4s\n" + "fmla v3.4s, v21.4s, v16.4s\n" + "fmla v2.4s, v28.4s, v10.4s\n" + "ldr q24, [x21, x24]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "ldr q21, [x26, x24]\n" + "fmla v6.4s, v19.4s, v15.4s\n" + "add x21, x21, #16\n" + "fmla v5.4s, v25.4s, v9.4s\n" + "add x26, x26, #16\n" + "fmla v2.4s, v25.4s, v12.4s\n" + "fmla v4.4s, v25.4s, v10.4s\n" + "fmla v1.4s, v25.4s, v13.4s\n" + "fmla v3.4s, v25.4s, v11.4s\n" + "fmla v0.4s, v25.4s, v14.4s\n" + "fmla v7.4s, v27.4s, v9.4s\n" + "fmla v4.4s, v27.4s, v12.4s\n" + "fmla v6.4s, v27.4s, v10.4s\n" + "fmla v1.4s, v27.4s, v15.4s\n" + "fmla v3.4s, v27.4s, v13.4s\n" + "fmla v0.4s, v27.4s, v16.4s\n" + "fmla v2.4s, v20.4s, v9.4s\n" + "fmla v6.4s, v26.4s, v12.4s\n" + "fmla v4.4s, v18.4s, v9.4s\n" + "fmla v3.4s, v26.4s, v15.4s\n" + "fmla v1.4s, v20.4s, v10.4s\n" + "fmla v0.4s, v20.4s, v11.4s\n" + "movi v30.16b, #0\n" + "fmla v6.4s, v22.4s, v9.4s\n" + "fmov v29.4s, #6.0\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "fmla v3.4s, v18.4s, v10.4s\n" + "fmla v0.4s, v18.4s, v13.4s\n" + "fmax v8.4s, v8.4s, v30.4s\n" + "fmax v7.4s, v7.4s, v30.4s\n" + "fmax v6.4s, v6.4s, v30.4s\n" + "fmla v3.4s, v22.4s, v12.4s\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "fmla v0.4s, v22.4s, v15.4s\n" + "fmin v8.4s, v8.4s, v29.4s\n" + "fmin v7.4s, v7.4s, v29.4s\n" + "fmin v6.4s, v6.4s, v29.4s\n" + "str q8, [%[outptr0]]\n" + "fmla v3.4s, v24.4s, v9.4s\n" + "str q7, [%[outptr0], %[output_col_stride1]]\n" + "fmla v0.4s, v23.4s, v10.4s\n" + "str q6, [%[outptr0], x27]\n" + "fmax v5.4s, v5.4s, v30.4s\n" + "fmax v4.4s, v4.4s, v30.4s\n" + "add %[outptr0], %[outptr0], #16\n" + "fmla v0.4s, v24.4s, v12.4s\n" + "fmin v5.4s, v5.4s, v29.4s\n" + "fmin v4.4s, v4.4s, v29.4s\n" + "fmax v3.4s, v3.4s, v30.4s\n" + "str q5, [x25]\n" + "fmax v2.4s, v2.4s, v30.4s\n" + "str q4, [x25, %[output_col_stride1]]\n" + "fmla v0.4s, v21.4s, v9.4s\n" + "fmin v3.4s, v3.4s, v29.4s\n" + "fmin v2.4s, v2.4s, v29.4s\n" + "fmax v1.4s, v1.4s, v30.4s\n" + "str q3, [x25, x27]\n" + "str q2, [x13]\n" + "fmin v1.4s, v1.4s, v29.4s\n" + "fmax v0.4s, v0.4s, v30.4s\n" + "add x25, x25, #16\n" + "str q1, [x13, %[output_col_stride1]]\n" + "fmin v0.4s, v0.4s, v29.4s\n" + "str q0, [x13, x27]\n" + "add x13, x13, #16\n" + "4:\n" + "cbz x19, 7f\n" + "ldr s19, [%[wbptr]]\n" + "mov v8.16b, v19.16b\n" + "ldr s17, [%[wbptr], #4]\n" + "mov v5.16b, v19.16b\n" + "ldr s16, [%[wbptr], #8]\n" + "mov v7.16b, v19.16b\n" + "ldr s15, [%[wbptr], #12]\n" + "mov v2.16b, v19.16b\n" + "ldr s14, [%[wbptr], #16]\n" + "mov v4.16b, v19.16b\n" + "ldr s13, [%[wbptr], #20]\n" + "mov v6.16b, v19.16b\n" + "ldr s12, [%[wbptr], #24]\n" + "mov v1.16b, v19.16b\n" + "ldr s11, [%[wbptr], #28]\n" + "mov v3.16b, v19.16b\n" + "ldr s10, [%[wbptr], #32]\n" + "mov v0.16b, v19.16b\n" + "ldr s9, [%[wbptr], #36]\n" + "ldr s25, [%[inptr0]]\n" + "subs x19, x19, #1\n" + "fmla v8.4s, v25.4s, v17.4s\n" + "ldr s27, [x17]\n" + "ldr s26, [%[inptr0], %[input_col_stride1]]\n" + "ldr s20, [x14]\n" + "ldr s22, [x17, %[input_col_stride1]]\n" + "ldr s28, [%[inptr0], x18]\n" + "fmla v8.4s, v27.4s, v14.4s\n" + "ldr s23, [x21]\n" + "ldr s18, [x14, %[input_col_stride1]]\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "prfm pldl1keep, [x17, #64]\n" + "prfm pldl1keep, [%[inptr0], x28]\n" + "prfm pldl1keep, [x14, #64]\n" + "prfm pldl1keep, [x17, x28]\n" + "prfm pldl1keep, [%[inptr0], x22]\n" + "prfm pldl1keep, [x21, #64]\n" + "prfm pldl1keep, [x14, x28]\n" + "beq 6f\n" + "5:\n" + "fmla v5.4s, v27.4s, v17.4s\n" + "ldr s27, [x17, x18]\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "ldr s30, [%[inptr0], x15]\n" + "fmla v7.4s, v26.4s, v17.4s\n" + "ldr s31, [x26]\n" + "fmla v5.4s, v20.4s, v14.4s\n" + "ldr s24, [x21, %[input_col_stride1]]\n" + "fmla v8.4s, v20.4s, v11.4s\n" + "prfm pldl1keep, [x17, x22]\n" + "fmla v2.4s, v20.4s, v17.4s\n" + "ldr s29, [x14, x18]\n" + "fmla v5.4s, v22.4s, v16.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "prfm pldl1keep, [x26, #64]\n" + "fmla v7.4s, v22.4s, v14.4s\n" + "prfm pldl1keep, [x21, x28]\n" + "fmla v4.4s, v22.4s, v17.4s\n" + "ldr s21, [x17, x15]\n" + "fmla v8.4s, v28.4s, v15.4s\n" + "prfm pldl1keep, [x14, x22]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "prfm pldl1keep, [x17, x16]\n" + "fmla v6.4s, v28.4s, v17.4s\n" + "ldr s19, [%[inptr0], x24]\n" + "fmla v5.4s, v23.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "fmla v2.4s, v23.4s, v14.4s\n" + "ldr s28, [x26, %[input_col_stride1]]\n" + "fmla v8.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x26, x28]\n" + "fmla v5.4s, v18.4s, v13.4s\n" + "prfm pldl1keep, [x21, x22]\n" + "fmla v7.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x14, x16]\n" + "fmla v2.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x17, x23]\n" + "fmla v4.4s, v18.4s, v14.4s\n" + "prfm pldl1keep, [x26, x22]\n" + "fmla v1.4s, v18.4s, v17.4s\n" + "ldr s25, [x21, x18]\n" + "fmla v8.4s, v27.4s, v12.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v5.4s, v27.4s, v15.4s\n" + "prfm pldl1keep, [x14, x23]\n" + "fmla v7.4s, v27.4s, v13.4s\n" + "prfm pldl1keep, [x26, x16]\n" + "fmla v4.4s, v27.4s, v16.4s\n" + "prfm pldl1keep, [x21, x23]\n" + "fmla v6.4s, v27.4s, v14.4s\n" + "prfm pldl1keep, [x26, x23]\n" + "fmla v3.4s, v27.4s, v17.4s\n" + "ldr s27, [x14, x15]\n" + "fmla v7.4s, v30.4s, v15.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v6.4s, v30.4s, v16.4s\n" + "ldr s26, [x17, x24]\n" + "fmla v2.4s, v31.4s, v11.4s\n" + "ldr s20, [x26, x18]\n" + "fmla v5.4s, v24.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v4.4s, v24.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #4\n" + "fmla v2.4s, v24.4s, v13.4s\n" + "prfm pldl1keep, [%[inptr0], #64]\n" + "fmla v1.4s, v24.4s, v14.4s\n" + "ldr s18, [x21, x15]\n" + "fmla v8.4s, v29.4s, v9.4s\n" + "prfm pldl1keep, [%[inptr0], x28]\n" + "fmla v5.4s, v29.4s, v12.4s\n" + "prfm pldl1keep, [%[inptr0], x22]\n" + "fmla v7.4s, v29.4s, v10.4s\n" + "add x17, x17, #4\n" + "fmla v2.4s, v29.4s, v15.4s\n" + "prfm pldl1keep, [x17, #64]\n" + "fmla v4.4s, v29.4s, v13.4s\n" + "prfm pldl1keep, [x17, x28]\n" + "fmla v6.4s, v29.4s, v11.4s\n" + "subs x19, x19, #1\n" + "fmla v1.4s, v29.4s, v16.4s\n" + "fmla v3.4s, v29.4s, v14.4s\n" + "fmla v0.4s, v29.4s, v17.4s\n" + "ldr s22, [x14, x24]\n" + "fmla v7.4s, v21.4s, v12.4s\n" + "ldr s23, [x26, x15]\n" + "fmla v4.4s, v21.4s, v15.4s\n" + "add x14, x14, #4\n" + "fmla v6.4s, v21.4s, v13.4s\n" + "prfm pldl1keep, [x14, #64]\n" + "fmla v3.4s, v21.4s, v16.4s\n" + "ldr s24, [x21, x24]\n" + "fmla v2.4s, v28.4s, v10.4s\n" + "prfm pldl1keep, [x14, x28]\n" + "fmla v6.4s, v19.4s, v15.4s\n" + "ldr s21, [x26, x24]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "ldr s19, [%[wbptr]]\n" + "fmla v5.4s, v25.4s, v9.4s\n" + "add x21, x21, #4\n" + "fmla v2.4s, v25.4s, v12.4s\n" + "prfm pldl1keep, [x21, #64]\n" + "fmla v4.4s, v25.4s, v10.4s\n" + "add x26, x26, #4\n" + "fmla v1.4s, v25.4s, v13.4s\n" + "fmla v3.4s, v25.4s, v11.4s\n" + "fmla v0.4s, v25.4s, v14.4s\n" + "ldr s17, [%[wbptr], #4]\n" + "fmla v7.4s, v27.4s, v9.4s\n" + "ldr s25, [%[inptr0]]\n" + "fmla v4.4s, v27.4s, v12.4s\n" + "fmla v6.4s, v27.4s, v10.4s\n" + "fmla v1.4s, v27.4s, v15.4s\n" + "fmla v3.4s, v27.4s, v13.4s\n" + "fmla v0.4s, v27.4s, v16.4s\n" + "ldr s14, [%[wbptr], #16]\n" + "fmla v6.4s, v26.4s, v12.4s\n" + "ldr s27, [x17]\n" + "fmla v3.4s, v26.4s, v15.4s\n" + "ldr s26, [%[inptr0], %[input_col_stride1]]\n" + "fmla v2.4s, v20.4s, v9.4s\n" + "fmla v1.4s, v20.4s, v10.4s\n" + "fmla v0.4s, v20.4s, v11.4s\n" + "ldr s16, [%[wbptr], #8]\n" + "fmla v4.4s, v18.4s, v9.4s\n" + "ldr s20, [x14]\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "fmla v3.4s, v18.4s, v10.4s\n" + "fmla v0.4s, v18.4s, v13.4s\n" + "ldr s11, [%[wbptr], #28]\n" + "fmla v6.4s, v22.4s, v9.4s\n" + "movi v30.16b, #0\n" + "fmla v3.4s, v22.4s, v12.4s\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "fmla v0.4s, v22.4s, v15.4s\n" + "ldr s13, [%[wbptr], #20]\n" + "fmov v29.4s, #6.0\n" + "fmax v8.4s, v8.4s, v30.4s\n" + "fmla v3.4s, v24.4s, v9.4s\n" + "fmax v7.4s, v7.4s, v30.4s\n" + "fmla v0.4s, v23.4s, v10.4s\n" + "ldr s15, [%[wbptr], #12]\n" + "fmin v8.4s, v8.4s, v29.4s\n" + "ldr s22, [x17, %[input_col_stride1]]\n" + "fmin v7.4s, v7.4s, v29.4s\n" + "fmax v6.4s, v6.4s, v30.4s\n" + "str s8, [%[outptr0]]\n" + "fmla v0.4s, v24.4s, v12.4s\n" + "str s7, [%[outptr0], %[output_col_stride1]]\n" + "fmin v6.4s, v6.4s, v29.4s\n" + "fmax v5.4s, v5.4s, v30.4s\n" + "ldr s10, [%[wbptr], #32]\n" + "str s6, [%[outptr0], x27]\n" + "fmla v0.4s, v21.4s, v9.4s\n" + "fmin v5.4s, v5.4s, v29.4s\n" + "ldr s12, [%[wbptr], #24]\n" + "fmax v4.4s, v4.4s, v30.4s\n" + "ldr s28, [%[inptr0], x18]\n" + "str s5, [x25]\n" + "fmax v3.4s, v3.4s, v30.4s\n" + "fmin v4.4s, v4.4s, v29.4s\n" + "ldr s9, [%[wbptr], #36]\n" + "fmin v3.4s, v3.4s, v29.4s\n" + "ldr s23, [x21]\n" + "str s4, [x25, %[output_col_stride1]]\n" + "fmax v2.4s, v2.4s, v30.4s\n" + "str s3, [x25, x27]\n" + "fmax v1.4s, v1.4s, v30.4s\n" + "fmin v2.4s, v2.4s, v29.4s\n" + "ldr s18, [x14, %[input_col_stride1]]\n" + "fmin v1.4s, v1.4s, v29.4s\n" + "add %[outptr0], %[outptr0], #4\n" + "str s2, [x13]\n" + "fmax v0.4s, v0.4s, v30.4s\n" + "str s1, [x13, %[output_col_stride1]]\n" + "mov v8.16b, v19.16b\n" + "fmin v0.4s, v0.4s, v29.4s\n" + "add x25, x25, #4\n" + "mov v5.16b, v19.16b\n" + "mov v7.16b, v19.16b\n" + "str s0, [x13, x27]\n" + "mov v2.16b, v19.16b\n" + "mov v4.16b, v19.16b\n" + "add x13, x13, #4\n" + "mov v6.16b, v19.16b\n" + "mov v1.16b, v19.16b\n" + "mov v3.16b, v19.16b\n" + "mov v0.16b, v19.16b\n" + "fmla v8.4s, v25.4s, v17.4s\n" + "fmla v8.4s, v27.4s, v14.4s\n" + "bne 5b\n" + "6:\n" + "fmla v5.4s, v27.4s, v17.4s\n" + "ldr s27, [x17, x18]\n" + "fmla v8.4s, v26.4s, v16.4s\n" + "ldr s30, [%[inptr0], x15]\n" + "fmla v7.4s, v26.4s, v17.4s\n" + "ldr s31, [x26]\n" + "fmla v5.4s, v20.4s, v14.4s\n" + "ldr s24, [x21, %[input_col_stride1]]\n" + "fmla v8.4s, v20.4s, v11.4s\n" + "prfm pldl1keep, [x17, x22]\n" + "fmla v2.4s, v20.4s, v17.4s\n" + "ldr s29, [x14, x18]\n" + "fmla v5.4s, v22.4s, v16.4s\n" + "prfm pldl1keep, [%[inptr0], x16]\n" + "fmla v8.4s, v22.4s, v13.4s\n" + "prfm pldl1keep, [x26, #64]\n" + "fmla v7.4s, v22.4s, v14.4s\n" + "prfm pldl1keep, [x21, x28]\n" + "fmla v4.4s, v22.4s, v17.4s\n" + "ldr s21, [x17, x15]\n" + "fmla v8.4s, v28.4s, v15.4s\n" + "prfm pldl1keep, [x14, x22]\n" + "fmla v7.4s, v28.4s, v16.4s\n" + "prfm pldl1keep, [x17, x16]\n" + "fmla v6.4s, v28.4s, v17.4s\n" + "ldr s19, [%[inptr0], x24]\n" + "fmla v5.4s, v23.4s, v11.4s\n" + "prfm pldl1keep, [%[inptr0], x23]\n" + "fmla v2.4s, v23.4s, v14.4s\n" + "ldr s28, [x26, %[input_col_stride1]]\n" + "fmla v8.4s, v18.4s, v10.4s\n" + "prfm pldl1keep, [x26, x28]\n" + "fmla v5.4s, v18.4s, v13.4s\n" + "prfm pldl1keep, [x21, x22]\n" + "fmla v7.4s, v18.4s, v11.4s\n" + "prfm pldl1keep, [x14, x16]\n" + "fmla v2.4s, v18.4s, v16.4s\n" + "prfm pldl1keep, [x17, x23]\n" + "fmla v4.4s, v18.4s, v14.4s\n" + "prfm pldl1keep, [x26, x22]\n" + "fmla v1.4s, v18.4s, v17.4s\n" + "ldr s25, [x21, x18]\n" + "fmla v8.4s, v27.4s, v12.4s\n" + "prfm pldl1keep, [x21, x16]\n" + "fmla v5.4s, v27.4s, v15.4s\n" + "prfm pldl1keep, [x14, x23]\n" + "fmla v7.4s, v27.4s, v13.4s\n" + "prfm pldl1keep, [x26, x16]\n" + "fmla v4.4s, v27.4s, v16.4s\n" + "prfm pldl1keep, [x21, x23]\n" + "fmla v6.4s, v27.4s, v14.4s\n" + "prfm pldl1keep, [x26, x23]\n" + "fmla v3.4s, v27.4s, v17.4s\n" + "ldr s27, [x14, x15]\n" + "fmla v7.4s, v30.4s, v15.4s\n" + "add %[wbptr], %[wbptr], #40\n" + "fmla v6.4s, v30.4s, v16.4s\n" + "ldr s26, [x17, x24]\n" + "fmla v2.4s, v31.4s, v11.4s\n" + "ldr s20, [x26, x18]\n" + "fmla v5.4s, v24.4s, v10.4s\n" + "prfm pldl1keep, [%[wbptr], #64]\n" + "fmla v4.4s, v24.4s, v11.4s\n" + "add %[inptr0], %[inptr0], #4\n" + "fmla v2.4s, v24.4s, v13.4s\n" + "add x17, x17, #4\n" + "fmla v1.4s, v24.4s, v14.4s\n" + "ldr s18, [x21, x15]\n" + "fmla v8.4s, v29.4s, v9.4s\n" + "fmla v5.4s, v29.4s, v12.4s\n" + "fmla v7.4s, v29.4s, v10.4s\n" + "fmla v2.4s, v29.4s, v15.4s\n" + "fmla v4.4s, v29.4s, v13.4s\n" + "fmla v6.4s, v29.4s, v11.4s\n" + "fmla v1.4s, v29.4s, v16.4s\n" + "fmla v3.4s, v29.4s, v14.4s\n" + "fmla v0.4s, v29.4s, v17.4s\n" + "ldr s22, [x14, x24]\n" + "fmla v7.4s, v21.4s, v12.4s\n" + "ldr s23, [x26, x15]\n" + "fmla v4.4s, v21.4s, v15.4s\n" + "add x14, x14, #4\n" + "fmla v6.4s, v21.4s, v13.4s\n" + "fmla v3.4s, v21.4s, v16.4s\n" + "fmla v2.4s, v28.4s, v10.4s\n" + "ldr s24, [x21, x24]\n" + "fmla v1.4s, v28.4s, v11.4s\n" + "ldr s21, [x26, x24]\n" + "fmla v6.4s, v19.4s, v15.4s\n" + "add x21, x21, #4\n" + "fmla v5.4s, v25.4s, v9.4s\n" + "add x26, x26, #4\n" + "fmla v2.4s, v25.4s, v12.4s\n" + "fmla v4.4s, v25.4s, v10.4s\n" + "fmla v1.4s, v25.4s, v13.4s\n" + "fmla v3.4s, v25.4s, v11.4s\n" + "fmla v0.4s, v25.4s, v14.4s\n" + "fmla v7.4s, v27.4s, v9.4s\n" + "fmla v4.4s, v27.4s, v12.4s\n" + "fmla v6.4s, v27.4s, v10.4s\n" + "fmla v1.4s, v27.4s, v15.4s\n" + "fmla v3.4s, v27.4s, v13.4s\n" + "fmla v0.4s, v27.4s, v16.4s\n" + "fmla v2.4s, v20.4s, v9.4s\n" + "fmla v6.4s, v26.4s, v12.4s\n" + "fmla v4.4s, v18.4s, v9.4s\n" + "fmla v3.4s, v26.4s, v15.4s\n" + "fmla v1.4s, v20.4s, v10.4s\n" + "fmla v0.4s, v20.4s, v11.4s\n" + "movi v30.16b, #0\n" + "fmla v6.4s, v22.4s, v9.4s\n" + "fmov v29.4s, #6.0\n" + "fmla v1.4s, v18.4s, v12.4s\n" + "fmla v3.4s, v18.4s, v10.4s\n" + "fmla v0.4s, v18.4s, v13.4s\n" + "fmax v8.4s, v8.4s, v30.4s\n" + "fmax v7.4s, v7.4s, v30.4s\n" + "fmax v6.4s, v6.4s, v30.4s\n" + "fmla v3.4s, v22.4s, v12.4s\n" + "fmla v1.4s, v23.4s, v9.4s\n" + "fmla v0.4s, v22.4s, v15.4s\n" + "fmin v8.4s, v8.4s, v29.4s\n" + "fmin v7.4s, v7.4s, v29.4s\n" + "fmin v6.4s, v6.4s, v29.4s\n" + "str s8, [%[outptr0]]\n" + "fmla v3.4s, v24.4s, v9.4s\n" + "str s7, [%[outptr0], %[output_col_stride1]]\n" + "fmla v0.4s, v23.4s, v10.4s\n" + "str s6, [%[outptr0], x27]\n" + "fmax v5.4s, v5.4s, v30.4s\n" + "fmax v4.4s, v4.4s, v30.4s\n" + "add %[outptr0], %[outptr0], #4\n" + "fmla v0.4s, v24.4s, v12.4s\n" + "fmin v5.4s, v5.4s, v29.4s\n" + "fmin v4.4s, v4.4s, v29.4s\n" + "fmax v3.4s, v3.4s, v30.4s\n" + "str s5, [x25]\n" + "fmax v2.4s, v2.4s, v30.4s\n" + "str s4, [x25, %[output_col_stride1]]\n" + "fmla v0.4s, v21.4s, v9.4s\n" + "fmin v3.4s, v3.4s, v29.4s\n" + "fmin v2.4s, v2.4s, v29.4s\n" + "fmax v1.4s, v1.4s, v30.4s\n" + "str s3, [x25, x27]\n" + "str s2, [x13]\n" + "fmin v1.4s, v1.4s, v29.4s\n" + "fmax v0.4s, v0.4s, v30.4s\n" + "add x25, x25, #4\n" + "str s1, [x13, %[output_col_stride1]]\n" + "fmin v0.4s, v0.4s, v29.4s\n" + "str s0, [x13, x27]\n" + "add x13, x13, #4\n" + "7:\n" + : [inptr0] "+r" (input), [outptr0] "+r" (output), [wbptr] "+r" (weight_bias_ptr) + : [input_col_stride1] "r" (input_col_stride * sizeof(float)), [input_row_stride] "r" (input_row_stride * sizeof(float)), [output_row_stride] "r" (output_row_stride * sizeof(float)), [output_col_stride1] "r" (output_col_stride * sizeof(float)), [n_channels] "r" ((long) n_channels) + : "cc", "v0", "v1", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v2", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v3", "v30", "v31", "v4", "v5", "v6", "v7", "v8", "v9", "x13", "x14", "x15", "x16", "x17", "x18", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "memory" + ); +} -template <> -const Conv::TileFn Conv::tilefn_right[n_in_pad_right_fns][n_out_pad_right_fns] = { - { - ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 1>, - ConvImpl::template process_tile<true, 0, 0, 0, 0, 0, 2>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 0, 1, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 0, 1, 0, 1>, - ConvImpl::template process_tile<true, 0, 0, 0, 1, 0, 2>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 0, 2, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 0, 2, 0, 1>, - ConvImpl::template process_tile<true, 0, 0, 0, 2, 0, 2>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 0, 3, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 0, 3, 0, 1>, - ConvImpl::template process_tile<true, 0, 0, 0, 3, 0, 2>, - }, - { - ConvImpl::template process_tile<true, 0, 0, 0, 4, 0, 0>, - ConvImpl::template process_tile<true, 0, 0, 0, 4, 0, 1>, - ConvImpl::template process_tile<true, 0, 0, 0, 4, 0, 2>, - }, -}; +#endif // __aarch64__ -template <> -const Conv::TileFn Conv::tilefn_generic = ConvImpl::template process_tile<false>; +template class DepthwiseConvolution<3, 3, 3, 3, 1, 1, float, float, float>; -template class DepthwiseConvolution<3, 3, 3, 3, 1, 1, float, float>; } // namespace depthwise |