From d216f570750b8ccde3754c4aef53fc20a90cb32d Mon Sep 17 00:00:00 2001 From: Freddie Liardet Date: Tue, 3 Aug 2021 15:57:32 +0100 Subject: Update cpu depthwise kernels Resolves: COMPMID-4688 Signed-off-by: Freddie Liardet Change-Id: I9e22f967f5b7ccaebff2fc49f0253f621d62d820 Reviewed-on: https://review.mlplatform.org/c/ml/ComputeLibrary/+/6030 Tested-by: Arm Jenkins Reviewed-by: Pablo Marquez Tello Reviewed-by: Georgios Pinitas Comments-Addressed: Arm Jenkins --- Android.bp | 1 + filelist.json | 1 + .../arm_conv/depthwise/depthwise_depthfirst.hpp | 153 +++-- .../kernels/arm_conv/depthwise/depthwise_fp16.cpp | 60 +- .../kernels/arm_conv/depthwise/depthwise_fp32.cpp | 38 +- .../depthwise_implementation_constraints.hpp | 8 +- .../arm_conv/depthwise/interleaves/8b_mla.cpp | 40 +- .../arm_conv/depthwise/interleaves/8b_mla.hpp | 72 +++ .../arm_conv/depthwise/interleaves/sve_8b_mla.cpp | 40 ++ ...4_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 +- .../generic_indirect.cpp | 4 +- ...4_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 +- .../generic_indirect.cpp | 4 +- ...4_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 +- .../generic_indirect.cpp | 4 +- ...4_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 +- .../generic_indirect.cpp | 4 +- ...4_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 +- .../generic_indirect.cpp | 4 +- ...4_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 + .../generic_indirect.cpp | 4 + ...4_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 + .../generic_indirect.cpp | 4 + ...4_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 + .../generic_indirect.cpp | 4 + ...4_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 + .../generic_indirect.cpp | 4 + ...4_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 4 + .../generic_indirect.cpp | 4 + .../generic.cpp | 2 +- .../generic.cpp | 2 +- .../generic.cpp | 2 +- ...e_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 152 +++-- .../generic_indirect.cpp | 266 ++++---- ...e_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 244 ++++---- .../generic_indirect.cpp | 410 ++++++------- ...e_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 370 ++++++------ .../generic_indirect.cpp | 596 +++++++++--------- ...e_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 200 +++--- .../generic_indirect.cpp | 310 +++++----- ...e_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 296 +++++---- .../generic_indirect.cpp | 458 +++++++------- ...e_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 326 +++++----- .../generic_indirect.cpp | 296 +++++---- ...e_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 422 ++++++------- .../generic_indirect.cpp | 406 ++++++------- ...e_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 192 +++--- .../generic_indirect.cpp | 670 +++++++++------------ ...e_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 390 +++++------- .../generic_indirect.cpp | 386 +++++------- ...e_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp | 71 ++- .../generic_direct.cpp | 44 +- .../generic_indirect.cpp | 510 +++++++--------- ...ve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp | 4 +- .../generic.cpp | 2 +- .../generic.cpp | 2 +- .../generic.cpp | 2 +- src/core/NEON/kernels/arm_gemm/utils.hpp | 2 +- 77 files changed, 4685 insertions(+), 4190 deletions(-) create mode 100644 src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp create mode 100644 src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp diff --git a/Android.bp b/Android.bp index 20bca3666b..92248fd450 100644 --- a/Android.bp +++ b/Android.bp @@ -712,6 +712,7 @@ cc_library_static { srcs: [ "src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_s8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/a64_u8q_3x3_dot.cpp", + "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp", diff --git a/filelist.json b/filelist.json index 8aa869a909..5171f39e12 100644 --- a/filelist.json +++ b/filelist.json @@ -950,6 +950,7 @@ ], "sve": { "all": [ + "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_s8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_u8q_3x3_dot.cpp", "src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp", diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp index 53ad5b5c6b..8d1a8698c3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_depthfirst.hpp @@ -35,19 +35,61 @@ namespace arm_conv { namespace depthwise { -template -class DepthwiseDepthfirst : public DepthwiseCommon +struct IDepthwiseDepthfirstStrategy { - using TInput = typename strategy::input_type; - using TWeight = typename strategy::weight_type; - using TOutput = typename strategy::return_type; - using TAccum = typename strategy::bias_type; + virtual arm_gemm::VLType get_vl_type() const = 0; + + virtual unsigned int get_input_rows() const = 0; + virtual unsigned int get_input_cols() const = 0; + + virtual unsigned int get_output_rows() const = 0; + virtual unsigned int get_output_cols() const = 0; + + virtual unsigned int get_kernel_rows() const = 0; + virtual unsigned int get_kernel_cols() const = 0; + + virtual unsigned int get_stride_rows() const = 0; + virtual unsigned int get_stride_cols() const = 0; + + virtual void indirect_kernel( + const void *const *const input_ptrs, + void *const *const output_ptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const = 0; + + virtual void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const = 0; + + virtual ~IDepthwiseDepthfirstStrategy() {} +}; + +template +class DepthwiseDepthfirst : public DepthwiseCommon +{ + const std::unique_ptr m_strat; + + size_t sizeof_inptr_array(void) const + { + return sizeof(TInput *) * m_strat->get_input_rows() * m_strat->get_input_cols(); + } size_t sizeof_input_buffer(unsigned int n_input_channels) const { - return sizeof(TInput) * n_input_channels; + return sizeof(TInput) * n_input_channels; + } + + size_t sizeof_outptr_array(void) const + { + return sizeof(TInput *) * m_strat->get_output_rows() * m_strat->get_output_cols(); } size_t sizeof_output_buffer(unsigned int n_output_channels) const @@ -56,8 +98,10 @@ class DepthwiseDepthfirst : public DepthwiseCommon(args) + DepthwiseDepthfirst( + IDepthwiseDepthfirstStrategy *const strat, + const DepthwiseArgs &args + ) : DepthwiseCommon(args), m_strat(strat) { } @@ -67,7 +111,7 @@ class DepthwiseDepthfirst : public DepthwiseCommon(strategy::vl_type); + const unsigned int vl = arm_gemm::utils::get_vector_length(m_strat->get_vl_type()); const auto rounded_channels = arm_gemm::roundup(this->m_args.input_channels, vl); return (1 + this->m_args.kernel_rows * this->m_args.kernel_cols) * rounded_channels * sizeof(TWeight); } @@ -81,7 +125,7 @@ class DepthwiseDepthfirst : public DepthwiseCommon(_biases); const TWeight *const weights = static_cast(_weights); - const unsigned int vl = arm_gemm::utils::get_vector_length(strategy::vl_type); + const unsigned int vl = arm_gemm::utils::get_vector_length(m_strat->get_vl_type()); ld_weight_col = (ld_weight_col == 0) ? this->m_args.input_channels : ld_weight_col; ld_weight_row = (ld_weight_row == 0) ? this->m_args.kernel_cols * ld_weight_col : ld_weight_row; @@ -121,10 +165,12 @@ class DepthwiseDepthfirst : public DepthwiseCommonm_args.channel_multiplier; - return n_threads * (sizeof_output_buffer(n_output_channels) + sizeof_input_buffer(n_channels)); + return n_threads * (sizeof_inptr_array() + sizeof_outptr_array() + + sizeof_output_buffer(n_output_channels) + + sizeof_input_buffer(n_channels)); } - using DepthwiseCommon::execute; + using DepthwiseCommon::execute; void execute( const unsigned int batches, const unsigned int input_height, @@ -147,7 +193,6 @@ class DepthwiseDepthfirst : public DepthwiseCommonm_args.cpu_info); #ifdef CYCLE_PROFILING arm_gemm::profiler prof; #endif @@ -177,18 +222,19 @@ class DepthwiseDepthfirst : public DepthwiseCommon(_input); TOutput *const outptr = static_cast(_output); - // Create an array for the input pointers - const TInput * _inptr_array[strategy::input_rows * strategy::input_cols]; - const TInput **const inptr_array = _inptr_array; + // Allocate portions of the working space + uint8_t *working_space = static_cast(_working_space) + get_working_size(thread_id, input_channels); - // Create an array for the output pointers - TOutput * _outptr_array[strategy::output_rows * strategy::output_cols]; - TOutput **const outptr_array = _outptr_array; + const void **const inptr_array = reinterpret_cast(working_space); + working_space += sizeof_inptr_array(); + + void **const outptr_array = reinterpret_cast(working_space); + working_space += sizeof_outptr_array(); - // Allocate portions of the working space - uint8_t *const working_space = static_cast(_working_space) + get_working_size(thread_id, input_channels); TOutput *const output_buffer = reinterpret_cast(working_space); - TInput *const input_buffer = reinterpret_cast(working_space + sizeof_output_buffer(input_channels * this->m_args.channel_multiplier)); + working_space += sizeof_output_buffer(input_channels * this->m_args.channel_multiplier); + + TInput *const input_buffer = reinterpret_cast(working_space); // Initialise the input buffer for (unsigned int c = 0; c < input_channels; c++) @@ -206,11 +252,11 @@ class DepthwiseDepthfirst : public DepthwiseCommon(strategy::output_rows)) + start_out_i += static_cast(m_strat->get_output_rows())) { - const int end_out_i = start_out_i + strategy::output_rows; - const int start_in_i = start_out_i * strategy::stride_rows - padding.top; - const int end_in_i = start_in_i + strategy::input_rows; + const int end_out_i = start_out_i + m_strat->get_output_rows(); + const int start_in_i = start_out_i * m_strat->get_stride_rows() - padding.top; + const int end_in_i = start_in_i + m_strat->get_input_rows(); // Compute top/bottom padding const auto pad_top = static_cast(-std::min(start_in_i, 0)); @@ -221,14 +267,14 @@ class DepthwiseDepthfirst : public DepthwiseCommonget_input_rows() * m_strat->get_input_cols(); index++) { inptr_array[index] = input_buffer; } for (int start_out_j = 0; start_out_j < static_cast(output_width);) { - const int start_in_j = start_out_j * strategy::stride_cols - this->m_args.padding.left; + const int start_in_j = start_out_j * m_strat->get_stride_cols() - this->m_args.padding.left; const int pad_left = -std::min(0, start_in_j); // Compute how many output tiles we can compute with the direct kernel. @@ -236,17 +282,17 @@ class DepthwiseDepthfirst : public DepthwiseCommonget_output_cols(); // Continue to reduce this number as required to avoid reading // padding on the right edge. - int end_in_j = start_in_j + n_direct_tiles * strategy::input_cols; + int end_in_j = start_in_j + n_direct_tiles * m_strat->get_input_cols(); int pad_right = std::max(0, end_in_j - static_cast(input_width)); while (pad_right && n_direct_tiles) { n_direct_tiles--; - end_in_j -= strategy::input_cols; + end_in_j -= m_strat->get_input_cols(); pad_right = std::max(0, end_in_j - static_cast(input_width)); } } @@ -256,21 +302,21 @@ class DepthwiseDepthfirst : public DepthwiseCommonget_output_cols(); #ifdef CYCLE_PROFILING auto p = prof.ScopedProfiler(PROFILE_KERNEL, 0); #endif - strat.direct_kernel(1, n_direct_tiles, - inptr, ld_input_row, ld_input_col, - outptr, ld_output_row, ld_output_col, - parameters, this->m_args.input_channels, - activation_min, activation_max); + m_strat->direct_kernel(1, n_direct_tiles, + inptr, ld_input_row, ld_input_col, + outptr, ld_output_row, ld_output_col, + parameters, this->m_args.input_channels, + &activation_min, &activation_max); continue; } - const int end_out_j = start_out_j + strategy::output_cols; - const int end_in_j = start_in_j + strategy::input_cols; + const int end_out_j = start_out_j + m_strat->get_output_cols(); + const int end_in_j = start_in_j + m_strat->get_input_cols(); const auto pad_right = static_cast(-std::min(static_cast(input_width) - end_in_j, 0)); const unsigned int valid_output_cols = std::min( @@ -280,26 +326,26 @@ class DepthwiseDepthfirst : public DepthwiseCommonget_input_rows() - pad_bottom; i++) { // Can skip over the left padding because we will have either the // same or less than the previous tile. unsigned int j = pad_left; const TInput *colptr = inptr_batch + (start_in_i + i) * ld_input_row + (start_in_j + j) * ld_input_col; - const TInput **ptrs = inptr_array + i * strategy::input_cols + j; - for (; j < strategy::input_cols - pad_right; j++) + const void **ptrs = inptr_array + i * m_strat->get_input_cols() + j; + for (; j < m_strat->get_input_cols() - pad_right; j++) { *(ptrs++) = colptr; colptr += ld_input_col; } - for (; j < strategy::input_cols; j++) + for (; j < m_strat->get_input_cols(); j++) { *(ptrs++) = input_buffer; } } // Construct the output pointer array. - TOutput **outptr_pos = outptr_array; + void **outptr_pos = outptr_array; for (auto i = 0u; i < valid_output_rows; i++) { unsigned int j = 0u; @@ -309,27 +355,28 @@ class DepthwiseDepthfirst : public DepthwiseCommonget_output_cols(); j++) { *(outptr_pos++) = output_buffer; } } - for (auto i = valid_output_rows; i < strategy::output_rows; i++) + for (auto i = valid_output_rows; i < m_strat->get_output_rows(); i++) { - for (auto j = 0u; j < strategy::output_cols; j++) + for (auto j = 0u; j < m_strat->get_output_cols(); j++) { *(outptr_pos++) = output_buffer; } } - start_out_j += strategy::output_cols; + start_out_j += m_strat->get_output_cols(); #ifdef CYCLE_PROFILING // TODO Work number auto p = prof.ScopedProfiler(PROFILE_KERNEL, (unsigned long)(0)); #endif - strat.indirect_kernel(inptr_array, outptr_array, parameters, - this->m_args.input_channels, activation_min, activation_max); + m_strat->indirect_kernel(inptr_array, outptr_array, parameters, + this->m_args.input_channels, + &activation_min, &activation_max); } } } diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp index 6ba7c78e97..934272a0ac 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp16.cpp @@ -32,6 +32,9 @@ #include "depthwise_implementation_constraints.hpp" +// This can only be built if the target/compiler supports FP16 arguments. +#if defined(__ARM_FP16_ARGS) + #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) #include "kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp" @@ -76,11 +79,9 @@ namespace { return args.channel_multiplier > 1 ? 0 : std::numeric_limits::max(); } -#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) } -#if defined(__ARM_FP16_ARGS) - static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = { #if defined(__aarch64__) #if defined(ARM_COMPUTE_ENABLE_SVE) @@ -92,7 +93,8 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { @@ -103,7 +105,8 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { @@ -114,7 +117,8 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { @@ -125,7 +129,8 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { @@ -136,7 +141,8 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, #endif // defined(ARM_COMPUTE_ENABLE_SVE) @@ -145,56 +151,66 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier), + has_no_channel_multiplier, + cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier), + has_no_channel_multiplier, + cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier), + has_no_channel_multiplier, + cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier), + has_no_channel_multiplier, + cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier), + has_no_channel_multiplier, + cpu_has_fp16), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst<__fp16, __fp16, __fp16, __fp16>(strat, args); }, }, { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_generic_output3x3_mla_depthfirst", - constraint(has_no_channel_multiplier), + constraint(has_no_channel_multiplier, cpu_has_fp16), not_preferred, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { return new DepthwiseDepthfirstGeneric(args); @@ -203,7 +219,7 @@ static const DepthwiseImplementation<__fp16, __fp16> depthwise_fp16_methods[] = { DepthwiseMethod::DEPTHFIRST, "a64_fp16_nhwc_generic_with_multiplier_output2x8_mla_depthfirst", - nullptr, + constraint(cpu_has_fp16), not_preferred_if_no_multiplier, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon<__fp16, __fp16, __fp16> * { return new DepthwiseDepthfirstGenericWithMultiplier(args); @@ -223,7 +239,7 @@ const DepthwiseImplementation<__fp16> *depthwise_implementation_list() template UniqueDepthwiseCommon<__fp16> depthwise(const DepthwiseArgs &, const Nothing &); template std::vector get_compatible_kernels<__fp16>(const DepthwiseArgs &, const Nothing &); -#endif // defined(__ARM_FP16_ARGS) - } // namespace depthwise } // namespace arm_conv + +#endif // defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp index ac43df979c..7a26ba4230 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_fp32.cpp @@ -94,7 +94,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -105,7 +106,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -116,7 +118,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -127,7 +130,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -138,7 +142,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { cpu_has_sve), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -153,7 +158,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_3x3_s2_with_multiplier_output3x3_mla_depthfirst", - constraint(is_supported, cpu_has_sve), + constraint(is_supported, + cpu_has_sve), not_preferred_if_no_multiplier, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { return new DepthwiseDepthfirstWithMultiplier(args); @@ -162,7 +168,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { { DepthwiseMethod::DEPTHFIRST, "sve_fp32_nhwc_5x5_s1_with_multiplier_output2x4_mla_depthfirst", - constraint(is_supported, cpu_has_sve), + constraint(is_supported, + cpu_has_sve), not_preferred_if_no_multiplier, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { return new DepthwiseDepthfirstWithMultiplier(args); @@ -185,7 +192,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { has_no_channel_multiplier), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -195,17 +203,19 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { has_no_channel_multiplier), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { DepthwiseMethod::DEPTHFIRST, "a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst", constraint(is_supported, - has_no_channel_multiplier), + has_no_channel_multiplier), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -215,7 +225,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { has_no_channel_multiplier), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { @@ -225,7 +236,8 @@ static const DepthwiseImplementation depthwise_fp32_methods[] = { has_no_channel_multiplier), cycle_estimate, [] (const DepthwiseArgs &args, const Nothing &) -> DepthwiseCommon * { - return new DepthwiseDepthfirst(args); + auto strat = new a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(args.cpu_info); + return new DepthwiseDepthfirst(strat, args); }, }, { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp index 6526d001b3..419872744d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/depthwise_implementation_constraints.hpp @@ -33,7 +33,7 @@ #pragma once #include "arm_gemm.hpp" -#include "depthwise.hpp" +#include "src/core/NEON/kernels/assembly/depthwise.hpp" namespace arm_conv { @@ -97,6 +97,12 @@ bool cpu_has_sve2(const DepthwiseArgs &args, const void *) return args.cpu_info->has_sve2(); } +bool cpu_has_fp16(const DepthwiseArgs &args, const void *) __attribute__ ((unused)); +bool cpu_has_fp16(const DepthwiseArgs &args, const void *) +{ + return args.cpu_info->has_fp16(); +} + bool has_no_channel_multiplier(const DepthwiseArgs &args, const void *) __attribute__ ((unused)); bool has_no_channel_multiplier(const DepthwiseArgs &args, const void *) { diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp index 04b904275c..d59d6b7e35 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.cpp @@ -22,13 +22,7 @@ * SOFTWARE. */ -#include "arm_gemm.hpp" -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" -#include "src/core/NEON/kernels/assembly/depthwise.hpp" -#include -#include - -using namespace arm_gemm; +#include "8b_mla.hpp" size_t generic_get_packed_size( const VLType vec_type, @@ -84,41 +78,9 @@ void generic_pack( } } - -#define ADD_IMPLEMENTATION(ARCH, TYPENAME, TYPE, VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS) \ -struct interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla \ -{ \ - static size_t get_packed_size(const DepthwiseArgs &args); \ - static void pack_parameters( \ - unsigned int n_channels, void *outptr, \ - const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row \ - ); \ -}; \ -\ -size_t interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::get_packed_size(const DepthwiseArgs &args) \ -{ \ - return generic_get_packed_size(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, args.input_channels); \ -} \ -\ -void interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::pack_parameters(unsigned int n_channels, void *outptr, \ - const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row) \ -{ \ - generic_pack(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, n_channels, outptr, weights, ld_weight_col, ld_weight_row); \ -} - - namespace arm_conv { namespace depthwise { -#if defined(ARM_COMPUTE_ENABLE_SVE) - -ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 3, 3) -ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 5, 5) -ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 3, 3) -ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 5, 5) - -#endif // defined(ARM_COMPUTE_ENABLE_SVE) - ADD_IMPLEMENTATION(a64, s8q, int8_t, None, 2, 3, 3) ADD_IMPLEMENTATION(a64, s8q, int8_t, None, 2, 5, 5) ADD_IMPLEMENTATION(a64, u8q, uint8_t, None, 2, 3, 3) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp new file mode 100644 index 0000000000..3176d1dedd --- /dev/null +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/8b_mla.hpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "arm_gemm.hpp" +#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "src/core/NEON/kernels/assembly/depthwise.hpp" +#include +#include + +using namespace arm_gemm; + +size_t generic_get_packed_size( + const VLType vec_type, + const unsigned int acc_depth, + const unsigned int kernel_rows, + const unsigned int kernel_cols, + const unsigned int n_input_channels +); + +void generic_pack( + const VLType vec_type, + const unsigned int acc_depth, + const unsigned int kernel_rows, + const unsigned int kernel_cols, + const unsigned int n_channels, + void *_outptr, + const void *_weights, + size_t ld_weight_col, + size_t ld_weight_row +); + +#define ADD_IMPLEMENTATION(ARCH, TYPENAME, TYPE, VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS) \ +struct interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla \ +{ \ + static size_t get_packed_size(const DepthwiseArgs &args); \ + static void pack_parameters( \ + unsigned int n_channels, void *outptr, \ + const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row \ + ); \ +}; \ +\ +size_t interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::get_packed_size(const DepthwiseArgs &args) \ +{ \ + return generic_get_packed_size(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, args.input_channels); \ +} \ +\ +void interleave_ ## ARCH ## _ ## TYPENAME ## _ ## KERN_ROWS ## x ## KERN_COLS ## _mla::pack_parameters(unsigned int n_channels, void *outptr, \ + const TYPE *weights, size_t ld_weight_col, size_t ld_weight_row) \ +{ \ + generic_pack(VLType::VEC_TYPE, ACC_DEPTH, KERN_ROWS, KERN_COLS, n_channels, outptr, weights, ld_weight_col, ld_weight_row); \ +} diff --git a/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp new file mode 100644 index 0000000000..de74ca5f43 --- /dev/null +++ b/src/core/NEON/kernels/arm_conv/depthwise/interleaves/sve_8b_mla.cpp @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2021 Arm Limited. + * + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +#include "8b_mla.hpp" + +namespace arm_conv { +namespace depthwise { + +#if defined(__ARM_FEATURE_SVE) + +ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 3, 3) +ADD_IMPLEMENTATION(sve, s8q, int8_t, SVE, 2, 5, 5) +ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 3, 3) +ADD_IMPLEMENTATION(sve, u8q, uint8_t, SVE, 2, 5, 5) + +#endif // defined(__ARM_FEATURE_SVE) + +} // namespace depthwise +} // namespace arm_conv diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index bb43d57018..be50d1c535 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst +class a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index 143070cc7e..a85e44360e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -721,4 +721,4 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 35a9714944..a0a44997d1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -696,4 +696,4 @@ void a64_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index 90db8703b5..39fa7f62fd 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst +class a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index c2d67661f6..9b4b3ee50d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1156,4 +1156,4 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index fc988913be..e0abca91a2 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1289,4 +1289,4 @@ void a64_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index df5328724d..1e0d922be4 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst +class a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index c403576cf1..b5bee7ae7c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1734,4 +1734,4 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 2e877285ae..e493104c03 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -2006,4 +2006,4 @@ void a64_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index ca367cc1af..d89ae0cd6d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst +class a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 9b2d77fa53..a5df51c4f9 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -893,4 +893,4 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index 4857077b05..61c58186f5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -896,4 +896,4 @@ void a64_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 53d2a3a8e1..6b5f91fa64 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst +class a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -60,13 +62,60 @@ struct a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index 027fd7f4c7..b08059db0a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1385,4 +1385,4 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 43eac0e6bc..5b086ec1ff 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#if defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) namespace arm_conv { namespace depthwise { @@ -1426,4 +1426,4 @@ void a64_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) +#endif // defined(__aarch64__) && defined(__ARM_FP16_ARGS) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 88f20bb125..a888eb5776 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst +class a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index fae208fbab..73c1e07420 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -522,3 +524,5 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 2f93a68c23..42931fba17 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -509,3 +511,5 @@ void a64_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index 6a882ec52f..01bb06a561 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst +class a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 401528aa59..6d185e7274 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -823,3 +825,5 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 39ec001ae1..aa7d35e3e1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -901,3 +903,5 @@ void a64_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 84bac12429..17084b57d5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst +class a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 616fd0d0e7..6faacf144a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -1227,3 +1229,5 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 51a5679bff..aeaf1049f1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -1393,3 +1395,5 @@ void a64_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 8eb560562b..f23862b699 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if __aarch64__ + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst +class a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 4466ec1974..65e487ddbb 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -610,3 +612,5 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index a5153019e7..4b24862eb2 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include #include +#if __aarch64__ + namespace arm_conv { namespace depthwise { @@ -625,3 +627,5 @@ void a64_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // __aarch64__ diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 314fe766de..e4bfbe6783 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,21 +28,25 @@ #pragma once +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst +class a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::None; @@ -58,11 +62,60 @@ struct a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index b74a3c9b7c..8b030ecc8b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,6 +25,8 @@ #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -986,3 +988,5 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index adc5170ef7..4754a6f6f1 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,6 +25,8 @@ #include #include +#if defined(__aarch64__) + namespace arm_conv { namespace depthwise { @@ -1039,3 +1041,5 @@ void a64_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( } // namespace depthwise } // namespace arm_conv + +#endif // defined(__aarch64__) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 14e113b776..b62ebb1687 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index ccdde41973..8d22836e5b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 699cc6c80c..b42f29ab0d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/a64_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index b8e59306d5..1cfea9dcbd 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst +class sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index a4c1a40100..955a02de57 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,149 +113,141 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "add x27, x10, x22, LSL #1\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x26, x27, x22, LSL #1\n" - "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x14]\n" "add x25, x26, x22, LSL #1\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" "add x24, x11, x11\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" "add x23, x24, x11\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" "mul x19, x17, x20\n" // offset = tile_i * ld_output_row - "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" "madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col - "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" "mul x19, x19, x15\n" // offset *= output_tile_size - "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" "add x28, x28, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) - "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1) - "ld1h { z10.h }, p2/Z, [x10]\n" // Load input point (0, 0) + "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" "add x22, x28, x20, LSL #1\n" - "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3) + "whilelt p2.h, XZR, %x[n_channels]\n" + "ld1h { z9.h }, p2/Z, [x27, x11, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x10]\n" "addvl x14, x14, #16\n" - "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2) + "ld1h { z11.h }, p2/Z, [x10, x23, LSL #1]\n" "cmp x12, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" "addvl x14, x14, #-6\n" - "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1) + "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x11, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z4.h, z9.h\n" - "ld1h { z16.h }, p3/Z, [x14]\n" // Load from weights and bias + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" "whilelt p1.h, x12, %x[n_channels]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" "inch x21\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0) + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x25]\n" "inch x13\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2) + "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" "inch x12\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" + "ld1h { z16.h }, p3/Z, [x14]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z31.h, p3/M, z5.h, z12.h\n" "fmla z30.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1) + "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2) + "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" "addvl x10, x10, #1\n" "fmla z28.h, p3/M, z3.h, z13.h\n" "fmla z31.h, p3/M, z7.h, z13.h\n" "fmla z30.h, p3/M, z6.h, z13.h\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" "addvl x27, x27, #1\n" "fmla z29.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x14, #5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0) - "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z9.h }, p2/Z, [x26]\n" + "ld1h { z1.h }, p3/Z, [x14, #2, MUL VL]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x14, #1, MUL VL]\n" "fmla z28.h, p3/M, z2.h, z12.h\n" - "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x14, #3, MUL VL]\n" "fmla z31.h, p3/M, z8.h, z10.h\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" "addvl x26, x26, #1\n" "fmla z29.h, p3/M, z3.h, z9.h\n" - "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n" // Load input point (2, 1) + "ld1h { z13.h }, p1/Z, [x26, x11, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x14, #4, MUL VL]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2) + "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" "whilelt p2.h, x13, %x[n_channels]\n" "fmla z29.h, p3/M, z7.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x14, #6, MUL VL]\n" "addvl x25, x25, #1\n" "fmla z31.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n" // Load input point (1, 1) + "ld1h { z9.h }, p1/Z, [x27, x11, LSL #1]\n" "cmp x12, %x[n_channels]\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x10]\n" // Load input point (0, 0) + "ld1h { z10.h }, p1/Z, [x10]\n" "fmla z28.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n" // Load input point (0, 3) - "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z11.h }, p1/Z, [x10, x23, LSL #1]\n" + "ld1h { z6.h }, p3/Z, [x14, #7, MUL VL]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" "addvl x14, x14, #16\n" "fmax z31.h, p3/M, z31.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z8.h }, p3/Z, [x14, #-7, MUL VL]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n" // Load input point (1, 2) + "ld1h { z12.h }, p1/Z, [x27, x24, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x14, #-8, MUL VL]\n" "addvl x14, x14, #-6\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0) - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x28]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z16.d\n" - "addvl x28, x28, #1\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "mov z29.d, z16.d\n" + "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" + "addvl x28, x28, #1\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1) - "mov z28.d, z16.d\n" + "st1h { z29.h }, p0, [x22]\n" + "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z4.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" "ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x17, #0x1\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x25]\n" // Load input point (3, 0) + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x25]\n" "add x16, x16, #0x1\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (2, 2) + "ld1h { z10.h }, p2/Z, [x26, x24, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x25, x23, LSL #1]\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "cmp x16, x19\n" @@ -263,10 +255,10 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "fmla z30.h, p3/M, z4.h, z12.h\n" "csel x16, x16, XZR, LT\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" // Load input point (0, 1) + "ld1h { z12.h }, p2/Z, [x10, x11, LSL #1]\n" "csel x17, x17, x21, LT\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (0, 2) + "ld1h { z9.h }, p2/Z, [x10, x24, LSL #1]\n" "cmp x17, x20\n" "fmla z31.h, p3/M, z7.h, z13.h\n" "fmla z30.h, p3/M, z6.h, z13.h\n" @@ -274,24 +266,24 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z29.h, p3/M, z5.h, z10.h\n" "fmla z31.h, p3/M, z2.h, z9.h\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26]\n" // Load input point (2, 0) + "ld1h { z9.h }, p2/Z, [x26]\n" "fmla z28.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" "fmla z31.h, p3/M, z8.h, z10.h\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x26, x23, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x25, x11, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" // Load input point (3, 2) + "ld1h { z12.h }, p2/Z, [x25, x24, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" "fmla z29.h, p3/M, z7.h, z11.h\n" "fmla z31.h, p3/M, z6.h, z9.h\n" @@ -303,14 +295,14 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( "fmla z28.h, p3/M, z7.h, z12.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x28]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x28]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x28, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) + "st1h { z29.h }, p0, [x22]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" // Store output point (1, 1) + "st1h { z28.h }, p0, [x22, x9, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -321,4 +313,4 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index a845e7c0c6..7cca6fbcbf 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -54,22 +54,22 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( const __fp16 max ) : outptrs(outptrs), params(params), min(min), max(max) { - inptrs[0] = input_ptrs[0]; - inptrs[1] = input_ptrs[1]; - inptrs[2] = input_ptrs[2]; - inptrs[3] = input_ptrs[3]; - inptrs[4] = input_ptrs[4]; - inptrs[5] = input_ptrs[5]; - inptrs[6] = input_ptrs[6]; - inptrs[7] = input_ptrs[7]; - inptrs[8] = input_ptrs[8]; - inptrs[9] = input_ptrs[9]; - inptrs[10] = input_ptrs[10]; - inptrs[11] = input_ptrs[11]; - inptrs[12] = input_ptrs[12]; - inptrs[13] = input_ptrs[13]; - inptrs[14] = input_ptrs[14]; - inptrs[15] = input_ptrs[15]; + inptrs[0] = input_ptrs[5]; + inptrs[1] = input_ptrs[0]; + inptrs[2] = input_ptrs[3]; + inptrs[3] = input_ptrs[6]; + inptrs[4] = input_ptrs[9]; + inptrs[5] = input_ptrs[12]; + inptrs[6] = input_ptrs[15]; + inptrs[7] = input_ptrs[1]; + inptrs[8] = input_ptrs[2]; + inptrs[9] = input_ptrs[10]; + inptrs[10] = input_ptrs[4]; + inptrs[11] = input_ptrs[7]; + inptrs[12] = input_ptrs[8]; + inptrs[13] = input_ptrs[11]; + inptrs[14] = input_ptrs[13]; + inptrs[15] = input_ptrs[14]; } }; @@ -78,207 +78,219 @@ void sve_fp16_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x3, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x4, [%x[params_struct], %[offsetof_args_params]]\n" - "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x5, #0x0\n" - "ldp x6, x7, [x19, #0x0]\n" - "cnth x8\n" - "ldp x17, x16, [x19, #0x10]\n" - "sub x15, XZR, x8\n" - "ldp x14, x13, [x19, #0x20]\n" - "whilelt p2.h, XZR, %x[n_channels]\n" - "ldp x12, x11, [x19, #0x30]\n" - "cmp x8, %x[n_channels]\n" - "ldp x10, x9, [x19, #0x40]\n" - "ldp x28, x27, [x19, #0x50]\n" - "ldp x26, x25, [x19, #0x60]\n" - "ldp x24, x23, [x19, #0x70]\n" - "ldp x22, x21, [x3, #0x0]\n" - "ldp x20, x19, [x3, #0x10]\n" + "mov x13, #0x0\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias - "addvl x4, x4, #16\n" - "ld1h { z9.h }, p2/Z, [x13, x5, LSL #1]\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias - "addvl x4, x4, #-6\n" - "ld1h { z10.h }, p2/Z, [x6, x5, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x9, x5, LSL #1]\n" + "cnth x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" + "whilelt p2.h, XZR, %x[n_channels]\n" + "ld1h { z16.h }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldr x22, [x14, #0x20]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z4.h, z9.h\n" - "ld1h { z16.h }, p3/Z, [x4]\n" // Load from weights and bias - "whilelt p1.h, x8, %x[n_channels]\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" - "inch x15\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" + "ldr x21, [x14, #0x28]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" + "ldr x20, [x14, #0x30]\n" + "inch x9\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" + "ldr x19, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" + "ldr x24, [x14, #0x50]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z12.h\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z13.h\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z7.h, z13.h\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z6.h, z13.h\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x9, x8, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z10.h\n" + "ldp x24, x23, [x14, #0x10]\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias + "ldr x22, [x14, #0x20]\n" "fmla z28.h, p3/M, z2.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z8.h, z10.h\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" + "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n" - "inch x5\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" + "inch x13\n" "fmla z29.h, p3/M, z7.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias - "whilelt p2.h, x5, %x[n_channels]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z31.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x13, x8, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n" "fmla z28.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x16, x8, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x6, x8, LSL #1]\n" - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" - "addvl x4, x4, #16\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x12, x8, LSL #1]\n" - "inch x8\n" + "addvl x15, x15, #16\n" "fmax z31.h, p3/M, z31.h, z18.h\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias - "cmp x8, %x[n_channels]\n" + "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n" + "inch x12\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias - "addvl x4, x4, #-6\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" - "mov z30.d, z16.d\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" - "mov z29.d, z16.d\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" - "mov z28.d, z16.d\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z4.h, z9.h\n" - "inch x15\n" - "fmla z30.h, p3/M, z3.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z4.h, z9.h\n" + "ldr x21, [x14, #0x28]\n" + "inch x9\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z3.h, z9.h\n" + "ldr x20, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z1.h, z9.h\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x5, LSL #1]\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z1.h, z9.h\n" + "ldr x19, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x24, [x14, #0x50]\n" "fmla z31.h, p3/M, z5.h, z12.h\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x7, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z6.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x17, x5, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z13.h\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z7.h, z13.h\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z6.h, z13.h\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" "fmla z31.h, p3/M, z2.h, z9.h\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" "fmla z28.h, p3/M, z2.h, z12.h\n" "fmla z31.h, p3/M, z8.h, z10.h\n" "fmla z30.h, p3/M, z7.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x25, x5, LSL #1]\n" - "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z28.h, p3/M, z5.h, z10.h\n" + "fmla z30.h, p3/M, z5.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z11.h\n" "fmla z31.h, p3/M, z6.h, z9.h\n" - "fmla z30.h, p3/M, z8.h, z10.h\n" "fmla z28.h, p3/M, z6.h, z11.h\n" + "fmla z30.h, p3/M, z8.h, z10.h\n" "fmla z29.h, p3/M, z8.h, z12.h\n" + "fmla z28.h, p3/M, z7.h, z12.h\n" "fmax z31.h, p3/M, z31.h, z18.h\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "fmla z28.h, p3/M, z7.h, z12.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index e1f23aae66..af8af1840a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst +class sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 0708f578a8..4126cefa34 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,107 +113,98 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "add x11, x14, x22, LSL #1\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x10, x11, x22, LSL #1\n" - "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x8]\n" "add x9, x10, x22, LSL #1\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" "add x28, x9, x22, LSL #1\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" "add x27, x15, x15\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" "add x26, x27, x15\n" - "mov z27.d, z16.d\n" - "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" "add x25, x26, x15\n" - "mov z26.d, z16.d\n" - "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" "mul x19, x6, x20\n" // offset = tile_i * ld_output_row - "mov z25.d, z16.d\n" - "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" "madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col - "mov z24.d, z16.d\n" + "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" "mul x19, x19, x23\n" // offset *= output_tile_size - "mov z23.d, z16.d\n" - "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x24, x13, x13\n" + "add x12, x12, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x23, x12, x20, LSL #1\n" "add x22, x23, x20, LSL #1\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2) - "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (0, 0) + "ld1h { z9.h }, p2/Z, [x10, x27, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x14]\n" "addvl x8, x8, #16\n" - "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4) + "ld1h { z11.h }, p2/Z, [x14, x25, LSL #1]\n" "cmp x16, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" "addvl x8, x8, #-6\n" - "ld1h { z12.h }, p2/Z, [x28]\n" // Load input point (4, 0) - "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2) + "ld1h { z12.h }, p2/Z, [x28]\n" + "ld1h { z13.h }, p2/Z, [x11, x27, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x16, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" "inch x21\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" "inch x17\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" "inch x16\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" + "ld1h { z16.h }, p3/Z, [x8]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4) + "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" "fmla z31.h, p3/M, z5.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z13.h\n" "fmla z27.h, p3/M, z1.h, z13.h\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" "fmla z30.h, p3/M, z6.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z27.h, p3/M, z3.h, z11.h\n" "fmla z25.h, p3/M, z1.h, z11.h\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x11]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4) + "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z5.h, z10.h\n" "fmla z26.h, p3/M, z4.h, z10.h\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0) + "ld1h { z12.h }, p2/Z, [x9]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z2.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1) + "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" @@ -223,173 +214,164 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3) + "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" "addvl x11, x11, #1\n" "fmla z31.h, p3/M, z4.h, z12.h\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2) + "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" "addvl x14, x14, #1\n" "fmla z24.h, p3/M, z8.h, z13.h\n" - "ld1h { z10.h }, p1/Z, [x14]\n" // Load input point (0, 0) + "ld1h { z10.h }, p1/Z, [x14]\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" "addvl x9, x9, #1\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z12.h }, p2/Z, [x10]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" - "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" "addvl x10, x10, #1\n" "fmla z27.h, p3/M, z8.h, z13.h\n" - "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n" // Load input point (2, 2) + "ld1h { z9.h }, p1/Z, [x10, x27, LSL #1]\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2) + "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" "whilelt p2.h, x17, %x[n_channels]\n" "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" "addvl x28, x28, #1\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" "cmp x16, %x[n_channels]\n" "fmla z25.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x28]\n" // Load input point (4, 0) + "ld1h { z12.h }, p1/Z, [x28]\n" "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z26.h, p3/M, z5.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z23.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n" // Load input point (0, 4) + "ld1h { z11.h }, p1/Z, [x14, x25, LSL #1]\n" "fmla z24.h, p3/M, z7.h, z13.h\n" - "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z25.h, p3/M, z8.h, z13.h\n" "fmax z31.h, p3/M, z31.h, z18.h\n" "fmla z23.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n" // Load input point (1, 2) + "ld1h { z13.h }, p1/Z, [x11, x27, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z6.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x8, x8, #16\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z8.h }, p3/Z, [x8, #-7, MUL VL]\n" "addvl x8, x8, #-6\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0) - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x12]\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z16.d\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2) - "mov z29.d, z16.d\n" - "addvl x12, x12, #1\n" + "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" "fmax z27.h, p3/M, z27.h, z18.h\n" "fmax z26.h, p3/M, z26.h, z18.h\n" + "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0) - "mov z28.d, z16.d\n" + "addvl x12, x12, #1\n" + "fmax z25.h, p3/M, z25.h, z18.h\n" + "st1h { z28.h }, p0, [x23]\n" "fmin z27.h, p3/M, z27.h, z17.h\n" - "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1) - "mov z27.d, z16.d\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2) - "mov z26.d, z16.d\n" - "addvl x23, x23, #1\n" - "fmax z25.h, p3/M, z25.h, z18.h\n" + "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" + "fmin z25.h, p3/M, z25.h, z17.h\n" "fmax z24.h, p3/M, z24.h, z18.h\n" + "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" + "addvl x23, x23, #1\n" "fmax z23.h, p3/M, z23.h, z18.h\n" - "fmin z25.h, p3/M, z25.h, z17.h\n" - "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0) - "mov z25.d, z16.d\n" + "st1h { z25.h }, p0, [x22]\n" "fmin z24.h, p3/M, z24.h, z17.h\n" - "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1) - "mov z24.d, z16.d\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2) - "mov z23.d, z16.d\n" + "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" "ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" "ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x6, #0x1\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x7, x7, #0x1\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" "cmp x7, x19\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" "csel x7, x7, XZR, LT\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" "csel x6, x6, x21, LT\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" "cmp x6, x20\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 3) + "ld1h { z10.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 4) + "ld1h { z12.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" "fmla z31.h, p3/M, z5.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z13.h\n" "fmla z27.h, p3/M, z1.h, z13.h\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z13.h }, p2/Z, [x14, x15, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x14, x26, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" "fmla z30.h, p3/M, z6.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z27.h, p3/M, z3.h, z11.h\n" "fmla z25.h, p3/M, z1.h, z11.h\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11]\n" // Load input point (1, 0) + "ld1h { z11.h }, p2/Z, [x11]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" // Load input point (1, 4) + "ld1h { z13.h }, p2/Z, [x11, x25, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z5.h, z10.h\n" "fmla z26.h, p3/M, z4.h, z10.h\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9]\n" // Load input point (3, 0) + "ld1h { z12.h }, p2/Z, [x9]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z2.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x27, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" // Load input point (1, 1) + "ld1h { z12.h }, p2/Z, [x11, x15, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" @@ -399,36 +381,36 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 3) + "ld1h { z13.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x11, x26, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z12.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (0, 2) + "ld1h { z11.h }, p2/Z, [x14, x27, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z13.h\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x9, x26, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z12.h }, p2/Z, [x10]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x10, x25, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z13.h\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" // Load input point (4, 2) + "ld1h { z13.h }, p2/Z, [x28, x27, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z25.h, p3/M, z0.h, z12.h\n" @@ -442,29 +424,29 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmax z30.h, p3/M, z30.h, z18.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x12]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x12]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x12, x13, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmax z27.h, p3/M, z27.h, z18.h\n" - "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" // Store output point (0, 2) + "st1h { z29.h }, p0, [x12, x24, LSL #1]\n" "fmax z26.h, p3/M, z26.h, z18.h\n" "fmax z25.h, p3/M, z25.h, z18.h\n" "fmax z24.h, p3/M, z24.h, z18.h\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x23]\n" // Store output point (1, 0) + "st1h { z28.h }, p0, [x23]\n" "fmin z27.h, p3/M, z27.h, z17.h\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" // Store output point (1, 1) + "st1h { z27.h }, p0, [x23, x13, LSL #1]\n" "fmin z25.h, p3/M, z25.h, z17.h\n" "fmin z24.h, p3/M, z24.h, z17.h\n" - "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" // Store output point (1, 2) + "st1h { z26.h }, p0, [x23, x24, LSL #1]\n" "fmax z23.h, p3/M, z23.h, z18.h\n" - "st1h { z25.h }, p0, [x22]\n" // Store output point (2, 0) + "st1h { z25.h }, p0, [x22]\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" // Store output point (2, 1) - "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" // Store output point (2, 2) + "st1h { z24.h }, p0, [x22, x13, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x24, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -475,4 +457,4 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 770576c5da..f79a36b2a3 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -87,130 +87,121 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x6, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x7, [%x[params_struct], %[offsetof_args_params]]\n" - "add x8, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x17, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x16\n" - "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias - "sub x15, XZR, x16\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias + "cnth x12\n" + "ld1h { z16.h }, p3/Z, [x15]\n" + "sub x11, XZR, x12\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias - "cmp x16, %x[n_channels]\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias - "mov z27.d, z16.d\n" - "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias - "mov z26.d, z16.d\n" - "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias - "mov z25.d, z16.d\n" - "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #16\n" - "mov z24.d, z16.d\n" - "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias - "mov z23.d, z16.d\n" - "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #-6\n" - "ldp x14, x13, [x8, #0x0]\n" - "ldp x12, x11, [x8, #0x10]\n" - "ldr x10, [x8, #0x20]\n" - "ld1h { z9.h }, p2/Z, [x14, x17, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x13, x17, LSL #1]\n" - "ld1h { z11.h }, p2/Z, [x12, x17, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x10, x17, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ldr x26, [x14, #0x20]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x9, [x8, #0x28]\n" - "whilelt p1.h, x16, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x28, [x8, #0x30]\n" - "inch x15\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x27, [x8, #0x38]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" + "inch x11\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" - "ldr x26, [x8, #0x40]\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" - "ldr x22, [x8, #0x48]\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "ldr x21, [x8, #0x50]\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" - "ldr x20, [x8, #0x58]\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" - "ldr x19, [x8, #0x60]\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" - "ldr x25, [x8, #0x68]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" - "ldr x24, [x8, #0x70]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x23, [x8, #0x78]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.h, p3/M, z3.h, z13.h\n" - "ldr x14, [x8, #0x80]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.h, p3/M, z2.h, z13.h\n" - "ldr x13, [x8, #0x88]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.h, p3/M, z1.h, z13.h\n" - "ldr x12, [x8, #0x90]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" - "ldr x11, [x8, #0x98]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.h, p3/M, z6.h, z11.h\n" - "ldr x10, [x8, #0xa0]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.h, p3/M, z4.h, z11.h\n" - "ldr x9, [x8, #0xa8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.h, p3/M, z3.h, z11.h\n" - "ldr x28, [x8, #0xb0]\n" + "ld1h { z16.h }, p3/Z, [x15]\n" "fmla z25.h, p3/M, z1.h, z11.h\n" - "ldr x27, [x8, #0xb8]\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" - "ldr x26, [x8, #0xc0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ldr x22, [x6, #0x0]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.h, p3/M, z5.h, z10.h\n" - "ldr x21, [x6, #0x8]\n" "fmla z26.h, p3/M, z4.h, z10.h\n" - "ldr x20, [x6, #0x10]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" - "ldr x19, [x6, #0x18]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.h, p3/M, z2.h, z10.h\n" - "ld1h { z16.h }, p3/Z, [x7]\n" // Load from weights and bias "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z8.h, z10.h\n" @@ -219,191 +210,182 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ldp x14, x13, [x8, #0x0]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" - "ld1h { z9.h }, p1/Z, [x14, x16, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n" - "fmla z30.h, p3/M, z5.h, z11.h\n" - "ld1h { z10.h }, p1/Z, [x13, x16, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ldp x12, x11, [x8, #0x10]\n" + "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z13.h\n" - "ldr x10, [x8, #0x20]\n" + "ldr x26, [x14, #0x20]\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" - "ld1h { z1.h }, p3/Z, [x7, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z13.h\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n" - "inch x17\n" + "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n" + "inch x13\n" "fmla z31.h, p3/M, z6.h, z12.h\n" - "ld1h { z4.h }, p3/Z, [x7, #5, MUL VL]\n" // Load from weights and bias - "whilelt p2.h, x17, %x[n_channels]\n" + "ldp x10, x9, [x14, #0x0]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x7, #4, MUL VL]\n" // Load from weights and bias + "ldp x28, x27, [x14, #0x10]\n" "fmla z25.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x11, x16, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z29.h, p3/M, z8.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x7, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n" "fmla z26.h, p3/M, z5.h, z11.h\n" - "ld1h { z5.h }, p3/Z, [x7, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n" "fmla z23.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x12, x16, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n" "fmla z25.h, p3/M, z8.h, z13.h\n" - "ld1h { z2.h }, p3/Z, [x7, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n" "fmla z24.h, p3/M, z7.h, z13.h\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" "fmax z31.h, p3/M, z31.h, z18.h\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z23.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x10, x16, LSL #1]\n" - "inch x16\n" + "ld1h { z13.h }, p1/Z, [x26, x12, LSL #1]\n" + "inch x12\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z6.h }, p3/Z, [x7, #7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #16\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "ld1h { z7.h }, p3/Z, [x7, #-8, MUL VL]\n" // Load from weights and bias - "cmp x16, %x[n_channels]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" - "ld1h { z8.h }, p3/Z, [x7, #-7, MUL VL]\n" // Load from weights and bias - "addvl x7, x7, #-6\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" - "mov z31.d, z16.d\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "fmax z27.h, p3/M, z27.h, z18.h\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "ldr x22, [x6, #0x20]\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" - "mov z30.d, z16.d\n" - "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" - "mov z29.d, z16.d\n" - "ldr x21, [x6, #0x28]\n" - "fmax z27.h, p3/M, z27.h, z18.h\n" - "ldr x20, [x6, #0x30]\n" + "ldr x22, [x16, #0x20]\n" "fmax z26.h, p3/M, z26.h, z18.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" - "mov z28.d, z16.d\n" - "ldr x19, [x6, #0x38]\n" - "fmax z25.h, p3/M, z25.h, z18.h\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" + "fmin z28.h, p3/M, z28.h, z17.h\n" "fmin z27.h, p3/M, z27.h, z17.h\n" - "st1h { z27.h }, p0, [x22, x15, LSL #1]\n" - "mov z27.d, z16.d\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "ldr x22, [x6, #0x40]\n" - "fmin z25.h, p3/M, z25.h, z17.h\n" - "st1h { z26.h }, p0, [x21, x15, LSL #1]\n" - "mov z26.d, z16.d\n" + "ldr x21, [x16, #0x28]\n" + "fmax z25.h, p3/M, z25.h, z18.h\n" + "ldr x20, [x16, #0x30]\n" "fmax z24.h, p3/M, z24.h, z18.h\n" - "st1h { z25.h }, p0, [x20, x15, LSL #1]\n" - "mov z25.d, z16.d\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmax z23.h, p3/M, z23.h, z18.h\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" + "fmin z25.h, p3/M, z25.h, z17.h\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.h, p3/M, z24.h, z17.h\n" - "st1h { z24.h }, p0, [x19, x15, LSL #1]\n" - "mov z24.d, z16.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z23.h }, p0, [x22, x15, LSL #1]\n" - "mov z23.d, z16.d\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x9, [x8, #0x28]\n" - "inch x15\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x28, [x8, #0x30]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" + "inch x11\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x27, [x8, #0x38]\n" - "fmla z28.h, p3/M, z5.h, z9.h\n" - "ldr x26, [x8, #0x40]\n" - "fmla z27.h, p3/M, z4.h, z9.h\n" - "ldr x22, [x8, #0x48]\n" - "fmla z26.h, p3/M, z3.h, z9.h\n" - "ldr x21, [x8, #0x50]\n" - "fmla z25.h, p3/M, z2.h, z9.h\n" - "ldr x20, [x8, #0x58]\n" - "fmla z24.h, p3/M, z1.h, z9.h\n" - "ldr x19, [x8, #0x60]\n" - "fmla z23.h, p3/M, z0.h, z9.h\n" - "ldr x25, [x8, #0x68]\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z5.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.h, p3/M, z4.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.h, p3/M, z3.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.h, p3/M, z2.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.h, p3/M, z1.h, z9.h\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.h, p3/M, z0.h, z9.h\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x22, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z25.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z13.h\n" - "ldr x24, [x8, #0x70]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x23, [x8, #0x78]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.h, p3/M, z3.h, z13.h\n" - "ldr x14, [x8, #0x80]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.h, p3/M, z2.h, z13.h\n" - "ldr x13, [x8, #0x88]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.h, p3/M, z1.h, z13.h\n" - "ldr x12, [x8, #0x90]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z8.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z11.h\n" - "ldr x11, [x8, #0x98]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.h, p3/M, z6.h, z11.h\n" - "ldr x10, [x8, #0xa0]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.h, p3/M, z4.h, z11.h\n" - "ldr x9, [x8, #0xa8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.h, p3/M, z3.h, z11.h\n" - "ldr x28, [x8, #0xb0]\n" "fmla z25.h, p3/M, z1.h, z11.h\n" - "ldr x27, [x8, #0xb8]\n" "fmla z24.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" - "ldr x26, [x8, #0xc0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x20, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ldr x22, [x6, #0x0]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.h, p3/M, z5.h, z10.h\n" - "ldr x21, [x6, #0x8]\n" "fmla z26.h, p3/M, z4.h, z10.h\n" - "ldr x20, [x6, #0x10]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z10.h\n" - "ldr x19, [x6, #0x18]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.h, p3/M, z2.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z30.h, p3/M, z8.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x17, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z13.h\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z25.h, p3/M, z3.h, z12.h\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z10.h\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.h, p3/M, z6.h, z10.h\n" "fmla z25.h, p3/M, z5.h, z10.h\n" "fmla z28.h, p3/M, z8.h, z10.h\n" @@ -412,36 +394,36 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "fmla z26.h, p3/M, z8.h, z11.h\n" "fmla z25.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z6.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z23.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z27.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" "fmla z30.h, p3/M, z5.h, z11.h\n" "fmla z26.h, p3/M, z1.h, z11.h\n" "fmla z27.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z13.h\n" "fmla z23.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z27.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" "fmla z24.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x17, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z11.h\n" "fmla z30.h, p3/M, z1.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x17, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z13.h\n" "fmla z26.h, p3/M, z7.h, z13.h\n" "fmla z24.h, p3/M, z5.h, z13.h\n" "fmla z23.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x17, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z25.h, p3/M, z0.h, z12.h\n" @@ -455,41 +437,41 @@ void sve_fp16_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "fmax z30.h, p3/M, z30.h, z18.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "ldr x22, [x6, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "st1h { z30.h }, p0, [x21, x15, LSL #1]\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" "fmax z27.h, p3/M, z27.h, z18.h\n" "fmax z26.h, p3/M, z26.h, z18.h\n" - "st1h { z29.h }, p0, [x20, x15, LSL #1]\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "ldr x21, [x6, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z25.h, p3/M, z25.h, z18.h\n" - "ldr x20, [x6, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmax z24.h, p3/M, z24.h, z18.h\n" - "st1h { z28.h }, p0, [x19, x15, LSL #1]\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z17.h\n" "fmin z26.h, p3/M, z26.h, z17.h\n" - "ldr x19, [x6, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z25.h, p3/M, z25.h, z17.h\n" - "st1h { z27.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" "fmin z24.h, p3/M, z24.h, z17.h\n" "fmax z23.h, p3/M, z23.h, z18.h\n" - "st1h { z26.h }, p0, [x21, x15, LSL #1]\n" - "st1h { z25.h }, p0, [x20, x15, LSL #1]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" "fmin z23.h, p3/M, z23.h, z17.h\n" - "st1h { z24.h }, p0, [x19, x15, LSL #1]\n" - "ldr x22, [x6, #0x40]\n" - "st1h { z23.h }, p0, [x22, x15, LSL #1]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" + "ldr x22, [x16, #0x40]\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index 7d035f0571..60234c8a86 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst +class sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 93e1908df7..3cfac06449 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,92 +113,75 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "add x15, x8, x22, LSL #1\n" "ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x14, x15, x22, LSL #1\n" - "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias - "mov z31.d, z13.d\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z13.h }, p3/Z, [x4]\n" "add x13, x14, x22, LSL #1\n" - "mov z30.d, z13.d\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" "add x12, x13, x22, LSL #1\n" - "mov z29.d, z13.d\n" - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" "add x11, x12, x22, LSL #1\n" - "mov z28.d, z13.d\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" "add x10, x7, x7\n" - "mov z27.d, z13.d\n" - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" "add x9, x10, x7\n" - "mov z26.d, z13.d\n" - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" "add x28, x9, x7\n" - "mov z25.d, z13.d\n" - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" "add x27, x28, x7\n" - "mov z24.d, z13.d\n" + "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" "mul x19, x2, x20\n" // offset = tile_i * ld_output_row - "mov z23.d, z13.d\n" + "add x26, x17, x17\n" "madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col - "mov z22.d, z13.d\n" "mul x19, x19, x23\n" // offset *= output_tile_size - "mov z21.d, z13.d\n" "add x16, x16, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) - "mov z20.d, z13.d\n" - "add x26, x16, x20, LSL #1\n" - "mov z19.d, z13.d\n" - "add x25, x26, x20, LSL #1\n" - "mov z18.d, z13.d\n" + "add x25, x16, x20, LSL #1\n" "add x24, x25, x20, LSL #1\n" - "mov z17.d, z13.d\n" - "add x23, x17, x17\n" - "mov z16.d, z13.d\n" - "add x22, x23, x17\n" + "add x23, x24, x20, LSL #1\n" + "add x22, x26, x17\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2) - "ld1h { z10.h }, p2/Z, [x8]\n" // Load input point (0, 0) + "ld1h { z9.h }, p2/Z, [x14, x10, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x8]\n" "addvl x4, x4, #16\n" - "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5) + "ld1h { z11.h }, p2/Z, [x8, x27, LSL #1]\n" "cmp x6, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" "addvl x4, x4, #-6\n" - "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p2/Z, [x14, x9, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ld1h { z13.h }, p3/Z, [x4]\n" // Load from weights and bias + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x6, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" "inch x21\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" "inch x5\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" "inch x6\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2) + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0) - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5) + "ld1h { z10.h }, p2/Z, [x11]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" "fmla z29.h, p3/M, z7.h, z12.h\n" "fmla z26.h, p3/M, z5.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" "fmla z22.h, p3/M, z2.h, z12.h\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1) - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3) - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4) + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" "fmla z26.h, p3/M, z7.h, z9.h\n" "fmla z25.h, p3/M, z6.h, z9.h\n" @@ -206,15 +189,16 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.h, p3/M, z4.h, z9.h\n" "fmla z21.h, p3/M, z3.h, z9.h\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0) + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" + "ld1h { z13.h }, p3/Z, [x4]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5) + "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0) + "ld1h { z11.h }, p2/Z, [x12]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" "fmla z25.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z6.h, z10.h\n" @@ -224,261 +208,245 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2) + "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5) + "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1) + "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1) + "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4) + "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4) + "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2) + "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" "addvl x8, x8, #1\n" "fmla z31.h, p3/M, z2.h, z10.h\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0) + "ld1h { z10.h }, p2/Z, [x14]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5) + "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" "addvl x14, x14, #1\n" "fmla z31.h, p3/M, z6.h, z10.h\n" - "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n" // Load input point (2, 2) + "ld1h { z9.h }, p1/Z, [x14, x10, LSL #1]\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0) + "ld1h { z10.h }, p2/Z, [x13]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2) + "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5) + "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" "addvl x13, x13, #1\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2) + "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3) + "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3) + "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" "addvl x11, x11, #1\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1) + "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4) + "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" "addvl x15, x15, #1\n" "fmla z18.h, p3/M, z8.h, z12.h\n" "fmla z31.h, p3/M, z4.h, z10.h\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1) + "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4) + "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" "whilelt p2.h, x5, %x[n_channels]\n" "fmla z29.h, p3/M, z5.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x4, #1, MUL VL]\n" "addvl x12, x12, #1\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "cmp x6, %x[n_channels]\n" "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x4, #3, MUL VL]\n" "fmla z24.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n" // Load input point (0, 5) + "ld1h { z11.h }, p1/Z, [x8, x27, LSL #1]\n" "fmla z23.h, p3/M, z7.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x4, #2, MUL VL]\n" "fmla z22.h, p3/M, z6.h, z12.h\n" - "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z6.h }, p3/Z, [x4, #7, MUL VL]\n" "fmla z19.h, p3/M, z4.h, z12.h\n" "fmla z18.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p1/Z, [x14, x9, LSL #1]\n" "fmla z21.h, p3/M, z8.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x4, #4, MUL VL]\n" "fmla z20.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z5.h, z10.h\n" - "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x4, #6, MUL VL]\n" "fmla z16.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x8]\n" // Load input point (0, 0) + "ld1h { z10.h }, p1/Z, [x8]\n" "fmax z31.h, p3/M, z31.h, z15.h\n" - "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x4, #5, MUL VL]\n" "addvl x4, x4, #16\n" "fmax z30.h, p3/M, z30.h, z15.h\n" - "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x4, #-8, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z15.h\n" - "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z8.h }, p3/Z, [x4, #-7, MUL VL]\n" "addvl x4, x4, #-6\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0) - "mov z31.d, z13.d\n" + "st1h { z31.h }, p0, [x16]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" - "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z13.d\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2) - "mov z29.d, z13.d\n" + "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmax z27.h, p3/M, z27.h, z15.h\n" + "st1h { z29.h }, p0, [x16, x26, LSL #1]\n" "fmax z26.h, p3/M, z26.h, z15.h\n" "fmax z25.h, p3/M, z25.h, z15.h\n" + "fmax z24.h, p3/M, z24.h, z15.h\n" "fmin z28.h, p3/M, z28.h, z14.h\n" - "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3) - "mov z28.d, z13.d\n" - "addvl x16, x16, #1\n" + "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" - "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0) - "mov z27.d, z13.d\n" + "addvl x16, x16, #1\n" "fmin z26.h, p3/M, z26.h, z14.h\n" - "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1) - "mov z26.d, z13.d\n" + "st1h { z27.h }, p0, [x25]\n" "fmin z25.h, p3/M, z25.h, z14.h\n" - "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2) - "mov z25.d, z13.d\n" - "fmax z24.h, p3/M, z24.h, z15.h\n" + "fmin z24.h, p3/M, z24.h, z14.h\n" + "st1h { z26.h }, p0, [x25, x17, LSL #1]\n" "fmax z23.h, p3/M, z23.h, z15.h\n" + "st1h { z25.h }, p0, [x25, x26, LSL #1]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" "fmax z21.h, p3/M, z21.h, z15.h\n" - "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3) - "mov z24.d, z13.d\n" - "addvl x26, x26, #1\n" + "st1h { z24.h }, p0, [x25, x22, LSL #1]\n" + "addvl x25, x25, #1\n" "fmin z23.h, p3/M, z23.h, z14.h\n" - "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0) - "mov z23.d, z13.d\n" + "st1h { z23.h }, p0, [x24]\n" "fmin z22.h, p3/M, z22.h, z14.h\n" - "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1) - "mov z22.d, z13.d\n" "fmin z21.h, p3/M, z21.h, z14.h\n" - "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2) - "mov z21.d, z13.d\n" + "st1h { z22.h }, p0, [x24, x17, LSL #1]\n" "fmax z20.h, p3/M, z20.h, z15.h\n" "fmax z19.h, p3/M, z19.h, z15.h\n" + "st1h { z21.h }, p0, [x24, x26, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" "fmax z17.h, p3/M, z17.h, z15.h\n" + "fmax z16.h, p3/M, z16.h, z15.h\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3) - "mov z20.d, z13.d\n" - "addvl x25, x25, #1\n" + "st1h { z20.h }, p0, [x24, x22, LSL #1]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0) - "mov z19.d, z13.d\n" + "addvl x24, x24, #1\n" "fmin z18.h, p3/M, z18.h, z14.h\n" - "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1) - "mov z18.d, z13.d\n" + "st1h { z19.h }, p0, [x23]\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2) - "mov z17.d, z13.d\n" - "fmax z16.h, p3/M, z16.h, z15.h\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3) - "mov z16.d, z13.d\n" - "addvl x24, x24, #1\n" + "st1h { z18.h }, p0, [x23, x17, LSL #1]\n" + "st1h { z17.h }, p0, [x23, x26, LSL #1]\n" + "st1h { z16.h }, p0, [x23, x22, LSL #1]\n" + "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" "ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" "ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x2, #0x1\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x3, x3, #0x1\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" "cmp x3, x19\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" "csel x3, x3, XZR, LT\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" "csel x2, x2, x21, LT\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" // Load input point (3, 2) + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x13, x10, LSL #1]\n" "cmp x2, x20\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11]\n" // Load input point (5, 0) - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" // Load input point (5, 5) + "ld1h { z10.h }, p2/Z, [x11]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x11, x27, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" "fmla z29.h, p3/M, z7.h, z12.h\n" "fmla z26.h, p3/M, z5.h, z12.h\n" "fmla z28.h, p3/M, z6.h, z12.h\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" "fmla z22.h, p3/M, z2.h, z12.h\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" // Load input point (0, 1) - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" // Load input point (3, 3) - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" // Load input point (0, 4) + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x8, x7, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x13, x9, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x8, x28, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" "fmla z26.h, p3/M, z7.h, z9.h\n" "fmla z25.h, p3/M, z6.h, z9.h\n" @@ -486,15 +454,15 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.h, p3/M, z4.h, z9.h\n" "fmla z21.h, p3/M, z3.h, z9.h\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x15]\n" // Load input point (1, 0) + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" // Load input point (1, 5) + "ld1h { z12.h }, p2/Z, [x15, x27, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12]\n" // Load input point (4, 0) + "ld1h { z11.h }, p2/Z, [x12]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" "fmla z25.h, p3/M, z7.h, z10.h\n" "fmla z24.h, p3/M, z6.h, z10.h\n" @@ -504,117 +472,117 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" // Load input point (1, 2) + "ld1h { z10.h }, p2/Z, [x15, x10, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" // Load input point (1, 3) + "ld1h { z12.h }, p2/Z, [x15, x9, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" // Load input point (4, 5) + "ld1h { z11.h }, p2/Z, [x12, x27, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" // Load input point (2, 1) + "ld1h { z10.h }, p2/Z, [x14, x7, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" // Load input point (5, 1) + "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" // Load input point (2, 4) + "ld1h { z12.h }, p2/Z, [x14, x28, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" // Load input point (5, 4) + "ld1h { z11.h }, p2/Z, [x11, x28, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" // Load input point (0, 2) + "ld1h { z10.h }, p2/Z, [x8, x10, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x13, x7, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p2/Z, [x8, x9, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14]\n" // Load input point (2, 0) + "ld1h { z10.h }, p2/Z, [x14]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" // Load input point (3, 4) + "ld1h { z11.h }, p2/Z, [x13, x28, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" // Load input point (2, 5) + "ld1h { z12.h }, p2/Z, [x14, x27, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x13]\n" // Load input point (3, 0) + "ld1h { z10.h }, p2/Z, [x13]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" // Load input point (4, 2) + "ld1h { z11.h }, p2/Z, [x12, x10, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" // Load input point (3, 5) + "ld1h { z12.h }, p2/Z, [x13, x27, LSL #1]\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" // Load input point (5, 2) + "ld1h { z10.h }, p2/Z, [x11, x10, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" // Load input point (4, 3) + "ld1h { z11.h }, p2/Z, [x12, x9, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" // Load input point (5, 3) + "ld1h { z12.h }, p2/Z, [x11, x9, LSL #1]\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" // Load input point (1, 1) + "ld1h { z10.h }, p2/Z, [x15, x7, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" // Load input point (1, 4) + "ld1h { z11.h }, p2/Z, [x15, x28, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z10.h\n" "fmla z18.h, p3/M, z8.h, z12.h\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" // Load input point (4, 1) + "ld1h { z12.h }, p2/Z, [x12, x7, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" // Load input point (4, 4) + "ld1h { z10.h }, p2/Z, [x12, x28, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z25.h, p3/M, z2.h, z11.h\n" @@ -632,49 +600,49 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmax z29.h, p3/M, z29.h, z15.h\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x16]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x16]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x16, x17, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z14.h\n" "fmax z27.h, p3/M, z27.h, z15.h\n" - "st1h { z29.h }, p0, [x16, x23, LSL #1]\n" // Store output point (0, 2) + "st1h { z29.h }, p0, [x16, x26, LSL #1]\n" "fmax z26.h, p3/M, z26.h, z15.h\n" - "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" // Store output point (0, 3) + "st1h { z28.h }, p0, [x16, x22, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" "fmax z25.h, p3/M, z25.h, z15.h\n" - "st1h { z27.h }, p0, [x26]\n" // Store output point (1, 0) + "st1h { z27.h }, p0, [x25]\n" "fmin z26.h, p3/M, z26.h, z14.h\n" "fmin z25.h, p3/M, z25.h, z14.h\n" - "st1h { z26.h }, p0, [x26, x17, LSL #1]\n" // Store output point (1, 1) + "st1h { z26.h }, p0, [x25, x17, LSL #1]\n" "fmax z24.h, p3/M, z24.h, z15.h\n" "fmax z23.h, p3/M, z23.h, z15.h\n" - "st1h { z25.h }, p0, [x26, x23, LSL #1]\n" // Store output point (1, 2) + "st1h { z25.h }, p0, [x25, x26, LSL #1]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" "fmax z21.h, p3/M, z21.h, z15.h\n" "fmax z20.h, p3/M, z20.h, z15.h\n" "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z24.h }, p0, [x26, x22, LSL #1]\n" // Store output point (1, 3) + "st1h { z24.h }, p0, [x25, x22, LSL #1]\n" "fmin z23.h, p3/M, z23.h, z14.h\n" "fmin z22.h, p3/M, z22.h, z14.h\n" - "st1h { z23.h }, p0, [x25]\n" // Store output point (2, 0) + "st1h { z23.h }, p0, [x24]\n" "fmin z21.h, p3/M, z21.h, z14.h\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "st1h { z22.h }, p0, [x25, x17, LSL #1]\n" // Store output point (2, 1) + "st1h { z22.h }, p0, [x24, x17, LSL #1]\n" "fmax z19.h, p3/M, z19.h, z15.h\n" - "st1h { z21.h }, p0, [x25, x23, LSL #1]\n" // Store output point (2, 2) + "st1h { z21.h }, p0, [x24, x26, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" "fmax z17.h, p3/M, z17.h, z15.h\n" - "st1h { z20.h }, p0, [x25, x22, LSL #1]\n" // Store output point (2, 3) + "st1h { z20.h }, p0, [x24, x22, LSL #1]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "st1h { z19.h }, p0, [x24]\n" // Store output point (3, 0) + "st1h { z19.h }, p0, [x23]\n" "fmin z18.h, p3/M, z18.h, z14.h\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "st1h { z18.h }, p0, [x24, x17, LSL #1]\n" // Store output point (3, 1) + "st1h { z18.h }, p0, [x23, x17, LSL #1]\n" "fmax z16.h, p3/M, z16.h, z15.h\n" - "st1h { z17.h }, p0, [x24, x23, LSL #1]\n" // Store output point (3, 2) + "st1h { z17.h }, p0, [x23, x26, LSL #1]\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z16.h }, p0, [x24, x22, LSL #1]\n" // Store output point (3, 3) + "st1h { z16.h }, p0, [x23, x22, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -685,4 +653,4 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 8eaf0a46d6..66f6c3bb7a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -98,570 +98,538 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n" - "add x4, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z15.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x5, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z14.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x6\n" - "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias - "mov z31.d, z13.d\n" - "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias - "sub x7, XZR, x6\n" - "mov z30.d, z13.d\n" - "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias + "cnth x12\n" + "ld1h { z13.h }, p3/Z, [x15]\n" + "sub x11, XZR, x12\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "mov z29.d, z13.d\n" - "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias - "cmp x6, %x[n_channels]\n" - "mov z28.d, z13.d\n" - "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias - "mov z27.d, z13.d\n" - "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias - "mov z26.d, z13.d\n" - "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias - "mov z25.d, z13.d\n" - "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #16\n" - "mov z24.d, z13.d\n" - "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias - "mov z23.d, z13.d\n" - "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #-6\n" - "mov z22.d, z13.d\n" - "ldp x8, x17, [x4, #0x0]\n" - "mov z21.d, z13.d\n" - "ldp x16, x15, [x4, #0x10]\n" - "mov z20.d, z13.d\n" - "ld1h { z9.h }, p2/Z, [x8, x5, LSL #1]\n" - "mov z19.d, z13.d\n" - "mov z18.d, z13.d\n" - "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n" - "mov z17.d, z13.d\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" - "mov z16.d, z13.d\n" - "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x10, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0x20]\n" - "whilelt p1.h, x6, %x[n_channels]\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0x28]\n" - "inch x7\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0x30]\n" + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x20]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" + "inch x11\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0x38]\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0x40]\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0x48]\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" - "ldr x28, [x4, #0x50]\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" - "ldr x27, [x4, #0x58]\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" - "ldr x26, [x4, #0x60]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.h, p3/M, z7.h, z12.h\n" - "ldr x25, [x4, #0x68]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.h, p3/M, z5.h, z12.h\n" - "ldr x24, [x4, #0x70]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ldr x23, [x4, #0x78]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "ldr x8, [x4, #0x80]\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "ldr x17, [x4, #0x88]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.h, p3/M, z2.h, z12.h\n" - "ldr x16, [x4, #0x90]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "ldr x15, [x4, #0x98]\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n" - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0xa0]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0xa8]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0xb0]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0xb8]\n" "fmla z22.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0xc0]\n" "fmla z21.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0xc8]\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "ldr x22, [x2, #0x0]\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x2, #0x8]\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n" + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" - "ldr x28, [x4, #0xd0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ldr x27, [x4, #0xd8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" - "ldr x26, [x4, #0xe0]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.h, p3/M, z7.h, z10.h\n" - "ldr x20, [x2, #0x10]\n" + "ld1h { z13.h }, p3/Z, [x15]\n" "fmla z24.h, p3/M, z6.h, z10.h\n" - "ldr x19, [x2, #0x18]\n" "fmla z22.h, p3/M, z5.h, z10.h\n" - "ld1h { z13.h }, p3/Z, [x3]\n" // Load from weights and bias "fmla z21.h, p3/M, z4.h, z10.h\n" "fmla z20.h, p3/M, z3.h, z10.h\n" "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ldr x25, [x4, #0xe8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" - "ldr x23, [x4, #0xf8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" - "ldr x24, [x4, #0xf0]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" - "ldr x17, [x4, #0x108]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ldr x8, [x4, #0x100]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" - "ldr x15, [x4, #0x118]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" - "ldr x16, [x4, #0x110]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z10.h\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z10.h\n" - "ldp x8, x17, [x4, #0x0]\n" + "ldp x10, x9, [x14, #0x0]\n" "fmla z18.h, p3/M, z8.h, z12.h\n" - "ld1h { z9.h }, p1/Z, [x8, x6, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x10, x12, LSL #1]\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n" - "inch x5\n" + "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n" + "inch x13\n" "fmla z29.h, p3/M, z5.h, z11.h\n" - "ldp x16, x15, [x4, #0x10]\n" - "whilelt p2.h, x5, %x[n_channels]\n" + "ldp x28, x27, [x14, #0x10]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z28.h, p3/M, z4.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x3, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z25.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x3, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z24.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x16, x6, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x28, x12, LSL #1]\n" "fmla z23.h, p3/M, z7.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x3, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z22.h, p3/M, z6.h, z12.h\n" - "ld1h { z6.h }, p3/Z, [x3, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" "fmla z19.h, p3/M, z4.h, z12.h\n" "fmla z18.h, p3/M, z3.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x15, x6, LSL #1]\n" + "ld1h { z12.h }, p1/Z, [x27, x12, LSL #1]\n" "fmla z21.h, p3/M, z8.h, z10.h\n" - "ld1h { z3.h }, p3/Z, [x3, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z20.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z5.h, z10.h\n" - "ld1h { z5.h }, p3/Z, [x3, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z16.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x17, x6, LSL #1]\n" - "inch x6\n" + "ld1h { z10.h }, p1/Z, [x9, x12, LSL #1]\n" + "inch x12\n" "fmax z31.h, p3/M, z31.h, z15.h\n" - "ld1h { z4.h }, p3/Z, [x3, #5, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #16\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z30.h, p3/M, z30.h, z15.h\n" - "ld1h { z7.h }, p3/Z, [x3, #-8, MUL VL]\n" // Load from weights and bias - "cmp x6, %x[n_channels]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.h, p3/M, z29.h, z15.h\n" - "ld1h { z8.h }, p3/Z, [x3, #-7, MUL VL]\n" // Load from weights and bias - "addvl x3, x3, #-6\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmax z27.h, p3/M, z27.h, z15.h\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x22, x7, LSL #1]\n" - "mov z31.d, z13.d\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" - "ldr x22, [x2, #0x20]\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "st1h { z30.h }, p0, [x21, x7, LSL #1]\n" - "mov z30.d, z13.d\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.h, p3/M, z28.h, z14.h\n" - "st1h { z29.h }, p0, [x20, x7, LSL #1]\n" - "mov z29.d, z13.d\n" - "ldr x21, [x2, #0x28]\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" - "ldr x20, [x2, #0x30]\n" "fmax z26.h, p3/M, z26.h, z15.h\n" - "st1h { z28.h }, p0, [x19, x7, LSL #1]\n" - "mov z28.d, z13.d\n" - "ldr x19, [x2, #0x38]\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmax z25.h, p3/M, z25.h, z15.h\n" - "st1h { z27.h }, p0, [x22, x7, LSL #1]\n" - "mov z27.d, z13.d\n" - "ldr x22, [x2, #0x40]\n" - "fmin z26.h, p3/M, z26.h, z14.h\n" - "st1h { z26.h }, p0, [x21, x7, LSL #1]\n" - "mov z26.d, z13.d\n" - "fmin z25.h, p3/M, z25.h, z14.h\n" - "ldr x21, [x2, #0x48]\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmax z24.h, p3/M, z24.h, z15.h\n" - "st1h { z25.h }, p0, [x20, x7, LSL #1]\n" - "mov z25.d, z13.d\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.h, p3/M, z23.h, z15.h\n" - "ldr x20, [x2, #0x50]\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" + "fmin z26.h, p3/M, z26.h, z14.h\n" + "ldr x20, [x16, #0x30]\n" + "fmin z25.h, p3/M, z25.h, z14.h\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z24.h }, p0, [x19, x7, LSL #1]\n" - "mov z24.d, z13.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.h, p3/M, z23.h, z14.h\n" - "ldr x19, [x2, #0x58]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" - "st1h { z23.h }, p0, [x22, x7, LSL #1]\n" - "mov z23.d, z13.d\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" "fmax z21.h, p3/M, z21.h, z15.h\n" - "ldr x22, [x2, #0x60]\n" - "fmin z22.h, p3/M, z22.h, z14.h\n" - "st1h { z22.h }, p0, [x21, x7, LSL #1]\n" - "mov z22.d, z13.d\n" - "fmin z21.h, p3/M, z21.h, z14.h\n" - "ldr x21, [x2, #0x68]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" "fmax z20.h, p3/M, z20.h, z15.h\n" - "st1h { z21.h }, p0, [x20, x7, LSL #1]\n" - "mov z21.d, z13.d\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" "fmax z19.h, p3/M, z19.h, z15.h\n" - "ldr x20, [x2, #0x70]\n" + "ldr x21, [x16, #0x48]\n" + "fmin z22.h, p3/M, z22.h, z14.h\n" + "ldr x20, [x16, #0x50]\n" + "fmin z21.h, p3/M, z21.h, z14.h\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "st1h { z20.h }, p0, [x19, x7, LSL #1]\n" - "mov z20.d, z13.d\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "ldr x19, [x2, #0x78]\n" + "st1h { z22.h }, p0, [x21, x11, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" - "st1h { z19.h }, p0, [x22, x7, LSL #1]\n" - "mov z19.d, z13.d\n" + "st1h { z21.h }, p0, [x20, x11, LSL #1]\n" "fmax z17.h, p3/M, z17.h, z15.h\n" + "st1h { z20.h }, p0, [x19, x11, LSL #1]\n" + "fmax z16.h, p3/M, z16.h, z15.h\n" + "st1h { z19.h }, p0, [x22, x11, LSL #1]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.h, p3/M, z18.h, z14.h\n" - "st1h { z18.h }, p0, [x21, x7, LSL #1]\n" - "mov z18.d, z13.d\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "st1h { z17.h }, p0, [x20, x7, LSL #1]\n" - "mov z17.d, z13.d\n" - "fmax z16.h, p3/M, z16.h, z15.h\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z16.h }, p0, [x19, x7, LSL #1]\n" - "mov z16.d, z13.d\n" + "st1h { z18.h }, p0, [x21, x11, LSL #1]\n" + "st1h { z17.h }, p0, [x20, x11, LSL #1]\n" + "st1h { z16.h }, p0, [x19, x11, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0x20]\n" - "inch x7\n" - "fmla z30.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0x28]\n" + "movprfx z31, z13\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x20]\n" + "inch x11\n" + "movprfx z30, z13\n fmla z30.h, p3/M, z7.h, z9.h\n" + "ldr x25, [x14, #0x28]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0x30]\n" - "fmla z27.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0x38]\n" - "fmla z26.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0x40]\n" - "fmla z25.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0x48]\n" - "fmla z23.h, p3/M, z2.h, z9.h\n" - "ldr x28, [x4, #0x50]\n" - "fmla z22.h, p3/M, z1.h, z9.h\n" - "ldr x27, [x4, #0x58]\n" - "fmla z21.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x5, LSL #1]\n" + "movprfx z29, z13\n fmla z29.h, p3/M, z6.h, z9.h\n" + "ldr x24, [x14, #0x30]\n" + "movprfx z27, z13\n fmla z27.h, p3/M, z5.h, z9.h\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.h, p3/M, z4.h, z9.h\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.h, p3/M, z3.h, z9.h\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.h, p3/M, z2.h, z9.h\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.h, p3/M, z1.h, z9.h\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" - "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" + "movprfx z28, z13\n fmla z28.h, p3/M, z2.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z12.h\n" - "ldr x26, [x4, #0x60]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.h, p3/M, z7.h, z12.h\n" - "ldr x25, [x4, #0x68]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.h, p3/M, z5.h, z12.h\n" - "ldr x24, [x4, #0x70]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.h, p3/M, z6.h, z12.h\n" - "ldr x23, [x4, #0x78]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.h, p3/M, z4.h, z12.h\n" - "ldr x8, [x4, #0x80]\n" - "fmla z24.h, p3/M, z3.h, z12.h\n" - "ldr x17, [x4, #0x88]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.h, p3/M, z3.h, z12.h\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.h, p3/M, z2.h, z12.h\n" - "ldr x16, [x4, #0x90]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.h, p3/M, z1.h, z12.h\n" - "ldr x15, [x4, #0x98]\n" - "fmla z20.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x11, x5, LSL #1]\n" - "fmla z19.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x5, LSL #1]\n" - "fmla z16.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "movprfx z20, z13\n fmla z20.h, p3/M, z0.h, z12.h\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "movprfx z19, z13\n fmla z19.h, p3/M, z6.h, z10.h\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" + "movprfx z16, z13\n fmla z16.h, p3/M, z8.h, z11.h\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z27.h, p3/M, z8.h, z9.h\n" - "ldr x14, [x4, #0xa0]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.h, p3/M, z7.h, z9.h\n" - "ldr x13, [x4, #0xa8]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.h, p3/M, z6.h, z9.h\n" - "ldr x12, [x4, #0xb0]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.h, p3/M, z5.h, z9.h\n" - "ldr x11, [x4, #0xb8]\n" "fmla z22.h, p3/M, z4.h, z9.h\n" - "ldr x10, [x4, #0xc0]\n" "fmla z21.h, p3/M, z3.h, z9.h\n" - "ldr x9, [x4, #0xc8]\n" "fmla z19.h, p3/M, z2.h, z9.h\n" - "ldr x22, [x2, #0x0]\n" - "fmla z18.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x2, #0x8]\n" - "fmla z17.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28, x5, LSL #1]\n" + "movprfx z18, z13\n fmla z18.h, p3/M, z1.h, z9.h\n" + "movprfx z17, z13\n fmla z17.h, p3/M, z0.h, z9.h\n" + "ld1h { z9.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z12.h\n" - "ldr x28, [x4, #0xd0]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z2.h, z11.h\n" - "ldr x27, [x4, #0xd8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z26.h, p3/M, z8.h, z10.h\n" - "ldr x26, [x4, #0xe0]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.h, p3/M, z7.h, z10.h\n" - "ldr x20, [x2, #0x10]\n" "fmla z24.h, p3/M, z6.h, z10.h\n" - "ldr x19, [x2, #0x18]\n" "fmla z22.h, p3/M, z5.h, z10.h\n" "fmla z21.h, p3/M, z4.h, z10.h\n" "fmla z20.h, p3/M, z3.h, z10.h\n" "fmla z18.h, p3/M, z2.h, z10.h\n" "fmla z17.h, p3/M, z1.h, z10.h\n" "fmla z16.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ldr x25, [x4, #0xe8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z23.h, p3/M, z6.h, z11.h\n" - "ldr x23, [x4, #0xf8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z10.h\n" - "ldr x24, [x4, #0xf0]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" "fmla z29.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z2.h, z10.h\n" "fmla z26.h, p3/M, z1.h, z10.h\n" "fmla z25.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z20.h, p3/M, z8.h, z11.h\n" - "ldr x17, [x4, #0x108]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.h, p3/M, z5.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ldr x8, [x4, #0x100]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" "fmla z26.h, p3/M, z2.h, z12.h\n" "fmla z25.h, p3/M, z1.h, z12.h\n" "fmla z24.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z19.h, p3/M, z7.h, z11.h\n" - "ldr x15, [x4, #0x118]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.h, p3/M, z6.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z10.h\n" - "ldr x16, [x4, #0x110]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.h, p3/M, z6.h, z10.h\n" "fmla z27.h, p3/M, z4.h, z10.h\n" "fmla z26.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z1.h, z10.h\n" "fmla z22.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x14, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z17.h, p3/M, z8.h, z11.h\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.h, p3/M, z7.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x13, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z12.h\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.h, p3/M, z7.h, z12.h\n" "fmla z25.h, p3/M, z5.h, z12.h\n" "fmla z24.h, p3/M, z4.h, z12.h\n" "fmla z21.h, p3/M, z2.h, z12.h\n" "fmla z20.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x12, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x11, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z27.h, p3/M, z7.h, z11.h\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z4.h, z11.h\n" "fmla z22.h, p3/M, z3.h, z11.h\n" "fmla z19.h, p3/M, z1.h, z11.h\n" "fmla z18.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z12.h\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.h, p3/M, z1.h, z12.h\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z6.h, z10.h\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.h, p3/M, z3.h, z10.h\n" "fmla z23.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z25.h, p3/M, z8.h, z11.h\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z5.h, z11.h\n" "fmla z20.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z2.h, z11.h\n" "fmla z16.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z12.h\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.h, p3/M, z5.h, z12.h\n" "fmla z20.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x26, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z27.h, p3/M, z6.h, z10.h\n" "fmla z23.h, p3/M, z3.h, z10.h\n" "fmla z19.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x25, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z22.h, p3/M, z7.h, z11.h\n" "fmla z21.h, p3/M, z6.h, z11.h\n" "fmla z23.h, p3/M, z8.h, z11.h\n" "fmla z19.h, p3/M, z5.h, z11.h\n" "fmla z18.h, p3/M, z4.h, z11.h\n" "fmla z17.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z24.h, p3/M, z8.h, z12.h\n" "fmla z20.h, p3/M, z5.h, z12.h\n" "fmla z16.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x23, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z19.h, p3/M, z8.h, z10.h\n" "fmla z18.h, p3/M, z7.h, z10.h\n" "fmla z17.h, p3/M, z6.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x8, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x10, x13, LSL #1]\n" "fmla z22.h, p3/M, z8.h, z11.h\n" "fmla z21.h, p3/M, z7.h, z11.h\n" "fmla z20.h, p3/M, z6.h, z11.h\n" "fmla z18.h, p3/M, z5.h, z11.h\n" "fmla z17.h, p3/M, z4.h, z11.h\n" "fmla z16.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x17, x5, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x9, x13, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z10.h\n" "fmla z18.h, p3/M, z8.h, z12.h\n" "fmla z17.h, p3/M, z7.h, z12.h\n" "fmla z16.h, p3/M, z6.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x16, x5, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z10.h\n" "fmla z27.h, p3/M, z1.h, z10.h\n" "fmla z26.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x15, x5, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z11.h\n" "fmla z28.h, p3/M, z4.h, z11.h\n" "fmla z25.h, p3/M, z2.h, z11.h\n" @@ -679,68 +647,68 @@ void sve_fp16_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "fmax z29.h, p3/M, z29.h, z15.h\n" "fmax z28.h, p3/M, z28.h, z15.h\n" "fmin z31.h, p3/M, z31.h, z14.h\n" - "st1h { z31.h }, p0, [x22, x7, LSL #1]\n" + "st1h { z31.h }, p0, [x22, x11, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z14.h\n" "fmin z29.h, p3/M, z29.h, z14.h\n" - "ldr x22, [x2, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.h, p3/M, z28.h, z14.h\n" - "st1h { z30.h }, p0, [x21, x7, LSL #1]\n" + "st1h { z30.h }, p0, [x21, x11, LSL #1]\n" "fmax z27.h, p3/M, z27.h, z15.h\n" "fmax z26.h, p3/M, z26.h, z15.h\n" - "st1h { z29.h }, p0, [x20, x7, LSL #1]\n" + "st1h { z29.h }, p0, [x20, x11, LSL #1]\n" "fmax z25.h, p3/M, z25.h, z15.h\n" - "st1h { z28.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z28.h }, p0, [x19, x11, LSL #1]\n" "fmax z24.h, p3/M, z24.h, z15.h\n" - "ldr x21, [x2, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.h, p3/M, z23.h, z15.h\n" - "ldr x20, [x2, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmin z27.h, p3/M, z27.h, z14.h\n" - "ldr x19, [x2, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z26.h, p3/M, z26.h, z14.h\n" - "st1h { z27.h }, p0, [x22, x7, LSL #1]\n" + "st1h { z27.h }, p0, [x22, x11, LSL #1]\n" "fmin z25.h, p3/M, z25.h, z14.h\n" "fmin z24.h, p3/M, z24.h, z14.h\n" - "st1h { z26.h }, p0, [x21, x7, LSL #1]\n" + "st1h { z26.h }, p0, [x21, x11, LSL #1]\n" "fmin z23.h, p3/M, z23.h, z14.h\n" - "ldr x22, [x2, #0x40]\n" + "ldr x22, [x16, #0x40]\n" "fmax z22.h, p3/M, z22.h, z15.h\n" - "ldr x21, [x2, #0x48]\n" + "ldr x21, [x16, #0x48]\n" "fmax z21.h, p3/M, z21.h, z15.h\n" - "st1h { z25.h }, p0, [x20, x7, LSL #1]\n" + "st1h { z25.h }, p0, [x20, x11, LSL #1]\n" "fmax z20.h, p3/M, z20.h, z15.h\n" - "st1h { z24.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z24.h }, p0, [x19, x11, LSL #1]\n" "fmax z19.h, p3/M, z19.h, z15.h\n" - "st1h { z23.h }, p0, [x22, x7, LSL #1]\n" + "st1h { z23.h }, p0, [x22, x11, LSL #1]\n" "fmin z22.h, p3/M, z22.h, z14.h\n" - "ldr x20, [x2, #0x50]\n" + "ldr x20, [x16, #0x50]\n" "fmin z21.h, p3/M, z21.h, z14.h\n" - "ldr x19, [x2, #0x58]\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.h, p3/M, z20.h, z14.h\n" - "ldr x22, [x2, #0x60]\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.h, p3/M, z19.h, z14.h\n" - "st1h { z22.h }, p0, [x21, x7, LSL #1]\n" + "st1h { z22.h }, p0, [x21, x11, LSL #1]\n" "fmax z18.h, p3/M, z18.h, z15.h\n" - "st1h { z21.h }, p0, [x20, x7, LSL #1]\n" + "st1h { z21.h }, p0, [x20, x11, LSL #1]\n" "fmax z17.h, p3/M, z17.h, z15.h\n" - "st1h { z20.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z20.h }, p0, [x19, x11, LSL #1]\n" "fmax z16.h, p3/M, z16.h, z15.h\n" - "st1h { z19.h }, p0, [x22, x7, LSL #1]\n" - "ldr x21, [x2, #0x68]\n" + "st1h { z19.h }, p0, [x22, x11, LSL #1]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.h, p3/M, z18.h, z14.h\n" - "ldr x20, [x2, #0x70]\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.h, p3/M, z17.h, z14.h\n" - "ldr x19, [x2, #0x78]\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.h, p3/M, z16.h, z14.h\n" - "st1h { z18.h }, p0, [x21, x7, LSL #1]\n" - "st1h { z17.h }, p0, [x20, x7, LSL #1]\n" - "st1h { z16.h }, p0, [x19, x7, LSL #1]\n" + "st1h { z18.h }, p0, [x21, x11, LSL #1]\n" + "st1h { z17.h }, p0, [x20, x11, LSL #1]\n" + "st1h { z16.h }, p0, [x19, x11, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index a6a4afb3b5..5968309927 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst +class sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 2238bf08cd..d20e9913ae 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,225 +113,217 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( "add x9, x21, x22, LSL #1\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x28, x9, x22, LSL #1\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias - "mov z31.d, z17.d\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z17.h }, p3/Z, [x16]\n" "add x27, x28, x22, LSL #1\n" - "mov z30.d, z17.d\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "add x26, x27, x22, LSL #1\n" - "mov z29.d, z17.d\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" "add x25, x13, x13\n" - "mov z28.d, z17.d\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" "add x24, x25, x13\n" - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" "add x23, x24, x13\n" - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" "mul x19, x7, x20\n" // offset = tile_i * ld_output_row - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" "madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col - "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2) - "ld1h { z10.h }, p2/Z, [x21]\n" // Load input point (0, 0) + "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" "mul x19, x19, x17\n" // offset *= output_tile_size - "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1) + "whilelt p2.h, XZR, %x[n_channels]\n" + "ld1h { z9.h }, p2/Z, [x28, x25, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21]\n" "add x10, x10, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) - "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3) + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "add x22, x10, x20, LSL #1\n" - "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4) + "ld1h { z12.h }, p2/Z, [x21, x24, LSL #1]\n" "addvl x16, x16, #16\n" - "ld1h { z14.h }, p2/Z, [x9]\n" // Load input point (1, 0) + "ld1h { z13.h }, p2/Z, [x21, x23, LSL #1]\n" "cmp x14, %x[n_channels]\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" - "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1) - "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2) + "ld1h { z14.h }, p2/Z, [x9]\n" + "ld1h { z15.h }, p2/Z, [x9, x13, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x25, LSL #1]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" "whilelt p1.h, x14, %x[n_channels]\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" "inch x12\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ld1h { z17.h }, p3/Z, [x16]\n" "inch x15\n" - "addvl x21, x21, #1\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x21]\n" // Load input point (0, 0) + "addvl x21, x21, #1\n" + "ld1h { z10.h }, p1/Z, [x21]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4) "inch x14\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2) - "addvl x9, x9, #1\n" + "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0) + "ld1h { z14.h }, p2/Z, [x27]\n" + "addvl x9, x9, #1\n" "fmla z30.h, p3/M, z0.h, z16.h\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4) + "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0) + "ld1h { z15.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1) + "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" "addvl x28, x28, #1\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n" // Load input point (2, 2) + "ld1h { z9.h }, p1/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" "fmla z31.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0) + "ld1h { z15.h }, p2/Z, [x26]\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2) + "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2) + "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" "addvl x27, x27, #1\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n" // Load input point (0, 3) + "ld1h { z12.h }, p1/Z, [x21, x24, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z11.h\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n" // Load input point (0, 4) + "ld1h { z13.h }, p1/Z, [x21, x23, LSL #1]\n" "fmax z31.h, p3/M, z31.h, z19.h\n" "fmla z28.h, p3/M, z5.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3) + "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z19.h\n" "fmla z29.h, p3/M, z5.h, z16.h\n" - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" "fmin z31.h, p3/M, z31.h, z18.h\n" - "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0) - "mov z31.d, z17.d\n" + "st1h { z31.h }, p0, [x10]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4) - "whilelt p2.h, x15, %x[n_channels]\n" "fmla z29.h, p3/M, z8.h, z15.h\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias - "addvl x26, x26, #1\n" + "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" + "whilelt p2.h, x15, %x[n_channels]\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z17.d\n" - "addvl x10, x10, #1\n" + "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" + "addvl x26, x26, #1\n" "fmla z28.h, p3/M, z3.h, z16.h\n" - "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n" // Load input point (0, 2) + "ld1h { z16.h }, p1/Z, [x21, x25, LSL #1]\n" "cmp x14, %x[n_channels]\n" "fmax z29.h, p3/M, z29.h, z19.h\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" + "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z14.h\n" - "ld1h { z14.h }, p1/Z, [x9]\n" // Load input point (1, 0) + "ld1h { z14.h }, p1/Z, [x9]\n" + "addvl x10, x10, #1\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "mov z29.d, z17.d\n" + "st1h { z29.h }, p0, [x22]\n" "fmla z28.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n" // Load input point (1, 1) - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z15.h }, p1/Z, [x9, x13, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n" // Load input point (0, 1) - "addvl x16, x16, #16\n" + "ld1h { z11.h }, p1/Z, [x21, x13, LSL #1]\n" + "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" "fmax z28.h, p3/M, z28.h, z19.h\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias + "addvl x16, x16, #16\n" + "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1) - "mov z28.d, z17.d\n" - "addvl x22, x22, #1\n" + "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" "addvl x16, x16, #-6\n" + "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" + "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" "ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x7, #0x1\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x8, x8, #0x1\n" "fmla z31.h, p3/M, z0.h, z10.h\n" "cmp x8, x19\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (1, 4) + "ld1h { z12.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (1, 3) + "ld1h { z11.h }, p2/Z, [x9, x24, LSL #1]\n" "csel x8, x8, XZR, LT\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (1, 2) + "ld1h { z13.h }, p2/Z, [x9, x25, LSL #1]\n" "csel x7, x7, x21, LT\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27]\n" // Load input point (3, 0) + "ld1h { z14.h }, p2/Z, [x27]\n" "cmp x7, x20\n" "fmla z30.h, p3/M, z0.h, z16.h\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (3, 4) + "ld1h { z14.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x28]\n" // Load input point (2, 0) + "ld1h { z15.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" // Load input point (3, 1) + "ld1h { z11.h }, p2/Z, [x27, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" // Load input point (2, 1) + "ld1h { z16.h }, p2/Z, [x28, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (2, 3) + "ld1h { z12.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (2, 4) + "ld1h { z11.h }, p2/Z, [x28, x23, LSL #1]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (3, 3) + "ld1h { z13.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" "fmla z31.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26]\n" // Load input point (4, 0) + "ld1h { z15.h }, p2/Z, [x26]\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" // Load input point (4, 2) + "ld1h { z15.h }, p2/Z, [x26, x25, LSL #1]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (3, 2) + "ld1h { z16.h }, p2/Z, [x27, x25, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z30.h, p3/M, z8.h, z11.h\n" "fmla z29.h, p3/M, z7.h, z13.h\n" "fmax z31.h, p3/M, z31.h, z19.h\n" "fmla z28.h, p3/M, z5.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" // Load input point (4, 3) + "ld1h { z14.h }, p2/Z, [x26, x24, LSL #1]\n" "fmax z30.h, p3/M, z30.h, z19.h\n" "fmla z29.h, p3/M, z5.h, z16.h\n" "fmin z31.h, p3/M, z31.h, z18.h\n" - "st1h { z31.h }, p0, [x10]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x10]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z8.h, z15.h\n" - "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" // Load input point (4, 4) + "ld1h { z11.h }, p2/Z, [x26, x23, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x10, x11, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z16.h\n" "fmax z29.h, p3/M, z29.h, z19.h\n" "fmla z28.h, p3/M, z7.h, z14.h\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) + "st1h { z29.h }, p0, [x22]\n" "fmla z28.h, p3/M, z6.h, z15.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" "fmax z28.h, p3/M, z28.h, z19.h\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" // Store output point (1, 1) + "st1h { z28.h }, p0, [x22, x11, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -342,4 +334,4 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index 0d5d4176aa..ceba36d897 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -89,257 +89,249 @@ void sve_fp16_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" - "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z19.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x14, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x13\n" - "ldp x12, x11, [x19, #0x0]\n" - "sub x10, XZR, x13\n" - "ldp x9, x28, [x19, #0x10]\n" + "cnth x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias - "mov z31.d, z17.d\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias - "cmp x13, %x[n_channels]\n" - "mov z30.d, z17.d\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias - "mov z29.d, z17.d\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias - "mov z28.d, z17.d\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #16\n" - "ldp x27, x26, [x15, #0x0]\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #-6\n" - "ld1h { z9.h }, p2/Z, [x27, x14, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldp x25, x23, [x15, #0x10]\n" - "ldp x22, x21, [x15, #0x20]\n" - "ldp x20, x19, [x15, #0x30]\n" - "ld1h { z11.h }, p2/Z, [x25, x14, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x23, x14, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x21, x14, LSL #1]\n" - "ld1h { z15.h }, p2/Z, [x20, x14, LSL #1]\n" - "ld1h { z16.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z17.h }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ld1h { z11.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x23, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x21, x13, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x20, x13, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x19, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x24, [x15, #0x40]\n" - "whilelt p1.h, x13, %x[n_channels]\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "ldr x20, [x15, #0x48]\n" - "inch x10\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" - "ldr x23, [x15, #0x50]\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x40]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" + "ldr x25, [x14, #0x48]\n" + "inch x9\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" + "ldr x24, [x14, #0x50]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ldr x19, [x15, #0x58]\n" - "ldr x22, [x15, #0x60]\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ldr x21, [x15, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x20, [x15, #0x70]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n" - "ldr x19, [x15, #0x78]\n" + "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.h, p3/M, z0.h, z16.h\n" - "ldr x27, [x15, #0x80]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ldr x26, [x15, #0x88]\n" - "ldr x25, [x15, #0x90]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" - "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x23, [x15, #0x98]\n" + "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n" - "ldr x22, [x15, #0xa0]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x21, [x15, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" - "ldr x20, [x15, #0xb0]\n" - "ldr x19, [x15, #0xb8]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.h, p3/M, z6.h, z15.h\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n" - "ldr x24, [x15, #0xc0]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ldp x27, x26, [x15, #0x0]\n" + "ld1h { z17.h }, p3/Z, [x15]\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z11.h\n" - "ldp x25, x23, [x15, #0x10]\n" - "ldp x22, x21, [x15, #0x20]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z28.h, p3/M, z5.h, z14.h\n" "fmax z31.h, p3/M, z31.h, z19.h\n" - "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z29.h, p3/M, z7.h, z13.h\n" - "ld1h { z9.h }, p1/Z, [x27, x13, LSL #1]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" "fmax z30.h, p3/M, z30.h, z19.h\n" - "ld1h { z10.h }, p1/Z, [x26, x13, LSL #1]\n" - "ld1h { z12.h }, p1/Z, [x23, x13, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "fmin z31.h, p3/M, z31.h, z18.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" - "inch x14\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" + "inch x13\n" "fmla z29.h, p3/M, z5.h, z16.h\n" - "ld1h { z13.h }, p1/Z, [x22, x13, LSL #1]\n" - "whilelt p2.h, x14, %x[n_channels]\n" + "ldp x26, x25, [x14, #0x0]\n" + "whilelt p2.h, x13, %x[n_channels]\n" + "fmin z31.h, p3/M, z31.h, z18.h\n" + "ldp x24, x23, [x14, #0x10]\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "ldp x20, x19, [x15, #0x30]\n" - "ld1h { z17.h }, p3/Z, [x16]\n" // Load from weights and bias + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" "fmla z28.h, p3/M, z3.h, z16.h\n" - "st1h { z31.h }, p0, [x12, x10, LSL #1]\n" - "mov z31.d, z17.d\n" - "ld1h { z16.h }, p1/Z, [x19, x13, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x26, x12, LSL #1]\n" "fmla z29.h, p3/M, z8.h, z15.h\n" - "st1h { z30.h }, p0, [x11, x10, LSL #1]\n" - "mov z30.d, z17.d\n" - "ld1h { z0.h }, p3/Z, [x16, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z10.h }, p1/Z, [x25, x12, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z14.h\n" - "ld1h { z14.h }, p1/Z, [x21, x13, LSL #1]\n" - "ld1h { z1.h }, p3/Z, [x16, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z12.h }, p1/Z, [x23, x12, LSL #1]\n" + "ld1h { z13.h }, p1/Z, [x22, x12, LSL #1]\n" "fmax z29.h, p3/M, z29.h, z19.h\n" - "ld1h { z2.h }, p3/Z, [x16, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z14.h }, p1/Z, [x21, x12, LSL #1]\n" "fmla z28.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p1/Z, [x20, x13, LSL #1]\n" - "ld1h { z3.h }, p3/Z, [x16, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z15.h }, p1/Z, [x20, x12, LSL #1]\n" + "ld1h { z16.h }, p1/Z, [x19, x12, LSL #1]\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "ld1h { z4.h }, p3/Z, [x16, #5, MUL VL]\n" // Load from weights and bias - "ld1h { z5.h }, p3/Z, [x16, #6, MUL VL]\n" // Load from weights and bias + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmla z28.h, p3/M, z8.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x25, x13, LSL #1]\n" - "inch x13\n" + "ld1h { z11.h }, p1/Z, [x24, x12, LSL #1]\n" + "inch x12\n" "fmax z28.h, p3/M, z28.h, z19.h\n" - "st1h { z29.h }, p0, [x9, x10, LSL #1]\n" - "cmp x13, %x[n_channels]\n" - "mov z29.d, z17.d\n" - "ld1h { z6.h }, p3/Z, [x16, #7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #16\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" + "cmp x12, %x[n_channels]\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "ld1h { z7.h }, p3/Z, [x16, #-8, MUL VL]\n" // Load from weights and bias - "ld1h { z8.h }, p3/Z, [x16, #-7, MUL VL]\n" // Load from weights and bias - "addvl x16, x16, #-6\n" - "st1h { z28.h }, p0, [x28, x10, LSL #1]\n" - "mov z28.d, z17.d\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z5.h }, p3/Z, [x15, #6, MUL VL]\n" + "ld1h { z6.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" + "ld1h { z7.h }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1h { z8.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z8.h, z9.h\n" - "ldr x24, [x15, #0x40]\n" - "inch x10\n" - "fmla z30.h, p3/M, z6.h, z9.h\n" - "ldr x20, [x15, #0x48]\n" + "movprfx z31, z17\n fmla z31.h, p3/M, z8.h, z9.h\n" + "ldr x26, [x14, #0x40]\n" + "inch x9\n" + "movprfx z30, z17\n fmla z30.h, p3/M, z6.h, z9.h\n" + "ldr x25, [x14, #0x48]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z2.h, z9.h\n" - "ldr x23, [x15, #0x50]\n" - "fmla z28.h, p3/M, z0.h, z9.h\n" - "ldr x19, [x15, #0x58]\n" - "ldr x22, [x15, #0x60]\n" + "movprfx z29, z17\n fmla z29.h, p3/M, z2.h, z9.h\n" + "ldr x24, [x14, #0x50]\n" + "movprfx z28, z17\n fmla z28.h, p3/M, z0.h, z9.h\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.h, p3/M, z0.h, z10.h\n" - "ldr x21, [x15, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" - "ldr x20, [x15, #0x70]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x23, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z31.h, p3/M, z3.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x14, LSL #1]\n" - "ldr x19, [x15, #0x78]\n" + "ld1h { z14.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.h, p3/M, z0.h, z16.h\n" - "ldr x27, [x15, #0x80]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.h, p3/M, z3.h, z14.h\n" - "ldr x26, [x15, #0x88]\n" - "ldr x25, [x15, #0x90]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.h, p3/M, z4.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x22, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z29.h, p3/M, z0.h, z15.h\n" - "ld1h { z14.h }, p2/Z, [x26, x14, LSL #1]\n" - "ldr x23, [x15, #0x98]\n" + "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.h, p3/M, z2.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z5.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x14, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x23, x14, LSL #1]\n" - "ldr x22, [x15, #0xa0]\n" + "ld1h { z11.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.h, p3/M, z5.h, z13.h\n" - "ldr x21, [x15, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z1.h, z16.h\n" - "ldr x20, [x15, #0xb0]\n" - "ldr x19, [x15, #0xb8]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.h, p3/M, z6.h, z15.h\n" "fmla z28.h, p3/M, z4.h, z13.h\n" - "ld1h { z15.h }, p2/Z, [x25, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z30.h, p3/M, z7.h, z12.h\n" - "ld1h { z13.h }, p2/Z, [x22, x14, LSL #1]\n" - "ldr x24, [x15, #0xc0]\n" + "ld1h { z13.h }, p2/Z, [x22, x13, LSL #1]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.h, p3/M, z7.h, z16.h\n" - "ld1h { z16.h }, p2/Z, [x21, x14, LSL #1]\n" + "ld1h { z16.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" "fmla z29.h, p3/M, z6.h, z15.h\n" - "ld1h { z15.h }, p2/Z, [x19, x14, LSL #1]\n" + "ld1h { z15.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z8.h, z11.h\n" "fmla z28.h, p3/M, z5.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x20, x14, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x20, x13, LSL #1]\n" "fmax z31.h, p3/M, z31.h, z19.h\n" "fmla z29.h, p3/M, z7.h, z13.h\n" "fmax z30.h, p3/M, z30.h, z19.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x24, x14, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x26, x13, LSL #1]\n" "fmin z31.h, p3/M, z31.h, z18.h\n" - "st1h { z31.h }, p0, [x12, x10, LSL #1]\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmla z29.h, p3/M, z5.h, z16.h\n" "fmla z28.h, p3/M, z3.h, z16.h\n" "fmin z30.h, p3/M, z30.h, z18.h\n" - "st1h { z30.h }, p0, [x11, x10, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmla z28.h, p3/M, z7.h, z14.h\n" "fmla z29.h, p3/M, z8.h, z15.h\n" "fmla z28.h, p3/M, z6.h, z15.h\n" "fmax z29.h, p3/M, z29.h, z19.h\n" "fmla z28.h, p3/M, z8.h, z11.h\n" "fmin z29.h, p3/M, z29.h, z18.h\n" - "st1h { z29.h }, p0, [x9, x10, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" "fmax z28.h, p3/M, z28.h, z19.h\n" "fmin z28.h, p3/M, z28.h, z18.h\n" - "st1h { z28.h }, p0, [x28, x10, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 236f9bf43a..4a9bd33a1e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); -struct sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst +class sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef __fp16 bias_type; - typedef __fp16 input_type; - typedef __fp16 weight_type; - typedef __fp16 return_type; - + private: typedef void (*indirect_kern_type)(const __fp16 *const *const, __fp16 *const *const, const void *, unsigned int, const __fp16, const __fp16); + indirect_kern_type m_indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const __fp16 *, int64_t, int64_t, __fp16 *, int64_t, int64_t, const void *, unsigned int, const __fp16, const __fp16); + direct_kern_type m_direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef __fp16 return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast<__fp16 *const *>(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast<__fp16 *>(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index 6b1564e6c9..1c2e1e27ad 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -113,390 +113,382 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "add x20, x13, x22, LSL #1\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x10, x20, x22, LSL #1\n" - "ld1h { z16.h }, p3/Z, [x8]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z16.h }, p3/Z, [x8]\n" "add x9, x10, x22, LSL #1\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #1, MUL VL]\n" "add x28, x9, x22, LSL #1\n" - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #2, MUL VL]\n" "add x27, x28, x22, LSL #1\n" - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #3, MUL VL]\n" "add x26, x15, x15\n" - "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #4, MUL VL]\n" "add x25, x26, x15\n" - "mul x19, x5, x21\n" // offset = tile_i * ld_output_row + "ld1h { z4.h }, p3/Z, [x8, #5, MUL VL]\n" "add x24, x25, x15\n" + "mul x19, x5, x21\n" // offset = tile_i * ld_output_row "add x23, x24, x15\n" "madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col "mul x19, x19, x7\n" // offset *= output_tile_size "add x11, x11, x19, LSL #1\n" // outptrs[0] += offset * sizeof(__fp16) "add x22, x11, x21, LSL #1\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z5.h }, p2/Z, [x13]\n" // Load input point (0, 0) - "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z5.h }, p2/Z, [x13]\n" + "ld1h { z6.h }, p2/Z, [x13, x15, LSL #1]\n" "cmp x16, %x[n_channels]\n" - "ld1h { z7.h }, p2/Z, [x20]\n" // Load input point (1, 0) + "ld1h { z7.h }, p2/Z, [x20]\n" "addvl x8, x8, #6\n" - "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1) - "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2) - "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2) - "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3) - "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4) - "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5) - "ld1h { z14.h }, p2/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z8.h }, p2/Z, [x20, x15, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x13, x26, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x20, x26, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x13, x25, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x13, x24, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x20, x23, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x10]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3) + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" "whilelt p1.h, x16, %x[n_channels]\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" "inch x14\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z0.h }, p3/Z, [x8]\n" "inch x17\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4) + "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" "addvl x20, x20, #1\n" "fmla z30.h, p3/M, z1.h, z9.h\n" "inch x16\n" "fmla z29.h, p3/M, z1.h, z8.h\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5) + "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" "addvl x13, x13, #1\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z5.h\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2) + "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3) + "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" - "ld1h { z7.h }, p1/Z, [x20]\n" // Load input point (1, 0) + "ld1h { z7.h }, p1/Z, [x20]\n" "fmla z30.h, p3/M, z0.h, z8.h\n" "fmla z29.h, p3/M, z0.h, z14.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5) + "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" "fmla z29.h, p3/M, z1.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4) + "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" "addvl x10, x10, #1\n" "fmla z30.h, p3/M, z2.h, z5.h\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x8, x8, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0) - "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z5.h }, p2/Z, [x9]\n" + "ld1h { z16.h }, p3/Z, [x8, #4, MUL VL]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5) + "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z5.h\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z6.h\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4) + "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" "addvl x9, x9, #1\n" "fmla z30.h, p3/M, z2.h, z9.h\n" "fmla z29.h, p3/M, z2.h, z10.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0) + "ld1h { z9.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4) + "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2) + "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3) + "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5) + "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" "addvl x28, x28, #1\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1) + "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" - "ld1h { z14.h }, p1/Z, [x10]\n" // Load input point (2, 0) + "ld1h { z14.h }, p1/Z, [x10]\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2) + "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3) + "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" - "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n" // Load input point (1, 2) + "ld1h { z13.h }, p1/Z, [x20, x26, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4) + "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" - "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z5.h\n" - "ld1h { z5.h }, p1/Z, [x13]\n" // Load input point (0, 0) + "ld1h { z5.h }, p1/Z, [x13]\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5) + "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" "whilelt p2.h, x17, %x[n_channels]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x27, x27, #1\n" "fmla z31.h, p3/M, z3.h, z6.h\n" - "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n" // Load input point (0, 1) + "ld1h { z6.h }, p1/Z, [x13, x15, LSL #1]\n" "addvl x8, x8, #16\n" "fmla z30.h, p3/M, z3.h, z8.h\n" "cmp x16, %x[n_channels]\n" "fmla z29.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n" // Load input point (0, 3) + "ld1h { z11.h }, p1/Z, [x13, x25, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n" // Load input point (1, 1) + "ld1h { z8.h }, p1/Z, [x20, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n" // Load input point (1, 5) + "ld1h { z10.h }, p1/Z, [x20, x23, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n" // Load input point (0, 4) + "ld1h { z12.h }, p1/Z, [x13, x24, LSL #1]\n" "fmla z28.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n" // Load input point (0, 2) - "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z9.h }, p1/Z, [x13, x26, LSL #1]\n" + "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" "fmax z31.h, p3/M, z31.h, z18.h\n" "addvl x8, x8, #-6\n" "fmax z30.h, p3/M, z30.h, z18.h\n" "fmax z29.h, p3/M, z29.h, z18.h\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0) - "mov z31.d, z16.d\n" + "st1h { z31.h }, p0, [x11]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1) - "mov z30.d, z16.d\n" - "addvl x11, x11, #1\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "mov z29.d, z16.d\n" + "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1) - "mov z28.d, z16.d\n" + "addvl x11, x11, #1\n" + "st1h { z29.h }, p0, [x22]\n" + "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" // Load input point (1, 3) + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ld1h { z5.h }, p2/Z, [x20, x25, LSL #1]\n" "mov p0.b, p2.b\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" "add x21, x5, #0x1\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z0.h }, p3/Z, [x8]\n" // Load from weights and bias + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z0.h }, p3/Z, [x8]\n" "add x6, x6, #0x1\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" // Load input point (1, 4) + "ld1h { z6.h }, p2/Z, [x20, x24, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "fmla z29.h, p3/M, z1.h, z8.h\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "cmp x6, x19\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" // Load input point (0, 5) + "ld1h { z9.h }, p2/Z, [x13, x23, LSL #1]\n" "csel x6, x6, XZR, LT\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "csel x5, x5, x21, LT\n" "fmla z29.h, p3/M, z2.h, z13.h\n" "cmp x5, x20\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" // Load input point (2, 1) + "ld1h { z11.h }, p2/Z, [x10, x15, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z5.h\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" // Load input point (2, 2) + "ld1h { z12.h }, p2/Z, [x10, x26, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" // Load input point (2, 3) + "ld1h { z9.h }, p2/Z, [x10, x25, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #4, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" "fmla z30.h, p3/M, z0.h, z8.h\n" "fmla z29.h, p3/M, z0.h, z14.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" // Load input point (2, 5) + "ld1h { z8.h }, p2/Z, [x10, x23, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" "fmla z29.h, p3/M, z1.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" // Load input point (2, 4) + "ld1h { z13.h }, p2/Z, [x10, x24, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z5.h\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #7, MUL VL]\n" "addvl x8, x8, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x9]\n" // Load input point (3, 0) + "ld1h { z5.h }, p2/Z, [x9]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" // Load input point (3, 1) + "ld1h { z6.h }, p2/Z, [x9, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" // Load input point (3, 2) + "ld1h { z10.h }, p2/Z, [x9, x26, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-7, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" // Load input point (3, 5) + "ld1h { z14.h }, p2/Z, [x9, x23, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" "fmla z29.h, p3/M, z0.h, z5.h\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" // Load input point (3, 3) + "ld1h { z11.h }, p2/Z, [x9, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" "fmla z29.h, p3/M, z1.h, z6.h\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" // Load input point (3, 4) + "ld1h { z12.h }, p2/Z, [x9, x24, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z9.h\n" "fmla z29.h, p3/M, z2.h, z10.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x28]\n" // Load input point (4, 0) + "ld1h { z9.h }, p2/Z, [x28]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" // Load input point (4, 1) + "ld1h { z13.h }, p2/Z, [x28, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" // Load input point (4, 4) + "ld1h { z8.h }, p2/Z, [x28, x24, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" // Load input point (4, 2) + "ld1h { z5.h }, p2/Z, [x28, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x8, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" // Load input point (4, 3) + "ld1h { z6.h }, p2/Z, [x28, x25, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x8]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x8]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" // Load input point (4, 5) + "ld1h { z10.h }, p2/Z, [x28, x23, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x8, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27]\n" // Load input point (5, 0) + "ld1h { z11.h }, p2/Z, [x27]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x8, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" // Load input point (5, 1) + "ld1h { z12.h }, p2/Z, [x27, x15, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x8, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" // Load input point (5, 2) + "ld1h { z9.h }, p2/Z, [x27, x26, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" // Load input point (5, 3) + "ld1h { z11.h }, p2/Z, [x27, x25, LSL #1]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" // Load input point (5, 4) + "ld1h { z12.h }, p2/Z, [x27, x24, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" "fmla z31.h, p3/M, z2.h, z5.h\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" // Load input point (5, 5) + "ld1h { z9.h }, p2/Z, [x27, x23, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" "fmla z31.h, p3/M, z3.h, z6.h\n" "fmla z30.h, p3/M, z3.h, z8.h\n" @@ -511,13 +503,13 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "fmax z29.h, p3/M, z29.h, z18.h\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x11]\n" // Store output point (0, 0) + "st1h { z31.h }, p0, [x11]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" // Store output point (0, 1) + "st1h { z30.h }, p0, [x11, x12, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z29.h }, p0, [x22]\n" // Store output point (1, 0) - "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" // Store output point (1, 1) + "st1h { z29.h }, p0, [x22]\n" + "st1h { z28.h }, p0, [x22, x12, LSL #1]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -528,4 +520,4 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index be128b4aff..a0640daeca 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) namespace arm_conv { namespace depthwise { @@ -100,432 +100,424 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x5, [%x[params_struct], %[offsetof_args_params]]\n" - "add x6, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rh { z18.h }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "mov x7, #0x0\n" + "mov x13, #0x0\n" "ld1rh { z17.h }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "cnth x8\n" - "ldp x17, x16, [x19, #0x0]\n" - "sub x15, XZR, x8\n" - "ldp x14, x13, [x19, #0x10]\n" + "cnth x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.h, XZR, %x[n_channels]\n" - "ld1h { z16.h }, p3/Z, [x5]\n" // Load from weights and bias - "mov z31.d, z16.d\n" - "ld1h { z0.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias - "cmp x8, %x[n_channels]\n" - "mov z30.d, z16.d\n" - "ld1h { z1.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias - "mov z29.d, z16.d\n" - "ld1h { z2.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias - "mov z28.d, z16.d\n" - "ld1h { z3.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias - "ld1h { z4.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias - "addvl x5, x5, #6\n" - "ldp x12, x11, [x6, #0x0]\n" - "ldp x10, x9, [x6, #0x10]\n" - "ldp x20, x28, [x6, #0x20]\n" - "ld1h { z5.h }, p2/Z, [x12, x7, LSL #1]\n" - "ld1h { z6.h }, p2/Z, [x11, x7, LSL #1]\n" - "ld1h { z7.h }, p2/Z, [x10, x7, LSL #1]\n" - "ld1h { z8.h }, p2/Z, [x9, x7, LSL #1]\n" - "ld1h { z9.h }, p2/Z, [x20, x7, LSL #1]\n" - "ld1h { z13.h }, p2/Z, [x28, x7, LSL #1]\n" - "ldp x27, x19, [x6, #0x30]\n" - "ldp x26, x25, [x6, #0x40]\n" - "ld1h { z11.h }, p2/Z, [x27, x7, LSL #1]\n" - "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n" - "ld1h { z10.h }, p2/Z, [x26, x7, LSL #1]\n" - "ld1h { z14.h }, p2/Z, [x25, x7, LSL #1]\n" + "ld1h { z16.h }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1h { z0.h }, p3/Z, [x15, #1, MUL VL]\n" + "ld1h { z1.h }, p3/Z, [x15, #2, MUL VL]\n" + "ld1h { z2.h }, p3/Z, [x15, #3, MUL VL]\n" + "ld1h { z3.h }, p3/Z, [x15, #4, MUL VL]\n" + "ld1h { z4.h }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #6\n" + "ldp x26, x25, [x14, #0x0]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ld1h { z5.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x25, x13, LSL #1]\n" + "ld1h { z7.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x23, x13, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ldp x26, x25, [x14, #0x40]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x26, x13, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x25, x13, LSL #1]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ldr x24, [x6, #0x50]\n" - "whilelt p1.h, x8, %x[n_channels]\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" - "ldr x23, [x6, #0x58]\n" - "inch x15\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" - "ldr x22, [x6, #0x60]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ldr x24, [x14, #0x50]\n" + "whilelt p1.h, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" + "ldr x23, [x14, #0x58]\n" + "inch x9\n" + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" + "ldr x22, [x14, #0x60]\n" "mov p0.b, p2.b\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x6, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.h, p3/M, z1.h, z8.h\n" - "ldr x20, [x6, #0x70]\n" + "ldr x20, [x14, #0x70]\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ldr x19, [x6, #0x78]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.h, p3/M, z2.h, z13.h\n" - "ldr x12, [x6, #0x80]\n" + "ldr x26, [x14, #0x80]\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" - "ldr x11, [x6, #0x88]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.h, p3/M, z3.h, z5.h\n" - "ldr x10, [x6, #0x90]\n" + "ldr x24, [x14, #0x90]\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" - "ldr x9, [x6, #0x98]\n" + "ldr x23, [x14, #0x98]\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" - "ldr x20, [x6, #0xa0]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.h, p3/M, z0.h, z8.h\n" - "ldr x28, [x6, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.h, p3/M, z0.h, z14.h\n" - "ldr x27, [x6, #0xb0]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" - "ldr x19, [x6, #0xb8]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.h, p3/M, z1.h, z11.h\n" - "ldr x26, [x6, #0xc0]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z5.h\n" - "ldr x25, [x6, #0xc8]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" - "ldr x24, [x6, #0xd0]\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "addvl x5, x5, #16\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" - "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n" - "ldr x23, [x6, #0xd8]\n" + "ld1h { z16.h }, p3/Z, [x15, #4, MUL VL]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" - "ldr x22, [x6, #0xe0]\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" - "ldr x21, [x6, #0xe8]\n" + "ldr x22, [x14, #0xe0]\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" - "ldr x20, [x6, #0xf0]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.h, p3/M, z0.h, z5.h\n" - "ldr x19, [x6, #0xf8]\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ldr x12, [x6, #0x100]\n" "fmla z29.h, p3/M, z1.h, z6.h\n" - "ldr x11, [x6, #0x108]\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.h, p3/M, z2.h, z9.h\n" - "ldr x10, [x6, #0x110]\n" "fmla z29.h, p3/M, z2.h, z10.h\n" - "ldr x9, [x6, #0x118]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" - "ld1h { z16.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" - "ldp x12, x11, [x6, #0x0]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" - "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" - "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z5.h\n" - "ld1h { z5.h }, p1/Z, [x12, x8, LSL #1]\n" + "ld1h { z5.h }, p1/Z, [x26, x12, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n" - "inch x7\n" + "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n" + "inch x13\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ldp x10, x9, [x6, #0x10]\n" - "whilelt p2.h, x7, %x[n_channels]\n" + "ldp x24, x23, [x14, #0x10]\n" + "whilelt p2.h, x13, %x[n_channels]\n" "fmla z31.h, p3/M, z3.h, z6.h\n" - "ld1h { z6.h }, p1/Z, [x11, x8, LSL #1]\n" - "ldp x20, x28, [x6, #0x20]\n" + "ld1h { z6.h }, p1/Z, [x25, x12, LSL #1]\n" + "ldp x22, x21, [x14, #0x20]\n" "fmla z30.h, p3/M, z3.h, z8.h\n" - "ldp x27, x19, [x6, #0x30]\n" + "ldp x20, x19, [x14, #0x30]\n" "fmla z29.h, p3/M, z3.h, z11.h\n" - "ld1h { z7.h }, p1/Z, [x10, x8, LSL #1]\n" + "ld1h { z7.h }, p1/Z, [x24, x12, LSL #1]\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z13.h }, p1/Z, [x28, x8, LSL #1]\n" + "ld1h { z13.h }, p1/Z, [x21, x12, LSL #1]\n" "fmla z31.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p1/Z, [x9, x8, LSL #1]\n" + "ld1h { z8.h }, p1/Z, [x23, x12, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z11.h }, p1/Z, [x27, x8, LSL #1]\n" + "ld1h { z11.h }, p1/Z, [x20, x12, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p1/Z, [x19, x8, LSL #1]\n" + "ld1h { z12.h }, p1/Z, [x19, x12, LSL #1]\n" "fmla z28.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p1/Z, [x20, x8, LSL #1]\n" + "ld1h { z9.h }, p1/Z, [x22, x12, LSL #1]\n" "fmax z31.h, p3/M, z31.h, z18.h\n" - "ldp x26, x25, [x6, #0x40]\n" + "ldp x26, x25, [x14, #0x40]\n" "fmax z30.h, p3/M, z30.h, z18.h\n" - "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n" "fmax z29.h, p3/M, z29.h, z18.h\n" - "addvl x5, x5, #16\n" + "addvl x15, x15, #16\n" "fmax z28.h, p3/M, z28.h, z18.h\n" - "ld1h { z10.h }, p1/Z, [x26, x8, LSL #1]\n" - "ld1h { z14.h }, p1/Z, [x25, x8, LSL #1]\n" + "ld1h { z10.h }, p1/Z, [x26, x12, LSL #1]\n" + "ld1h { z14.h }, p1/Z, [x25, x12, LSL #1]\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "inch x8\n" + "inch x12\n" "fmin z30.h, p3/M, z30.h, z17.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias - "cmp x8, %x[n_channels]\n" + "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias - "addvl x5, x5, #-6\n" + "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z31.h }, p0, [x17, x15, LSL #1]\n" - "mov z31.d, z16.d\n" - "st1h { z30.h }, p0, [x16, x15, LSL #1]\n" - "mov z30.d, z16.d\n" - "st1h { z29.h }, p0, [x14, x15, LSL #1]\n" - "mov z29.d, z16.d\n" - "st1h { z28.h }, p0, [x13, x15, LSL #1]\n" - "mov z28.d, z16.d\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.h, p3/M, z0.h, z5.h\n" - "ldr x24, [x6, #0x50]\n" - "inch x15\n" - "fmla z30.h, p3/M, z0.h, z6.h\n" - "ldr x23, [x6, #0x58]\n" + "movprfx z31, z16\n fmla z31.h, p3/M, z0.h, z5.h\n" + "ldr x24, [x14, #0x50]\n" + "inch x9\n" + "movprfx z30, z16\n fmla z30.h, p3/M, z0.h, z6.h\n" + "ldr x23, [x14, #0x58]\n" "mov p0.b, p2.b\n" - "fmla z29.h, p3/M, z0.h, z7.h\n" - "ldr x22, [x6, #0x60]\n" - "fmla z28.h, p3/M, z0.h, z8.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" - "ld1h { z0.h }, p3/Z, [x5]\n" // Load from weights and bias + "movprfx z29, z16\n fmla z29.h, p3/M, z0.h, z7.h\n" + "ldr x22, [x14, #0x60]\n" + "movprfx z28, z16\n fmla z28.h, p3/M, z0.h, z8.h\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ld1h { z0.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z9.h\n" - "ldr x21, [x6, #0x68]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.h, p3/M, z1.h, z8.h\n" "fmla z28.h, p3/M, z1.h, z13.h\n" - "ld1h { z1.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias - "ldr x20, [x6, #0x70]\n" + "ld1h { z1.h }, p3/Z, [x15, #1, MUL VL]\n" + "ldr x20, [x14, #0x70]\n" "fmla z31.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" - "ldr x19, [x6, #0x78]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.h, p3/M, z2.h, z13.h\n" "fmla z28.h, p3/M, z2.h, z5.h\n" - "ld1h { z2.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias - "ldr x12, [x6, #0x80]\n" + "ld1h { z2.h }, p3/Z, [x15, #2, MUL VL]\n" + "ldr x26, [x14, #0x80]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" - "ldr x11, [x6, #0x88]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.h, p3/M, z3.h, z5.h\n" "fmla z28.h, p3/M, z3.h, z6.h\n" - "ld1h { z3.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias - "ldr x10, [x6, #0x90]\n" + "ld1h { z3.h }, p3/Z, [x15, #3, MUL VL]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z6.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #4, MUL VL]\n" // Load from weights and bias - "ldr x9, [x6, #0x98]\n" + "ld1h { z4.h }, p3/Z, [x15, #4, MUL VL]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.h, p3/M, z0.h, z7.h\n" - "ldr x20, [x6, #0xa0]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.h, p3/M, z0.h, z8.h\n" - "ldr x28, [x6, #0xa8]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.h, p3/M, z0.h, z14.h\n" "fmla z28.h, p3/M, z0.h, z11.h\n" - "ld1h { z0.h }, p3/Z, [x5, #5, MUL VL]\n" // Load from weights and bias - "ldr x27, [x6, #0xb0]\n" + "ld1h { z0.h }, p3/Z, [x15, #5, MUL VL]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z31.h, p3/M, z1.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x11, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z30.h, p3/M, z1.h, z13.h\n" - "ldr x19, [x6, #0xb8]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.h, p3/M, z1.h, z11.h\n" "fmla z28.h, p3/M, z1.h, z12.h\n" - "ld1h { z1.h }, p3/Z, [x5, #6, MUL VL]\n" // Load from weights and bias - "ldr x26, [x6, #0xc0]\n" + "ld1h { z1.h }, p3/Z, [x15, #6, MUL VL]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z31.h, p3/M, z2.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z5.h\n" - "ldr x25, [x6, #0xc8]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.h, p3/M, z2.h, z12.h\n" "fmla z28.h, p3/M, z2.h, z9.h\n" - "ld1h { z2.h }, p3/Z, [x5, #7, MUL VL]\n" // Load from weights and bias - "addvl x5, x5, #16\n" + "ld1h { z2.h }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.h, p3/M, z3.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x10, x7, LSL #1]\n" - "ldr x24, [x6, #0xd0]\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.h, p3/M, z3.h, z6.h\n" - "ldr x23, [x6, #0xd8]\n" "fmla z29.h, p3/M, z3.h, z9.h\n" "fmla z28.h, p3/M, z3.h, z13.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-8, MUL VL]\n" // Load from weights and bias - "ldr x22, [x6, #0xe0]\n" + "ld1h { z3.h }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x9, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.h, p3/M, z4.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z13.h\n" "fmla z28.h, p3/M, z4.h, z8.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-7, MUL VL]\n" // Load from weights and bias - "ldr x21, [x6, #0xe8]\n" + "ld1h { z4.h }, p3/Z, [x15, #-7, MUL VL]\n" + "ldr x22, [x14, #0xe0]\n" "fmla z31.h, p3/M, z0.h, z14.h\n" - "ld1h { z14.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z14.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z11.h\n" - "ldr x20, [x6, #0xf0]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.h, p3/M, z0.h, z5.h\n" "fmla z28.h, p3/M, z0.h, z6.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-6, MUL VL]\n" // Load from weights and bias - "ldr x19, [x6, #0xf8]\n" + "ld1h { z0.h }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x28, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x21, x13, LSL #1]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.h, p3/M, z1.h, z12.h\n" - "ldr x12, [x6, #0x100]\n" "fmla z29.h, p3/M, z1.h, z6.h\n" "fmla z28.h, p3/M, z1.h, z10.h\n" - "ld1h { z1.h }, p3/Z, [x5, #-5, MUL VL]\n" // Load from weights and bias - "ldr x11, [x6, #0x108]\n" + "ld1h { z1.h }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.h, p3/M, z2.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x27, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x20, x13, LSL #1]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.h, p3/M, z2.h, z9.h\n" - "ldr x10, [x6, #0x110]\n" "fmla z29.h, p3/M, z2.h, z10.h\n" "fmla z28.h, p3/M, z2.h, z11.h\n" - "ld1h { z2.h }, p3/Z, [x5, #-4, MUL VL]\n" // Load from weights and bias - "ldr x9, [x6, #0x118]\n" + "ld1h { z2.h }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x26, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.h, p3/M, z3.h, z13.h\n" "fmla z29.h, p3/M, z3.h, z11.h\n" "fmla z28.h, p3/M, z3.h, z12.h\n" - "ld1h { z3.h }, p3/Z, [x5, #-3, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z13.h\n" - "ld1h { z13.h }, p2/Z, [x25, x7, LSL #1]\n" + "ld1h { z13.h }, p2/Z, [x25, x13, LSL #1]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.h, p3/M, z4.h, z8.h\n" - "ld1h { z8.h }, p2/Z, [x22, x7, LSL #1]\n" + "ld1h { z8.h }, p2/Z, [x22, x13, LSL #1]\n" "fmla z29.h, p3/M, z4.h, z12.h\n" "fmla z28.h, p3/M, z4.h, z14.h\n" - "ld1h { z4.h }, p3/Z, [x5, #-2, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z5.h\n" - "ld1h { z5.h }, p2/Z, [x24, x7, LSL #1]\n" + "ld1h { z5.h }, p2/Z, [x24, x13, LSL #1]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.h, p3/M, z0.h, z6.h\n" "fmla z29.h, p3/M, z0.h, z9.h\n" "fmla z28.h, p3/M, z0.h, z13.h\n" - "ld1h { z0.h }, p3/Z, [x5, #-1, MUL VL]\n" // Load from weights and bias + "ld1h { z0.h }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.h, p3/M, z1.h, z6.h\n" - "ld1h { z6.h }, p2/Z, [x23, x7, LSL #1]\n" + "ld1h { z6.h }, p2/Z, [x23, x13, LSL #1]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.h, p3/M, z1.h, z10.h\n" "fmla z29.h, p3/M, z1.h, z13.h\n" "fmla z28.h, p3/M, z1.h, z5.h\n" - "ld1h { z1.h }, p3/Z, [x5]\n" // Load from weights and bias + "ld1h { z1.h }, p3/Z, [x15]\n" "fmla z31.h, p3/M, z2.h, z10.h\n" - "ld1h { z10.h }, p2/Z, [x21, x7, LSL #1]\n" + "ld1h { z10.h }, p2/Z, [x21, x13, LSL #1]\n" "fmla z30.h, p3/M, z2.h, z11.h\n" "fmla z29.h, p3/M, z2.h, z5.h\n" "fmla z28.h, p3/M, z2.h, z6.h\n" - "ld1h { z2.h }, p3/Z, [x5, #1, MUL VL]\n" // Load from weights and bias + "ld1h { z2.h }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.h, p3/M, z3.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x20, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x20, x13, LSL #1]\n" "fmla z30.h, p3/M, z3.h, z12.h\n" "fmla z29.h, p3/M, z3.h, z6.h\n" "fmla z28.h, p3/M, z3.h, z8.h\n" - "ld1h { z3.h }, p3/Z, [x5, #2, MUL VL]\n" // Load from weights and bias + "ld1h { z3.h }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.h, p3/M, z4.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x19, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x19, x13, LSL #1]\n" "fmla z30.h, p3/M, z4.h, z14.h\n" "fmla z29.h, p3/M, z4.h, z8.h\n" "fmla z28.h, p3/M, z4.h, z10.h\n" - "ld1h { z4.h }, p3/Z, [x5, #3, MUL VL]\n" // Load from weights and bias + "ld1h { z4.h }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.h, p3/M, z0.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x12, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x26, x13, LSL #1]\n" "fmla z30.h, p3/M, z0.h, z13.h\n" "fmla z29.h, p3/M, z0.h, z11.h\n" - "ld1h { z11.h }, p2/Z, [x11, x7, LSL #1]\n" + "ld1h { z11.h }, p2/Z, [x25, x13, LSL #1]\n" "fmla z28.h, p3/M, z0.h, z12.h\n" "fmla z31.h, p3/M, z1.h, z13.h\n" "fmla z30.h, p3/M, z1.h, z5.h\n" "fmla z29.h, p3/M, z1.h, z12.h\n" - "ld1h { z12.h }, p2/Z, [x10, x7, LSL #1]\n" + "ld1h { z12.h }, p2/Z, [x24, x13, LSL #1]\n" "fmla z28.h, p3/M, z1.h, z9.h\n" "fmla z31.h, p3/M, z2.h, z5.h\n" "fmla z30.h, p3/M, z2.h, z6.h\n" "fmla z29.h, p3/M, z2.h, z9.h\n" - "ld1h { z9.h }, p2/Z, [x9, x7, LSL #1]\n" + "ld1h { z9.h }, p2/Z, [x23, x13, LSL #1]\n" "fmla z28.h, p3/M, z2.h, z11.h\n" "fmla z31.h, p3/M, z3.h, z6.h\n" "fmla z30.h, p3/M, z3.h, z8.h\n" @@ -540,20 +532,20 @@ void sve_fp16_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "fmax z29.h, p3/M, z29.h, z18.h\n" "fmax z28.h, p3/M, z28.h, z18.h\n" "fmin z31.h, p3/M, z31.h, z17.h\n" - "st1h { z31.h }, p0, [x17, x15, LSL #1]\n" + "st1h { z31.h }, p0, [x11, x9, LSL #1]\n" "fmin z30.h, p3/M, z30.h, z17.h\n" "fmin z29.h, p3/M, z29.h, z17.h\n" - "st1h { z30.h }, p0, [x16, x15, LSL #1]\n" + "st1h { z30.h }, p0, [x10, x9, LSL #1]\n" "fmin z28.h, p3/M, z28.h, z17.h\n" - "st1h { z29.h }, p0, [x14, x15, LSL #1]\n" - "st1h { z28.h }, p0, [x13, x15, LSL #1]\n" + "st1h { z29.h }, p0, [x28, x9, LSL #1]\n" + "st1h { z28.h }, p0, [x27, x9, LSL #1]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) && defined(__ARM_FP16_ARGS) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index 05e82d4e76..e07e6314f8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst +class sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 4; constexpr static unsigned int input_cols = 4; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp index eb632eb4fe..571246be3e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -88,277 +88,229 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_direct_impl( __asm__ __volatile__( "ptrue p3.b\n" - "mov x8, #0x0\n" "mov x17, #0x0\n" + "mov x16, #0x0\n" "1:" // Tile loop - "str x8, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "mov x21, #0x2\n" - "str x17, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "cntb x16\n" - "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" - "add x16, x16, XZR, LSL #4\n" - "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" - "cntb x14\n" - "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" - "cntb x12\n" - "ldr x11, [%x[params_struct], %[offsetof_args_inptr]]\n" - "mul x19, x8, x20\n" // offset = tile_i * ld_input_row - "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" - "madd x19, x17, x13, x19\n" // offset += tile_j * ld_input_col - "ldr x10, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" - "mul x19, x19, x21\n" // offset *= kernel_stride * output_size - "ldr x9, [%x[params_struct], %[offsetof_args_outptr]]\n" - "add x11, x11, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) + "str x17, [%x[params_struct], %[offsetof_args_tile_i]]\n" + "mov x23, #0x2\n" + "str x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "mov x15, #0x2\n" + "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x13, #0x0\n" + "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" + "cntw x12\n" + "ldr x11, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" + "sub x21, XZR, x12\n" + "ldr x10, [%x[params_struct], %[offsetof_args_inptr]]\n" + "mul x19, x17, x22\n" // offset = tile_i * ld_input_row + "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" + "madd x19, x16, x11, x19\n" // offset += tile_j * ld_input_col + "ldr x9, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" + "mul x19, x19, x23\n" // offset *= kernel_stride * output_size + "ldr x28, [%x[params_struct], %[offsetof_args_outptr]]\n" + "add x10, x10, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "add x28, x11, x20, LSL #2\n" + "add x27, x10, x22, LSL #2\n" "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "add x27, x28, x20, LSL #2\n" - "ld1w { z16.s }, p3/Z, [x15]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" - "add x26, x27, x20, LSL #2\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" - "add x25, x13, x13\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" - "add x24, x25, x13\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" - "add x14, x14, x13, LSL #4\n" - "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" - "add x12, x12, x25, LSL #4\n" - "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" - "cntb x23\n" - "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" - "add x23, x23, x24, LSL #4\n" - "prfm pldl1keep, [x28, x14]\n" - "mov x20, #0x2\n" - "prfm pldl1keep, [x11, x16]\n" - "mul x19, x8, x22\n" // offset = tile_i * ld_output_row - "prfm pldl1keep, [x11, x23]\n" - "madd x19, x17, x10, x19\n" // offset += tile_j * ld_output_col - "prfm pldl1keep, [x28, x12]\n" - "mul x19, x19, x20\n" // offset *= output_tile_size - "prfm pldl1keep, [x27, x14]\n" - "add x9, x9, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "mov x21, #0x0\n" - "add x22, x9, x22, LSL #2\n" - "cntw x20\n" - "sub x19, XZR, x20\n" + "add x26, x27, x22, LSL #2\n" + "ld1w { z16.s }, p3/Z, [x14]\n" + "add x25, x26, x22, LSL #2\n" + "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" + "add x24, x11, x11\n" + "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" + "add x23, x24, x11\n" + "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" + "mul x19, x17, x20\n" // offset = tile_i * ld_output_row + "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" + "madd x19, x16, x9, x19\n" // offset += tile_j * ld_output_col + "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" + "mul x19, x19, x15\n" // offset *= output_tile_size + "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" + "add x28, x28, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) + "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" + "add x22, x28, x20, LSL #2\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x11]\n" - "addvl x15, x15, #16\n" - "ld1w { z11.s }, p2/Z, [x11, x24, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" - "addvl x15, x15, #-6\n" - "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x27, x11, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10]\n" + "addvl x14, x14, #16\n" + "ld1w { z11.s }, p2/Z, [x10, x23, LSL #2]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" + "addvl x14, x14, #-6\n" + "ld1w { z12.s }, p2/Z, [x27, x24, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x26, x11, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x16]\n" - "whilelt p1.s, x20, %x[n_channels]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x23]\n" - "incw x19\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x14]\n" - "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" "incw x21\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "mov p0.b, p2.b\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x25]\n" + "incw x13\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x11, x12]\n" - "incw x20\n" + "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n" + "incw x12\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x27, x12]\n" + "ld1w { z16.s }, p3/Z, [x14]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x28, x16]\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x28, x23]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n" + "addvl x10, x10, #1\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "addvl x11, x11, #1\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x16]\n" - "prfm pldl1keep, [x27, x23]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" - "prfm pldl1keep, [x26, x14]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x26, x12]\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28]\n" + "ld1w { z11.s }, p2/Z, [x27]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z16.s }, p3/Z, [x15]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n" + "addvl x27, x27, #1\n" "fmla z29.s, p3/M, z5.s, z10.s\n" - "addvl x28, x28, #1\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x28, x14]\n" - "prfm pldl1keep, [x11, x16]\n" + "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x11, x23]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x27]\n" + "ld1w { z9.s }, p2/Z, [x26]\n" + "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "prfm pldl1keep, [x28, x12]\n" + "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" "fmla z28.s, p3/M, z2.s, z12.s\n" - "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" "fmla z31.s, p3/M, z8.s, z10.s\n" - "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n" + "addvl x26, x26, #1\n" "fmla z29.s, p3/M, z3.s, z9.s\n" - "addvl x27, x27, #1\n" + "ld1w { z13.s }, p1/Z, [x26, x11, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" - "ld1w { z13.s }, p1/Z, [x27, x13, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n" - "whilelt p2.s, x21, %x[n_channels]\n" + "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z29.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x27, x14]\n" - "addvl x26, x26, #1\n" + "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" + "addvl x25, x25, #1\n" "fmla z31.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p1/Z, [x28, x13, LSL #2]\n" - "cmp x20, %x[n_channels]\n" + "ld1w { z9.s }, p1/Z, [x27, x11, LSL #2]\n" + "cmp x12, %x[n_channels]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p1/Z, [x11]\n" + "ld1w { z10.s }, p1/Z, [x10]\n" "fmla z28.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x11, x24, LSL #2]\n" - "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z11.s }, p1/Z, [x10, x23, LSL #2]\n" + "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "addvl x14, x14, #16\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" + "ld1w { z12.s }, p1/Z, [x27, x24, LSL #2]\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x28, x25, LSL #2]\n" - "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" + "addvl x14, x14, #-6\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "addvl x15, x15, #16\n" + "st1w { z31.s }, p0, [x28]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" - "addvl x15, x15, #-6\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z31.s }, p0, [x9]\n" - "mov z31.d, z16.d\n" + "st1w { z30.s }, p0, [x28, x9, LSL #2]\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" + "addvl x28, x28, #1\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z30.s }, p0, [x9, x10, LSL #2]\n" - "addvl x9, x9, #1\n" - "mov z30.d, z16.d\n" "st1w { z29.s }, p0, [x22]\n" - "mov z29.d, z16.d\n" - "st1w { z28.s }, p0, [x22, x10, LSL #2]\n" - "mov z28.d, z16.d\n" + "st1w { z28.s }, p0, [x22, x9, LSL #2]\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x16]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "ldr x17, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x23]\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x14]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26]\n" - "prfm pldl1keep, [x11, x12]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" + "ldr x16, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "add x21, x17, #0x1\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x25]\n" + "add x16, x16, #0x1\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x25, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x24, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x23, LSL #2]\n" + "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x26, x24, LSL #2]\n" - "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x27, x12]\n" - "prfm pldl1keep, [x28, x16]\n" + "cmp x16, x19\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x28, x23]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" + "csel x16, x16, XZR, LT\n" + "fmla z28.s, p3/M, z1.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x10, x11, LSL #2]\n" + "csel x17, x17, x21, LT\n" "fmla z29.s, p3/M, z6.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x11, x13, LSL #2]\n" - "fmla z28.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x11, x25, LSL #2]\n" - "prfm pldl1keep, [x27, x16]\n" + "ld1w { z9.s }, p2/Z, [x10, x24, LSL #2]\n" + "cmp x17, x20\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x23]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" + "fmla z28.s, p3/M, z3.s, z13.s\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x26, x14]\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28]\n" - "prfm pldl1keep, [x26, x12]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ldr x8, [%x[params_struct], %[offsetof_args_tile_i]]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "add x21, x8, #0x1\n" + "ld1w { z12.s }, p2/Z, [x27, x23, LSL #2]\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x27]\n" "fmla z29.s, p3/M, z5.s, z10.s\n" - "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "ldr x17, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "add x17, x17, #0x1\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x27]\n" - "fmla z31.s, p3/M, z8.s, z10.s\n" - "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "ld1w { z9.s }, p2/Z, [x26]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "cmp x17, x19\n" - "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x24, LSL #2]\n" - "fmla z31.s, p3/M, z3.s, z11.s\n" - "csel x17, x17, XZR, LT\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" - "csel x8, x8, x21, LT\n" + "fmla z31.s, p3/M, z8.s, z10.s\n" + "fmla z30.s, p3/M, z7.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x26, x23, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z12.s\n" - "cmp x8, x20\n" - "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x25, LSL #2]\n" - "fmla z31.s, p3/M, z6.s, z9.s\n" "fmla z29.s, p3/M, z3.s, z9.s\n" + "fmla z31.s, p3/M, z3.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x11, LSL #2]\n" + "fmla z30.s, p3/M, z5.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x25, x24, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" - "fmla z30.s, p3/M, z8.s, z10.s\n" "fmla z29.s, p3/M, z7.s, z11.s\n" - "fmax z31.s, p3/M, z31.s, z18.s\n" + "fmla z31.s, p3/M, z6.s, z9.s\n" + "fmla z30.s, p3/M, z8.s, z10.s\n" "fmla z28.s, p3/M, z6.s, z11.s\n" - "fmax z30.s, p3/M, z30.s, z18.s\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x9]\n" + "fmax z31.s, p3/M, z31.s, z18.s\n" + "fmax z30.s, p3/M, z30.s, z18.s\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "fmin z30.s, p3/M, z30.s, z17.s\n" - "st1w { z30.s }, p0, [x9, x10, LSL #2]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" + "fmin z31.s, p3/M, z31.s, z17.s\n" + "st1w { z31.s }, p0, [x28]\n" + "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" + "st1w { z30.s }, p0, [x28, x9, LSL #2]\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" "st1w { z29.s }, p0, [x22]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z28.s }, p0, [x22, x10, LSL #2]\n" + "st1w { z28.s }, p0, [x22, x9, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp index fb41ca0754..77a6c683b0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -54,22 +54,22 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( const float max ) : outptrs(outptrs), params(params), min(min), max(max) { - inptrs[0] = input_ptrs[0]; - inptrs[1] = input_ptrs[1]; - inptrs[2] = input_ptrs[2]; - inptrs[3] = input_ptrs[3]; - inptrs[4] = input_ptrs[4]; - inptrs[5] = input_ptrs[5]; - inptrs[6] = input_ptrs[6]; - inptrs[7] = input_ptrs[7]; - inptrs[8] = input_ptrs[8]; - inptrs[9] = input_ptrs[9]; - inptrs[10] = input_ptrs[10]; - inptrs[11] = input_ptrs[11]; - inptrs[12] = input_ptrs[12]; - inptrs[13] = input_ptrs[13]; - inptrs[14] = input_ptrs[14]; - inptrs[15] = input_ptrs[15]; + inptrs[0] = input_ptrs[5]; + inptrs[1] = input_ptrs[0]; + inptrs[2] = input_ptrs[3]; + inptrs[3] = input_ptrs[6]; + inptrs[4] = input_ptrs[9]; + inptrs[5] = input_ptrs[12]; + inptrs[6] = input_ptrs[15]; + inptrs[7] = input_ptrs[1]; + inptrs[8] = input_ptrs[2]; + inptrs[9] = input_ptrs[10]; + inptrs[10] = input_ptrs[4]; + inptrs[11] = input_ptrs[7]; + inptrs[12] = input_ptrs[8]; + inptrs[13] = input_ptrs[11]; + inptrs[14] = input_ptrs[13]; + inptrs[15] = input_ptrs[14]; } }; @@ -78,216 +78,194 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x2, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x3, [%x[params_struct], %[offsetof_args_params]]\n" - "add x19, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x4, ALL, MUL #2\n" - "ldp x5, x6, [x19, #0x0]\n" - "mov x7, #0x0\n" - "ldp x8, x17, [x19, #0x10]\n" - "cntw x16\n" - "ldp x15, x14, [x19, #0x20]\n" - "sub x13, XZR, x16\n" - "ldp x12, x11, [x19, #0x30]\n" - "whilelt p2.s, XZR, %x[n_channels]\n" - "ldp x10, x9, [x19, #0x40]\n" - "cmp x16, %x[n_channels]\n" - "ldp x28, x27, [x19, #0x50]\n" - "ldp x26, x25, [x19, #0x60]\n" - "ldp x24, x23, [x19, #0x70]\n" - "ldp x22, x21, [x2, #0x0]\n" - "ldp x20, x19, [x2, #0x10]\n" + "mov x13, #0x0\n" "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "ld1w { z16.s }, p3/Z, [x3]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n" - "addvl x3, x3, #16\n" - "ld1w { z9.s }, p2/Z, [x14, x7, LSL #2]\n" - "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n" - "addvl x3, x3, #-6\n" - "prfm pldl1keep, [x14, x4]\n" - "ld1w { z10.s }, p2/Z, [x5, x7, LSL #2]\n" - "prfm pldl1keep, [x5, x4]\n" - "ld1w { z11.s }, p2/Z, [x17, x7, LSL #2]\n" - "prfm pldl1keep, [x17, x4]\n" - "ld1w { z12.s }, p2/Z, [x12, x7, LSL #2]\n" - "prfm pldl1keep, [x12, x4]\n" - "ld1w { z13.s }, p2/Z, [x9, x7, LSL #2]\n" - "prfm pldl1keep, [x9, x4]\n" + "cntw x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" + "whilelt p2.s, XZR, %x[n_channels]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldr x22, [x14, #0x20]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x4]\n" - "whilelt p1.s, x16, %x[n_channels]\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x23, x4]\n" - "incw x13\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x6, x4]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "ldr x21, [x14, #0x28]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x14, #0x30]\n" + "incw x9\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "ldr x19, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n" - "prfm pldl1keep, [x8, x4]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x28, x4]\n" + "ldr x24, [x14, #0x50]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x15, x4]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x11, x4]\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x10, x4]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x4]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" - "prfm pldl1keep, [x25, x4]\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x24, x4]\n" + "ldr x19, [x14, #0x78]\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "addvl x4, x4, #1\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n" - "prfm pldl1keep, [x14, x4]\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x5, x4]\n" + "ldp x24, x23, [x14, #0x10]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x17, x4]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x12, x4]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z13.s }, p1/Z, [x9, x16, LSL #2]\n" + "ldr x22, [x14, #0x20]\n" "fmla z28.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x9, x4]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z8.s, z10.s\n" - "ld1w { z16.s }, p3/Z, [x3]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" - "ld1w { z0.s }, p3/Z, [x3, #1, MUL VL]\n" + "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" - "ld1w { z1.s }, p3/Z, [x3, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n" - "incw x7\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "incw x13\n" "fmla z29.s, p3/M, z7.s, z11.s\n" - "ld1w { z2.s }, p3/Z, [x3, #3, MUL VL]\n" - "whilelt p2.s, x7, %x[n_channels]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z31.s, p3/M, z6.s, z9.s\n" - "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n" "fmla z28.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x17, x16, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p1/Z, [x5, x16, LSL #2]\n" - "ld1w { z3.s }, p3/Z, [x3, #4, MUL VL]\n" + "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "ld1w { z4.s }, p3/Z, [x3, #5, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" + "addvl x15, x15, #16\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x12, x16, LSL #2]\n" - "incw x16\n" + "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n" + "incw x12\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z5.s }, p3/Z, [x3, #6, MUL VL]\n" - "cmp x16, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x3, #7, MUL VL]\n" - "addvl x3, x3, #16\n" - "fmin z31.s, p3/M, z31.s, z17.s\n" - "ld1w { z7.s }, p3/Z, [x3, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x3, #-7, MUL VL]\n" - "addvl x3, x3, #-6\n" + "fmin z31.s, p3/M, z31.s, z17.s\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "st1w { z31.s }, p0, [x22, x13, LSL #2]\n" - "mov z31.d, z16.d\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x21, x13, LSL #2]\n" - "mov z30.d, z16.d\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z29.s }, p0, [x20, x13, LSL #2]\n" - "mov z29.d, z16.d\n" - "st1w { z28.s }, p0, [x19, x13, LSL #2]\n" - "mov z28.d, z16.d\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x26, x4]\n" - "incw x13\n" - "fmla z30.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x23, x4]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z4.s, z9.s\n" + "ldr x21, [x14, #0x28]\n" + "incw x9\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z3.s, z9.s\n" + "ldr x20, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x6, x4]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x26, x7, LSL #2]\n" - "prfm pldl1keep, [x8, x4]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z1.s, z9.s\n" + "ldr x19, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x26, [x14, #0x40]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x7, LSL #2]\n" + "ldr x25, [x14, #0x48]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x28, x4]\n" - "prfm pldl1keep, [x15, x4]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x24, [x14, #0x50]\n" "fmla z31.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x11, x4]\n" + "ldr x23, [x14, #0x58]\n" "fmla z30.s, p3/M, z4.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x6, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z6.s, z9.s\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "ld1w { z9.s }, p2/Z, [x8, x7, LSL #2]\n" - "prfm pldl1keep, [x10, x4]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x27, x4]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z6.s, z13.s\n" - "prfm pldl1keep, [x25, x4]\n" + "ldr x20, [x14, #0x70]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x15, x7, LSL #2]\n" - "prfm pldl1keep, [x24, x4]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x19, [x14, #0x78]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z10.s\n" "fmla z28.s, p3/M, z4.s, z10.s\n" "fmla z31.s, p3/M, z2.s, z9.s\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x10, x7, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" "fmla z28.s, p3/M, z2.s, z12.s\n" "fmla z31.s, p3/M, z8.s, z10.s\n" "fmla z30.s, p3/M, z7.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x27, x7, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x25, x7, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z5.s, z10.s\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x24, x7, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z11.s\n" "fmla z31.s, p3/M, z6.s, z9.s\n" "fmla z28.s, p3/M, z6.s, z11.s\n" @@ -298,21 +276,21 @@ void sve_fp32_nhwc_3x3_s1_output2x2_mla_depthfirst_indirect_impl( "fmax z30.s, p3/M, z30.s, z18.s\n" "fmax z29.s, p3/M, z29.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x22, x13, LSL #2]\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x21, x13, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "st1w { z29.s }, p0, [x20, x13, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z28.s }, p0, [x19, x13, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp index ef5f4187f9..eb9de9fa01 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst +class sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp index 6bc333be41..c485b7dde0 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -88,167 +88,123 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( __asm__ __volatile__( "ptrue p3.b\n" - "mov x3, #0x0\n" - "mov x4, #0x0\n" + "mov x6, #0x0\n" + "mov x7, #0x0\n" "1:" // Tile loop - "str x3, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "mov x22, #0x3\n" - "str x4, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "cntb x5\n" - "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n" - "add x5, x5, XZR, LSL #4\n" - "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" - "cntb x7\n" - "ldr x8, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" - "cntb x17\n" - "ldr x16, [%x[params_struct], %[offsetof_args_inptr]]\n" - "mul x19, x3, x20\n" // offset = tile_i * ld_input_row - "ldr x21, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" - "madd x19, x4, x8, x19\n" // offset += tile_j * ld_input_col - "ldr x15, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" - "mul x19, x19, x22\n" // offset *= kernel_stride * output_size - "ldr x14, [%x[params_struct], %[offsetof_args_outptr]]\n" - "add x16, x16, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) + "str x6, [%x[params_struct], %[offsetof_args_tile_i]]\n" + "mov x24, #0x3\n" + "str x7, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "mov x23, #0x3\n" + "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x17, #0x0\n" + "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" + "cntw x16\n" + "ldr x15, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" + "sub x21, XZR, x16\n" + "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n" + "mul x19, x6, x22\n" // offset = tile_i * ld_input_row + "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" + "madd x19, x7, x15, x19\n" // offset += tile_j * ld_input_col + "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" + "mul x19, x19, x24\n" // offset *= kernel_stride * output_size + "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n" + "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "add x13, x16, x20, LSL #2\n" + "add x11, x14, x22, LSL #2\n" "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "add x12, x13, x20, LSL #2\n" - "ld1w { z16.s }, p3/Z, [x6]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" - "add x11, x12, x20, LSL #2\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" - "add x10, x11, x20, LSL #2\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" - "add x9, x8, x8\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" - "add x28, x9, x8\n" - "mov z27.d, z16.d\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" - "add x27, x28, x8\n" - "mov z26.d, z16.d\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" - "add x7, x7, x8, LSL #4\n" - "mov z25.d, z16.d\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" - "add x17, x17, x9, LSL #4\n" - "mov z24.d, z16.d\n" - "prfm pldl1keep, [x12, x17]\n" - "cntb x26\n" - "mov z23.d, z16.d\n" - "prfm pldl1keep, [x16, x5]\n" - "add x26, x26, x28, LSL #4\n" - "cntb x25\n" - "mov x20, #0x3\n" - "add x25, x25, x27, LSL #4\n" - "prfm pldl1keep, [x16, x25]\n" - "prfm pldl1keep, [x10, x5]\n" - "mul x19, x3, x21\n" // offset = tile_i * ld_output_row - "prfm pldl1keep, [x13, x17]\n" - "madd x19, x4, x15, x19\n" // offset += tile_j * ld_output_col - "add x24, x15, x15\n" - "mul x19, x19, x20\n" // offset *= output_tile_size - "add x14, x14, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "add x23, x14, x21, LSL #2\n" - "add x22, x23, x21, LSL #2\n" - "mov x21, #0x0\n" - "cntw x20\n" - "sub x19, XZR, x20\n" + "add x10, x11, x22, LSL #2\n" + "ld1w { z16.s }, p3/Z, [x8]\n" + "add x9, x10, x22, LSL #2\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" + "add x28, x9, x22, LSL #2\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" + "add x27, x15, x15\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" + "add x26, x27, x15\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "add x25, x26, x15\n" + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "mul x19, x6, x20\n" // offset = tile_i * ld_output_row + "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" + "madd x19, x7, x13, x19\n" // offset += tile_j * ld_output_col + "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" + "mul x19, x19, x23\n" // offset *= output_tile_size + "add x24, x13, x13\n" + "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) + "add x23, x12, x20, LSL #2\n" + "add x22, x23, x20, LSL #2\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z9.s }, p2/Z, [x12, x9, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x16]\n" - "addvl x6, x6, #16\n" - "ld1w { z11.s }, p2/Z, [x16, x27, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" - "ld1w { z12.s }, p2/Z, [x10]\n" - "ld1w { z13.s }, p2/Z, [x13, x9, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x14]\n" + "addvl x8, x8, #16\n" + "ld1w { z11.s }, p2/Z, [x14, x25, LSL #2]\n" + "cmp x16, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" + "addvl x8, x8, #-6\n" + "ld1w { z12.s }, p2/Z, [x28]\n" + "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x10, x25]\n" - "whilelt p1.s, x20, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "prfm pldl1keep, [x12, x7]\n" - "incw x19\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x16, x7]\n" - "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x16, x26]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "whilelt p1.s, x16, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" "incw x21\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x12, x26]\n" - "incw x20\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x13, x5]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x13, x25]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x5]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x11, x17]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "mov p0.b, p2.b\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "incw x17\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "incw x16\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "ld1w { z16.s }, p3/Z, [x8]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x11, x25]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x10, x7]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x13, x7]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "prfm pldl1keep, [x13, x26]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x26]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x7]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x16, x17]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x11, x26]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x12, x5]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x12, x25]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13]\n" + "ld1w { z11.s }, p2/Z, [x11]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x17]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ld1w { z16.s }, p3/Z, [x6]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" "fmla z26.s, p3/M, z4.s, z10.s\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11]\n" + "ld1w { z12.s }, p2/Z, [x9]\n" "fmla z29.s, p3/M, z7.s, z10.s\n" "fmla z24.s, p3/M, z2.s, z10.s\n" "fmla z23.s, p3/M, z1.s, z10.s\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" "fmla z26.s, p3/M, z6.s, z10.s\n" "fmla z25.s, p3/M, z5.s, z10.s\n" @@ -258,198 +214,164 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.s, p3/M, z8.s, z11.s\n" "fmla z25.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n" - "addvl x13, x13, #1\n" + "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" + "addvl x11, x11, #1\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "fmla z30.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" "fmla z30.s, p3/M, z5.s, z11.s\n" "fmla z26.s, p3/M, z1.s, z11.s\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n" - "addvl x16, x16, #1\n" + "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n" + "addvl x14, x14, #1\n" "fmla z24.s, p3/M, z8.s, z13.s\n" - "ld1w { z10.s }, p1/Z, [x16]\n" + "ld1w { z10.s }, p1/Z, [x14]\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n" - "addvl x11, x11, #1\n" + "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" + "addvl x9, x9, #1\n" "fmla z28.s, p3/M, z7.s, z12.s\n" "fmla z27.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12]\n" + "ld1w { z12.s }, p2/Z, [x10]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" "fmla z30.s, p3/M, z1.s, z11.s\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n" - "addvl x12, x12, #1\n" + "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" + "addvl x10, x10, #1\n" "fmla z27.s, p3/M, z8.s, z13.s\n" - "ld1w { z9.s }, p1/Z, [x12, x9, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n" "fmla z26.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x12, x17]\n" "fmla z24.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x16, x5]\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n" - "whilelt p2.s, x21, %x[n_channels]\n" + "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n" + "whilelt p2.s, x17, %x[n_channels]\n" "fmla z31.s, p3/M, z6.s, z12.s\n" - "prfm pldl1keep, [x16, x25]\n" - "addvl x10, x10, #1\n" + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "addvl x28, x28, #1\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x10, x5]\n" - "cmp x20, %x[n_channels]\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "cmp x16, %x[n_channels]\n" "fmla z25.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p1/Z, [x10]\n" + "ld1w { z12.s }, p1/Z, [x28]\n" "fmla z29.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x13, x17]\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" "fmla z26.s, p3/M, z5.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" "fmla z23.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x16, x27, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x14, x25, LSL #2]\n" "fmla z24.s, p3/M, z7.s, z13.s\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" "fmla z25.s, p3/M, z8.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" "fmla z23.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p1/Z, [x13, x9, LSL #2]\n" + "ld1w { z13.s }, p1/Z, [x11, x27, LSL #2]\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" - "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" - "addvl x6, x6, #16\n" + "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" + "addvl x8, x8, #16\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" + "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" + "fmax z29.s, p3/M, z29.s, z18.s\n" + "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" + "addvl x8, x8, #-6\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "st1w { z31.s }, p0, [x14]\n" - "mov z31.d, z16.d\n" + "st1w { z31.s }, p0, [x12]\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x14, x15, LSL #2]\n" - "mov z30.d, z16.d\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z29.s }, p0, [x14, x24, LSL #2]\n" - "mov z29.d, z16.d\n" - "addvl x14, x14, #1\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" + "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" "fmax z27.s, p3/M, z27.s, z18.s\n" - "st1w { z28.s }, p0, [x23]\n" - "mov z28.d, z16.d\n" "fmax z26.s, p3/M, z26.s, z18.s\n" + "st1w { z29.s }, p0, [x12, x24, LSL #2]\n" + "fmin z28.s, p3/M, z28.s, z17.s\n" + "addvl x12, x12, #1\n" "fmax z25.s, p3/M, z25.s, z18.s\n" - "fmax z24.s, p3/M, z24.s, z18.s\n" + "st1w { z28.s }, p0, [x23]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" - "st1w { z27.s }, p0, [x23, x15, LSL #2]\n" - "mov z27.d, z16.d\n" "fmin z26.s, p3/M, z26.s, z17.s\n" + "st1w { z27.s }, p0, [x23, x13, LSL #2]\n" + "fmin z25.s, p3/M, z25.s, z17.s\n" + "fmax z24.s, p3/M, z24.s, z18.s\n" "st1w { z26.s }, p0, [x23, x24, LSL #2]\n" - "mov z26.d, z16.d\n" "addvl x23, x23, #1\n" - "fmin z25.s, p3/M, z25.s, z17.s\n" + "fmax z23.s, p3/M, z23.s, z18.s\n" "st1w { z25.s }, p0, [x22]\n" - "mov z25.d, z16.d\n" "fmin z24.s, p3/M, z24.s, z17.s\n" - "st1w { z24.s }, p0, [x22, x15, LSL #2]\n" - "mov z24.d, z16.d\n" - "fmax z23.s, p3/M, z23.s, z18.s\n" "fmin z23.s, p3/M, z23.s, z17.s\n" + "st1w { z24.s }, p0, [x22, x13, LSL #2]\n" "st1w { z23.s }, p0, [x22, x24, LSL #2]\n" - "mov z23.d, z16.d\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x10, x25]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x6, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "prfm pldl1keep, [x12, x7]\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x16, x7]\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x16, x26]\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x12, x26]\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x13, x5]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x13, x25]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x11, x5]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x11, x17]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x7, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "add x21, x6, #0x1\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" + "add x7, x7, #0x1\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "cmp x7, x19\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "csel x7, x7, XZR, LT\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "csel x6, x6, x21, LT\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "cmp x6, x20\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x12, x28, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x26, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x8, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x15, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x27, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x25, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x11, x25]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x10, x7]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x13, x7]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "prfm pldl1keep, [x13, x26]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x26]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x16, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x14, x15, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x16, x28, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x7]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x16, x17]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x11, x26]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x12, x5]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x12, x25]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13]\n" + "ld1w { z11.s }, p2/Z, [x11]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x10, x17]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x13, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x11, x25, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "ldr x3, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "add x21, x3, #0x1\n" - "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11]\n" - "fmla z29.s, p3/M, z7.s, z10.s\n" - "ldr x4, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "add x4, x4, #0x1\n" - "fmla z30.s, p3/M, z8.s, z10.s\n" - "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" - "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "cmp x4, x19\n" "fmla z26.s, p3/M, z4.s, z10.s\n" + "fmla z30.s, p3/M, z2.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x9]\n" + "fmla z29.s, p3/M, z7.s, z10.s\n" "fmla z24.s, p3/M, z2.s, z10.s\n" - "csel x4, x4, XZR, LT\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x9, LSL #2]\n" - "csel x3, x3, x21, LT\n" + "fmla z30.s, p3/M, z8.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x9, x27, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "cmp x3, x20\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x25, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x8, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x15, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x13, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x11, x15, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" "fmla z26.s, p3/M, z6.s, z10.s\n" "fmla z25.s, p3/M, z5.s, z10.s\n" @@ -459,36 +381,36 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmla z26.s, p3/M, z8.s, z11.s\n" "fmla z25.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x28, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x26, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x28, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "fmla z30.s, p3/M, z3.s, z12.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x8, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x15, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" "fmla z30.s, p3/M, z5.s, z11.s\n" "fmla z26.s, p3/M, z1.s, z11.s\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x16, x9, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x14, x27, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z13.s\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x28, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" "fmla z27.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12]\n" + "ld1w { z12.s }, p2/Z, [x10]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" "fmla z30.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x27, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z13.s\n" "fmla z26.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z5.s, z13.s\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x10, x9, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x27, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z12.s\n" "fmla z28.s, p3/M, z3.s, z12.s\n" "fmla z25.s, p3/M, z0.s, z12.s\n" @@ -502,13 +424,13 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "fmax z30.s, p3/M, z30.s, z18.s\n" "fmax z29.s, p3/M, z29.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x14]\n" + "st1w { z31.s }, p0, [x12]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x14, x15, LSL #2]\n" + "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z18.s\n" "fmax z27.s, p3/M, z27.s, z18.s\n" - "st1w { z29.s }, p0, [x14, x24, LSL #2]\n" + "st1w { z29.s }, p0, [x12, x24, LSL #2]\n" "fmax z26.s, p3/M, z26.s, z18.s\n" "fmax z25.s, p3/M, z25.s, z18.s\n" "fmax z24.s, p3/M, z24.s, z18.s\n" @@ -516,23 +438,23 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_direct_impl( "st1w { z28.s }, p0, [x23]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" "fmin z26.s, p3/M, z26.s, z17.s\n" - "st1w { z27.s }, p0, [x23, x15, LSL #2]\n" + "st1w { z27.s }, p0, [x23, x13, LSL #2]\n" "fmin z25.s, p3/M, z25.s, z17.s\n" "fmin z24.s, p3/M, z24.s, z17.s\n" "st1w { z26.s }, p0, [x23, x24, LSL #2]\n" "fmax z23.s, p3/M, z23.s, z18.s\n" "st1w { z25.s }, p0, [x22]\n" "fmin z23.s, p3/M, z23.s, z17.s\n" - "st1w { z24.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z24.s }, p0, [x22, x13, LSL #2]\n" "st1w { z23.s }, p0, [x22, x24, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp index 3877ae2f03..72b182679d 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -87,413 +87,343 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x17, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" - "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x14, ALL, MUL #2\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "mov x13, #0x0\n" - "ld1w { z16.s }, p3/Z, [x16]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "cntw x12\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" "sub x11, XZR, x12\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "cmp x12, %x[n_channels]\n" - "mov z27.d, z16.d\n" - "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" - "mov z26.d, z16.d\n" - "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" - "mov z25.d, z16.d\n" - "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" - "addvl x16, x16, #16\n" - "mov z24.d, z16.d\n" - "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" - "mov z23.d, z16.d\n" - "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" - "addvl x16, x16, #-6\n" - "ldp x10, x22, [x15, #0x0]\n" - "ldp x9, x28, [x15, #0x10]\n" - "ldr x24, [x15, #0x20]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n" - "prfm pldl1keep, [x10, x14]\n" - "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n" - "prfm pldl1keep, [x22, x14]\n" - "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" - "prfm pldl1keep, [x9, x14]\n" - "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" - "prfm pldl1keep, [x28, x14]\n" - "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n" - "prfm pldl1keep, [x24, x14]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ldr x26, [x14, #0x20]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x15, #0x28]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" "whilelt p1.s, x12, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x23, [x15, #0x30]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" "incw x11\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x26, [x15, #0x38]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x27, x14]\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x23, x14]\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x14]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "ldr x25, [x15, #0x40]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "ldr x19, [x15, #0x48]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "ldr x24, [x15, #0x50]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x24, x14]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "ldr x23, [x15, #0x58]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x22, [x15, #0x60]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x23, x14]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x15, #0x68]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x15, #0x70]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x15, #0x78]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z26.s, p3/M, z4.s, z10.s\n" - "ldr x10, [x15, #0x80]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z10.s\n" - "ldr x22, [x15, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x10, x14]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x9, [x15, #0x90]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x9, x14]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" - "ldr x28, [x15, #0x98]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" - "ldr x24, [x15, #0xa0]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x28, x14]\n" "fmla z25.s, p3/M, z5.s, z10.s\n" - "ldr x27, [x15, #0xa8]\n" "fmla z28.s, p3/M, z8.s, z10.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z24.s, p3/M, z4.s, z10.s\n" - "ldr x23, [x15, #0xb0]\n" "fmla z23.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x27, x14]\n" "fmla z26.s, p3/M, z8.s, z11.s\n" - "ldr x26, [x15, #0xb8]\n" "fmla z25.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "ldr x25, [x15, #0xc0]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ldp x10, x22, [x15, #0x0]\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x25, x14]\n" - "addvl x14, x14, #1\n" "fmla z30.s, p3/M, z5.s, z11.s\n" - "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n" "fmla z26.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x10, x14]\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z13.s\n" - "ld1w { z10.s }, p1/Z, [x22, x12, LSL #2]\n" + "ldr x26, [x14, #0x20]\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x22, x14]\n" "fmla z27.s, p3/M, z6.s, z12.s\n" - "ldp x9, x28, [x15, #0x10]\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "ldr x24, [x15, #0x20]\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x9, x14]\n" "fmla z30.s, p3/M, z1.s, z11.s\n" - "prfm pldl1keep, [x28, x14]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z13.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z26.s, p3/M, z7.s, z13.s\n" - "ldr x22, [x17, #0x0]\n" "fmla z24.s, p3/M, z5.s, z13.s\n" - "ldr x21, [x17, #0x8]\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n" "incw x13\n" "fmla z31.s, p3/M, z6.s, z12.s\n" - "ldr x20, [x17, #0x10]\n" + "ldp x10, x9, [x14, #0x0]\n" "whilelt p2.s, x13, %x[n_channels]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ldr x19, [x17, #0x18]\n" + "ldp x28, x27, [x14, #0x10]\n" "fmla z25.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p1/Z, [x28, x12, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z29.s, p3/M, z8.s, z11.s\n" - "ld1w { z16.s }, p3/Z, [x16]\n" + "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n" "fmla z26.s, p3/M, z5.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n" "fmla z23.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x9, x12, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n" "fmla z25.s, p3/M, z8.s, z13.s\n" - "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n" "fmla z24.s, p3/M, z7.s, z13.s\n" - "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z23.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p1/Z, [x24, x12, LSL #2]\n" + "ld1w { z13.s }, p1/Z, [x26, x12, LSL #2]\n" "incw x12\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" "cmp x12, %x[n_channels]\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" - "addvl x16, x16, #16\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" "fmax z27.s, p3/M, z27.s, z18.s\n" - "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" - "addvl x16, x16, #-6\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" - "mov z31.d, z16.d\n" - "ldr x22, [x17, #0x20]\n" "fmin z29.s, p3/M, z29.s, z17.s\n" + "ldr x22, [x16, #0x20]\n" + "fmax z26.s, p3/M, z26.s, z18.s\n" "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" - "mov z30.d, z16.d\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" - "mov z29.d, z16.d\n" - "ldr x21, [x17, #0x28]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" - "ldr x20, [x17, #0x30]\n" - "fmax z26.s, p3/M, z26.s, z18.s\n" - "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" - "mov z28.d, z16.d\n" - "ldr x19, [x17, #0x38]\n" + "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" + "fmin z26.s, p3/M, z26.s, z17.s\n" + "ldr x21, [x16, #0x28]\n" "fmax z25.s, p3/M, z25.s, z18.s\n" + "ldr x20, [x16, #0x30]\n" + "fmax z24.s, p3/M, z24.s, z18.s\n" + "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" + "fmax z23.s, p3/M, z23.s, z18.s\n" "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" - "mov z27.d, z16.d\n" - "ldr x22, [x17, #0x40]\n" - "fmin z26.s, p3/M, z26.s, z17.s\n" "st1w { z26.s }, p0, [x21, x11, LSL #2]\n" - "mov z26.d, z16.d\n" "fmin z25.s, p3/M, z25.s, z17.s\n" - "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" - "mov z25.d, z16.d\n" - "fmax z24.s, p3/M, z24.s, z18.s\n" - "fmax z23.s, p3/M, z23.s, z18.s\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.s, p3/M, z24.s, z17.s\n" - "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" - "mov z24.d, z16.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.s, p3/M, z23.s, z17.s\n" + "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" + "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" - "mov z23.d, z16.d\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x27, [x15, #0x28]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" "incw x11\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x23, [x15, #0x30]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x26, [x15, #0x38]\n" - "fmla z28.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x27, x14]\n" - "fmla z27.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x23, x14]\n" - "fmla z26.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x26, x14]\n" - "fmla z25.s, p3/M, z2.s, z9.s\n" - "ldr x25, [x15, #0x40]\n" - "fmla z24.s, p3/M, z1.s, z9.s\n" - "ldr x19, [x15, #0x48]\n" - "fmla z23.s, p3/M, z0.s, z9.s\n" - "ldr x24, [x15, #0x50]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z5.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z27, z16\n fmla z27.s, p3/M, z4.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z26, z16\n fmla z26.s, p3/M, z3.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z25, z16\n fmla z25.s, p3/M, z2.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z24, z16\n fmla z24.s, p3/M, z1.s, z9.s\n" + "ldr x26, [x14, #0x60]\n" + "movprfx z23, z16\n fmla z23.s, p3/M, z0.s, z9.s\n" + "ldr x22, [x16, #0x0]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z25.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z13.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x25, [x14, #0x68]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x24, [x14, #0x70]\n" "fmla z29.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x24, x14]\n" + "ldr x9, [x14, #0x88]\n" "fmla z28.s, p3/M, z2.s, z13.s\n" - "ldr x23, [x15, #0x58]\n" + "ldr x21, [x16, #0x8]\n" "fmla z27.s, p3/M, z1.s, z13.s\n" - "ldr x22, [x15, #0x60]\n" + "ldr x20, [x16, #0x10]\n" "fmla z26.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z8.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x23, x14]\n" + "ldr x23, [x14, #0x78]\n" "fmla z30.s, p3/M, z6.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x10, [x14, #0x80]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x15, #0x68]\n" + "ldr x19, [x16, #0x18]\n" "fmla z27.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x15, #0x70]\n" "fmla z25.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x15, #0x78]\n" "fmla z24.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x27, [x14, #0x98]\n" "fmla z27.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z26.s, p3/M, z4.s, z10.s\n" - "ldr x10, [x15, #0x80]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z10.s\n" - "ldr x22, [x15, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z24.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x10, x14]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x9, [x15, #0x90]\n" "fmla z30.s, p3/M, z8.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x22, x14]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x9, x14]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z26.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z25.s, p3/M, z3.s, z12.s\n" - "ldr x28, [x15, #0x98]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z10.s\n" - "ldr x24, [x15, #0xa0]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z26.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x28, x14]\n" "fmla z25.s, p3/M, z5.s, z10.s\n" - "ldr x27, [x15, #0xa8]\n" "fmla z28.s, p3/M, z8.s, z10.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z24.s, p3/M, z4.s, z10.s\n" - "ldr x23, [x15, #0xb0]\n" "fmla z23.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x27, x14]\n" "fmla z26.s, p3/M, z8.s, z11.s\n" - "ldr x26, [x15, #0xb8]\n" "fmla z25.s, p3/M, z7.s, z13.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z24.s, p3/M, z6.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z23.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "ldr x25, [x15, #0xc0]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ldr x22, [x17, #0x0]\n" "fmla z27.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x25, x14]\n" "fmla z30.s, p3/M, z5.s, z11.s\n" - "ldr x21, [x17, #0x8]\n" "fmla z26.s, p3/M, z1.s, z11.s\n" - "ldr x20, [x17, #0x10]\n" "fmla z27.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z13.s\n" - "ldr x19, [x17, #0x18]\n" "fmla z23.s, p3/M, z7.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x27, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" "fmla z27.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" "fmla z24.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z11.s\n" "fmla z30.s, p3/M, z1.s, z11.s\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z13.s\n" "fmla z26.s, p3/M, z7.s, z13.s\n" "fmla z24.s, p3/M, z5.s, z13.s\n" "fmla z23.s, p3/M, z4.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z12.s\n" "fmla z28.s, p3/M, z3.s, z12.s\n" "fmla z25.s, p3/M, z0.s, z12.s\n" @@ -510,21 +440,21 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "ldr x22, [x17, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmax z28.s, p3/M, z28.s, z18.s\n" "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" "fmax z27.s, p3/M, z27.s, z18.s\n" "fmax z26.s, p3/M, z26.s, z18.s\n" "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "ldr x21, [x17, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z25.s, p3/M, z25.s, z18.s\n" - "ldr x20, [x17, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmax z24.s, p3/M, z24.s, z18.s\n" "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" "fmin z27.s, p3/M, z27.s, z17.s\n" "fmin z26.s, p3/M, z26.s, z17.s\n" - "ldr x19, [x17, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z25.s, p3/M, z25.s, z17.s\n" "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" "fmin z24.s, p3/M, z24.s, z17.s\n" @@ -533,15 +463,15 @@ void sve_fp32_nhwc_3x3_s1_output3x3_mla_depthfirst_indirect_impl( "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" "fmin z23.s, p3/M, z23.s, z17.s\n" "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" - "ldr x22, [x17, #0x40]\n" + "ldr x22, [x16, #0x40]\n" "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z16", "z17", "z18", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp index fc9588cd58..d7be9b122e 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst +class sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp index 7df8e481c0..84b4b3b72b 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -114,45 +114,29 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x14, x15, x22, LSL #2\n" "ld1w { z13.s }, p3/Z, [x4]\n" - "mov z31.d, z13.d\n" - "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n" "add x13, x14, x22, LSL #2\n" - "mov z30.d, z13.d\n" - "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x4, #1, MUL VL]\n" "add x12, x13, x22, LSL #2\n" - "mov z29.d, z13.d\n" - "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x4, #2, MUL VL]\n" "add x11, x12, x22, LSL #2\n" - "mov z28.d, z13.d\n" - "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x4, #3, MUL VL]\n" "add x10, x7, x7\n" - "mov z27.d, z13.d\n" - "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x4, #4, MUL VL]\n" "add x9, x10, x7\n" - "mov z26.d, z13.d\n" - "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x4, #5, MUL VL]\n" "add x28, x9, x7\n" - "mov z25.d, z13.d\n" - "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x4, #6, MUL VL]\n" "add x27, x28, x7\n" - "mov z24.d, z13.d\n" + "ld1w { z6.s }, p3/Z, [x4, #7, MUL VL]\n" "mul x19, x2, x20\n" // offset = tile_i * ld_output_row - "mov z23.d, z13.d\n" + "add x26, x17, x17\n" "madd x19, x3, x17, x19\n" // offset += tile_j * ld_output_col - "mov z22.d, z13.d\n" "mul x19, x19, x23\n" // offset *= output_tile_size - "mov z21.d, z13.d\n" "add x16, x16, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "mov z20.d, z13.d\n" - "add x26, x16, x20, LSL #2\n" - "mov z19.d, z13.d\n" - "add x25, x26, x20, LSL #2\n" - "mov z18.d, z13.d\n" + "add x25, x16, x20, LSL #2\n" "add x24, x25, x20, LSL #2\n" - "mov z17.d, z13.d\n" - "add x23, x17, x17\n" - "mov z16.d, z13.d\n" - "add x22, x23, x17\n" + "add x23, x24, x20, LSL #2\n" + "add x22, x26, x17\n" "whilelt p2.s, XZR, %x[n_channels]\n" "ld1w { z9.s }, p2/Z, [x14, x10, LSL #2]\n" "ld1w { z10.s }, p2/Z, [x8]\n" @@ -165,39 +149,38 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "ld1w { z12.s }, p2/Z, [x14, x9, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ld1w { z13.s }, p3/Z, [x4]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" "whilelt p1.s, x6, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" "incw x21\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" "mov p0.b, p2.b\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" "incw x5\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" "incw x6\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x11]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" "fmla z29.s, p3/M, z7.s, z12.s\n" "fmla z26.s, p3/M, z5.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" "fmla z22.s, p3/M, z2.s, z12.s\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" "fmla z26.s, p3/M, z7.s, z9.s\n" @@ -206,10 +189,11 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.s, p3/M, z4.s, z9.s\n" "fmla z21.s, p3/M, z3.s, z9.s\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" + "ld1w { z13.s }, p3/Z, [x4]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x15, x27, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" @@ -375,109 +359,93 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "addvl x4, x4, #-6\n" "fmin z31.s, p3/M, z31.s, z14.s\n" "st1w { z31.s }, p0, [x16]\n" - "mov z31.d, z13.d\n" "fmin z30.s, p3/M, z30.s, z14.s\n" - "st1w { z30.s }, p0, [x16, x17, LSL #2]\n" - "mov z30.d, z13.d\n" "fmin z29.s, p3/M, z29.s, z14.s\n" - "st1w { z29.s }, p0, [x16, x23, LSL #2]\n" - "mov z29.d, z13.d\n" + "st1w { z30.s }, p0, [x16, x17, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z15.s\n" "fmax z27.s, p3/M, z27.s, z15.s\n" + "st1w { z29.s }, p0, [x16, x26, LSL #2]\n" "fmax z26.s, p3/M, z26.s, z15.s\n" "fmax z25.s, p3/M, z25.s, z15.s\n" + "fmax z24.s, p3/M, z24.s, z15.s\n" "fmin z28.s, p3/M, z28.s, z14.s\n" "st1w { z28.s }, p0, [x16, x22, LSL #2]\n" - "mov z28.d, z13.d\n" - "addvl x16, x16, #1\n" "fmin z27.s, p3/M, z27.s, z14.s\n" - "st1w { z27.s }, p0, [x26]\n" - "mov z27.d, z13.d\n" + "addvl x16, x16, #1\n" "fmin z26.s, p3/M, z26.s, z14.s\n" - "st1w { z26.s }, p0, [x26, x17, LSL #2]\n" - "mov z26.d, z13.d\n" + "st1w { z27.s }, p0, [x25]\n" "fmin z25.s, p3/M, z25.s, z14.s\n" - "st1w { z25.s }, p0, [x26, x23, LSL #2]\n" - "mov z25.d, z13.d\n" - "fmax z24.s, p3/M, z24.s, z15.s\n" + "fmin z24.s, p3/M, z24.s, z14.s\n" + "st1w { z26.s }, p0, [x25, x17, LSL #2]\n" "fmax z23.s, p3/M, z23.s, z15.s\n" + "st1w { z25.s }, p0, [x25, x26, LSL #2]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" "fmax z21.s, p3/M, z21.s, z15.s\n" - "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z24.s }, p0, [x26, x22, LSL #2]\n" - "mov z24.d, z13.d\n" - "addvl x26, x26, #1\n" + "st1w { z24.s }, p0, [x25, x22, LSL #2]\n" + "addvl x25, x25, #1\n" "fmin z23.s, p3/M, z23.s, z14.s\n" - "st1w { z23.s }, p0, [x25]\n" - "mov z23.d, z13.d\n" + "st1w { z23.s }, p0, [x24]\n" "fmin z22.s, p3/M, z22.s, z14.s\n" - "st1w { z22.s }, p0, [x25, x17, LSL #2]\n" - "mov z22.d, z13.d\n" "fmin z21.s, p3/M, z21.s, z14.s\n" - "st1w { z21.s }, p0, [x25, x23, LSL #2]\n" - "mov z21.d, z13.d\n" + "st1w { z22.s }, p0, [x24, x17, LSL #2]\n" "fmax z20.s, p3/M, z20.s, z15.s\n" "fmax z19.s, p3/M, z19.s, z15.s\n" + "st1w { z21.s }, p0, [x24, x26, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" "fmax z17.s, p3/M, z17.s, z15.s\n" + "fmax z16.s, p3/M, z16.s, z15.s\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "st1w { z20.s }, p0, [x25, x22, LSL #2]\n" - "mov z20.d, z13.d\n" - "addvl x25, x25, #1\n" + "st1w { z20.s }, p0, [x24, x22, LSL #2]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "st1w { z19.s }, p0, [x24]\n" - "mov z19.d, z13.d\n" + "addvl x24, x24, #1\n" "fmin z18.s, p3/M, z18.s, z14.s\n" - "st1w { z18.s }, p0, [x24, x17, LSL #2]\n" - "mov z18.d, z13.d\n" + "st1w { z19.s }, p0, [x23]\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "st1w { z17.s }, p0, [x24, x23, LSL #2]\n" - "mov z17.d, z13.d\n" - "fmax z16.s, p3/M, z16.s, z15.s\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z16.s }, p0, [x24, x22, LSL #2]\n" - "mov z16.d, z13.d\n" - "addvl x24, x24, #1\n" + "st1w { z18.s }, p0, [x23, x17, LSL #2]\n" + "st1w { z17.s }, p0, [x23, x26, LSL #2]\n" + "st1w { z16.s }, p0, [x23, x22, LSL #2]\n" + "addvl x23, x23, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" "ldr x2, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" "ldr x3, [%x[params_struct], %[offsetof_args_tile_j]]\n" "add x21, x2, #0x1\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" "add x3, x3, #0x1\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" "cmp x3, x19\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" "csel x3, x3, XZR, LT\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" "csel x2, x2, x21, LT\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x13, x10, LSL #2]\n" "cmp x2, x20\n" "fmla z31.s, p3/M, z0.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x11]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x11, x27, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" "fmla z29.s, p3/M, z7.s, z12.s\n" "fmla z26.s, p3/M, z5.s, z12.s\n" "fmla z28.s, p3/M, z6.s, z12.s\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" "fmla z22.s, p3/M, z2.s, z12.s\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x8, x7, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" "ld1w { z10.s }, p2/Z, [x13, x9, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" "ld1w { z11.s }, p2/Z, [x8, x28, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" "fmla z26.s, p3/M, z7.s, z9.s\n" @@ -486,8 +454,8 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "fmla z22.s, p3/M, z4.s, z9.s\n" "fmla z21.s, p3/M, z3.s, z9.s\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" "ld1w { z9.s }, p2/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" "fmla z30.s, p3/M, z0.s, z12.s\n" @@ -638,43 +606,43 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( "st1w { z30.s }, p0, [x16, x17, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z14.s\n" "fmax z27.s, p3/M, z27.s, z15.s\n" - "st1w { z29.s }, p0, [x16, x23, LSL #2]\n" + "st1w { z29.s }, p0, [x16, x26, LSL #2]\n" "fmax z26.s, p3/M, z26.s, z15.s\n" "st1w { z28.s }, p0, [x16, x22, LSL #2]\n" "fmin z27.s, p3/M, z27.s, z14.s\n" "fmax z25.s, p3/M, z25.s, z15.s\n" - "st1w { z27.s }, p0, [x26]\n" + "st1w { z27.s }, p0, [x25]\n" "fmin z26.s, p3/M, z26.s, z14.s\n" "fmin z25.s, p3/M, z25.s, z14.s\n" - "st1w { z26.s }, p0, [x26, x17, LSL #2]\n" + "st1w { z26.s }, p0, [x25, x17, LSL #2]\n" "fmax z24.s, p3/M, z24.s, z15.s\n" "fmax z23.s, p3/M, z23.s, z15.s\n" - "st1w { z25.s }, p0, [x26, x23, LSL #2]\n" + "st1w { z25.s }, p0, [x25, x26, LSL #2]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" "fmax z21.s, p3/M, z21.s, z15.s\n" "fmax z20.s, p3/M, z20.s, z15.s\n" "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z24.s }, p0, [x26, x22, LSL #2]\n" + "st1w { z24.s }, p0, [x25, x22, LSL #2]\n" "fmin z23.s, p3/M, z23.s, z14.s\n" "fmin z22.s, p3/M, z22.s, z14.s\n" - "st1w { z23.s }, p0, [x25]\n" + "st1w { z23.s }, p0, [x24]\n" "fmin z21.s, p3/M, z21.s, z14.s\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "st1w { z22.s }, p0, [x25, x17, LSL #2]\n" + "st1w { z22.s }, p0, [x24, x17, LSL #2]\n" "fmax z19.s, p3/M, z19.s, z15.s\n" - "st1w { z21.s }, p0, [x25, x23, LSL #2]\n" + "st1w { z21.s }, p0, [x24, x26, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" "fmax z17.s, p3/M, z17.s, z15.s\n" - "st1w { z20.s }, p0, [x25, x22, LSL #2]\n" + "st1w { z20.s }, p0, [x24, x22, LSL #2]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "st1w { z19.s }, p0, [x24]\n" + "st1w { z19.s }, p0, [x23]\n" "fmin z18.s, p3/M, z18.s, z14.s\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "st1w { z18.s }, p0, [x24, x17, LSL #2]\n" + "st1w { z18.s }, p0, [x23, x17, LSL #2]\n" "fmax z16.s, p3/M, z16.s, z15.s\n" - "st1w { z17.s }, p0, [x24, x23, LSL #2]\n" + "st1w { z17.s }, p0, [x23, x26, LSL #2]\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z16.s }, p0, [x24, x22, LSL #2]\n" + "st1w { z16.s }, p0, [x23, x22, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) @@ -685,4 +653,4 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp index 22e12a7b9a..158d44046c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -98,644 +98,538 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( activation_min, activation_max); __asm__ __volatile__( - "ldr x5, [%x[params_struct], %[offsetof_args_outptrs]]\n" + "ldr x16, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x6, [%x[params_struct], %[offsetof_args_params]]\n" - "add x7, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z15.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x8, ALL, MUL #2\n" + "mov x13, #0x0\n" "ld1rw { z14.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "mov x17, #0x0\n" - "ld1w { z13.s }, p3/Z, [x6]\n" - "mov z31.d, z13.d\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" - "cntw x16\n" - "mov z30.d, z13.d\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" - "sub x15, XZR, x16\n" - "mov z29.d, z13.d\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" + "cntw x12\n" + "ld1w { z13.s }, p3/Z, [x15]\n" + "sub x11, XZR, x12\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z28.d, z13.d\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" - "cmp x16, %x[n_channels]\n" - "mov z27.d, z13.d\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" - "mov z26.d, z13.d\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" - "mov z25.d, z13.d\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" - "addvl x6, x6, #16\n" - "mov z24.d, z13.d\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "mov z23.d, z13.d\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" - "mov z22.d, z13.d\n" - "ldp x14, x13, [x7, #0x0]\n" - "mov z21.d, z13.d\n" - "ldp x12, x11, [x7, #0x10]\n" - "mov z20.d, z13.d\n" - "ld1w { z9.s }, p2/Z, [x14, x17, LSL #2]\n" - "mov z19.d, z13.d\n" - "mov z18.d, z13.d\n" - "prfm pldl1keep, [x14, x8]\n" - "mov z17.d, z13.d\n" - "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n" - "mov z16.d, z13.d\n" - "prfm pldl1keep, [x13, x8]\n" - "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n" - "prfm pldl1keep, [x12, x8]\n" - "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n" - "prfm pldl1keep, [x11, x8]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x10, x9, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1w { z9.s }, p2/Z, [x10, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "ldp x28, x27, [x14, #0x10]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x7, #0x20]\n" - "whilelt p1.s, x16, %x[n_channels]\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x21, [x7, #0x28]\n" - "incw x15\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x20, [x7, #0x30]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x20]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" + "incw x11\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" "mov p0.b, p2.b\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x20, x8]\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" - "ldr x19, [x7, #0x38]\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" - "ldr x10, [x7, #0x40]\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x19, x8]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.s, p3/M, z5.s, z12.s\n" - "ldr x9, [x7, #0x48]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ldr x28, [x7, #0x50]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "ldr x27, [x7, #0x58]\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x27, x8]\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" - "ldr x26, [x7, #0x60]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.s, p3/M, z7.s, z9.s\n" - "ldr x25, [x7, #0x68]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.s, p3/M, z6.s, z9.s\n" - "ldr x24, [x7, #0x70]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x26, x8]\n" "fmla z22.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z21.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x24, x8]\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x7, #0x78]\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "ldr x14, [x7, #0x80]\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x14, x8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z26.s, p3/M, z8.s, z10.s\n" - "ldr x13, [x7, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.s, p3/M, z7.s, z10.s\n" - "ldr x12, [x7, #0x90]\n" + "ld1w { z13.s }, p3/Z, [x15]\n" "fmla z24.s, p3/M, z6.s, z10.s\n" - "ldr x11, [x7, #0x98]\n" "fmla z22.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" "fmla z21.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z20.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x11, x8]\n" "fmla z18.s, p3/M, z2.s, z10.s\n" - "ldr x22, [x7, #0xa0]\n" "fmla z17.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x7, #0xa8]\n" "fmla z16.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" "fmla z28.s, p3/M, z5.s, z12.s\n" - "ldr x20, [x7, #0xb0]\n" "fmla z24.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z6.s, z11.s\n" - "ldr x19, [x7, #0xb8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x20, x8]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x19, x8]\n" "fmla z29.s, p3/M, z3.s, z10.s\n" - "ldr x10, [x7, #0xc0]\n" "fmla z27.s, p3/M, z2.s, z10.s\n" - "ldr x9, [x7, #0xc8]\n" "fmla z26.s, p3/M, z1.s, z10.s\n" - "ldr x28, [x7, #0xd0]\n" "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z20.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ldr x27, [x7, #0xd8]\n" "fmla z26.s, p3/M, z2.s, z12.s\n" - "ldr x26, [x7, #0xe0]\n" "fmla z25.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x7, #0xe8]\n" "fmla z24.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z19.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x27, x8]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z10.s\n" - "prfm pldl1keep, [x26, x8]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z27.s, p3/M, z4.s, z10.s\n" - "ldr x24, [x7, #0xf0]\n" "fmla z26.s, p3/M, z3.s, z10.s\n" - "ldr x23, [x7, #0xf8]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x14, [x7, #0x100]\n" "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z17.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x24, x8]\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x14, x8]\n" "fmla z25.s, p3/M, z5.s, z12.s\n" - "ldr x13, [x7, #0x108]\n" "fmla z24.s, p3/M, z4.s, z12.s\n" - "ldr x12, [x7, #0x110]\n" "fmla z21.s, p3/M, z2.s, z12.s\n" - "ldr x11, [x7, #0x118]\n" "fmla z20.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z29.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x8]\n" - "addvl x8, x8, #1\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.s, p3/M, z6.s, z11.s\n" - "ldr x22, [x5, #0x0]\n" "fmla z23.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x5, #0x8]\n" "fmla z22.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x5, #0x10]\n" "fmla z19.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x5, #0x18]\n" "fmla z18.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" - "ld1w { z13.s }, p3/Z, [x6]\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z10.s\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.s, p3/M, z3.s, z10.s\n" "fmla z23.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z25.s, p3/M, z8.s, z11.s\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z5.s, z11.s\n" "fmla z20.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z2.s, z11.s\n" "fmla z16.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z28.s, p3/M, z8.s, z12.s\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.s, p3/M, z5.s, z12.s\n" "fmla z20.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z27.s, p3/M, z6.s, z10.s\n" "fmla z23.s, p3/M, z3.s, z10.s\n" "fmla z19.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z22.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z6.s, z11.s\n" "fmla z23.s, p3/M, z8.s, z11.s\n" "fmla z19.s, p3/M, z5.s, z11.s\n" "fmla z18.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z12.s\n" "fmla z20.s, p3/M, z5.s, z12.s\n" "fmla z16.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z19.s, p3/M, z8.s, z10.s\n" "fmla z18.s, p3/M, z7.s, z10.s\n" "fmla z17.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z22.s, p3/M, z8.s, z11.s\n" "fmla z21.s, p3/M, z7.s, z11.s\n" "fmla z20.s, p3/M, z6.s, z11.s\n" "fmla z18.s, p3/M, z5.s, z11.s\n" "fmla z17.s, p3/M, z4.s, z11.s\n" "fmla z16.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z10.s\n" - "ldp x14, x13, [x7, #0x0]\n" + "ldp x10, x9, [x14, #0x0]\n" "fmla z18.s, p3/M, z8.s, z12.s\n" - "ld1w { z9.s }, p1/Z, [x14, x16, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x10, x12, LSL #2]\n" "fmla z17.s, p3/M, z7.s, z12.s\n" "fmla z16.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x14, x8]\n" "fmla z27.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" "fmla z26.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n" - "incw x17\n" + "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n" + "incw x13\n" "fmla z29.s, p3/M, z5.s, z11.s\n" - "ldp x12, x11, [x7, #0x10]\n" - "whilelt p2.s, x17, %x[n_channels]\n" + "ldp x28, x27, [x14, #0x10]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z28.s, p3/M, z4.s, z11.s\n" - "ld1w { z0.s }, p3/Z, [x6, #1, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z25.s, p3/M, z2.s, z11.s\n" - "ld1w { z2.s }, p3/Z, [x6, #3, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z24.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x12, x16, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x28, x12, LSL #2]\n" "fmla z23.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x12, x8]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z22.s, p3/M, z6.s, z12.s\n" - "prfm pldl1keep, [x11, x8]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" "fmla z19.s, p3/M, z4.s, z12.s\n" - "ld1w { z1.s }, p3/Z, [x6, #2, MUL VL]\n" "fmla z18.s, p3/M, z3.s, z12.s\n" - "ld1w { z12.s }, p1/Z, [x11, x16, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x27, x12, LSL #2]\n" "fmla z21.s, p3/M, z8.s, z10.s\n" - "ld1w { z3.s }, p3/Z, [x6, #4, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z20.s, p3/M, z7.s, z10.s\n" - "ld1w { z6.s }, p3/Z, [x6, #7, MUL VL]\n" "fmla z17.s, p3/M, z5.s, z10.s\n" - "ld1w { z5.s }, p3/Z, [x6, #6, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z16.s, p3/M, z4.s, z10.s\n" - "ld1w { z10.s }, p1/Z, [x13, x16, LSL #2]\n" - "incw x16\n" + "ld1w { z10.s }, p1/Z, [x9, x12, LSL #2]\n" + "incw x12\n" "fmax z31.s, p3/M, z31.s, z15.s\n" - "ld1w { z4.s }, p3/Z, [x6, #5, MUL VL]\n" - "addvl x6, x6, #16\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #16\n" "fmax z30.s, p3/M, z30.s, z15.s\n" - "ld1w { z7.s }, p3/Z, [x6, #-8, MUL VL]\n" - "cmp x16, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmax z29.s, p3/M, z29.s, z15.s\n" - "ld1w { z8.s }, p3/Z, [x6, #-7, MUL VL]\n" - "addvl x6, x6, #-6\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmax z28.s, p3/M, z28.s, z15.s\n" "fmax z27.s, p3/M, z27.s, z15.s\n" "fmin z31.s, p3/M, z31.s, z14.s\n" - "st1w { z31.s }, p0, [x22, x15, LSL #2]\n" - "mov z31.d, z13.d\n" + "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z14.s\n" - "ldr x22, [x5, #0x20]\n" "fmin z29.s, p3/M, z29.s, z14.s\n" - "st1w { z30.s }, p0, [x21, x15, LSL #2]\n" - "mov z30.d, z13.d\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.s, p3/M, z28.s, z14.s\n" - "st1w { z29.s }, p0, [x20, x15, LSL #2]\n" - "mov z29.d, z13.d\n" - "ldr x21, [x5, #0x28]\n" + "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" "fmin z27.s, p3/M, z27.s, z14.s\n" - "ldr x20, [x5, #0x30]\n" "fmax z26.s, p3/M, z26.s, z15.s\n" - "st1w { z28.s }, p0, [x19, x15, LSL #2]\n" - "mov z28.d, z13.d\n" - "ldr x19, [x5, #0x38]\n" + "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" "fmax z25.s, p3/M, z25.s, z15.s\n" - "st1w { z27.s }, p0, [x22, x15, LSL #2]\n" - "mov z27.d, z13.d\n" - "ldr x22, [x5, #0x40]\n" - "fmin z26.s, p3/M, z26.s, z14.s\n" - "st1w { z26.s }, p0, [x21, x15, LSL #2]\n" - "mov z26.d, z13.d\n" - "fmin z25.s, p3/M, z25.s, z14.s\n" - "ldr x21, [x5, #0x48]\n" + "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" "fmax z24.s, p3/M, z24.s, z15.s\n" - "st1w { z25.s }, p0, [x20, x15, LSL #2]\n" - "mov z25.d, z13.d\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.s, p3/M, z23.s, z15.s\n" - "ldr x20, [x5, #0x50]\n" + "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" + "fmin z26.s, p3/M, z26.s, z14.s\n" + "ldr x20, [x16, #0x30]\n" + "fmin z25.s, p3/M, z25.s, z14.s\n" + "ldr x19, [x16, #0x38]\n" "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z24.s }, p0, [x19, x15, LSL #2]\n" - "mov z24.d, z13.d\n" + "ldr x22, [x16, #0x40]\n" "fmin z23.s, p3/M, z23.s, z14.s\n" - "ldr x19, [x5, #0x58]\n" + "st1w { z26.s }, p0, [x21, x11, LSL #2]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" - "st1w { z23.s }, p0, [x22, x15, LSL #2]\n" - "mov z23.d, z13.d\n" + "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" "fmax z21.s, p3/M, z21.s, z15.s\n" - "ldr x22, [x5, #0x60]\n" - "fmin z22.s, p3/M, z22.s, z14.s\n" - "st1w { z22.s }, p0, [x21, x15, LSL #2]\n" - "mov z22.d, z13.d\n" - "fmin z21.s, p3/M, z21.s, z14.s\n" - "ldr x21, [x5, #0x68]\n" + "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" "fmax z20.s, p3/M, z20.s, z15.s\n" - "st1w { z21.s }, p0, [x20, x15, LSL #2]\n" - "mov z21.d, z13.d\n" + "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" "fmax z19.s, p3/M, z19.s, z15.s\n" - "ldr x20, [x5, #0x70]\n" + "ldr x21, [x16, #0x48]\n" + "fmin z22.s, p3/M, z22.s, z14.s\n" + "ldr x20, [x16, #0x50]\n" + "fmin z21.s, p3/M, z21.s, z14.s\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "st1w { z20.s }, p0, [x19, x15, LSL #2]\n" - "mov z20.d, z13.d\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "ldr x19, [x5, #0x78]\n" + "st1w { z22.s }, p0, [x21, x11, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" - "st1w { z19.s }, p0, [x22, x15, LSL #2]\n" - "mov z19.d, z13.d\n" + "st1w { z21.s }, p0, [x20, x11, LSL #2]\n" "fmax z17.s, p3/M, z17.s, z15.s\n" + "st1w { z20.s }, p0, [x19, x11, LSL #2]\n" + "fmax z16.s, p3/M, z16.s, z15.s\n" + "st1w { z19.s }, p0, [x22, x11, LSL #2]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.s, p3/M, z18.s, z14.s\n" - "st1w { z18.s }, p0, [x21, x15, LSL #2]\n" - "mov z18.d, z13.d\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "st1w { z17.s }, p0, [x20, x15, LSL #2]\n" - "mov z17.d, z13.d\n" - "fmax z16.s, p3/M, z16.s, z15.s\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z16.s }, p0, [x19, x15, LSL #2]\n" - "mov z16.d, z13.d\n" + "st1w { z18.s }, p0, [x21, x11, LSL #2]\n" + "st1w { z17.s }, p0, [x20, x11, LSL #2]\n" + "st1w { z16.s }, p0, [x19, x11, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x7, #0x20]\n" - "incw x15\n" - "fmla z30.s, p3/M, z7.s, z9.s\n" - "ldr x21, [x7, #0x28]\n" + "movprfx z31, z13\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x20]\n" + "incw x11\n" + "movprfx z30, z13\n fmla z30.s, p3/M, z7.s, z9.s\n" + "ldr x25, [x14, #0x28]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z6.s, z9.s\n" - "ldr x20, [x7, #0x30]\n" - "fmla z27.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" - "fmla z26.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" - "fmla z25.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x20, x8]\n" - "fmla z23.s, p3/M, z2.s, z9.s\n" - "ldr x19, [x7, #0x38]\n" - "fmla z22.s, p3/M, z1.s, z9.s\n" - "ldr x10, [x7, #0x40]\n" - "fmla z21.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x17, LSL #2]\n" + "movprfx z29, z13\n fmla z29.s, p3/M, z6.s, z9.s\n" + "ldr x24, [x14, #0x30]\n" + "movprfx z27, z13\n fmla z27.s, p3/M, z5.s, z9.s\n" + "ldr x23, [x14, #0x38]\n" + "movprfx z26, z13\n fmla z26.s, p3/M, z4.s, z9.s\n" + "ldr x10, [x14, #0x40]\n" + "movprfx z25, z13\n fmla z25.s, p3/M, z3.s, z9.s\n" + "ldr x9, [x14, #0x48]\n" + "movprfx z23, z13\n fmla z23.s, p3/M, z2.s, z9.s\n" + "ldr x28, [x14, #0x50]\n" + "movprfx z22, z13\n fmla z22.s, p3/M, z1.s, z9.s\n" + "ldr x27, [x14, #0x58]\n" + "movprfx z21, z13\n fmla z21.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" + "movprfx z28, z13\n fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x19, x8]\n" + "ldr x26, [x14, #0x60]\n" "fmla z29.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x25, [x14, #0x68]\n" "fmla z26.s, p3/M, z5.s, z12.s\n" - "ldr x9, [x7, #0x48]\n" + "ldr x24, [x14, #0x70]\n" "fmla z28.s, p3/M, z6.s, z12.s\n" - "ldr x28, [x7, #0x50]\n" + "ldr x22, [x16, #0x0]\n" "fmla z25.s, p3/M, z4.s, z12.s\n" - "ldr x27, [x7, #0x58]\n" - "fmla z24.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x21, [x16, #0x8]\n" + "movprfx z24, z13\n fmla z24.s, p3/M, z3.s, z12.s\n" + "ldr x20, [x16, #0x10]\n" "fmla z22.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" + "ldr x19, [x16, #0x18]\n" "fmla z21.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x27, x8]\n" - "fmla z20.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x19, x17, LSL #2]\n" - "fmla z19.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x9, x17, LSL #2]\n" - "fmla z16.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "movprfx z20, z13\n fmla z20.s, p3/M, z0.s, z12.s\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "movprfx z19, z13\n fmla z19.s, p3/M, z6.s, z10.s\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" + "movprfx z16, z13\n fmla z16.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z27.s, p3/M, z8.s, z9.s\n" - "ldr x26, [x7, #0x60]\n" + "ldr x23, [x14, #0x78]\n" "fmla z26.s, p3/M, z7.s, z9.s\n" - "ldr x25, [x7, #0x68]\n" + "ldr x10, [x14, #0x80]\n" "fmla z25.s, p3/M, z6.s, z9.s\n" - "ldr x24, [x7, #0x70]\n" + "ldr x9, [x14, #0x88]\n" "fmla z23.s, p3/M, z5.s, z9.s\n" - "prfm pldl1keep, [x26, x8]\n" "fmla z22.s, p3/M, z4.s, z9.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z21.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x24, x8]\n" "fmla z19.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x7, #0x78]\n" - "fmla z18.s, p3/M, z1.s, z9.s\n" - "ldr x14, [x7, #0x80]\n" - "fmla z17.s, p3/M, z0.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x28, x17, LSL #2]\n" + "movprfx z18, z13\n fmla z18.s, p3/M, z1.s, z9.s\n" + "movprfx z17, z13\n fmla z17.s, p3/M, z0.s, z9.s\n" + "ld1w { z9.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x28, [x14, #0x90]\n" "fmla z30.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x14, x8]\n" + "ldr x27, [x14, #0x98]\n" "fmla z28.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z26.s, p3/M, z8.s, z10.s\n" - "ldr x13, [x7, #0x88]\n" + "ldr x26, [x14, #0xa0]\n" "fmla z25.s, p3/M, z7.s, z10.s\n" - "ldr x12, [x7, #0x90]\n" "fmla z24.s, p3/M, z6.s, z10.s\n" - "ldr x11, [x7, #0x98]\n" "fmla z22.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" "fmla z21.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z20.s, p3/M, z3.s, z10.s\n" - "prfm pldl1keep, [x11, x8]\n" "fmla z18.s, p3/M, z2.s, z10.s\n" - "ldr x22, [x7, #0xa0]\n" "fmla z17.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x7, #0xa8]\n" "fmla z16.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x22, x8]\n" + "ldr x25, [x14, #0xa8]\n" "fmla z27.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x21, x8]\n" "fmla z28.s, p3/M, z5.s, z12.s\n" - "ldr x20, [x7, #0xb0]\n" "fmla z24.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z23.s, p3/M, z6.s, z11.s\n" - "ldr x19, [x7, #0xb8]\n" + "ldr x23, [x14, #0xb8]\n" "fmla z19.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z10.s\n" - "prfm pldl1keep, [x20, x8]\n" + "ldr x24, [x14, #0xb0]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x19, x8]\n" "fmla z29.s, p3/M, z3.s, z10.s\n" - "ldr x10, [x7, #0xc0]\n" "fmla z27.s, p3/M, z2.s, z10.s\n" - "ldr x9, [x7, #0xc8]\n" "fmla z26.s, p3/M, z1.s, z10.s\n" - "ldr x28, [x7, #0xd0]\n" "fmla z25.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z20.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x10, x8]\n" + "ldr x9, [x14, #0xc8]\n" "fmla z16.s, p3/M, z5.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x9, x8]\n" + "ldr x10, [x14, #0xc0]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x28, x8]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ldr x27, [x7, #0xd8]\n" "fmla z26.s, p3/M, z2.s, z12.s\n" - "ldr x26, [x7, #0xe0]\n" "fmla z25.s, p3/M, z1.s, z12.s\n" - "ldr x25, [x7, #0xe8]\n" "fmla z24.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z19.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x27, x8]\n" + "ldr x27, [x14, #0xd8]\n" "fmla z18.s, p3/M, z6.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z10.s\n" - "prfm pldl1keep, [x26, x8]\n" + "ldr x28, [x14, #0xd0]\n" "fmla z30.s, p3/M, z6.s, z10.s\n" - "prfm pldl1keep, [x25, x8]\n" "fmla z27.s, p3/M, z4.s, z10.s\n" - "ldr x24, [x7, #0xf0]\n" "fmla z26.s, p3/M, z3.s, z10.s\n" - "ldr x23, [x7, #0xf8]\n" "fmla z23.s, p3/M, z1.s, z10.s\n" - "ldr x14, [x7, #0x100]\n" "fmla z22.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x22, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z17.s, p3/M, z8.s, z11.s\n" - "prfm pldl1keep, [x24, x8]\n" + "ldr x26, [x14, #0xe0]\n" "fmla z16.s, p3/M, z7.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x21, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z12.s\n" - "prfm pldl1keep, [x23, x8]\n" + "ldr x25, [x14, #0xe8]\n" "fmla z28.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x14, x8]\n" "fmla z25.s, p3/M, z5.s, z12.s\n" - "ldr x13, [x7, #0x108]\n" "fmla z24.s, p3/M, z4.s, z12.s\n" - "ldr x12, [x7, #0x110]\n" "fmla z21.s, p3/M, z2.s, z12.s\n" - "ldr x11, [x7, #0x118]\n" "fmla z20.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x20, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "prfm pldl1keep, [x13, x8]\n" + "ldr x24, [x14, #0xf0]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x12, x8]\n" "fmla z29.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x19, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z27.s, p3/M, z7.s, z11.s\n" - "prfm pldl1keep, [x11, x8]\n" + "ldr x23, [x14, #0xf8]\n" "fmla z26.s, p3/M, z6.s, z11.s\n" - "ldr x22, [x5, #0x0]\n" "fmla z23.s, p3/M, z4.s, z11.s\n" - "ldr x21, [x5, #0x8]\n" "fmla z22.s, p3/M, z3.s, z11.s\n" - "ldr x20, [x5, #0x10]\n" "fmla z19.s, p3/M, z1.s, z11.s\n" - "ldr x19, [x5, #0x18]\n" "fmla z18.s, p3/M, z0.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z12.s\n" + "ldr x10, [x14, #0x100]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x9, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z6.s, z10.s\n" + "ldr x9, [x14, #0x108]\n" "fmla z27.s, p3/M, z3.s, z10.s\n" "fmla z23.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x28, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z25.s, p3/M, z8.s, z11.s\n" + "ldr x28, [x14, #0x110]\n" "fmla z24.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z5.s, z11.s\n" "fmla z20.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z2.s, z11.s\n" "fmla z16.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x27, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z28.s, p3/M, z8.s, z12.s\n" + "ldr x27, [x14, #0x118]\n" "fmla z24.s, p3/M, z5.s, z12.s\n" "fmla z20.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x26, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z27.s, p3/M, z6.s, z10.s\n" "fmla z23.s, p3/M, z3.s, z10.s\n" "fmla z19.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x25, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z22.s, p3/M, z7.s, z11.s\n" "fmla z21.s, p3/M, z6.s, z11.s\n" "fmla z23.s, p3/M, z8.s, z11.s\n" "fmla z19.s, p3/M, z5.s, z11.s\n" "fmla z18.s, p3/M, z4.s, z11.s\n" "fmla z17.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x24, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z24.s, p3/M, z8.s, z12.s\n" "fmla z20.s, p3/M, z5.s, z12.s\n" "fmla z16.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x23, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z19.s, p3/M, z8.s, z10.s\n" "fmla z18.s, p3/M, z7.s, z10.s\n" "fmla z17.s, p3/M, z6.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x14, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x10, x13, LSL #2]\n" "fmla z22.s, p3/M, z8.s, z11.s\n" "fmla z21.s, p3/M, z7.s, z11.s\n" "fmla z20.s, p3/M, z6.s, z11.s\n" "fmla z18.s, p3/M, z5.s, z11.s\n" "fmla z17.s, p3/M, z4.s, z11.s\n" "fmla z16.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x13, x17, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x9, x13, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z10.s\n" "fmla z18.s, p3/M, z8.s, z12.s\n" "fmla z17.s, p3/M, z7.s, z12.s\n" "fmla z16.s, p3/M, z6.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x12, x17, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z10.s\n" "fmla z27.s, p3/M, z1.s, z10.s\n" "fmla z26.s, p3/M, z0.s, z10.s\n" - "ld1w { z10.s }, p2/Z, [x11, x17, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z11.s\n" "fmla z28.s, p3/M, z4.s, z11.s\n" "fmla z25.s, p3/M, z2.s, z11.s\n" @@ -753,68 +647,68 @@ void sve_fp32_nhwc_3x3_s1_output4x4_mla_depthfirst_indirect_impl( "fmax z29.s, p3/M, z29.s, z15.s\n" "fmax z28.s, p3/M, z28.s, z15.s\n" "fmin z31.s, p3/M, z31.s, z14.s\n" - "st1w { z31.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z31.s }, p0, [x22, x11, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z14.s\n" "fmin z29.s, p3/M, z29.s, z14.s\n" - "ldr x22, [x5, #0x20]\n" + "ldr x22, [x16, #0x20]\n" "fmin z28.s, p3/M, z28.s, z14.s\n" - "st1w { z30.s }, p0, [x21, x15, LSL #2]\n" + "st1w { z30.s }, p0, [x21, x11, LSL #2]\n" "fmax z27.s, p3/M, z27.s, z15.s\n" "fmax z26.s, p3/M, z26.s, z15.s\n" - "st1w { z29.s }, p0, [x20, x15, LSL #2]\n" + "st1w { z29.s }, p0, [x20, x11, LSL #2]\n" "fmax z25.s, p3/M, z25.s, z15.s\n" - "st1w { z28.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z28.s }, p0, [x19, x11, LSL #2]\n" "fmax z24.s, p3/M, z24.s, z15.s\n" - "ldr x21, [x5, #0x28]\n" + "ldr x21, [x16, #0x28]\n" "fmax z23.s, p3/M, z23.s, z15.s\n" - "ldr x20, [x5, #0x30]\n" + "ldr x20, [x16, #0x30]\n" "fmin z27.s, p3/M, z27.s, z14.s\n" - "ldr x19, [x5, #0x38]\n" + "ldr x19, [x16, #0x38]\n" "fmin z26.s, p3/M, z26.s, z14.s\n" - "st1w { z27.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z27.s }, p0, [x22, x11, LSL #2]\n" "fmin z25.s, p3/M, z25.s, z14.s\n" "fmin z24.s, p3/M, z24.s, z14.s\n" - "st1w { z26.s }, p0, [x21, x15, LSL #2]\n" + "st1w { z26.s }, p0, [x21, x11, LSL #2]\n" "fmin z23.s, p3/M, z23.s, z14.s\n" - "ldr x22, [x5, #0x40]\n" + "ldr x22, [x16, #0x40]\n" "fmax z22.s, p3/M, z22.s, z15.s\n" - "ldr x21, [x5, #0x48]\n" + "ldr x21, [x16, #0x48]\n" "fmax z21.s, p3/M, z21.s, z15.s\n" - "st1w { z25.s }, p0, [x20, x15, LSL #2]\n" + "st1w { z25.s }, p0, [x20, x11, LSL #2]\n" "fmax z20.s, p3/M, z20.s, z15.s\n" - "st1w { z24.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z24.s }, p0, [x19, x11, LSL #2]\n" "fmax z19.s, p3/M, z19.s, z15.s\n" - "st1w { z23.s }, p0, [x22, x15, LSL #2]\n" + "st1w { z23.s }, p0, [x22, x11, LSL #2]\n" "fmin z22.s, p3/M, z22.s, z14.s\n" - "ldr x20, [x5, #0x50]\n" + "ldr x20, [x16, #0x50]\n" "fmin z21.s, p3/M, z21.s, z14.s\n" - "ldr x19, [x5, #0x58]\n" + "ldr x19, [x16, #0x58]\n" "fmin z20.s, p3/M, z20.s, z14.s\n" - "ldr x22, [x5, #0x60]\n" + "ldr x22, [x16, #0x60]\n" "fmin z19.s, p3/M, z19.s, z14.s\n" - "st1w { z22.s }, p0, [x21, x15, LSL #2]\n" + "st1w { z22.s }, p0, [x21, x11, LSL #2]\n" "fmax z18.s, p3/M, z18.s, z15.s\n" - "st1w { z21.s }, p0, [x20, x15, LSL #2]\n" + "st1w { z21.s }, p0, [x20, x11, LSL #2]\n" "fmax z17.s, p3/M, z17.s, z15.s\n" - "st1w { z20.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z20.s }, p0, [x19, x11, LSL #2]\n" "fmax z16.s, p3/M, z16.s, z15.s\n" - "st1w { z19.s }, p0, [x22, x15, LSL #2]\n" - "ldr x21, [x5, #0x68]\n" + "st1w { z19.s }, p0, [x22, x11, LSL #2]\n" + "ldr x21, [x16, #0x68]\n" "fmin z18.s, p3/M, z18.s, z14.s\n" - "ldr x20, [x5, #0x70]\n" + "ldr x20, [x16, #0x70]\n" "fmin z17.s, p3/M, z17.s, z14.s\n" - "ldr x19, [x5, #0x78]\n" + "ldr x19, [x16, #0x78]\n" "fmin z16.s, p3/M, z16.s, z14.s\n" - "st1w { z18.s }, p0, [x21, x15, LSL #2]\n" - "st1w { z17.s }, p0, [x20, x15, LSL #2]\n" - "st1w { z16.s }, p0, [x19, x15, LSL #2]\n" + "st1w { z18.s }, p0, [x21, x11, LSL #2]\n" + "st1w { z17.s }, p0, [x20, x11, LSL #2]\n" + "st1w { z16.s }, p0, [x19, x11, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp index 2119c06965..28d44d0b5f 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst +class sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 5; constexpr static unsigned int input_cols = 5; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl; - sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp index 78e67e1be1..ac33dcbce5 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -88,318 +88,250 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_direct_impl( __asm__ __volatile__( "ptrue p3.b\n" - "mov x5, #0x0\n" - "mov x6, #0x0\n" + "mov x7, #0x0\n" + "mov x8, #0x0\n" "1:" // Tile loop - "str x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "mov x21, #0x4\n" - "str x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "cntb x7\n" - "ldr x8, [%x[params_struct], %[offsetof_args_params]]\n" - "add x7, x7, XZR, LSL #4\n" - "ldr x20, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" - "cntb x17\n" - "ldr x16, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" - "cntb x15\n" - "ldr x14, [%x[params_struct], %[offsetof_args_inptr]]\n" - "mul x19, x5, x20\n" // offset = tile_i * ld_input_row - "ldr x22, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" - "madd x19, x6, x16, x19\n" // offset += tile_j * ld_input_col - "ldr x13, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" - "mul x19, x19, x21\n" // offset *= kernel_stride * output_size - "ldr x12, [%x[params_struct], %[offsetof_args_outptr]]\n" - "add x14, x14, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) + "str x7, [%x[params_struct], %[offsetof_args_tile_i]]\n" + "mov x23, #0x4\n" + "str x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "mov x17, #0x2\n" + "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" + "mov x15, #0x0\n" + "ldr x22, [%x[params_struct], %[offsetof_args_ld_input_row]]\n" + "cntw x14\n" + "ldr x13, [%x[params_struct], %[offsetof_args_ld_input_col]]\n" + "sub x12, XZR, x14\n" + "ldr x21, [%x[params_struct], %[offsetof_args_inptr]]\n" + "mul x19, x7, x22\n" // offset = tile_i * ld_input_row + "ldr x20, [%x[params_struct], %[offsetof_args_ld_output_row]]\n" + "madd x19, x8, x13, x19\n" // offset += tile_j * ld_input_col + "ldr x11, [%x[params_struct], %[offsetof_args_ld_output_col]]\n" + "mul x19, x19, x23\n" // offset *= kernel_stride * output_size + "ldr x10, [%x[params_struct], %[offsetof_args_outptr]]\n" + "add x21, x21, x19, LSL #2\n" // inptr[0] += offset * sizeof(float) "ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "add x11, x14, x20, LSL #2\n" + "add x9, x21, x22, LSL #2\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "add x10, x11, x20, LSL #2\n" - "ld1w { z17.s }, p3/Z, [x8]\n" - "mov z31.d, z17.d\n" - "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" - "add x9, x10, x20, LSL #2\n" - "mov z30.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" - "add x28, x9, x20, LSL #2\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" - "add x27, x16, x16\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" - "add x26, x27, x16\n" - "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" - "add x25, x26, x16\n" - "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" - "add x17, x17, x16, LSL #4\n" - "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" - "add x15, x15, x27, LSL #4\n" - "cntb x24\n" - "prfm pldl1keep, [x10, x15]\n" - "prfm pldl1keep, [x14, x7]\n" - "add x24, x24, x26, LSL #4\n" - "prfm pldl1keep, [x14, x17]\n" - "cntb x23\n" - "prfm pldl1keep, [x14, x24]\n" - "add x23, x23, x25, LSL #4\n" - "mov x20, #0x2\n" - "prfm pldl1keep, [x14, x23]\n" - "prfm pldl1keep, [x11, x7]\n" - "mul x19, x5, x22\n" // offset = tile_i * ld_output_row - "prfm pldl1keep, [x11, x17]\n" - "madd x19, x6, x13, x19\n" // offset += tile_j * ld_output_col - "prfm pldl1keep, [x14, x15]\n" - "mul x19, x19, x20\n" // offset *= output_tile_size - "mov x21, #0x0\n" - "add x12, x12, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) - "add x22, x12, x22, LSL #2\n" - "cntw x20\n" - "sub x19, XZR, x20\n" + "add x28, x9, x22, LSL #2\n" + "ld1w { z17.s }, p3/Z, [x16]\n" + "add x27, x28, x22, LSL #2\n" + "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "add x26, x27, x22, LSL #2\n" + "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "add x25, x13, x13\n" + "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "add x24, x25, x13\n" + "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "add x23, x24, x13\n" + "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" + "mul x19, x7, x20\n" // offset = tile_i * ld_output_row + "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" + "madd x19, x8, x11, x19\n" // offset += tile_j * ld_output_col + "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" + "mul x19, x19, x17\n" // offset *= output_tile_size "whilelt p2.s, XZR, %x[n_channels]\n" - "ld1w { z9.s }, p2/Z, [x10, x27, LSL #2]\n" - "ld1w { z10.s }, p2/Z, [x14]\n" - "addvl x8, x8, #16\n" - "ld1w { z11.s }, p2/Z, [x14, x16, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" - "addvl x8, x8, #-6\n" - "ld1w { z12.s }, p2/Z, [x14, x26, LSL #2]\n" - "ld1w { z13.s }, p2/Z, [x14, x25, LSL #2]\n" - "ld1w { z14.s }, p2/Z, [x11]\n" - "ld1w { z15.s }, p2/Z, [x11, x16, LSL #2]\n" - "ld1w { z16.s }, p2/Z, [x14, x27, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x28, x25, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x21]\n" + "add x10, x10, x19, LSL #2\n" // outptrs[0] += offset * sizeof(float) + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" + "add x22, x10, x20, LSL #2\n" + "ld1w { z12.s }, p2/Z, [x21, x24, LSL #2]\n" + "addvl x16, x16, #16\n" + "ld1w { z13.s }, p2/Z, [x21, x23, LSL #2]\n" + "cmp x14, %x[n_channels]\n" + "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" + "addvl x16, x16, #-6\n" + "ld1w { z14.s }, p2/Z, [x9]\n" + "ld1w { z15.s }, p2/Z, [x9, x13, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x25, LSL #2]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x11, x24]\n" - "whilelt p1.s, x20, %x[n_channels]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x11, x23]\n" - "incw x19\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x11, x15]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "whilelt p1.s, x14, %x[n_channels]\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "incw x12\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x9, x7]\n" - "incw x21\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ld1w { z17.s }, p3/Z, [x16]\n" + "incw x15\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x10, x7]\n" - "addvl x14, x14, #1\n" + "addvl x21, x21, #1\n" + "ld1w { z10.s }, p1/Z, [x21]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n" - "incw x20\n" + "incw x14\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" - "prfm pldl1keep, [x9, x17]\n" + "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x9]\n" - "addvl x11, x11, #1\n" + "ld1w { z14.s }, p2/Z, [x27]\n" + "addvl x9, x9, #1\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "prfm pldl1keep, [x10, x17]\n" - "prfm pldl1keep, [x9, x24]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "prfm pldl1keep, [x10, x24]\n" - "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x10]\n" + "ld1w { z15.s }, p2/Z, [x28]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "prfm pldl1keep, [x9, x23]\n" - "prfm pldl1keep, [x28, x7]\n" + "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" - "prfm pldl1keep, [x10, x23]\n" + "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n" + "addvl x28, x28, #1\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "addvl x10, x10, #1\n" + "ld1w { z9.s }, p1/Z, [x28, x25, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" - "prfm pldl1keep, [x28, x17]\n" + "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "prfm pldl1keep, [x9, x15]\n" - "prfm pldl1keep, [x28, x24]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "ld1w { z15.s }, p2/Z, [x28]\n" + "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n" - "prfm pldl1keep, [x28, x15]\n" + "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" + "fmla z29.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n" - "addvl x9, x9, #1\n" + "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n" + "addvl x27, x27, #1\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x28, x23]\n" - "fmla z29.s, p3/M, z6.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n" + "ld1w { z12.s }, p1/Z, [x21, x24, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ld1w { z9.s }, p1/Z, [x10, x27, LSL #2]\n" - "prfm pldl1keep, [x10, x15]\n" + "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" + "fmla z29.s, p3/M, z7.s, z13.s\n" + "ld1w { z13.s }, p1/Z, [x21, x23, LSL #2]\n" "fmax z31.s, p3/M, z31.s, z19.s\n" - "ld1w { z10.s }, p1/Z, [x14]\n" "fmla z28.s, p3/M, z5.s, z14.s\n" - "fmla z29.s, p3/M, z7.s, z13.s\n" - "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n" "fmax z30.s, p3/M, z30.s, z19.s\n" - "prfm pldl1keep, [x14, x7]\n" - "prfm pldl1keep, [x14, x17]\n" + "fmla z29.s, p3/M, z5.s, z16.s\n" + "ld1w { z5.s }, p3/Z, [x16, #6, MUL VL]\n" "fmin z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x14, x26, LSL #2]\n" + "st1w { z31.s }, p0, [x10]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "fmla z29.s, p3/M, z5.s, z16.s\n" - "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n" - "whilelt p2.s, x21, %x[n_channels]\n" + "fmla z29.s, p3/M, z8.s, z15.s\n" + "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n" + "whilelt p2.s, x15, %x[n_channels]\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "prfm pldl1keep, [x14, x24]\n" - "addvl x28, x28, #1\n" + "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" + "addvl x26, x26, #1\n" "fmla z28.s, p3/M, z3.s, z16.s\n" - "ld1w { z13.s }, p1/Z, [x14, x25, LSL #2]\n" - "cmp x20, %x[n_channels]\n" - "fmla z29.s, p3/M, z8.s, z15.s\n" - "prfm pldl1keep, [x14, x23]\n" - "prfm pldl1keep, [x11, x7]\n" - "fmla z28.s, p3/M, z7.s, z14.s\n" - "ld1w { z14.s }, p1/Z, [x11]\n" - "prfm pldl1keep, [x11, x17]\n" + "ld1w { z16.s }, p1/Z, [x21, x25, LSL #2]\n" + "cmp x14, %x[n_channels]\n" "fmax z29.s, p3/M, z29.s, z19.s\n" - "ld1w { z16.s }, p1/Z, [x14, x27, LSL #2]\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ld1w { z15.s }, p1/Z, [x11, x16, LSL #2]\n" - "prfm pldl1keep, [x14, x15]\n" + "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" + "st1w { z30.s }, p0, [x10, x11, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z14.s\n" + "ld1w { z14.s }, p1/Z, [x9]\n" + "addvl x10, x10, #1\n" "fmin z29.s, p3/M, z29.s, z18.s\n" - "st1w { z31.s }, p0, [x12]\n" + "st1w { z29.s }, p0, [x22]\n" + "fmla z28.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p1/Z, [x9, x13, LSL #2]\n" "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z11.s }, p1/Z, [x14, x16, LSL #2]\n" - "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" + "ld1w { z11.s }, p1/Z, [x21, x13, LSL #2]\n" + "ld1w { z6.s }, p3/Z, [x16, #7, MUL VL]\n" "fmax z28.s, p3/M, z28.s, z19.s\n" - "st1w { z29.s }, p0, [x22]\n" - "addvl x12, x12, #1\n" + "addvl x16, x16, #16\n" + "ld1w { z7.s }, p3/Z, [x16, #-8, MUL VL]\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z17.s }, p3/Z, [x8]\n" - "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" - "mov z31.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" - "mov z30.d, z17.d\n" - "st1w { z28.s }, p0, [x22, x13, LSL #2]\n" + "ld1w { z8.s }, p3/Z, [x16, #-7, MUL VL]\n" + "addvl x16, x16, #-6\n" + "st1w { z28.s }, p0, [x22, x11, LSL #2]\n" "addvl x22, x22, #1\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x8, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x8, #7, MUL VL]\n" - "addvl x8, x8, #16\n" - "ld1w { z7.s }, p3/Z, [x8, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x8, #-7, MUL VL]\n" - "addvl x8, x8, #-6\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "prfm pldl1keep, [x11, x24]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x7, [%x[params_struct], %[offsetof_args_tile_i]]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "prfm pldl1keep, [x11, x23]\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "prfm pldl1keep, [x11, x15]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x9, x7]\n" - "prfm pldl1keep, [x10, x7]\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "ldr x8, [%x[params_struct], %[offsetof_args_tile_j]]\n" + "add x21, x7, #0x1\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" + "add x8, x8, #0x1\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x9, x17]\n" + "cmp x8, x19\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x11, x25, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x9, x23, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x11, x26, LSL #2]\n" - "prfm pldl1keep, [x10, x17]\n" + "ld1w { z11.s }, p2/Z, [x9, x24, LSL #2]\n" + "csel x8, x8, XZR, LT\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x11, x27, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x9, x25, LSL #2]\n" + "csel x7, x7, x21, LT\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x9]\n" - "prfm pldl1keep, [x9, x24]\n" + "ld1w { z14.s }, p2/Z, [x27]\n" + "cmp x7, x20\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "prfm pldl1keep, [x10, x24]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x9, x25, LSL #2]\n" - "prfm pldl1keep, [x9, x23]\n" + "ld1w { z14.s }, p2/Z, [x27, x23, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x10]\n" + "ld1w { z15.s }, p2/Z, [x28]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x9, x16, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x27, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "prfm pldl1keep, [x28, x7]\n" - "prfm pldl1keep, [x10, x23]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x10, x16, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x28, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x10, x26, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x28, x24, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x10, x25, LSL #2]\n" - "prfm pldl1keep, [x28, x17]\n" + "ld1w { z11.s }, p2/Z, [x28, x23, LSL #2]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "prfm pldl1keep, [x9, x15]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x9, x26, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x27, x24, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "prfm pldl1keep, [x28, x24]\n" - "prfm pldl1keep, [x28, x15]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "ld1w { z15.s }, p2/Z, [x28]\n" + "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "ld1w { z13.s }, p2/Z, [x28, x16, LSL #2]\n" - "prfm pldl1keep, [x28, x23]\n" + "fmla z29.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p2/Z, [x26, x25, LSL #2]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x9, x27, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x27, x25, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" - "add x21, x5, #0x1\n" - "fmla z28.s, p3/M, z5.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x28, x26, LSL #2]\n" - "fmla z29.s, p3/M, z6.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x28, x27, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "add x6, x6, #0x1\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x28, x25, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z13.s\n" - "ldr x20, [%x[params_struct], %[offsetof_args_n_tile_rows]]\n" "fmax z31.s, p3/M, z31.s, z19.s\n" - "ldr x19, [%x[params_struct], %[offsetof_args_n_tile_cols]]\n" - "cmp x6, x19\n" + "fmla z28.s, p3/M, z5.s, z14.s\n" + "ld1w { z14.s }, p2/Z, [x26, x24, LSL #2]\n" + "fmax z30.s, p3/M, z30.s, z19.s\n" "fmla z29.s, p3/M, z5.s, z16.s\n" - "fmla z28.s, p3/M, z3.s, z16.s\n" - "csel x6, x6, XZR, LT\n" "fmin z31.s, p3/M, z31.s, z18.s\n" - "st1w { z31.s }, p0, [x12]\n" - "fmla z28.s, p3/M, z7.s, z14.s\n" - "csel x5, x5, x21, LT\n" + "st1w { z31.s }, p0, [x10]\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" "fmla z29.s, p3/M, z8.s, z15.s\n" - "cmp x5, x20\n" - "fmax z30.s, p3/M, z30.s, z19.s\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" + "ld1w { z11.s }, p2/Z, [x26, x23, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "st1w { z30.s }, p0, [x12, x13, LSL #2]\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" + "st1w { z30.s }, p0, [x10, x11, LSL #2]\n" + "fmla z28.s, p3/M, z3.s, z16.s\n" "fmax z29.s, p3/M, z29.s, z19.s\n" + "fmla z28.s, p3/M, z7.s, z14.s\n" "fmin z29.s, p3/M, z29.s, z18.s\n" "st1w { z29.s }, p0, [x22]\n" + "fmla z28.s, p3/M, z6.s, z15.s\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" "fmax z28.s, p3/M, z28.s, z19.s\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "st1w { z28.s }, p0, [x22, x13, LSL #2]\n" + "st1w { z28.s }, p0, [x22, x11, LSL #2]\n" "blt 1b\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_args_inptr] "I" (offsetof(Args, inptr)), [offsetof_args_ld_input_col] "I" (offsetof(Args, ld_input_col)), [offsetof_args_ld_input_row] "I" (offsetof(Args, ld_input_row)), [offsetof_args_ld_output_col] "I" (offsetof(Args, ld_output_col)), [offsetof_args_ld_output_row] "I" (offsetof(Args, ld_output_row)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_n_tile_cols] "I" (offsetof(Args, n_tile_cols)), [offsetof_args_n_tile_rows] "I" (offsetof(Args, n_tile_rows)), [offsetof_args_outptr] "I" (offsetof(Args, outptr)), [offsetof_args_params] "I" (offsetof(Args, params)), [offsetof_args_tile_i] "I" (offsetof(Args, tile_i)), [offsetof_args_tile_j] "I" (offsetof(Args, tile_j)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x5", "x6", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x7", "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp index 8555cfea7c..829b0ff2c7 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -89,309 +89,249 @@ void sve_fp32_nhwc_3x3_s2_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x14, [%x[params_struct], %[offsetof_args_params]]\n" - "add x13, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z19.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x12, ALL, MUL #2\n" + "mov x13, #0x0\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" - "mov x11, #0x0\n" - "ldp x10, x9, [x19, #0x0]\n" - "cntw x28\n" - "ldp x27, x26, [x19, #0x10]\n" - "sub x25, XZR, x28\n" - "ld1w { z17.s }, p3/Z, [x14]\n" - "mov z31.d, z17.d\n" - "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" + "cntw x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z30.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" - "cmp x28, %x[n_channels]\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" - "addvl x14, x14, #16\n" - "ldp x21, x20, [x13, #0x0]\n" - "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" - "addvl x14, x14, #-6\n" - "ld1w { z9.s }, p2/Z, [x21, x11, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z10.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" - "ldp x24, x23, [x13, #0x10]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldp x20, x19, [x13, #0x30]\n" - "ld1w { z11.s }, p2/Z, [x24, x11, LSL #2]\n" - "prfm pldl1keep, [x24, x12]\n" - "ld1w { z12.s }, p2/Z, [x23, x11, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" - "ld1w { z14.s }, p2/Z, [x21, x11, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z15.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" - "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z17.s }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "ldp x26, x25, [x14, #0x0]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ld1w { z11.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x19, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x13, #0x40]\n" - "whilelt p1.s, x28, %x[n_channels]\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "ldr x21, [x13, #0x48]\n" - "incw x25\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x20, [x13, #0x50]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x40]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "ldr x25, [x14, #0x48]\n" + "incw x9\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x24, [x14, #0x50]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x22, x12]\n" - "ldr x19, [x13, #0x58]\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "ldr x21, [x13, #0x60]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "ldr x20, [x13, #0x68]\n" - "ldr x19, [x13, #0x70]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ldr x19, [x13, #0x78]\n" - "ldr x21, [x13, #0x80]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "ldr x20, [x13, #0x88]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x24, [x13, #0x90]\n" + "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x24, x12]\n" - "ldr x23, [x13, #0x98]\n" + "ld1w { z17.s }, p3/Z, [x15]\n" "fmla z29.s, p3/M, z6.s, z15.s\n" - "ldr x22, [x13, #0xa0]\n" - "fmax z31.s, p3/M, z31.s, z19.s\n" - "ldr x21, [x13, #0xa8]\n" - "fmla z28.s, p3/M, z5.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "fmin z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" + "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "fmla z28.s, p3/M, z5.s, z14.s\n" + "fmax z31.s, p3/M, z31.s, z19.s\n" + "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z29.s, p3/M, z7.s, z13.s\n" - "ldr x20, [x13, #0xb0]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" "fmax z30.s, p3/M, z30.s, z19.s\n" - "ldr x19, [x13, #0xb8]\n" - "ldr x22, [x13, #0xc0]\n" - "fmla z28.s, p3/M, z3.s, z16.s\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "incw x13\n" "fmla z29.s, p3/M, z5.s, z16.s\n" - "prfm pldl1keep, [x20, x12]\n" + "ldp x26, x25, [x14, #0x0]\n" + "whilelt p2.s, x13, %x[n_channels]\n" + "fmin z31.s, p3/M, z31.s, z18.s\n" + "ldp x24, x23, [x14, #0x10]\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" - "fmla z28.s, p3/M, z7.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" + "ldp x22, x21, [x14, #0x20]\n" + "ldp x20, x19, [x14, #0x30]\n" + "fmla z28.s, p3/M, z3.s, z16.s\n" + "ld1w { z9.s }, p1/Z, [x26, x12, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z15.s\n" - "prfm pldl1keep, [x22, x12]\n" - "incw x11\n" - "fmla z28.s, p3/M, z6.s, z15.s\n" - "ldp x21, x20, [x13, #0x0]\n" - "whilelt p2.s, x11, %x[n_channels]\n" + "ld1w { z10.s }, p1/Z, [x25, x12, LSL #2]\n" + "fmla z28.s, p3/M, z7.s, z14.s\n" + "ld1w { z12.s }, p1/Z, [x23, x12, LSL #2]\n" + "ld1w { z13.s }, p1/Z, [x22, x12, LSL #2]\n" "fmax z29.s, p3/M, z29.s, z19.s\n" - "ldp x24, x23, [x13, #0x10]\n" - "addvl x12, x12, #1\n" - "fmla z28.s, p3/M, z8.s, z11.s\n" - "ld1w { z9.s }, p1/Z, [x21, x28, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z14.s }, p1/Z, [x21, x12, LSL #2]\n" + "fmla z28.s, p3/M, z6.s, z15.s\n" + "ld1w { z15.s }, p1/Z, [x20, x12, LSL #2]\n" + "ld1w { z16.s }, p1/Z, [x19, x12, LSL #2]\n" "fmin z29.s, p3/M, z29.s, z18.s\n" - "ld1w { z10.s }, p1/Z, [x20, x28, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" + "fmla z28.s, p3/M, z8.s, z11.s\n" + "ld1w { z11.s }, p1/Z, [x24, x12, LSL #2]\n" + "incw x12\n" "fmax z28.s, p3/M, z28.s, z19.s\n" - "ld1w { z11.s }, p1/Z, [x24, x28, LSL #2]\n" - "prfm pldl1keep, [x24, x12]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" + "cmp x12, %x[n_channels]\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z12.s }, p1/Z, [x23, x28, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "ldp x22, x21, [x13, #0x20]\n" - "ldp x20, x19, [x13, #0x30]\n" - "st1w { z31.s }, p0, [x10, x25, LSL #2]\n" - "ld1w { z13.s }, p1/Z, [x22, x28, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" - "ld1w { z14.s }, p1/Z, [x21, x28, LSL #2]\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z15.s }, p1/Z, [x20, x28, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" - "ld1w { z16.s }, p1/Z, [x19, x28, LSL #2]\n" - "incw x28\n" - "prfm pldl1keep, [x19, x12]\n" - "cmp x28, %x[n_channels]\n" - "st1w { z30.s }, p0, [x9, x25, LSL #2]\n" - "st1w { z29.s }, p0, [x27, x25, LSL #2]\n" - "st1w { z28.s }, p0, [x26, x25, LSL #2]\n" - "ld1w { z17.s }, p3/Z, [x14]\n" - "mov z31.d, z17.d\n" - "ld1w { z0.s }, p3/Z, [x14, #1, MUL VL]\n" - "mov z30.d, z17.d\n" - "ld1w { z1.s }, p3/Z, [x14, #2, MUL VL]\n" - "mov z29.d, z17.d\n" - "ld1w { z2.s }, p3/Z, [x14, #3, MUL VL]\n" - "mov z28.d, z17.d\n" - "ld1w { z3.s }, p3/Z, [x14, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x14, #5, MUL VL]\n" - "ld1w { z5.s }, p3/Z, [x14, #6, MUL VL]\n" - "ld1w { z6.s }, p3/Z, [x14, #7, MUL VL]\n" - "addvl x14, x14, #16\n" - "ld1w { z7.s }, p3/Z, [x14, #-8, MUL VL]\n" - "ld1w { z8.s }, p3/Z, [x14, #-7, MUL VL]\n" - "addvl x14, x14, #-6\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z5.s }, p3/Z, [x15, #6, MUL VL]\n" + "ld1w { z6.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" + "ld1w { z7.s }, p3/Z, [x15, #-8, MUL VL]\n" + "ld1w { z8.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z8.s, z9.s\n" - "ldr x22, [x13, #0x40]\n" - "incw x25\n" - "fmla z30.s, p3/M, z6.s, z9.s\n" - "ldr x21, [x13, #0x48]\n" + "movprfx z31, z17\n fmla z31.s, p3/M, z8.s, z9.s\n" + "ldr x26, [x14, #0x40]\n" + "incw x9\n" + "movprfx z30, z17\n fmla z30.s, p3/M, z6.s, z9.s\n" + "ldr x25, [x14, #0x48]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x20, [x13, #0x50]\n" - "fmla z28.s, p3/M, z0.s, z9.s\n" - "prfm pldl1keep, [x22, x12]\n" - "ldr x19, [x13, #0x58]\n" + "movprfx z29, z17\n fmla z29.s, p3/M, z2.s, z9.s\n" + "ldr x24, [x14, #0x50]\n" + "movprfx z28, z17\n fmla z28.s, p3/M, z0.s, z9.s\n" + "ldr x23, [x14, #0x58]\n" + "ldr x22, [x14, #0x60]\n" "fmla z31.s, p3/M, z0.s, z10.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ldr x21, [x14, #0x68]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x20, [x14, #0x70]\n" "fmla z30.s, p3/M, z2.s, z13.s\n" - "ld1w { z13.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z31.s, p3/M, z3.s, z14.s\n" - "ld1w { z14.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z14.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x19, [x14, #0x78]\n" "fmla z30.s, p3/M, z0.s, z16.s\n" - "ldr x21, [x13, #0x60]\n" + "ldr x26, [x14, #0x80]\n" "fmla z29.s, p3/M, z3.s, z14.s\n" - "ldr x20, [x13, #0x68]\n" - "ldr x19, [x13, #0x70]\n" + "ldr x25, [x14, #0x88]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.s, p3/M, z4.s, z15.s\n" - "ld1w { z15.s }, p2/Z, [x21, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z29.s, p3/M, z0.s, z15.s\n" - "ld1w { z11.s }, p2/Z, [x20, x11, LSL #2]\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.s, p3/M, z2.s, z16.s\n" - "ld1w { z16.s }, p2/Z, [x19, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z5.s, z12.s\n" - "prfm pldl1keep, [x19, x12]\n" + "ld1w { z12.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z11.s\n" - "ldr x19, [x13, #0x78]\n" - "ldr x21, [x13, #0x80]\n" + "ld1w { z11.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z31.s, p3/M, z5.s, z13.s\n" - "ldr x20, [x13, #0x88]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x24, [x13, #0x90]\n" + "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z16.s\n" - "ld1w { z13.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "ldr x20, [x14, #0xb0]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z31.s, p3/M, z6.s, z15.s\n" - "ld1w { z12.s }, p2/Z, [x21, x11, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x21, x12]\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "ld1w { z15.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z30.s, p3/M, z7.s, z12.s\n" - "prfm pldl1keep, [x20, x12]\n" + "ld1w { z13.s }, p2/Z, [x22, x13, LSL #2]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z31.s, p3/M, z7.s, z16.s\n" - "ld1w { z15.s }, p2/Z, [x24, x11, LSL #2]\n" + "ld1w { z16.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x24, x12]\n" - "ldr x23, [x13, #0x98]\n" "fmla z29.s, p3/M, z6.s, z15.s\n" - "ldr x22, [x13, #0xa0]\n" - "fmax z31.s, p3/M, z31.s, z19.s\n" - "ldr x21, [x13, #0xa8]\n" - "fmla z28.s, p3/M, z5.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x23, x11, LSL #2]\n" - "prfm pldl1keep, [x23, x12]\n" - "fmin z31.s, p3/M, z31.s, z18.s\n" - "ld1w { z13.s }, p2/Z, [x22, x11, LSL #2]\n" - "prfm pldl1keep, [x22, x12]\n" + "ld1w { z15.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z8.s, z11.s\n" - "ld1w { z16.s }, p2/Z, [x21, x11, LSL #2]\n" - "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x12]\n" + "fmla z28.s, p3/M, z5.s, z14.s\n" + "ld1w { z14.s }, p2/Z, [x20, x13, LSL #2]\n" + "fmax z31.s, p3/M, z31.s, z19.s\n" "fmla z29.s, p3/M, z7.s, z13.s\n" - "ldr x20, [x13, #0xb0]\n" "fmax z30.s, p3/M, z30.s, z19.s\n" - "ldr x19, [x13, #0xb8]\n" - "ldr x22, [x13, #0xc0]\n" - "fmla z28.s, p3/M, z3.s, z16.s\n" - "ld1w { z14.s }, p2/Z, [x20, x11, LSL #2]\n" + "fmla z28.s, p3/M, z2.s, z11.s\n" + "ld1w { z11.s }, p2/Z, [x26, x13, LSL #2]\n" + "fmin z31.s, p3/M, z31.s, z18.s\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmla z29.s, p3/M, z5.s, z16.s\n" - "prfm pldl1keep, [x20, x12]\n" + "fmla z28.s, p3/M, z3.s, z16.s\n" "fmin z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z15.s }, p2/Z, [x19, x11, LSL #2]\n" - "prfm pldl1keep, [x19, x12]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmla z28.s, p3/M, z7.s, z14.s\n" - "ld1w { z11.s }, p2/Z, [x22, x11, LSL #2]\n" "fmla z29.s, p3/M, z8.s, z15.s\n" - "prfm pldl1keep, [x22, x12]\n" - "st1w { z31.s }, p0, [x10, x25, LSL #2]\n" "fmla z28.s, p3/M, z6.s, z15.s\n" - "st1w { z30.s }, p0, [x9, x25, LSL #2]\n" "fmax z29.s, p3/M, z29.s, z19.s\n" "fmla z28.s, p3/M, z8.s, z11.s\n" "fmin z29.s, p3/M, z29.s, z18.s\n" - "st1w { z29.s }, p0, [x27, x25, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" "fmax z28.s, p3/M, z28.s, z19.s\n" "fmin z28.s, p3/M, z28.s, z18.s\n" - "st1w { z28.s }, p0, [x26, x25, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp index 6f1f187818..751874ffbf 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst.hpp @@ -28,7 +28,7 @@ #pragma once -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -36,15 +36,17 @@ namespace depthwise { void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); -struct sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst +class sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst : public IDepthwiseDepthfirstStrategy { - typedef float bias_type; - typedef float input_type; - typedef float weight_type; - typedef float return_type; - + private: typedef void (*indirect_kern_type)(const float *const *const, float *const *const, const void *, unsigned int, const float, const float); + indirect_kern_type m_indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; + typedef void (*direct_kern_type)(const unsigned int, const unsigned int, const float *, int64_t, int64_t, float *, int64_t, int64_t, const void *, unsigned int, const float, const float); + direct_kern_type m_direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; + + public: + typedef float return_type; constexpr static arm_gemm::VLType vl_type = arm_gemm::VLType::SVE; @@ -60,13 +62,60 @@ struct sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst constexpr static unsigned int input_rows = 6; constexpr static unsigned int input_cols = 6; - indirect_kern_type indirect_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl; - direct_kern_type direct_kernel = sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl; - sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst(const CPUInfo *) {} + + arm_gemm::VLType get_vl_type(void) const override { return vl_type; } + + unsigned int get_kernel_rows(void) const override { return kernel_rows; } + unsigned int get_kernel_cols(void) const override { return kernel_cols; } + + unsigned int get_stride_rows(void) const override { return stride_rows; } + unsigned int get_stride_cols(void) const override { return stride_cols; } + + unsigned int get_output_rows(void) const override { return output_rows; } + unsigned int get_output_cols(void) const override { return output_cols; } + + unsigned int get_input_rows(void) const override { return input_rows; } + unsigned int get_input_cols(void) const override { return input_cols; } + + void indirect_kernel( + const void *const *const input_ptrs, + void *const *const outptrs, + const void *params, + unsigned int n_channels, + const void *activation_min, + const void *activation_max + ) const override + { + m_indirect_kernel( + reinterpret_cast(input_ptrs), + reinterpret_cast(outptrs), + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } + + void direct_kernel( + const unsigned int n_tile_rows, const unsigned int n_tile_cols, + const void *inptr, int64_t ld_input_row, int64_t ld_input_col, + void *outptr, int64_t ld_output_row, int64_t ld_output_col, + const void *params, unsigned int n_channels, + const void *activation_min, const void *activation_max + ) const override + { + m_direct_kernel( + n_tile_rows, n_tile_cols, + static_cast(inptr), ld_input_row, ld_input_col, + static_cast(outptr), ld_output_row, ld_output_col, + params, n_channels, + *static_cast(activation_min), + *static_cast(activation_max) + ); + } }; } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp index edafe82770..ea8bbbd7e8 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_direct.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -114,22 +114,18 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "add x10, x20, x22, LSL #2\n" "ld1w { z16.s }, p3/Z, [x8]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" "add x9, x10, x22, LSL #2\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x8, #1, MUL VL]\n" "add x28, x9, x22, LSL #2\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x8, #2, MUL VL]\n" "add x27, x28, x22, LSL #2\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x8, #3, MUL VL]\n" "add x26, x15, x15\n" - "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x8, #4, MUL VL]\n" "add x25, x26, x15\n" - "mul x19, x5, x21\n" // offset = tile_i * ld_output_row + "ld1w { z4.s }, p3/Z, [x8, #5, MUL VL]\n" "add x24, x25, x15\n" + "mul x19, x5, x21\n" // offset = tile_i * ld_output_row "add x23, x24, x15\n" "madd x19, x6, x12, x19\n" // offset += tile_j * ld_output_col "mul x19, x19, x7\n" // offset *= output_tile_size @@ -150,14 +146,14 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "ld1w { z14.s }, p2/Z, [x10]\n" "bge 3f\n" "2:" // Tile loop: Channel loop - "fmla z31.s, p3/M, z0.s, z5.s\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n" "whilelt p1.s, x16, %x[n_channels]\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" "incw x14\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" "ld1w { z0.s }, p3/Z, [x8]\n" "incw x17\n" "fmla z31.s, p3/M, z1.s, z6.s\n" @@ -334,29 +330,25 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( "fmax z28.s, p3/M, z28.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" "st1w { z31.s }, p0, [x11]\n" - "mov z31.d, z16.d\n" "fmin z30.s, p3/M, z30.s, z17.s\n" + "fmin z29.s, p3/M, z29.s, z17.s\n" "st1w { z30.s }, p0, [x11, x12, LSL #2]\n" - "mov z30.d, z16.d\n" + "fmin z28.s, p3/M, z28.s, z17.s\n" "addvl x11, x11, #1\n" - "fmin z29.s, p3/M, z29.s, z17.s\n" "st1w { z29.s }, p0, [x22]\n" - "mov z29.d, z16.d\n" - "fmin z28.s, p3/M, z28.s, z17.s\n" "st1w { z28.s }, p0, [x22, x12, LSL #2]\n" - "mov z28.d, z16.d\n" "addvl x22, x22, #1\n" "blt 2b\n" "3:" // Tile loop: Channel tail - "fmla z31.s, p3/M, z0.s, z5.s\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x20, x25, LSL #2]\n" "mov p0.b, p2.b\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" "ldr x5, [%x[params_struct], %[offsetof_args_tile_i]]\n" "add x21, x5, #0x1\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" "ldr x6, [%x[params_struct], %[offsetof_args_tile_j]]\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" "ld1w { z0.s }, p3/Z, [x8]\n" "add x6, x6, #0x1\n" "fmla z31.s, p3/M, z1.s, z6.s\n" @@ -528,4 +520,4 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_direct_impl( } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp index 1bfe7eb09c..070270764c 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst/generic_indirect.cpp @@ -25,7 +25,7 @@ #include #include -#if defined(ARM_COMPUTE_ENABLE_SVE) +#if __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -100,511 +100,429 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( __asm__ __volatile__( "ldr x19, [%x[params_struct], %[offsetof_args_outptrs]]\n" "ptrue p3.b\n" - "ldr x16, [%x[params_struct], %[offsetof_args_params]]\n" - "add x15, %x[params_struct], %[offsetof_Args_inptrs]\n" + "ldr x15, [%x[params_struct], %[offsetof_args_params]]\n" + "add x14, %x[params_struct], %[offsetof_Args_inptrs]\n" "ld1rw { z18.s }, p3/Z, [%x[params_struct], %[offsetof_args_min]]\n" - "cntb x14, ALL, MUL #2\n" - "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" "mov x13, #0x0\n" - "ldp x12, x11, [x19, #0x0]\n" - "cntw x10\n" - "ldp x9, x28, [x19, #0x10]\n" - "sub x27, XZR, x10\n" - "ld1w { z16.s }, p3/Z, [x16]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1rw { z17.s }, p3/Z, [%x[params_struct], %[offsetof_args_max]]\n" + "cntw x12\n" + "ldp x11, x10, [x19, #0x0]\n" + "sub x9, XZR, x12\n" + "ldp x28, x27, [x19, #0x10]\n" "whilelt p2.s, XZR, %x[n_channels]\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x16, #2, MUL VL]\n" - "cmp x10, %x[n_channels]\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x16, #3, MUL VL]\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x16, #4, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x16, #5, MUL VL]\n" - "addvl x16, x16, #6\n" - "ldp x26, x25, [x15, #0x0]\n" - "ldp x24, x23, [x15, #0x10]\n" - "ldp x20, x19, [x15, #0x20]\n" + "ld1w { z16.s }, p3/Z, [x15]\n" + "cmp x12, %x[n_channels]\n" + "ld1w { z0.s }, p3/Z, [x15, #1, MUL VL]\n" + "ld1w { z1.s }, p3/Z, [x15, #2, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #3, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #4, MUL VL]\n" + "ld1w { z4.s }, p3/Z, [x15, #5, MUL VL]\n" + "addvl x15, x15, #6\n" + "ldp x26, x25, [x14, #0x0]\n" + "ldp x24, x23, [x14, #0x10]\n" + "ldp x22, x21, [x14, #0x20]\n" "ld1w { z5.s }, p2/Z, [x26, x13, LSL #2]\n" - "prfm pldl1keep, [x26, x14]\n" "ld1w { z6.s }, p2/Z, [x25, x13, LSL #2]\n" - "prfm pldl1keep, [x25, x14]\n" "ld1w { z7.s }, p2/Z, [x24, x13, LSL #2]\n" - "prfm pldl1keep, [x24, x14]\n" "ld1w { z8.s }, p2/Z, [x23, x13, LSL #2]\n" - "prfm pldl1keep, [x23, x14]\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" - "prfm pldl1keep, [x19, x14]\n" - "ldp x22, x21, [x15, #0x30]\n" - "ldp x20, x19, [x15, #0x40]\n" - "ld1w { z11.s }, p2/Z, [x22, x13, LSL #2]\n" - "prfm pldl1keep, [x22, x14]\n" - "ld1w { z12.s }, p2/Z, [x21, x13, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" - "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n" - "prfm pldl1keep, [x19, x14]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z13.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldp x20, x19, [x14, #0x30]\n" + "ldp x26, x25, [x14, #0x40]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z10.s }, p2/Z, [x26, x13, LSL #2]\n" + "ld1w { z14.s }, p2/Z, [x25, x13, LSL #2]\n" "bge 2f\n" "1:" // Channel loop - "fmla z31.s, p3/M, z0.s, z5.s\n" - "ldr x21, [x15, #0x50]\n" - "whilelt p1.s, x10, %x[n_channels]\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0x58]\n" - "incw x27\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" - "ldr x20, [x15, #0x60]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" + "ldr x24, [x14, #0x50]\n" + "whilelt p1.s, x12, %x[n_channels]\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" + "ldr x23, [x14, #0x58]\n" + "incw x9\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" + "ldr x22, [x14, #0x60]\n" "mov p0.b, p2.b\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.s, p3/M, z1.s, z8.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x20, [x14, #0x70]\n" "fmla z28.s, p3/M, z1.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x16]\n" + "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0x68]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.s, p3/M, z2.s, z13.s\n" - "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n" + "ldr x26, [x14, #0x80]\n" "fmla z28.s, p3/M, z2.s, z5.s\n" - "ldr x20, [x15, #0x70]\n" + "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.s, p3/M, z3.s, z5.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ldr x24, [x14, #0x90]\n" "fmla z28.s, p3/M, z3.s, z6.s\n" - "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z9.s\n" - "ldr x19, [x15, #0x78]\n" + "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n" + "ldr x23, [x14, #0x98]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "ldr x26, [x15, #0x80]\n" + "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z7.s\n" - "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.s, p3/M, z0.s, z8.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.s, p3/M, z0.s, z14.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z8.s\n" - "ldr x25, [x15, #0x88]\n" + "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.s, p3/M, z1.s, z11.s\n" - "ldr x24, [x15, #0x90]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z13.s\n" "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z5.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z28.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x15, #0x98]\n" + "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.s, p3/M, z3.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.s, p3/M, z3.s, z6.s\n" - "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n" + "ld1w { z16.s }, p3/Z, [x15, #4, MUL VL]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "ldr x20, [x15, #0xa0]\n" + "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z6.s\n" "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "addvl x16, x16, #16\n" + "ldr x22, [x14, #0xe0]\n" "fmla z28.s, p3/M, z4.s, z8.s\n" - "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z14.s\n" - "ldr x19, [x15, #0xa8]\n" + "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z11.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.s, p3/M, z0.s, z5.s\n" - "ldr x22, [x15, #0xb0]\n" "fmla z28.s, p3/M, z0.s, z6.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x22, x14]\n" "fmla z29.s, p3/M, z1.s, z6.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x15, #0xb8]\n" + "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.s, p3/M, z2.s, z9.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z10.s\n" - "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "ldr x20, [x15, #0xc0]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x19, [x15, #0xc8]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.s, p3/M, z4.s, z8.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z28.s, p3/M, z4.s, z14.s\n" - "ldr x21, [x15, #0xd0]\n" + "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z5.s\n" - "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0xd8]\n" "fmla z29.s, p3/M, z0.s, z9.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z1.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z5.s\n" - "ldr x20, [x15, #0xe0]\n" + "ld1w { z1.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0xe8]\n" "fmla z29.s, p3/M, z2.s, z5.s\n" - "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z6.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z3.s, z6.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z8.s\n" - "ldr x20, [x15, #0xf0]\n" + "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "ldr x19, [x15, #0xf8]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z14.s\n" - "ld1w { z1.s }, p3/Z, [x16]\n" "fmla z29.s, p3/M, z4.s, z8.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ldr x26, [x15, #0x100]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldp x26, x25, [x14, #0x0]\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "ldr x25, [x15, #0x108]\n" "fmla z30.s, p3/M, z1.s, z5.s\n" - "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" + "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z5.s\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z5.s }, p1/Z, [x26, x12, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x24, [x15, #0x110]\n" - "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" + "incw x13\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "ldr x23, [x15, #0x118]\n" + "ldp x24, x23, [x14, #0x10]\n" + "whilelt p2.s, x13, %x[n_channels]\n" "fmla z31.s, p3/M, z3.s, z6.s\n" + "ld1w { z6.s }, p1/Z, [x25, x12, LSL #2]\n" + "ldp x22, x21, [x14, #0x20]\n" "fmla z30.s, p3/M, z3.s, z8.s\n" - "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldp x20, x19, [x14, #0x30]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x24, x14]\n" - "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" + "ld1w { z7.s }, p1/Z, [x24, x12, LSL #2]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x23, x14]\n" - "incw x13\n" + "ld1w { z13.s }, p1/Z, [x21, x12, LSL #2]\n" "fmla z31.s, p3/M, z4.s, z8.s\n" - "ldp x26, x25, [x15, #0x0]\n" - "whilelt p2.s, x13, %x[n_channels]\n" + "ld1w { z8.s }, p1/Z, [x23, x12, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "ldp x24, x23, [x15, #0x10]\n" - "addvl x14, x14, #1\n" + "ld1w { z11.s }, p1/Z, [x20, x12, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "ldp x20, x19, [x15, #0x20]\n" - "ldp x22, x21, [x15, #0x30]\n" + "ld1w { z12.s }, p1/Z, [x19, x12, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z9.s\n" - "ld1w { z5.s }, p1/Z, [x26, x10, LSL #2]\n" + "ld1w { z9.s }, p1/Z, [x22, x12, LSL #2]\n" "fmax z31.s, p3/M, z31.s, z18.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ldp x26, x25, [x14, #0x40]\n" "fmax z30.s, p3/M, z30.s, z18.s\n" - "ld1w { z6.s }, p1/Z, [x25, x10, LSL #2]\n" + "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n" "fmax z29.s, p3/M, z29.s, z18.s\n" - "prfm pldl1keep, [x25, x14]\n" + "addvl x15, x15, #16\n" "fmax z28.s, p3/M, z28.s, z18.s\n" - "ld1w { z7.s }, p1/Z, [x24, x10, LSL #2]\n" + "ld1w { z10.s }, p1/Z, [x26, x12, LSL #2]\n" + "ld1w { z14.s }, p1/Z, [x25, x12, LSL #2]\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "prfm pldl1keep, [x24, x14]\n" + "incw x12\n" "fmin z30.s, p3/M, z30.s, z17.s\n" - "ld1w { z8.s }, p1/Z, [x23, x10, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n" + "cmp x12, %x[n_channels]\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "prfm pldl1keep, [x23, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n" + "addvl x15, x15, #-6\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "ld1w { z9.s }, p1/Z, [x20, x10, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z13.s }, p1/Z, [x19, x10, LSL #2]\n" - "prfm pldl1keep, [x19, x14]\n" - "ld1w { z11.s }, p1/Z, [x22, x10, LSL #2]\n" - "prfm pldl1keep, [x22, x14]\n" - "ld1w { z12.s }, p1/Z, [x21, x10, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" - "ldp x20, x19, [x15, #0x40]\n" - "st1w { z31.s }, p0, [x12, x27, LSL #2]\n" - "st1w { z30.s }, p0, [x11, x27, LSL #2]\n" - "ld1w { z10.s }, p1/Z, [x20, x10, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z14.s }, p1/Z, [x19, x10, LSL #2]\n" - "incw x10\n" - "prfm pldl1keep, [x19, x14]\n" - "cmp x10, %x[n_channels]\n" - "st1w { z29.s }, p0, [x9, x27, LSL #2]\n" - "st1w { z28.s }, p0, [x28, x27, LSL #2]\n" - "ld1w { z16.s }, p3/Z, [x16, #4, MUL VL]\n" - "mov z31.d, z16.d\n" - "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n" - "mov z30.d, z16.d\n" - "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n" - "mov z29.d, z16.d\n" - "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n" - "addvl x16, x16, #16\n" - "mov z28.d, z16.d\n" - "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n" - "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n" - "addvl x16, x16, #-6\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" "blt 1b\n" "2:" // Channel tail - "fmla z31.s, p3/M, z0.s, z5.s\n" - "ldr x21, [x15, #0x50]\n" - "incw x27\n" - "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0x58]\n" + "movprfx z31, z16\n fmla z31.s, p3/M, z0.s, z5.s\n" + "ldr x24, [x14, #0x50]\n" + "incw x9\n" + "movprfx z30, z16\n fmla z30.s, p3/M, z0.s, z6.s\n" + "ldr x23, [x14, #0x58]\n" "mov p0.b, p2.b\n" - "fmla z29.s, p3/M, z0.s, z7.s\n" - "ldr x20, [x15, #0x60]\n" - "fmla z28.s, p3/M, z0.s, z8.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" - "prfm pldl1keep, [x21, x14]\n" + "movprfx z29, z16\n fmla z29.s, p3/M, z0.s, z7.s\n" + "ldr x22, [x14, #0x60]\n" + "movprfx z28, z16\n fmla z28.s, p3/M, z0.s, z8.s\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ld1w { z0.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z9.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0x68]\n" "fmla z29.s, p3/M, z1.s, z8.s\n" "fmla z28.s, p3/M, z1.s, z13.s\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z0.s }, p3/Z, [x16]\n" + "ld1w { z1.s }, p3/Z, [x15, #1, MUL VL]\n" + "ldr x20, [x14, #0x70]\n" "fmla z31.s, p3/M, z2.s, z9.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0x68]\n" + "ldr x19, [x14, #0x78]\n" "fmla z29.s, p3/M, z2.s, z13.s\n" "fmla z28.s, p3/M, z2.s, z5.s\n" - "ld1w { z1.s }, p3/Z, [x16, #1, MUL VL]\n" - "ldr x20, [x15, #0x70]\n" + "ld1w { z2.s }, p3/Z, [x15, #2, MUL VL]\n" + "ldr x26, [x14, #0x80]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x25, [x14, #0x88]\n" "fmla z29.s, p3/M, z3.s, z5.s\n" "fmla z28.s, p3/M, z3.s, z6.s\n" - "prfm pldl1keep, [x20, x14]\n" - "ld1w { z2.s }, p3/Z, [x16, #2, MUL VL]\n" + "ld1w { z3.s }, p3/Z, [x15, #3, MUL VL]\n" + "ldr x24, [x14, #0x90]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z9.s\n" - "ldr x19, [x15, #0x78]\n" + "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z6.s\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "ld1w { z3.s }, p3/Z, [x16, #3, MUL VL]\n" - "ldr x26, [x15, #0x80]\n" + "ld1w { z4.s }, p3/Z, [x15, #4, MUL VL]\n" + "ldr x23, [x14, #0x98]\n" "fmla z31.s, p3/M, z0.s, z7.s\n" - "ld1w { z9.s }, p2/Z, [x19, x13, LSL #2]\n" + "ldr x22, [x14, #0xa0]\n" "fmla z30.s, p3/M, z0.s, z8.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ldr x21, [x14, #0xa8]\n" "fmla z29.s, p3/M, z0.s, z14.s\n" "fmla z28.s, p3/M, z0.s, z11.s\n" - "prfm pldl1keep, [x26, x14]\n" - "ld1w { z4.s }, p3/Z, [x16, #4, MUL VL]\n" + "ld1w { z0.s }, p3/Z, [x15, #5, MUL VL]\n" + "ldr x20, [x14, #0xb0]\n" "fmla z31.s, p3/M, z1.s, z8.s\n" - "ldr x25, [x15, #0x88]\n" + "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z30.s, p3/M, z1.s, z13.s\n" - "ld1w { z0.s }, p3/Z, [x16, #5, MUL VL]\n" + "ldr x19, [x14, #0xb8]\n" "fmla z29.s, p3/M, z1.s, z11.s\n" "fmla z28.s, p3/M, z1.s, z12.s\n" - "ld1w { z8.s }, p2/Z, [x25, x13, LSL #2]\n" - "prfm pldl1keep, [x25, x14]\n" + "ld1w { z1.s }, p3/Z, [x15, #6, MUL VL]\n" + "ldr x25, [x14, #0xc8]\n" "fmla z31.s, p3/M, z2.s, z13.s\n" "ld1w { z13.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z5.s\n" - "ldr x24, [x15, #0x90]\n" + "ldr x26, [x14, #0xc0]\n" "fmla z29.s, p3/M, z2.s, z12.s\n" "fmla z28.s, p3/M, z2.s, z9.s\n" - "ldr x23, [x15, #0x98]\n" - "ld1w { z1.s }, p3/Z, [x16, #6, MUL VL]\n" + "ld1w { z2.s }, p3/Z, [x15, #7, MUL VL]\n" + "addvl x15, x15, #16\n" "fmla z31.s, p3/M, z3.s, z5.s\n" "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0xd0]\n" "fmla z30.s, p3/M, z3.s, z6.s\n" - "prfm pldl1keep, [x24, x14]\n" "fmla z29.s, p3/M, z3.s, z9.s\n" "fmla z28.s, p3/M, z3.s, z13.s\n" - "prfm pldl1keep, [x23, x14]\n" - "ldr x20, [x15, #0xa0]\n" + "ld1w { z3.s }, p3/Z, [x15, #-8, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z6.s\n" "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0xd8]\n" "fmla z30.s, p3/M, z4.s, z10.s\n" - "ld1w { z2.s }, p3/Z, [x16, #7, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z13.s\n" - "addvl x16, x16, #16\n" "fmla z28.s, p3/M, z4.s, z8.s\n" - "ld1w { z10.s }, p2/Z, [x20, x13, LSL #2]\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #-7, MUL VL]\n" + "ldr x22, [x14, #0xe0]\n" "fmla z31.s, p3/M, z0.s, z14.s\n" - "ldr x19, [x15, #0xa8]\n" + "ld1w { z14.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z11.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-8, MUL VL]\n" + "ldr x19, [x14, #0xf8]\n" "fmla z29.s, p3/M, z0.s, z5.s\n" - "ldr x22, [x15, #0xb0]\n" "fmla z28.s, p3/M, z0.s, z6.s\n" - "prfm pldl1keep, [x19, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-6, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z11.s\n" - "ld1w { z11.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x21, x13, LSL #2]\n" + "ldr x21, [x14, #0xe8]\n" "fmla z30.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x22, x14]\n" "fmla z29.s, p3/M, z1.s, z6.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-7, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z10.s\n" - "ldr x21, [x15, #0xb8]\n" + "ld1w { z1.s }, p3/Z, [x15, #-5, MUL VL]\n" "fmla z31.s, p3/M, z2.s, z12.s\n" - "ld1w { z12.s }, p2/Z, [x22, x13, LSL #2]\n" + "ld1w { z12.s }, p2/Z, [x20, x13, LSL #2]\n" + "ldr x20, [x14, #0xf0]\n" "fmla z30.s, p3/M, z2.s, z9.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-6, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z10.s\n" - "ld1w { z14.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #-4, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z9.s\n" - "ldr x20, [x15, #0xc0]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" + "ldr x26, [x14, #0x100]\n" "fmla z30.s, p3/M, z3.s, z13.s\n" - "ldr x19, [x15, #0xc8]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "ld1w { z1.s }, p3/Z, [x16, #-5, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "ld1w { z9.s }, p2/Z, [x20, x13, LSL #2]\n" + "ld1w { z3.s }, p3/Z, [x15, #-3, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z13.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z13.s }, p2/Z, [x25, x13, LSL #2]\n" + "ldr x25, [x14, #0x108]\n" "fmla z30.s, p3/M, z4.s, z8.s\n" - "ld1w { z13.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z8.s }, p2/Z, [x22, x13, LSL #2]\n" "fmla z29.s, p3/M, z4.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z28.s, p3/M, z4.s, z14.s\n" - "ldr x21, [x15, #0xd0]\n" + "ld1w { z4.s }, p3/Z, [x15, #-2, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z5.s\n" - "ld1w { z2.s }, p3/Z, [x16, #-4, MUL VL]\n" + "ld1w { z5.s }, p2/Z, [x24, x13, LSL #2]\n" + "ldr x24, [x14, #0x110]\n" "fmla z30.s, p3/M, z0.s, z6.s\n" - "ldr x19, [x15, #0xd8]\n" "fmla z29.s, p3/M, z0.s, z9.s\n" - "ld1w { z5.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z28.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x21, x14]\n" + "ld1w { z0.s }, p3/Z, [x15, #-1, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z6.s\n" - "ld1w { z6.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z6.s }, p2/Z, [x23, x13, LSL #2]\n" + "ldr x23, [x14, #0x118]\n" "fmla z30.s, p3/M, z1.s, z10.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z1.s, z13.s\n" - "ld1w { z3.s }, p3/Z, [x16, #-3, MUL VL]\n" "fmla z28.s, p3/M, z1.s, z5.s\n" - "ldr x20, [x15, #0xe0]\n" + "ld1w { z1.s }, p3/Z, [x15]\n" "fmla z31.s, p3/M, z2.s, z10.s\n" - "ld1w { z4.s }, p3/Z, [x16, #-2, MUL VL]\n" + "ld1w { z10.s }, p2/Z, [x21, x13, LSL #2]\n" "fmla z30.s, p3/M, z2.s, z11.s\n" - "ldr x19, [x15, #0xe8]\n" "fmla z29.s, p3/M, z2.s, z5.s\n" - "ld1w { z8.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z6.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z2.s }, p3/Z, [x15, #1, MUL VL]\n" "fmla z31.s, p3/M, z3.s, z11.s\n" - "ld1w { z10.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z30.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z3.s, z6.s\n" - "ld1w { z0.s }, p3/Z, [x16, #-1, MUL VL]\n" "fmla z28.s, p3/M, z3.s, z8.s\n" - "ldr x20, [x15, #0xf0]\n" + "ld1w { z3.s }, p3/Z, [x15, #2, MUL VL]\n" "fmla z31.s, p3/M, z4.s, z12.s\n" - "ldr x19, [x15, #0xf8]\n" + "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" "fmla z30.s, p3/M, z4.s, z14.s\n" - "ld1w { z1.s }, p3/Z, [x16]\n" "fmla z29.s, p3/M, z4.s, z8.s\n" - "ld1w { z11.s }, p2/Z, [x20, x13, LSL #2]\n" "fmla z28.s, p3/M, z4.s, z10.s\n" - "prfm pldl1keep, [x20, x14]\n" + "ld1w { z4.s }, p3/Z, [x15, #3, MUL VL]\n" "fmla z31.s, p3/M, z0.s, z9.s\n" - "ld1w { z12.s }, p2/Z, [x19, x13, LSL #2]\n" + "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z30.s, p3/M, z0.s, z13.s\n" - "prfm pldl1keep, [x19, x14]\n" "fmla z29.s, p3/M, z0.s, z11.s\n" - "ldr x26, [x15, #0x100]\n" + "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z28.s, p3/M, z0.s, z12.s\n" - "ld1w { z2.s }, p3/Z, [x16, #1, MUL VL]\n" "fmla z31.s, p3/M, z1.s, z13.s\n" - "ldr x25, [x15, #0x108]\n" "fmla z30.s, p3/M, z1.s, z5.s\n" - "ld1w { z9.s }, p2/Z, [x26, x13, LSL #2]\n" "fmla z29.s, p3/M, z1.s, z12.s\n" - "prfm pldl1keep, [x26, x14]\n" + "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z28.s, p3/M, z1.s, z9.s\n" - "ld1w { z11.s }, p2/Z, [x25, x13, LSL #2]\n" "fmla z31.s, p3/M, z2.s, z5.s\n" - "prfm pldl1keep, [x25, x14]\n" "fmla z30.s, p3/M, z2.s, z6.s\n" - "ld1w { z3.s }, p3/Z, [x16, #2, MUL VL]\n" "fmla z29.s, p3/M, z2.s, z9.s\n" - "ldr x24, [x15, #0x110]\n" - "ld1w { z4.s }, p3/Z, [x16, #3, MUL VL]\n" + "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z28.s, p3/M, z2.s, z11.s\n" - "ldr x23, [x15, #0x118]\n" "fmla z31.s, p3/M, z3.s, z6.s\n" "fmla z30.s, p3/M, z3.s, z8.s\n" - "ld1w { z12.s }, p2/Z, [x24, x13, LSL #2]\n" "fmla z29.s, p3/M, z3.s, z11.s\n" - "prfm pldl1keep, [x24, x14]\n" - "ld1w { z9.s }, p2/Z, [x23, x13, LSL #2]\n" "fmla z28.s, p3/M, z3.s, z12.s\n" - "prfm pldl1keep, [x23, x14]\n" "fmla z31.s, p3/M, z4.s, z8.s\n" "fmla z30.s, p3/M, z4.s, z10.s\n" "fmla z29.s, p3/M, z4.s, z12.s\n" @@ -612,22 +530,22 @@ void sve_fp32_nhwc_5x5_s1_output2x2_mla_depthfirst_indirect_impl( "fmax z31.s, p3/M, z31.s, z18.s\n" "fmax z30.s, p3/M, z30.s, z18.s\n" "fmax z29.s, p3/M, z29.s, z18.s\n" + "fmax z28.s, p3/M, z28.s, z18.s\n" "fmin z31.s, p3/M, z31.s, z17.s\n" - "st1w { z31.s }, p0, [x12, x27, LSL #2]\n" + "st1w { z31.s }, p0, [x11, x9, LSL #2]\n" "fmin z30.s, p3/M, z30.s, z17.s\n" "fmin z29.s, p3/M, z29.s, z17.s\n" - "st1w { z30.s }, p0, [x11, x27, LSL #2]\n" - "fmax z28.s, p3/M, z28.s, z18.s\n" - "st1w { z29.s }, p0, [x9, x27, LSL #2]\n" + "st1w { z30.s }, p0, [x10, x9, LSL #2]\n" "fmin z28.s, p3/M, z28.s, z17.s\n" - "st1w { z28.s }, p0, [x28, x27, LSL #2]\n" + "st1w { z29.s }, p0, [x28, x9, LSL #2]\n" + "st1w { z28.s }, p0, [x27, x9, LSL #2]\n" : : [n_channels] "r" ((unsigned long) n_channels), [offsetof_Args_inptrs] "I" (offsetof(Args, inptrs)), [offsetof_args_max] "I" (offsetof(Args, max)), [offsetof_args_min] "I" (offsetof(Args, min)), [offsetof_args_outptrs] "I" (offsetof(Args, outptrs)), [offsetof_args_params] "I" (offsetof(Args, params)), [params_struct] "r" (¶ms_struct) - : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" + : "cc", "memory", "p0", "p1", "p2", "p3", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x19", "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x27", "x28", "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", "z10", "z11", "z12", "z13", "z14", "z16", "z17", "z18", "z28", "z29", "z30", "z31" ); } } // namespace depthwise } // namespace arm_conv -#endif // defined(ARM_COMPUTE_ENABLE_SVE) +#endif // __aarch64__ && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp index c501c67a5b..b524fd7c93 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst.hpp @@ -29,7 +29,7 @@ #pragma once -#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2) +#if defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) namespace arm_conv { namespace depthwise { @@ -72,4 +72,4 @@ struct sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst } // namespace depthwise } // namespace arm_conv -#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) && defined(ARM_COMPUTE_ENABLE_SVE2) +#endif // defined(__aarch64__) && defined(ARM_COMPUTE_ENABLE_SVE) diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp index 40220ad84e..52dc4683ad 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp index 39ab3534f5..34ba8ece12 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_3x3_s2_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include #include diff --git a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp index 7f4272672c..441da6da7a 100644 --- a/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp +++ b/src/core/NEON/kernels/arm_conv/depthwise/kernels/sve_u8q_nhwc_5x5_s1_output2x2_mla_depthfirst/generic.cpp @@ -22,7 +22,7 @@ * SOFTWARE. */ -#include "src/core/NEON/kernels/arm_gemm/utils.hpp" +#include "arm_gemm.hpp" #include #include diff --git a/src/core/NEON/kernels/arm_gemm/utils.hpp b/src/core/NEON/kernels/arm_gemm/utils.hpp index 0aca0e6e34..18e124b83e 100644 --- a/src/core/NEON/kernels/arm_gemm/utils.hpp +++ b/src/core/NEON/kernels/arm_gemm/utils.hpp @@ -27,8 +27,8 @@ #include "arm_gemm.hpp" #include -#include #include +#include // Macro for unreachable code (e.g. impossible default cases on switch) #define UNREACHABLE(why) __builtin_unreachable() -- cgit v1.2.1